diff --git a/Cargo.lock b/Cargo.lock index 242c45e66d..077b3044c9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7074,6 +7074,7 @@ dependencies = [ "chrono", "comfy-table", "csv", + "flate2", "futures", "num-bigint 0.4.8", "proptest", diff --git a/core/crates/core/Cargo.toml b/core/crates/core/Cargo.toml index 687b37c42f..812a3fb3fd 100644 --- a/core/crates/core/Cargo.toml +++ b/core/crates/core/Cargo.toml @@ -10,6 +10,7 @@ async-trait.workspace = true chrono.workspace = true comfy-table.workspace = true csv.workspace = true +flate2.workspace = true futures.workspace = true num-bigint.workspace = true rust_xlsxwriter.workspace = true diff --git a/core/crates/core/src/connection.rs b/core/crates/core/src/connection.rs index c030ffcd62..f8a89968cf 100644 --- a/core/crates/core/src/connection.rs +++ b/core/crates/core/src/connection.rs @@ -8,6 +8,7 @@ use crate::column::ColumnInfo; use crate::commit::{CommitResolution, CommitToken}; use crate::ddl::{CharacterSet, InboundForeignKey, OutboundForeignKey, TableDefinition}; use crate::dialect::SqlDialect; +use crate::dump::{DumpCatalog, DumpScope}; use crate::editor_session::{EditorSession, EditorSessionOptions}; use crate::error::DriverError; use crate::foreign_key_info::ForeignKeyInfo; @@ -118,6 +119,15 @@ pub trait Connection: Send + Sync { }) } + /// What a dump of `scope` writes: each object's SQL in the server's + /// own text where the engine keeps one, and the tables whose rows go + /// in. Read as one operation, so it has the call's whole deadline. + async fn read_dump_catalog(&self, _scope: &DumpScope, _options: CallOptions) -> Result { + Err(DriverError::Unsupported { + feature: UnsupportedFeature::SqlDump, + }) + } + /// The server's databases the login can open, the connection's own /// marked as the default and the engine's own as system. async fn list_databases(&self, _options: CallOptions) -> Result, DriverError> { diff --git a/core/crates/core/src/ddl/column_attributes.rs b/core/crates/core/src/ddl/column_attributes.rs index 0add42112a..cd69f991fb 100644 --- a/core/crates/core/src/ddl/column_attributes.rs +++ b/core/crates/core/src/ddl/column_attributes.rs @@ -1,6 +1,6 @@ use super::{ - CollationName, ColumnCompression, ColumnKeyRoles, ColumnStorage, ColumnVisibility, GeneratedColumn, IdentityStyle, - UnmodeledAttribute, + CollationName, ColumnCompression, ColumnKeyRoles, ColumnStorage, ColumnVisibility, GeneratedColumn, IdentitySeed, + IdentityStyle, UnmodeledAttribute, }; use crate::column::SqlExpression; @@ -42,6 +42,9 @@ pub struct ColumnAttributes { /// method rather than following the server's default. pub compression: Option, pub identity: IdentityStyle, + /// SQL Server's identity seed and increment, which a CREATE has to + /// state to number the rows as the table does. + pub identity_seed: Option, /// The SQL Server constraint that holds the default. A type change /// has to drop it by name first and add it back afterwards. pub default_constraint_name: Option, diff --git a/core/crates/core/src/ddl/identity_seed.rs b/core/crates/core/src/ddl/identity_seed.rs new file mode 100644 index 0000000000..5b8793903b --- /dev/null +++ b/core/crates/core/src/ddl/identity_seed.rs @@ -0,0 +1,43 @@ +/// SQL Server's `IDENTITY(seed, increment)` as the catalog lists it. Text, +/// since a decimal identity's numbers go past any integer type. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct IdentitySeed { + seed: String, + increment: String, +} + +impl IdentitySeed { + /// `None` unless both are whole numbers, the only form an identity + /// takes, so nothing else reaches a statement. + pub fn new(seed: &str, increment: &str) -> Option { + (whole_number(seed) && whole_number(increment)).then(|| Self { + seed: seed.to_owned(), + increment: increment.to_owned(), + }) + } + + pub fn seed(&self) -> &str { + &self.seed + } + + pub fn increment(&self) -> &str { + &self.increment + } +} + +fn whole_number(text: &str) -> bool { + let digits = text.strip_prefix('-').unwrap_or(text); + !digits.is_empty() && digits.bytes().all(|byte| byte.is_ascii_digit()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn only_whole_numbers_make_a_seed() { + assert!(IdentitySeed::new("-3", "1").is_some()); + assert_eq!(IdentitySeed::new("1)", "1"), None); + assert_eq!(IdentitySeed::new("", "1"), None); + } +} diff --git a/core/crates/core/src/ddl/mod.rs b/core/crates/core/src/ddl/mod.rs index 9e14707b41..8d78e5f3e1 100644 --- a/core/crates/core/src/ddl/mod.rs +++ b/core/crates/core/src/ddl/mod.rs @@ -30,6 +30,7 @@ mod draft_table_options; mod draft_type; mod drop_table_options; mod generated_column; +mod identity_seed; mod identity_style; mod inbound_foreign_key; mod order_renames; @@ -79,6 +80,7 @@ pub use draft_table_options::DraftTableOptions; pub use draft_type::{DraftType, ResolvedType}; pub use drop_table_options::DropTableOptions; pub use generated_column::{GeneratedColumn, GeneratedStorage}; +pub use identity_seed::IdentitySeed; pub use identity_style::IdentityStyle; pub use inbound_foreign_key::InboundForeignKey; pub use order_renames::order_renames; diff --git a/core/crates/core/src/dialect/sql_dialect.rs b/core/crates/core/src/dialect/sql_dialect.rs index 79bdb80e61..fdc2fefba1 100644 --- a/core/crates/core/src/dialect/sql_dialect.rs +++ b/core/crates/core/src/dialect/sql_dialect.rs @@ -1,5 +1,6 @@ use crate::column::{ColumnInfo, ColumnType, SqlExpression}; use crate::ddl::DdlDialect; +use crate::dump::DumpDialect; use crate::edit::CellExpression; use crate::filter::{Combinator, FilterOp, operators_for}; use crate::meta::{EngineRowId, EngineRowIdPart, TableRef}; @@ -89,6 +90,12 @@ pub trait SqlDialect: Send + Sync + std::fmt::Debug { None } + /// How the engine writes a SQL dump and runs one back, or `None` + /// where the app has no dump for it. + fn dump(&self) -> Option<&dyn DumpDialect> { + None + } + /// How a page of a table is read. Where the server pages, a page /// carries no ORDER BY and no window, so a user sort is not offered. fn table_paging(&self) -> TablePaging { diff --git a/core/crates/core/src/dump/dump_catalog.rs b/core/crates/core/src/dump/dump_catalog.rs new file mode 100644 index 0000000000..886b59ca5c --- /dev/null +++ b/core/crates/core/src/dump/dump_catalog.rs @@ -0,0 +1,236 @@ +use crate::meta::TableRef; + +/// What a dump covers. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum DumpScope { + /// Every schema of the connection's database, the engine's own left + /// out. On an engine whose databases are its schemas, the database the + /// connection is in. + Database, + Schema(String), +} + +/// Where an object goes in the file, in the order a restore needs. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum DumpSection { + /// Schemas, extensions, types, sequences, routines a table can name, + /// and the tables themselves. + BeforeData, + /// What would slow the rows down or refuse them in the wrong order: + /// sequence positions, keys, indexes, foreign keys, comments. + AfterData, + /// Views and routines, ordered by the names their text mentions. + Programs, + /// Last, so loading the rows fires none of them. + Triggers, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum DumpObjectKind { + Schema, + Extension, + Collation, + Type, + Sequence, + Function, + Procedure, + Table, + Index, + Constraint, + ForeignKey, + SequenceValue, + Comment, + View, + MaterializedView, + Trigger, + Dictionary, +} + +impl DumpObjectKind { + /// The word the file's comment line names the object with. + pub fn label(self) -> &'static str { + match self { + Self::Schema => "Schema", + Self::Extension => "Extension", + Self::Collation => "Collation", + Self::Type => "Type", + Self::Sequence => "Sequence", + Self::Function => "Function", + Self::Procedure => "Procedure", + Self::Table => "Table", + Self::Index => "Index", + Self::Constraint => "Constraint", + Self::ForeignKey => "Foreign key", + Self::SequenceValue => "Sequence value", + Self::Comment => "Comment", + Self::View => "View", + Self::MaterializedView => "Materialized view", + Self::Trigger => "Trigger", + Self::Dictionary => "Dictionary", + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DumpStatement { + /// The statement with no terminator. + pub sql: String, + /// It holds a routine body with semicolons of its own, which MySQL's + /// client reads as one statement only inside a `DELIMITER` block. + pub body: bool, +} + +impl DumpStatement { + pub fn new(sql: impl Into) -> Self { + Self { + sql: sql.into(), + body: false, + } + } + + pub fn body(sql: impl Into) -> Self { + Self { + sql: sql.into(), + body: true, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DumpObject { + pub kind: DumpObjectKind, + pub section: DumpSection, + pub schema: Option, + pub name: String, + pub statements: Vec, + /// The table a CREATE TABLE makes, so it follows the order the rows + /// load in. + pub table: Option, + /// What to write instead in a dump without rows, such as a + /// materialized view made `WITH NO DATA`. + pub without_rows: Option>, +} + +impl DumpObject { + pub fn new( + kind: DumpObjectKind, + section: DumpSection, + schema: Option, + name: impl Into, + statements: Vec, + ) -> Self { + Self { + kind, + section, + schema, + name: name.into(), + statements, + table: None, + without_rows: None, + } + } + + /// A CREATE TABLE, placed in the table order. + pub fn table(table: TableRef, statements: Vec) -> Self { + Self { + kind: DumpObjectKind::Table, + section: DumpSection::BeforeData, + schema: table.schema.clone(), + name: table.name.clone(), + statements, + table: Some(table), + without_rows: None, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DumpTable { + /// As the connection reads it. + pub table: TableRef, + /// The tables it must come after: those its foreign keys point at + /// where the engine checks them as rows load, and a partition's + /// parent. + pub references: Vec, + /// Whether its rows are read. A partitioned table's rows are its + /// partitions'. + pub rows: bool, + /// A row can point at a row of its own table and the engine checks + /// that as each statement ends, so each row goes in a statement of + /// its own. + pub one_row_per_insert: bool, + /// Columns the server fills on its own that the file's CREATE makes + /// ordinary, so their values are written: SQL Server's period + /// columns. + pub plain_columns: Vec, +} + +impl DumpTable { + pub fn new(table: TableRef) -> Self { + Self { + table, + references: Vec::new(), + rows: true, + one_row_per_insert: false, + plain_columns: Vec::new(), + } + } +} + +/// An object the dump leaves out. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DumpSkip { + pub kind: DumpObjectKind, + pub schema: Option, + pub name: String, + pub reason: DumpSkipReason, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum DumpSkipReason { + /// The server gave no text for it, as for an object the login may + /// not read. + NoDefinition, + /// Its code is encrypted on the server. + Encrypted, + /// Its code is a .NET assembly, not SQL. + ClrRoutine, + /// Its text holds a line a client reads as the end of a batch, such + /// as a SQL Server line that is only `GO`, so no file can carry it + /// whole. + SplitsApart, + /// Its text hides a secret the server will not show, such as a + /// password. + HiddenSecret, + /// The dump writes no object of its kind, such as a foreign table or + /// an aggregate. + NotSupported, + /// Its definition is in the file but not the rows it stores, such as + /// a materialized view that refills only from rows inserted later. + RowsNotCarried, + /// Its rows live on another server, which a load would write to. + RemoteRows, + /// Row-level security shows the login only some of its rows, so none + /// go in the file. + RowsFiltered, + /// The login may not read it, such as a sequence's position. + NotReadable, + /// A partition whose generated column's expression differs from its + /// parent's, which `PARTITION OF` makes it take: the file's rows get + /// the parent's values. + ExpressionNotCarried, + /// A SQL Server table with a system-time period, written as an + /// ordinary table with its period columns' values. Its history table + /// is an ordinary table too. + VersioningNotCarried, +} + +/// What a dump of one scope writes, read from the catalog in one go. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct DumpCatalog { + pub objects: Vec, + pub tables: Vec, + pub skipped: Vec, +} diff --git a/core/crates/core/src/dump/dump_dialect.rs b/core/crates/core/src/dump/dump_dialect.rs new file mode 100644 index 0000000000..bd97177dbb --- /dev/null +++ b/core/crates/core/src/dump/dump_dialect.rs @@ -0,0 +1,58 @@ +use super::DumpStatement; + +/// How one engine writes a SQL dump and runs one back. +/// +/// Reached through `SqlDialect::dump`, which is `None` for an engine the +/// app does not dump. A file written here loads with the engine's own +/// client as well as with TablePro's import. +pub trait DumpDialect: Send + Sync + std::fmt::Debug { + /// What follows TablePro's comment lines at the top of the file. + fn header(&self) -> String { + String::new() + } + + fn footer(&self) -> String { + String::new() + } + + /// The comment line that names the object after it, `text` free of + /// line breaks already. + fn comment(&self, text: &str) -> String { + format!("\n-- {text}\n") + } + + /// One statement as the engine's client reads it from a file. + fn statement(&self, statement: &DumpStatement) -> String { + format!("{};\n", statement.sql) + } + + /// The statement that lets an INSERT set the table's identity column, + /// or stops letting it. + fn identity_insert(&self, _table_sql: &str, _on: bool) -> Option { + None + } + + /// Whether a CREATE or DROP rolls back with its transaction, so an + /// import can run as one. + fn transactional_ddl(&self) -> bool { + false + } + + fn begin(&self) -> &'static str { + "BEGIN" + } + + fn commit(&self) -> &'static str { + "COMMIT" + } + + fn rollback(&self) -> &'static str { + "ROLLBACK" + } + + /// What runs right after `begin` in an import that is one + /// transaction. + fn after_begin(&self) -> &'static [&'static str] { + &[] + } +} diff --git a/core/crates/core/src/dump/dump_order.rs b/core/crates/core/src/dump/dump_order.rs new file mode 100644 index 0000000000..b4d00a2305 --- /dev/null +++ b/core/crates/core/src/dump/dump_order.rs @@ -0,0 +1,580 @@ +use std::collections::{BTreeMap, BTreeSet, HashMap}; + +use crate::meta::TableRef; +use crate::sql_syntax::SqlGrammar; +use crate::sql_syntax::script::{LexicalSettings, ScriptPlan, ScriptTokenKind}; + +use super::{DumpObject, DumpObjectKind, DumpTable}; + +/// The tables' positions with every table after those it references, +/// and the tables a reference cycle left in the order they came. +pub(super) fn order_tables(tables: &[DumpTable]) -> (Vec, Vec) { + let index: HashMap<&TableRef, usize> = tables + .iter() + .enumerate() + .map(|(position, table)| (&table.table, position)) + .collect(); + let edges: Vec> = tables + .iter() + .enumerate() + .map(|(position, table)| { + table + .references + .iter() + .filter_map(|reference| index.get(reference).copied()) + .filter(|parent| *parent != position) + .collect() + }) + .collect(); + let (order, stuck) = stable_topological(&edges); + let unordered = stuck.iter().map(|&position| tables[position].table.clone()).collect(); + (order, unordered) +} + +/// The objects' positions with each one after the others its text names. +/// A name written with its schema matches only that schema's object; a +/// bare one matches any of that name, so a name that is also a column's +/// can add an edge that is not there. Within a cycle that makes, only +/// the names read where a relation or a call stands hold. +#[cfg(test)] +fn order_programs(objects: &[&DumpObject], grammar: SqlGrammar) -> Vec { + ordered(&mention_edges(objects, grammar)) +} + +/// [`stable_topological`] over mention edges. A cycle the edges make is +/// tried again without the edges of names read anywhere else, such as a +/// column a view selects that shares a view's name. +fn ordered(edges: &[Vec]) -> Vec { + let every: Vec> = edges + .iter() + .map(|list| list.iter().map(|mention| mention.parent).collect()) + .collect(); + let (order, stuck) = stable_topological(&every); + if stuck.is_empty() { + return order; + } + let stuck: BTreeSet = stuck.into_iter().collect(); + let kept: Vec> = edges + .iter() + .enumerate() + .map(|(child, list)| { + list.iter() + .filter(|mention| mention.placed || !(stuck.contains(&child) && stuck.contains(&mention.parent))) + .map(|mention| mention.parent) + .collect() + }) + .collect(); + stable_topological(&kept).0 +} + +/// An object another one's text names, and whether the name stands where +/// only a relation or a routine can: after FROM, JOIN, INTO and the like, +/// or before a call's `(`. +#[derive(Debug, Clone, Copy)] +struct Mention { + parent: usize, + placed: bool, +} + +/// The words after which a name is a relation or a routine. +const RELATION_WORDS: [&str; 9] = [ + "FROM", + "JOIN", + "INTO", + "UPDATE", + "TABLE", + "REFERENCES", + "EXEC", + "EXECUTE", + "CALL", +]; + +/// What is made before the rows, and what after the keys and indexes. +/// +/// `before` is every object of the pre-data section, the tables among +/// them in the order their rows load, and `after` the keys, checks and +/// indexes the rows come before. A view or routine one of those names, +/// through a DEFAULT, a CHECK, a generated column, a domain or an index +/// expression, is made with the tables, and so is whatever that one +/// names in turn. A materialized view never is: its rows come +/// from the tables', and ClickHouse's would take each row loaded after +/// it again. Each group is put in the order its objects name each other +/// in, ties kept as they came. +pub(super) fn creation_order<'a>( + before: &[&'a DumpObject], + programs: &[&'a DumpObject], + after: &[&'a DumpObject], + grammar: SqlGrammar, +) -> (Vec<&'a DumpObject>, Vec<&'a DumpObject>) { + let all: Vec<&DumpObject> = before.iter().chain(programs).chain(after).copied().collect(); + let made = before.len() + programs.len(); + let parents = mention_edges(&all, grammar); + let mut early: Vec = (0..made).map(|position| position < before.len()).collect(); + let mut pending: Vec = (0..before.len()).chain(made..all.len()).collect(); + while let Some(node) = pending.pop() { + for &Mention { parent, .. } in &parents[node] { + let movable = parent < made && all[parent].kind != DumpObjectKind::MaterializedView; + if movable && !early[parent] { + early[parent] = true; + pending.push(parent); + } + } + } + let group = |wanted: bool| { + let members: Vec = (0..made).filter(|&position| early[position] == wanted).collect(); + let local: HashMap = members + .iter() + .enumerate() + .map(|(at, &position)| (position, at)) + .collect(); + let edges: Vec> = members + .iter() + .map(|&position| { + parents[position] + .iter() + .filter_map(|mention| { + local.get(&mention.parent).map(|&parent| Mention { + parent, + placed: mention.placed, + }) + }) + .collect() + }) + .collect(); + ordered(&edges) + .into_iter() + .map(|at| all[members[at]]) + .collect::>() + }; + (group(true), group(false)) +} + +/// For each object, the others its text names. +fn mention_edges(objects: &[&DumpObject], grammar: SqlGrammar) -> Vec> { + let mut by_name: HashMap> = HashMap::new(); + for (position, object) in objects.iter().enumerate() { + by_name.entry(object.name.to_lowercase()).or_default().push(position); + } + let schema_of = |position: usize| objects[position].schema.as_deref().map(str::to_lowercase); + objects + .iter() + .enumerate() + .map(|(position, object)| { + let mut parents: BTreeMap = BTreeMap::new(); + for (schema, name, placed) in mentions(object, grammar) { + for &other in by_name.get(&name).into_iter().flatten() { + let same_schema = schema.is_none() || schema == schema_of(other); + if other != position && same_schema { + *parents.entry(other).or_default() |= placed; + } + } + } + parents + .into_iter() + .map(|(parent, placed)| Mention { parent, placed }) + .collect() + }) + .collect() +} + +/// The names the object's statements hold, each with the schema written +/// before it, unquoted and in lower case. Left out: the object's own name +/// where its CREATE states it, a name an `AS` gives (a column or table +/// alias), a common table expression's name, and a schema written +/// before a dot. Each says whether it stands where only a relation or a +/// routine can. +fn mentions(object: &DumpObject, grammar: SqlGrammar) -> Vec<(Option, String, bool)> { + let own = object.name.to_lowercase(); + let own_schema = object.schema.as_deref().map(str::to_lowercase); + let mut found = Vec::new(); + let mut declared = false; + for statement in &object.statements { + let plan = ScriptPlan::build(&statement.sql, grammar, LexicalSettings::default_for(grammar)); + let tokens: Vec<(ScriptTokenKind, &str)> = plan + .tokens(0..statement.sql.len()) + .filter(|token| !token.kind.is_trivia()) + .filter_map(|token| statement.sql.get(token.range.clone()).map(|text| (token.kind, text))) + .collect(); + let name_at = |at: usize| match tokens.get(at) { + Some((ScriptTokenKind::Word, text)) => Some(text.to_lowercase()), + Some((ScriptTokenKind::QuotedIdentifier, text)) => Some(unquote(text).to_lowercase()), + _ => None, + }; + let is_dot = |at: usize| matches!(tokens.get(at), Some((ScriptTokenKind::Punct, "."))); + let is_as = |at: usize| matches!(tokens.get(at), Some((ScriptTokenKind::Word, text)) if text.eq_ignore_ascii_case("AS")); + let local = common_table_names(&tokens); + for at in 0..tokens.len() { + let Some(name) = name_at(at) else { + continue; + }; + if is_dot(at + 1) || (at > 0 && is_as(at - 1)) { + continue; + } + let schema = (at >= 2 && is_dot(at - 1)).then(|| name_at(at - 2)).flatten(); + if schema.is_none() && local.contains(&name) { + continue; + } + if !declared && name == own && (schema.is_none() || schema == own_schema) { + declared = true; + continue; + } + let head = match schema { + Some(_) => at.checked_sub(3), + None => at.checked_sub(1), + }; + let after_relation_word = head.and_then(|head| tokens.get(head)).is_some_and(|(kind, text)| { + *kind == ScriptTokenKind::Word && RELATION_WORDS.iter().any(|word| text.eq_ignore_ascii_case(word)) + }); + let called = matches!(tokens.get(at + 1), Some((ScriptTokenKind::OpenParen, _))); + found.push((schema, name, after_relation_word || called)); + } + } + found +} + +/// The names a WITH gives its common table expressions, `name AS (` or +/// `name (columns) AS (`, in lower case. A query reads its own one by +/// that name, not the catalog's. +fn common_table_names(tokens: &[(ScriptTokenKind, &str)]) -> BTreeSet { + let word = |at: usize, wanted: &str| matches!(tokens.get(at), Some((ScriptTokenKind::Word, text)) if text.eq_ignore_ascii_case(wanted)); + let open = |at: usize| matches!(tokens.get(at), Some((ScriptTokenKind::OpenParen, _))); + let mut names = BTreeSet::new(); + for at in 1..tokens.len() { + let after_list = matches!(tokens[at - 1].0, ScriptTokenKind::Punct if tokens[at - 1].1 == ",") + || word(at - 1, "WITH") + || word(at - 1, "RECURSIVE"); + if !after_list { + continue; + } + let name = match tokens[at] { + (ScriptTokenKind::Word, text) => text.to_lowercase(), + (ScriptTokenKind::QuotedIdentifier, text) => unquote(text).to_lowercase(), + _ => continue, + }; + let mut next = at + 1; + if open(next) { + let mut depth = 0usize; + while let Some((kind, _)) = tokens.get(next) { + match kind { + ScriptTokenKind::OpenParen => depth += 1, + ScriptTokenKind::CloseParen => depth = depth.saturating_sub(1), + _ => {} + } + next += 1; + if depth == 0 { + break; + } + } + } + if word(next, "AS") && (open(next + 1) || (word(next + 1, "MATERIALIZED") || word(next + 1, "NOT"))) { + names.insert(name); + } + } + names +} + +/// The name a quoted identifier spells, its doubled quote read as one. +fn unquote(text: &str) -> String { + let mut chars = text.chars(); + let close = match (chars.next(), chars.next_back()) { + (Some('"'), Some('"')) => '"', + (Some('`'), Some('`')) => '`', + (Some('['), Some(']')) => ']', + _ => return text.to_owned(), + }; + let inner = text.get(1..text.len() - 1).unwrap_or(text); + inner.replace(&format!("{close}{close}"), &close.to_string()) +} + +/// Kahn's order, taking the earliest ready position each step so an +/// order with nothing to fix stays as it came. `parents[i]` must come +/// before `i`. The positions a cycle leaves follow in their own order. +fn stable_topological(parents: &[Vec]) -> (Vec, Vec) { + let count = parents.len(); + let mut waiting: Vec = parents.iter().map(Vec::len).collect(); + let mut children: Vec> = vec![Vec::new(); count]; + for (child, list) in parents.iter().enumerate() { + for &parent in list { + children[parent].push(child); + } + } + let mut ready: BTreeSet = (0..count).filter(|&position| waiting[position] == 0).collect(); + let mut order = Vec::with_capacity(count); + while let Some(next) = ready.pop_first() { + order.push(next); + for &child in &children[next] { + waiting[child] -= 1; + if waiting[child] == 0 { + ready.insert(child); + } + } + } + let placed: BTreeSet = order.iter().copied().collect(); + let stuck: Vec = (0..count).filter(|position| !placed.contains(position)).collect(); + order.extend(stuck.iter().copied()); + (order, stuck) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::dump::{DumpObjectKind, DumpSection, DumpStatement}; + + fn table(name: &str, references: &[&str]) -> DumpTable { + let mut table = DumpTable::new(TableRef::unqualified(name)); + table.references = references.iter().map(|name| TableRef::unqualified(*name)).collect(); + table + } + + fn names(tables: &[DumpTable], order: &[usize]) -> Vec { + order + .iter() + .map(|&position| tables[position].table.name.clone()) + .collect() + } + + #[test] + fn a_parent_comes_before_the_tables_that_reference_it() { + let tables = [ + table("lines", &["orders", "products"]), + table("orders", &["customers"]), + table("customers", &[]), + table("products", &[]), + ]; + + let (order, unordered) = order_tables(&tables); + + assert_eq!(names(&tables, &order), ["customers", "orders", "products", "lines"]); + assert!(unordered.is_empty()); + } + + #[test] + fn a_self_reference_and_an_unknown_table_do_not_hold_a_table_back() { + let tables = [table("tree", &["tree", "elsewhere"]), table("a", &[])]; + + let (order, unordered) = order_tables(&tables); + + assert_eq!(names(&tables, &order), ["tree", "a"]); + assert!(unordered.is_empty()); + } + + #[test] + fn a_cycle_keeps_its_tables_in_their_order_and_names_them() { + let tables = [table("a", &["b"]), table("b", &["a"]), table("c", &[])]; + + let (order, unordered) = order_tables(&tables); + + assert_eq!(names(&tables, &order), ["c", "a", "b"]); + assert_eq!(unordered, [TableRef::unqualified("a"), TableRef::unqualified("b")]); + } + + fn view(name: &str, sql: &str) -> DumpObject { + DumpObject::new( + DumpObjectKind::View, + DumpSection::Programs, + None, + name, + vec![DumpStatement::new(sql)], + ) + } + + #[test] + fn a_view_follows_the_views_and_routines_it_names() { + let objects = [ + view("top", "CREATE VIEW top AS SELECT * FROM \"Middle\""), + view("Middle", "CREATE VIEW \"Middle\" AS SELECT double_it(n) FROM base"), + view( + "double_it", + "CREATE FUNCTION double_it(x int) RETURNS int AS 'SELECT x * 2'", + ), + view("base", "CREATE VIEW base AS SELECT 1 AS n"), + ]; + let refs: Vec<&DumpObject> = objects.iter().collect(); + + let order = order_programs(&refs, SqlGrammar::PostgreSql); + + let named: Vec<&str> = order.iter().map(|&position| objects[position].name.as_str()).collect(); + assert_eq!(named, ["double_it", "base", "Middle", "top"]); + } + + #[test] + fn a_view_named_like_one_in_another_schema_follows_the_one_it_reads() { + let schema_view = |schema: &str, sql: &str| { + DumpObject::new( + DumpObjectKind::View, + DumpSection::Programs, + Some(schema.to_owned()), + "v", + vec![DumpStatement::new(sql)], + ) + }; + let objects = [ + schema_view("a", "CREATE VIEW a.v AS SELECT * FROM z.v"), + schema_view("z", "CREATE VIEW z.v AS SELECT 1 AS n"), + ]; + let refs: Vec<&DumpObject> = objects.iter().collect(); + + assert_eq!(order_programs(&refs, SqlGrammar::PostgreSql), [1, 0]); + } + + #[test] + fn a_routine_a_table_calls_is_made_with_the_tables_after_what_it_reads() { + let routine = DumpObject::new( + DumpObjectKind::Function, + DumpSection::Programs, + None, + "total", + vec![DumpStatement::new("CREATE MACRO total() AS (SELECT sum(i) FROM base)")], + ); + let base = DumpObject::table( + TableRef::unqualified("base"), + vec![DumpStatement::new("CREATE TABLE base (i int)")], + ); + let uses = DumpObject::table( + TableRef::unqualified("uses"), + vec![DumpStatement::new("CREATE TABLE uses (n int DEFAULT total())")], + ); + let other = view("other", "CREATE VIEW other AS SELECT * FROM base"); + + let (early, late) = creation_order(&[&uses, &base], &[&routine, &other], &[], SqlGrammar::DuckDb); + + let named = |objects: &[&DumpObject]| objects.iter().map(|object| object.name.clone()).collect::>(); + assert_eq!(named(&early), ["base", "total", "uses"]); + assert_eq!(named(&late), ["other"]); + } + + #[test] + fn a_common_table_expression_is_not_the_view_of_its_name() { + let objects = [ + view("a", "CREATE VIEW a AS SELECT n FROM z"), + view( + "z", + "CREATE VIEW z AS WITH a AS (SELECT 1 AS n), b (m) AS MATERIALIZED (SELECT 2) SELECT n FROM a, b", + ), + ]; + let refs: Vec<&DumpObject> = objects.iter().collect(); + + assert_eq!(order_programs(&refs, SqlGrammar::PostgreSql), [1, 0]); + } + + #[test] + fn a_qualified_name_a_common_table_expression_shares_is_the_catalogs() { + let objects = [ + view("a", "CREATE VIEW a AS WITH z AS (SELECT * FROM main.z) SELECT * FROM z"), + DumpObject::new( + DumpObjectKind::View, + DumpSection::Programs, + Some("main".to_owned()), + "z", + vec![DumpStatement::new("CREATE VIEW z AS SELECT 1 AS n")], + ), + ]; + let refs: Vec<&DumpObject> = objects.iter().collect(); + + assert_eq!(order_programs(&refs, SqlGrammar::DuckDb), [1, 0]); + } + + #[test] + fn a_doubled_quote_in_a_name_is_one_quote() { + let objects = [ + view("a", "CREATE VIEW a AS SELECT * FROM \"z\"\"base\""), + view("z\"base", "CREATE VIEW \"z\"\"base\" AS SELECT 1 AS n"), + ]; + let refs: Vec<&DumpObject> = objects.iter().collect(); + + assert_eq!(order_programs(&refs, SqlGrammar::DuckDb), [1, 0]); + } + + #[test] + fn a_column_named_like_a_view_does_not_hold_the_view_back() { + let objects = [ + view("email", "CREATE VIEW email AS SELECT * FROM summary"), + view("summary", "CREATE VIEW summary AS SELECT email FROM users"), + ]; + let refs: Vec<&DumpObject> = objects.iter().collect(); + + assert_eq!(order_programs(&refs, SqlGrammar::DuckDb), [1, 0]); + } + + #[test] + fn a_view_a_macro_before_the_rows_reads_is_made_before_it() { + let reads = DumpObject::new( + DumpObjectKind::Function, + DumpSection::BeforeData, + None, + "m", + vec![DumpStatement::new("CREATE MACRO m() AS (SELECT n FROM v)")], + ); + let source = view("v", "CREATE VIEW v AS SELECT 1 AS n"); + + let (early, late) = creation_order(&[&reads], &[&source], &[], SqlGrammar::DuckDb); + + let named = |objects: &[&DumpObject]| objects.iter().map(|object| object.name.clone()).collect::>(); + assert_eq!(named(&early), ["v", "m"]); + assert!(late.is_empty()); + } + + #[test] + fn a_routine_a_check_after_the_rows_calls_is_made_before_the_rows() { + let routine = DumpObject::new( + DumpObjectKind::Function, + DumpSection::Programs, + Some("dbo".to_owned()), + "in_stock", + vec![DumpStatement::new( + "CREATE FUNCTION dbo.in_stock(@id int) RETURNS bit WITH SCHEMABINDING AS \ + BEGIN RETURN (SELECT 1 FROM dbo.stock WHERE id = @id) END", + )], + ); + let stock = DumpObject::table( + TableRef::new(Some("dbo".to_owned()), "stock".to_owned()), + vec![DumpStatement::new("CREATE TABLE dbo.stock (id int)")], + ); + let check = DumpObject::new( + DumpObjectKind::Constraint, + DumpSection::AfterData, + Some("dbo".to_owned()), + "ck_stock", + vec![DumpStatement::new( + "ALTER TABLE dbo.orders ADD CONSTRAINT ck_stock CHECK (dbo.in_stock(item) = 1)", + )], + ); + let totals = DumpObject::new( + DumpObjectKind::MaterializedView, + DumpSection::Programs, + Some("dbo".to_owned()), + "item", + vec![DumpStatement::new( + "CREATE MATERIALIZED VIEW dbo.item AS SELECT id FROM dbo.stock", + )], + ); + + let (early, late) = creation_order(&[&stock], &[&totals, &routine], &[&check], SqlGrammar::MsSql); + + let named = |objects: &[&DumpObject]| objects.iter().map(|object| object.name.clone()).collect::>(); + assert_eq!(named(&early), ["stock", "in_stock"]); + assert_eq!(named(&late), ["item"]); + } + + #[test] + fn a_column_alias_is_not_a_reference() { + let objects = [ + view("a", "CREATE VIEW a AS SELECT * FROM z"), + view("z", "CREATE VIEW z AS SELECT 1 AS a"), + ]; + let refs: Vec<&DumpObject> = objects.iter().collect(); + + assert_eq!(order_programs(&refs, SqlGrammar::DuckDb), [1, 0]); + } + + #[test] + fn a_name_inside_a_string_is_not_a_reference() { + let objects = [ + view("a", "CREATE VIEW a AS SELECT 'b' AS label"), + view("b", "CREATE VIEW b AS SELECT * FROM a"), + ]; + let refs: Vec<&DumpObject> = objects.iter().collect(); + + assert_eq!(order_programs(&refs, SqlGrammar::PostgreSql), [0, 1]); + } +} diff --git a/core/crates/core/src/dump/dump_writer.rs b/core/crates/core/src/dump/dump_writer.rs new file mode 100644 index 0000000000..7127400071 --- /dev/null +++ b/core/crates/core/src/dump/dump_writer.rs @@ -0,0 +1,543 @@ +use std::sync::Arc; + +use async_trait::async_trait; +use thiserror::Error; +use tokio::sync::mpsc; + +use crate::browse::{BrowseSqlError, DeclaredPrecisionSink, ExportRequest, build_export}; +use crate::call_options::CallOptions; +use crate::column::{ColumnInfo, ResultColumn}; +use crate::connection::Connection; +use crate::dialect::{DatabaseLayout, SqlDialect}; +use crate::error::DriverError; +use crate::filter::FilterSet; +use crate::meta::TableRef; +use crate::read_statement::ReadStatement; +use crate::result_event::ResultEvent; +use crate::result_sink::{ResultSink, SinkFlow}; +use crate::unsupported_feature::UnsupportedFeature; +use crate::value::Value; + +use super::dump_order::{creation_order, order_tables}; +use super::insert_batch::InsertBatch; +use super::{ + DumpCatalog, DumpDialect, DumpObject, DumpObjectKind, DumpScope, DumpSection, DumpSkip, DumpSkipReason, + DumpStatement, DumpTable, +}; + +/// Chunks between a table's read and the file. Few, so a slow disk holds +/// the read back. +const CHUNKS_IN_FLIGHT: usize = 4; + +/// Text a chunk holds before it goes to the file. +const CHUNK_BYTES: usize = 256 * 1024; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DumpRequest { + pub scope: DumpScope, + /// Whether the tables' rows go in, or only their structure. + pub data: bool, +} + +/// Where a dump's bytes go. +#[async_trait] +pub trait DumpOutput: Send { + async fn write(&mut self, bytes: Vec) -> Result<(), DumpOutputClosed>; + + fn progress(&mut self, _progress: DumpProgress) {} +} + +/// The file stopped taking bytes, and said why where it was written. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DumpOutputClosed; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DumpProgress { + pub tables_done: usize, + pub tables_total: usize, + pub rows: u64, +} + +/// Values with no SQL literal on the engine, written as NULL. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OmittedValues { + pub table: TableRef, + pub column: String, + pub count: u64, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct DumpReport { + pub tables: usize, + pub rows: u64, + pub objects: usize, + pub omitted: Vec, + pub skipped: Vec, + /// Tables whose foreign keys point at each other, written in the + /// order they were listed. + pub unordered: Vec, +} + +#[derive(Debug, Clone, PartialEq, Error)] +pub enum DumpError { + #[error(transparent)] + Driver(#[from] DriverError), + #[error("a table's read could not be built: {0}")] + Read(#[from] BrowseSqlError), + #[error("the file stopped taking bytes")] + OutputClosed, +} + +impl DumpError { + pub fn code(&self) -> &'static str { + match self { + Self::Driver(error) => error.code(), + Self::Read(_) => "dump.read", + Self::OutputClosed => "dump.output_closed", + } + } +} + +impl From for DumpError { + fn from(_: DumpOutputClosed) -> Self { + Self::OutputClosed + } +} + +/// Write `request`'s scope as one SQL file into `output`: the catalog in +/// the order a restore needs it, and each table's rows as INSERTs read +/// with `stream_read`. Tables are read one after another, each in a +/// transaction of its own, not at one snapshot. +pub async fn write_dump( + connection: &dyn Connection, + request: &DumpRequest, + output: &mut dyn DumpOutput, + options: CallOptions, +) -> Result { + let dialect = connection.dialect(); + if dialect.dump().is_none() { + return Err(DriverError::Unsupported { + feature: UnsupportedFeature::SqlDump, + } + .into()); + } + let catalog = connection.read_dump_catalog(&request.scope, options.clone()).await?; + let mut writer = Writer { + dialect: Arc::clone(&dialect), + output, + report: DumpReport { + tables: catalog.tables.len(), + skipped: catalog.skipped.clone(), + ..DumpReport::default() + }, + tables_done: 0, + tables_read: catalog.tables.iter().filter(|table| table.rows).count(), + data: request.data, + }; + writer.run(connection, &catalog, request.data, options).await?; + Ok(writer.report) +} + +struct Writer<'a> { + dialect: Arc, + output: &'a mut dyn DumpOutput, + report: DumpReport, + tables_done: usize, + /// The tables whose rows are read, which progress counts. + tables_read: usize, + data: bool, +} + +impl Writer<'_> { + fn dump(&self) -> &dyn DumpDialect { + self.dialect.dump().unwrap_or(&NO_DUMP) + } + + async fn run( + &mut self, + connection: &dyn Connection, + catalog: &DumpCatalog, + data: bool, + options: CallOptions, + ) -> Result<(), DumpError> { + let (table_order, unordered) = order_tables(&catalog.tables); + self.report.unordered = unordered; + let tables: Vec<&DumpTable> = table_order.iter().map(|&position| &catalog.tables[position]).collect(); + + let mut head = preamble(connection.server_info()); + head.push_str(&self.dump().header()); + self.output.write(head.into_bytes()).await?; + + let section = |wanted: DumpSection| catalog.objects.iter().filter(move |object| object.section == wanted); + let mut before: Vec<&DumpObject> = section(DumpSection::BeforeData) + .filter(|object| object.table.is_none()) + .collect(); + before.extend(tables_in_order(&catalog.objects, &tables)); + let programs: Vec<&DumpObject> = section(DumpSection::Programs).collect(); + let after: Vec<&DumpObject> = section(DumpSection::AfterData).collect(); + let (early, late) = creation_order(&before, &programs, &after, self.dialect.grammar()); + self.objects(&early, &options).await?; + + if data { + for table in tables.iter().filter(|table| table.rows) { + options.cancel().cancelled_check()?; + self.rows(connection, table, &options).await?; + self.tables_done += 1; + self.output.progress(self.progress()); + } + } + + self.objects(&after, &options).await?; + self.objects(&late, &options).await?; + let triggers: Vec<&DumpObject> = section(DumpSection::Triggers).collect(); + self.objects(&triggers, &options).await?; + + let mut tail = self.dump().footer(); + tail.push_str("\n-- Dump completed\n"); + self.output.write(tail.into_bytes()).await?; + Ok(()) + } + + fn progress(&self) -> DumpProgress { + DumpProgress { + tables_done: self.tables_done, + tables_total: self.tables_read, + rows: self.report.rows, + } + } + + async fn objects(&mut self, objects: &[&DumpObject], options: &CallOptions) -> Result<(), DumpError> { + let mut text = String::new(); + for object in objects.iter().filter(|object| !object.statements.is_empty()) { + options.cancel().cancelled_check()?; + text.push_str(&self.dump().comment(&format!( + "{}: {}", + object.kind.label(), + comment_safe(&qualified(object.schema.as_deref(), &object.name)) + ))); + let statements = match (&object.without_rows, self.data) { + (Some(without_rows), false) => without_rows, + _ => &object.statements, + }; + for statement in statements { + text.push_str(&self.dump().statement(&terminated_off(statement))); + } + self.report.objects += 1; + if text.len() >= CHUNK_BYTES { + self.output.write(std::mem::take(&mut text).into_bytes()).await?; + } + } + if !text.is_empty() { + self.output.write(text.into_bytes()).await?; + } + Ok(()) + } + + async fn rows( + &mut self, + connection: &dyn Connection, + table: &DumpTable, + options: &CallOptions, + ) -> Result<(), DumpError> { + let mut columns = connection.fetch_columns(&table.table, options.clone()).await?; + for column in &mut columns { + if table.plain_columns.contains(&column.name) { + column.is_generated = false; + } + } + // PostgreSQL allows a table of no columns, which no SELECT list + // can read and whose rows only DEFAULT VALUES can write. + if columns.is_empty() { + self.report.skipped.push(DumpSkip { + kind: DumpObjectKind::Table, + schema: table.table.schema.clone(), + name: table.table.name.clone(), + reason: DumpSkipReason::RowsNotCarried, + }); + return Ok(()); + } + let columns: Arc<[ColumnInfo]> = columns.into(); + let identity = connection.row_identity(&table.table, options.clone()).await?; + let read = build_export( + &*self.dialect, + &ExportRequest { + table: &table.table, + columns: &columns, + filter: &FilterSet::default(), + sort: &[], + identity: &identity, + }, + )?; + let written_as = match self.dialect.database_layout() { + DatabaseLayout::AsSchemas => TableRef::unqualified(table.table.name.clone()), + DatabaseLayout::Single | DatabaseLayout::WithSchemas => table.table.clone(), + }; + let (sender, mut received) = mpsc::channel(CHUNKS_IN_FLIGHT); + let sink = RowsSink { + dialect: Arc::clone(&self.dialect), + table_sql: self.dialect.portable_name(&written_as), + info: Arc::clone(&columns), + one_row_per_insert: table.one_row_per_insert, + sender, + batch: None, + columns: Arc::from([]), + rows: 0, + }; + let sink = Box::new(DeclaredPrecisionSink::new(Box::new(sink), Arc::clone(&columns))); + let header = self.dump().comment(&format!( + "Data: {}", + comment_safe(&qualified(table.table.schema.as_deref(), &table.table.name)) + )); + self.output.write(header.into_bytes()).await?; + + let read = ReadStatement::Generated(read); + let read = connection.stream_read(&read, sink, options.clone()); + let tables_total = self.tables_read; + let rows_before = self.report.rows; + let tables_done = self.tables_done; + let output = &mut *self.output; + let drain = async move { + let mut omitted = Vec::new(); + let mut closed = false; + while let Some(message) = received.recv().await { + match message { + RowsMessage::Chunk { text, rows } => { + if output.write(text.into_bytes()).await.is_err() { + closed = true; + break; + } + output.progress(DumpProgress { + tables_done, + tables_total, + rows: rows_before + rows, + }); + } + RowsMessage::Finished { rows, omitted: columns } => { + omitted = columns; + return (closed, rows, omitted); + } + } + } + (closed, 0, omitted) + }; + let (read, (closed, rows, omitted)) = futures::future::join(read, drain).await; + if closed { + return Err(DumpError::OutputClosed); + } + read?; + self.report.rows += rows; + self.report + .omitted + .extend(omitted.into_iter().map(|(column, count)| OmittedValues { + table: table.table.clone(), + column, + count, + })); + Ok(()) + } +} + +/// The CREATE TABLE objects in the order the tables load, then any for a +/// table the catalog lists no rows for. +fn tables_in_order<'a>(objects: &'a [DumpObject], tables: &[&DumpTable]) -> Vec<&'a DumpObject> { + let creates: Vec<&DumpObject> = objects + .iter() + .filter(|object| object.section == DumpSection::BeforeData && object.table.is_some()) + .collect(); + let mut ordered: Vec<&DumpObject> = Vec::with_capacity(creates.len()); + for table in tables { + ordered.extend( + creates + .iter() + .filter(|object| object.table.as_ref() == Some(&table.table)) + .copied(), + ); + } + let listed = |object: &&DumpObject| tables.iter().any(|table| object.table.as_ref() == Some(&table.table)); + ordered.extend(creates.iter().filter(|object| !listed(object)).copied()); + ordered +} + +/// A statement without the `;` the server's own text may end with, which +/// the dialect adds back in its own form. A routine body keeps its text. +fn terminated_off(statement: &DumpStatement) -> DumpStatement { + let sql = statement.sql.trim_end(); + let sql = match statement.body { + true => sql, + false => sql.trim_end_matches(';').trim_end(), + }; + DumpStatement { + sql: sql.to_owned(), + body: statement.body, + } +} + +fn qualified(schema: Option<&str>, name: &str) -> String { + match schema { + Some(schema) if !schema.is_empty() => format!("{schema}.{name}"), + _ => name.to_owned(), + } +} + +/// A name in a `--` comment, with nothing that would end the comment and +/// make the rest of the name SQL, or a client command, on a line of its +/// own. +fn comment_safe(text: &str) -> String { + text.chars() + .map(|character| match character.is_control() { + true => ' ', + false => character, + }) + .collect() +} + +fn preamble(server: &crate::server_info::ServerInfo) -> String { + format!( + "-- TablePro SQL dump\n-- Server: {}\n", + comment_safe(&format!("{} {}", server.product, server.version)) + ) +} + +#[derive(Debug)] +struct NoDump; +impl DumpDialect for NoDump {} +static NO_DUMP: NoDump = NoDump; + +trait CancelCheck { + fn cancelled_check(&self) -> Result<(), DumpError>; +} + +impl CancelCheck for tokio_util::sync::CancellationToken { + fn cancelled_check(&self) -> Result<(), DumpError> { + match self.is_cancelled() { + true => Err(DriverError::Cancelled.into()), + false => Ok(()), + } + } +} + +enum RowsMessage { + Chunk { text: String, rows: u64 }, + Finished { rows: u64, omitted: Vec<(String, u64)> }, +} + +/// A table's rows as INSERT text, handed to the file a chunk at a time. +struct RowsSink { + dialect: Arc, + table_sql: String, + info: Arc<[ColumnInfo]>, + one_row_per_insert: bool, + sender: mpsc::Sender, + batch: Option, + columns: Arc<[ResultColumn]>, + rows: u64, +} + +impl RowsSink { + fn text(&self, statements: Vec) -> String { + let dump = self.dialect.dump().unwrap_or(&NO_DUMP); + statements.iter().map(|statement| dump.statement(statement)).collect() + } + + async fn send(&mut self, statements: Vec) -> SinkFlow { + if statements.is_empty() { + return SinkFlow::Continue; + } + let text = self.text(statements); + let message = RowsMessage::Chunk { text, rows: self.rows }; + match self.sender.send(message).await { + Ok(()) => SinkFlow::Continue, + Err(_) => SinkFlow::ConsumerGone, + } + } + + fn encode(&mut self, rows: &[Vec]) -> Vec { + let mut out = Vec::new(); + let dump = self.dialect.dump().unwrap_or(&NO_DUMP); + if let Some(batch) = &mut self.batch { + for row in rows { + batch.row(&*self.dialect, dump, &self.columns, row, &mut out); + } + } + self.rows += rows.len() as u64; + out + } +} + +#[async_trait] +impl ResultSink for RowsSink { + async fn accept(&mut self, event: ResultEvent) -> SinkFlow { + match event { + ResultEvent::ResultSetStarted { columns } | ResultEvent::ColumnsResolved { columns, .. } => { + if self.batch.is_none() { + self.batch = InsertBatch::new( + &*self.dialect, + self.table_sql.clone(), + &columns, + &self.info, + self.one_row_per_insert, + ); + } + self.columns = columns; + SinkFlow::Continue + } + ResultEvent::Rows(rows) => { + let statements = self.encode(&rows); + self.send(statements).await + } + ResultEvent::ResultSetFinished { .. } => { + let mut out = Vec::new(); + let dump = self.dialect.dump().unwrap_or(&NO_DUMP); + if let Some(batch) = &mut self.batch { + batch.finish(dump, &mut out); + } + let flow = self.send(out).await; + let omitted = match &self.batch { + Some(batch) => batch + .omitted(&self.columns) + .map(|(column, count)| (column.to_owned(), count)) + .collect(), + // Rows with no column to name and no DEFAULT VALUES + // spelling go in as nothing, which the report says. + None if self.rows > 0 => vec![("*".to_owned(), self.rows)], + None => Vec::new(), + }; + let finished = RowsMessage::Finished { + rows: self.rows, + omitted, + }; + match (flow, self.sender.send(finished).await) { + (SinkFlow::Continue, Ok(())) => SinkFlow::Continue, + _ => SinkFlow::ConsumerGone, + } + } + ResultEvent::RowLimitReached { .. } | ResultEvent::CommandCompleted { .. } | ResultEvent::Notice(_) => { + SinkFlow::Continue + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_name_with_a_line_break_stays_inside_its_comment() { + let line = format!("\n-- Table: {}\n", comment_safe("t\nDROP TABLE users;\r\\! rm")); + + assert_eq!(line.lines().filter(|line| !line.is_empty()).count(), 1, "{line}"); + } + + #[test] + fn the_servers_own_terminator_comes_off_a_statement_but_not_a_body() { + assert_eq!( + terminated_off(&DumpStatement::new("CREATE VIEW v AS SELECT 1; \n")).sql, + "CREATE VIEW v AS SELECT 1" + ); + assert_eq!( + terminated_off(&DumpStatement::body("BEGIN SELECT 1; END;")).sql, + "BEGIN SELECT 1; END;" + ); + } +} diff --git a/core/crates/core/src/dump/gzip_chunks.rs b/core/crates/core/src/dump/gzip_chunks.rs new file mode 100644 index 0000000000..4c626e3905 --- /dev/null +++ b/core/crates/core/src/dump/gzip_chunks.rs @@ -0,0 +1,63 @@ +use std::io::Write as _; + +use flate2::Compression; +use flate2::write::GzEncoder; + +/// A file's bytes gzipped as they arrive, so a `.sql.gz` is written as it +/// is made, never held whole. +pub struct GzipChunks { + encoder: GzEncoder>, +} + +impl Default for GzipChunks { + fn default() -> Self { + Self::new() + } +} + +impl GzipChunks { + pub fn new() -> Self { + Self { + encoder: GzEncoder::new(Vec::new(), Compression::default()), + } + } + + /// The compressed bytes `bytes` made so far, which may be none. + pub fn compress(&mut self, bytes: &[u8]) -> std::io::Result> { + self.encoder.write_all(bytes)?; + Ok(std::mem::take(self.encoder.get_mut())) + } + + /// The rest of the stream and its trailer. + pub fn finish(self) -> std::io::Result> { + self.encoder.finish() + } +} + +#[cfg(test)] +mod tests { + use std::io::Read as _; + + use flate2::read::MultiGzDecoder; + + use super::*; + + #[test] + fn the_chunks_together_are_one_gzip_stream_of_the_input() { + let mut gzip = GzipChunks::new(); + let mut file = Vec::new(); + let mut expected = Vec::new(); + for part in 0..200 { + let text = format!("INSERT INTO t VALUES ({part});\n"); + expected.extend_from_slice(text.as_bytes()); + file.extend(gzip.compress(text.as_bytes()).expect("compress")); + } + file.extend(gzip.finish().expect("finish")); + + let mut read = String::new(); + MultiGzDecoder::new(file.as_slice()) + .read_to_string(&mut read) + .expect("a gzip stream"); + assert_eq!(read.as_bytes(), expected.as_slice()); + } +} diff --git a/core/crates/core/src/dump/insert_batch.rs b/core/crates/core/src/dump/insert_batch.rs new file mode 100644 index 0000000000..305a3d5e58 --- /dev/null +++ b/core/crates/core/src/dump/insert_batch.rs @@ -0,0 +1,308 @@ +use crate::column::{ColumnInfo, ResultColumn}; +use crate::dialect::SqlDialect; +use crate::value::Value; + +use super::{DumpDialect, DumpStatement}; + +/// The most rows one INSERT carries, under every engine's own limit: +/// SQL Server takes 1,000 rows in a VALUES list. +const MAX_ROWS: usize = 1_000; + +/// The most bytes one INSERT carries, mysqldump's `net_buffer_length`, +/// well under every `max_allowed_packet` MySQL and MariaDB ship with. +pub(crate) const MAX_STATEMENT_BYTES: usize = 1_046_528; + +/// A table's rows as multi-row INSERTs with a column list, cut at a row +/// count and a size. +pub(super) struct InsertBatch { + /// For a table whose every column the server computes, the statement + /// that inserts one row of defaults, written once per row. + default_row: Option, + start: String, + table_sql: String, + /// For each result column, whether the INSERT names it. A column the + /// server computes is left out: every engine refuses a value there. + written: Vec, + writes_numbered: bool, + max_rows: usize, + tuples: String, + rows: usize, + identity_on: bool, + /// Per result column, the values that had no literal and went in as + /// NULL. + omitted: Vec, +} + +impl InsertBatch { + /// `None` when no column can be written and the engine has no INSERT + /// of defaults alone. + pub(super) fn new( + dialect: &dyn SqlDialect, + table_sql: String, + columns: &[ResultColumn], + info: &[ColumnInfo], + one_row_per_insert: bool, + ) -> Option { + let described = |name: &str| info.iter().find(|column| column.name == name); + let written: Vec = columns + .iter() + .map(|column| described(&column.name).is_none_or(ColumnInfo::is_writable)) + .collect(); + let names: Vec = columns + .iter() + .zip(&written) + .filter(|(_, written)| **written) + .map(|(column, _)| dialect.quote_identifier(&column.name)) + .collect(); + if names.is_empty() { + let default_row = dialect.insert_default_row(&table_sql)?; + return Some(Self { + default_row: Some(default_row), + start: String::new(), + table_sql, + omitted: vec![0; written.len()], + written, + writes_numbered: false, + max_rows: 1, + tuples: String::new(), + rows: 0, + identity_on: false, + }); + } + let writes_numbered = columns + .iter() + .zip(&written) + .any(|(column, written)| *written && described(&column.name).is_some_and(|info| info.is_auto_increment)); + let mut start = format!("INSERT INTO {table_sql} ({}) ", names.join(", ")); + if let Some(clause) = dialect.insert_overriding_clause().filter(|_| writes_numbered) { + start.push_str(clause); + start.push(' '); + } + start.push_str("VALUES\n"); + let max_rows = match one_row_per_insert { + true => 1, + false => dialect.insert_batch_limits().rows.clamp(1, MAX_ROWS), + }; + Some(Self { + default_row: None, + start, + table_sql, + omitted: vec![0; written.len()], + written, + writes_numbered, + max_rows, + tuples: String::new(), + rows: 0, + identity_on: false, + }) + } + + pub(super) fn row( + &mut self, + dialect: &dyn SqlDialect, + dump: &dyn DumpDialect, + columns: &[ResultColumn], + row: &[Value], + out: &mut Vec, + ) { + if let Some(default_row) = &self.default_row { + out.push(DumpStatement::new(default_row.clone())); + return; + } + let mut tuple = String::from("("); + let mut first = true; + for (index, column) in columns.iter().enumerate() { + if !self.written.get(index).copied().unwrap_or(false) { + continue; + } + if !first { + tuple.push_str(", "); + } + first = false; + match row.get(index).map(|value| dialect.literal(value, &column.column_type)) { + Some(Ok(literal)) => tuple.push_str(&literal), + Some(Err(_)) => { + self.omitted[index] += 1; + tuple.push_str("NULL"); + } + None => tuple.push_str("NULL"), + } + } + tuple.push(')'); + let fits = self.tuples.len() + tuple.len() + 2 <= MAX_STATEMENT_BYTES.saturating_sub(self.start.len()); + if self.rows > 0 && !fits { + self.flush(dump, out); + } + if self.rows > 0 { + self.tuples.push_str(",\n"); + } + self.tuples.push_str(&tuple); + self.rows += 1; + if self.rows >= self.max_rows { + self.flush(dump, out); + } + } + + /// The last statement, and the identity insert turned off. + pub(super) fn finish(&mut self, dump: &dyn DumpDialect, out: &mut Vec) { + self.flush(dump, out); + if self.identity_on { + self.identity_on = false; + out.extend(dump.identity_insert(&self.table_sql, false).map(DumpStatement::new)); + } + } + + /// The columns whose values went in as NULL for want of a literal, + /// with how many each. + pub(super) fn omitted<'a>(&'a self, columns: &'a [ResultColumn]) -> impl Iterator + 'a { + columns + .iter() + .zip(&self.omitted) + .filter(|(_, count)| **count > 0) + .map(|(column, count)| (column.name.as_str(), *count)) + } + + fn flush(&mut self, dump: &dyn DumpDialect, out: &mut Vec) { + if self.rows == 0 { + return; + } + if self.writes_numbered && !self.identity_on { + self.identity_on = true; + out.extend(dump.identity_insert(&self.table_sql, true).map(DumpStatement::new)); + } + let mut sql = String::with_capacity(self.start.len() + self.tuples.len()); + sql.push_str(&self.start); + sql.push_str(&self.tuples); + out.push(DumpStatement::new(sql)); + self.tuples.clear(); + self.rows = 0; + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::column::{ColumnInfo, ColumnType}; + use crate::dialect::TestDialect; + + #[derive(Debug)] + struct Plain; + impl DumpDialect for Plain { + fn identity_insert(&self, table_sql: &str, on: bool) -> Option { + Some(format!("IDENTITY {table_sql} {on}")) + } + } + + fn text(name: &str) -> ResultColumn { + ResultColumn::new(name, ColumnType::untyped_text("text")) + } + + fn info(name: &str, generated: bool, numbered: bool) -> ColumnInfo { + ColumnInfo { + name: name.to_owned(), + column_type: ColumnType::untyped_text("text"), + nullable: true, + primary_key: false, + is_auto_increment: numbered, + is_generated: generated, + partition_scoped: false, + default: crate::column::ColumnDefault::None, + comment: None, + } + } + + /// `count` rows whose first value is `width` characters long. A format + /// width past `u16::MAX` panics, so the padding is built by hand. + fn rows(count: usize, width: usize) -> Vec> { + (0..count) + .map(|row| { + let digits = row.to_string(); + let padded = format!("{}{digits}", "0".repeat(width.saturating_sub(digits.len()))); + vec![Value::Text(padded), Value::Text("x".to_owned())] + }) + .collect() + } + + fn run(batch: &mut InsertBatch, columns: &[ResultColumn], rows: &[Vec]) -> Vec { + let mut out = Vec::new(); + for row in rows { + batch.row(&TestDialect::default(), &Plain, columns, row, &mut out); + } + batch.finish(&Plain, &mut out); + out.into_iter().map(|statement| statement.sql).collect() + } + + #[test] + fn a_generated_column_is_left_out_and_rows_share_a_statement() { + let columns = [text("a"), text("b")]; + let info = [info("a", false, false), info("b", true, false)]; + let mut batch = + InsertBatch::new(&TestDialect::default(), "t".to_owned(), &columns, &info, false).expect("a column"); + + let statements = run(&mut batch, &columns, &rows(3, 1)); + + assert_eq!(statements.len(), 1, "{statements:?}"); + assert!( + statements[0].starts_with("INSERT INTO t (\"a\") VALUES\n"), + "{}", + statements[0] + ); + assert_eq!(statements[0].matches("),\n(").count(), 2, "{}", statements[0]); + } + + #[test] + fn a_statement_closes_at_the_size_limit_and_one_wide_row_still_goes() { + let columns = [text("a"), text("b")]; + let mut batch = + InsertBatch::new(&TestDialect::default(), "t".to_owned(), &columns, &[], false).expect("a column"); + + let statements = run(&mut batch, &columns, &rows(4, MAX_STATEMENT_BYTES / 3)); + + assert_eq!(statements.len(), 2, "two rows fit a statement"); + let wide = run( + &mut InsertBatch::new(&TestDialect::default(), "t".to_owned(), &columns, &[], false).expect("a column"), + &columns, + &rows(1, MAX_STATEMENT_BYTES * 2), + ); + assert_eq!(wide.len(), 1); + } + + #[test] + fn a_numbered_column_turns_the_identity_insert_on_for_the_table_and_off_after() { + let columns = [text("id"), text("b")]; + let info = [info("id", false, true), info("b", false, false)]; + let mut batch = + InsertBatch::new(&TestDialect::default(), "t".to_owned(), &columns, &info, true).expect("a column"); + + let statements = run(&mut batch, &columns, &rows(2, 1)); + + assert_eq!(statements.first().map(String::as_str), Some("IDENTITY t true")); + assert_eq!(statements.last().map(String::as_str), Some("IDENTITY t false")); + assert_eq!(statements.len(), 4, "one row a statement: {statements:?}"); + } + + #[test] + fn a_row_of_computed_columns_goes_in_as_defaults() { + let columns = [text("n")]; + let info = [info("n", true, false)]; + let mut batch = + InsertBatch::new(&TestDialect::default(), "t".to_owned(), &columns, &info, false).expect("a default row"); + + let statements = run(&mut batch, &columns, &[vec![Value::Int(42)], vec![Value::Int(42)]]); + + assert_eq!( + statements, + ["INSERT INTO t DEFAULT VALUES", "INSERT INTO t DEFAULT VALUES"] + ); + } + + #[test] + fn an_empty_table_writes_nothing() { + let columns = [text("id")]; + let info = [info("id", false, true)]; + let mut batch = + InsertBatch::new(&TestDialect::default(), "t".to_owned(), &columns, &info, false).expect("a column"); + + assert!(run(&mut batch, &columns, &[]).is_empty()); + } +} diff --git a/core/crates/core/src/dump/mod.rs b/core/crates/core/src/dump/mod.rs new file mode 100644 index 0000000000..9b9bbc3f1a --- /dev/null +++ b/core/crates/core/src/dump/mod.rs @@ -0,0 +1,20 @@ +//! A whole database or schema as one SQL file: what each engine's +//! catalog says, in the order a restore needs it, and every table's rows +//! as INSERTs. The engine's parts are its driver's `read_dump_catalog` +//! and its dialect's `DumpDialect`. + +mod dump_catalog; +mod dump_dialect; +mod dump_order; +mod dump_writer; +mod gzip_chunks; +mod insert_batch; + +pub use dump_catalog::{ + DumpCatalog, DumpObject, DumpObjectKind, DumpScope, DumpSection, DumpSkip, DumpSkipReason, DumpStatement, DumpTable, +}; +pub use dump_dialect::DumpDialect; +pub use dump_writer::{ + DumpError, DumpOutput, DumpOutputClosed, DumpProgress, DumpReport, DumpRequest, OmittedValues, write_dump, +}; +pub use gzip_chunks::GzipChunks; diff --git a/core/crates/core/src/import/mod.rs b/core/crates/core/src/import/mod.rs index 756d3da261..2fea197687 100644 --- a/core/crates/core/src/import/mod.rs +++ b/core/crates/core/src/import/mod.rs @@ -1,11 +1,13 @@ -//! Reading a file or pasted text into a table, which is the other -//! direction from `export`. +//! Reading a file or pasted text into a table, or a SQL file into a +//! connection, which is the other direction from `export` and `dump`. mod csv_import; mod import_column; mod import_plan; mod pasted_rows; mod pasted_table; +mod sql_file; +mod sql_import; pub use csv_import::{ CsvImportOptions, CsvRowError, CsvSheet, ImportError, MAX_CSV_FILE_BYTES, MAX_FIELDS, MAX_PREVIEW_ROWS, read_csv, @@ -15,3 +17,8 @@ pub use import_column::ImportColumn; pub use import_plan::{ImportPlan, plan_import}; pub use pasted_rows::PastedRows; pub use pasted_table::{MAX_PASTED_CELLS, MAX_PASTED_ROWS, PastedTable}; +pub use sql_file::{SqlFileError, SqlFileReader}; +pub use sql_import::{ + MAX_KEPT_FAILURES, SqlImportEnd, SqlImportErrorPolicy, SqlImportFailure, SqlImportFault, SqlImportOptions, + SqlImportProgress, SqlImportReport, TransactionOutcome, run_sql_import, +}; diff --git a/core/crates/core/src/import/sql_file.rs b/core/crates/core/src/import/sql_file.rs new file mode 100644 index 0000000000..eadcb8bacf --- /dev/null +++ b/core/crates/core/src/import/sql_file.rs @@ -0,0 +1,245 @@ +use std::collections::VecDeque; +use std::io::{BufRead, BufReader, Read}; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; + +use flate2::bufread::MultiGzDecoder; + +use crate::sql_syntax::SqlGrammar; +use crate::sql_syntax::script::{LexicalSettings, ScriptReadError, ScriptReader, ScriptUnit}; + +/// Bytes read from the file at a time. +const READ_BYTES: usize = 256 * 1024; + +const GZIP_MAGIC: [u8; 2] = [0x1f, 0x8b]; + +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum SqlFileError { + #[error("the file could not be read: {0}")] + Read(String), + /// Bytes on the line that are not UTF-8, as a dump of binary values + /// written without hex has. + #[error("line {line} is not UTF-8 text")] + NotUtf8 { line: u64 }, + #[error(transparent)] + Script(#[from] ScriptReadError), +} + +impl SqlFileError { + pub fn code(&self) -> &'static str { + match self { + Self::Read(_) => "sql_file.read", + Self::NotUtf8 { .. } => "sql_file.not_utf8", + Self::Script(ScriptReadError::TooLong { .. }) => "sql_file.statement_too_long", + } + } +} + +/// A SQL file's units, read as they are needed: gzip when the file starts +/// with gzip's magic bytes, UTF-8 with or without a byte order mark. +pub struct SqlFileReader { + source: Box, + reader: Option, + units: VecDeque, + /// Bytes of the file read so far, before any gunzip, for progress + /// against its size. + consumed: Arc, + /// A character cut by the end of a read, finished by the next. + partial: Vec, + line: u64, + first: bool, + failed: bool, + /// Set by whoever stopped taking units, so a long run of lines that + /// make none, such as a refused COPY block, is not read to its end. + stop: Arc, +} + +impl SqlFileReader { + pub fn new(file: Box, grammar: SqlGrammar, copy_overriding: Option<&str>) -> Self { + let consumed = Arc::new(AtomicU64::new(0)); + let counted = Counted { + inner: file, + consumed: Arc::clone(&consumed), + }; + let mut buffered = BufReader::with_capacity(READ_BYTES, counted); + let gzipped = buffered + .fill_buf() + .map(|head| head.starts_with(&GZIP_MAGIC)) + .unwrap_or(false); + let source: Box = match gzipped { + true => Box::new(BufReader::with_capacity(READ_BYTES, MultiGzDecoder::new(buffered))), + false => Box::new(buffered), + }; + Self { + source, + reader: Some(ScriptReader::new(grammar).with_copy_overriding(copy_overriding)), + units: VecDeque::new(), + consumed, + partial: Vec::new(), + line: 1, + first: true, + failed: false, + stop: Arc::new(AtomicBool::new(false)), + } + } + + /// The bytes of the file read so far, shared so another thread can + /// show progress. + /// How the session reads a string as the file starts. + pub fn with_settings(mut self, settings: LexicalSettings) -> Self { + self.reader = self.reader.map(|reader| reader.with_settings(settings)); + self + } + + /// The flag that ends the reading once set. + pub fn stop(&self) -> Arc { + Arc::clone(&self.stop) + } + + pub fn consumed(&self) -> Arc { + Arc::clone(&self.consumed) + } + + fn fill(&mut self) -> Result { + let mut bytes = vec![0; READ_BYTES]; + let read = loop { + match self.source.read(&mut bytes) { + Ok(read) => break read, + Err(error) if error.kind() == std::io::ErrorKind::Interrupted => continue, + Err(error) => return Err(SqlFileError::Read(error.to_string())), + } + }; + if read == 0 { + if !self.partial.is_empty() { + return Err(SqlFileError::NotUtf8 { line: self.line }); + } + let Some(reader) = self.reader.take() else { + return Ok(false); + }; + self.units.extend(reader.finish()?); + return Ok(true); + } + if self.reader.is_none() { + return Ok(false); + } + bytes.truncate(read); + let mut chunk = std::mem::take(&mut self.partial); + chunk.extend_from_slice(&bytes); + if self.first { + self.first = false; + if chunk.starts_with(b"\xEF\xBB\xBF") { + chunk.drain(..3); + } + } + let valid = match std::str::from_utf8(&chunk) { + Ok(_) => chunk.len(), + Err(error) if error.error_len().is_none() => error.valid_up_to(), + Err(error) => { + let before = chunk.get(..error.valid_up_to()).unwrap_or_default(); + let line = self.line + before.iter().filter(|byte| **byte == b'\n').count() as u64; + return Err(SqlFileError::NotUtf8 { line }); + } + }; + self.partial = chunk.split_off(valid); + let text = String::from_utf8(chunk).map_err(|_| SqlFileError::NotUtf8 { line: self.line })?; + self.line += text.matches('\n').count() as u64; + let Some(reader) = self.reader.as_mut() else { + return Ok(false); + }; + self.units.extend(reader.push(&text)?); + Ok(true) + } +} + +impl Iterator for SqlFileReader { + type Item = Result; + + fn next(&mut self) -> Option { + loop { + if let Some(unit) = self.units.pop_front() { + return Some(Ok(unit)); + } + if self.failed || self.stop.load(Ordering::Relaxed) { + return None; + } + match self.fill() { + Ok(true) => {} + Ok(false) => return None, + Err(error) => { + self.failed = true; + return Some(Err(error)); + } + } + } + } +} + +struct Counted { + inner: Box, + consumed: Arc, +} + +impl Read for Counted { + fn read(&mut self, buffer: &mut [u8]) -> std::io::Result { + let read = self.inner.read(buffer)?; + self.consumed.fetch_add(read as u64, Ordering::Relaxed); + Ok(read) + } +} + +#[cfg(test)] +mod tests { + use std::io::Write as _; + + use flate2::Compression; + use flate2::write::GzEncoder; + + use super::*; + + fn units(bytes: Vec) -> Vec> { + SqlFileReader::new(Box::new(std::io::Cursor::new(bytes)), SqlGrammar::Sqlite, None).collect() + } + + fn texts(bytes: Vec) -> Vec { + units(bytes) + .into_iter() + .map(|unit| unit.expect("a unit").text) + .collect() + } + + #[test] + fn a_gzipped_file_reads_as_its_text() { + let text = (0..5000) + .map(|n| format!("INSERT INTO t VALUES ({n}, 'é');\n")) + .collect::(); + let mut gzip = GzEncoder::new(Vec::new(), Compression::fast()); + gzip.write_all(text.as_bytes()).expect("write"); + let plain = texts(text.clone().into_bytes()); + + let read = texts(gzip.finish().expect("finish")); + + assert_eq!(read.len(), 5000); + assert_eq!(read, plain); + } + + #[test] + fn a_byte_order_mark_is_passed_over() { + assert_eq!(texts(b"\xEF\xBB\xBFSELECT 1;".to_vec()), ["SELECT 1"]); + } + + #[test] + fn bytes_that_are_not_utf8_name_their_line() { + let read = units(b"SELECT 1;\nSELECT 2;\nSELECT X'00', '\xff';\n".to_vec()); + + assert_eq!(read.last(), Some(&Err(SqlFileError::NotUtf8 { line: 3 }))); + } + + #[test] + fn a_character_cut_between_reads_is_whole() { + // The read ends between the two bytes of the é. + let mut text = "-".repeat(READ_BYTES - 10); + text.push_str("\nSELECT 'é';\n"); + + assert_eq!(texts(text.into_bytes()), ["SELECT 'é'"]); + } +} diff --git a/core/crates/core/src/import/sql_import.rs b/core/crates/core/src/import/sql_import.rs new file mode 100644 index 0000000000..c4c0ddbbf7 --- /dev/null +++ b/core/crates/core/src/import/sql_import.rs @@ -0,0 +1,778 @@ +use std::io::Read; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use async_trait::async_trait; +use tokio::sync::mpsc; +use tokio_util::sync::CancellationToken; + +use crate::call_options::CallOptions; +use crate::dialect::SqlDialect; +use crate::dump::DumpDialect; +use crate::editor_session::EditorSession; +use crate::error::DriverError; +use crate::result_event::{ResultEvent, RunEnd}; +use crate::result_sink::{ResultSink, SinkFlow}; +use crate::row_limit::RowLimit; +use crate::run_options::RunOptions; +use crate::sql_syntax::SqlGrammar; +use crate::sql_syntax::script::{ + LexicalSettings, RefusedUnit, ScriptPlan, ScriptTokenKind, ScriptUnit, ScriptUnitKind, TransactionControl, + transaction_control, +}; +use crate::timeout_phase::TimeoutPhase; +use crate::transaction_state::TransactionState; +use crate::user_sql::UserSql; +use crate::value::Value; + +use super::sql_file::{SqlFileError, SqlFileReader}; + +/// Failures a report keeps with their text. A run that continues past +/// errors counts the rest. +pub const MAX_KEPT_FAILURES: usize = 1_000; + +/// Characters of a failed statement a report keeps. +const KEPT_STATEMENT_CHARS: usize = 2_000; + +/// Units read ahead of the one running. +const UNITS_AHEAD: usize = 8; + +/// How long a rollback or commit after the file may take. +const FINISH_DEADLINE: Duration = Duration::from_secs(120); + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SqlImportErrorPolicy { + /// Stop at the first statement that fails, as the mysql client and + /// psql with `ON_ERROR_STOP` do. + Stop, + /// Note it and go on, as `mysql --force` does. + Continue, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SqlImportOptions { + pub on_error: SqlImportErrorPolicy, + /// Run the whole file as one transaction, on an engine whose DDL rolls + /// back. Only with `Stop`: a statement that failed leaves a + /// PostgreSQL transaction refusing the rest. + pub one_transaction: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct SqlImportProgress { + /// Bytes of the file read, before any gunzip. + pub bytes_read: u64, + pub statements: u64, + pub failed: u64, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct SqlImportFailure { + pub line: u64, + pub statement: String, + pub fault: SqlImportFault, +} + +#[derive(Debug, Clone, PartialEq)] +pub enum SqlImportFault { + /// The server refused the statement. + Statement(DriverError), + /// A line nothing here can run. + Refused(RefusedUnit), + /// The file ends the transaction the import runs it in, with a + /// ROLLBACK that would undo everything before it, or a SQL Server + /// COMMIT or ROLLBACK that leaves the rest outside it. + FileEndsTransaction, +} + +#[derive(Debug, Clone, PartialEq)] +pub enum SqlImportEnd { + Completed, + /// A statement failed and the import stopped there. + Stopped, + Cancelled, + File(SqlFileError), + /// The editor session itself failed, as on a lost connection. + Session(DriverError), +} + +#[derive(Debug, Clone, PartialEq)] +pub enum TransactionOutcome { + /// Every statement committed as it ran, or the file's own + /// transactions decided. + AsWritten, + Committed, + RolledBack, + /// The file left a transaction of its own open, and it was rolled + /// back, as a client closing its connection does. + OpenRolledBack, + /// The commit or rollback failed, so what the server kept is unknown. + Unknown(DriverError), +} + +#[derive(Debug, Clone, PartialEq)] +pub struct SqlImportReport { + pub end: SqlImportEnd, + /// Statements the server ran, each `GO n` repeat counted. + pub statements: u64, + /// Lines passed over: psql-only lines, and the file's own transaction + /// statements inside the import's transaction. + pub skipped: u64, + pub failures: Vec, + pub failed: u64, + pub transaction: TransactionOutcome, + pub elapsed: Duration, +} + +/// Run a SQL file on `editor`, a unit at a time, as the engine's own +/// client would. +/// +/// The file is read on a blocking thread a piece at a time and never held +/// whole. `editor` should be a session of the import's own, so what the +/// file sets on its session stays there. +pub async fn run_sql_import( + editor: &dyn EditorSession, + dialect: &dyn SqlDialect, + file: Box, + options: SqlImportOptions, + progress: &mut (dyn FnMut(SqlImportProgress) + Send), + cancel: &CancellationToken, +) -> SqlImportReport { + let started = Instant::now(); + let grammar = dialect.grammar(); + let settings = session_settings(editor, grammar, cancel).await; + let reader = SqlFileReader::new(file, grammar, dialect.insert_overriding_clause()).with_settings(settings); + let mut file = FileUnits::start(reader); + let transaction = dialect.dump().filter(|dump| { + options.one_transaction && dump.transactional_ddl() && options.on_error == SqlImportErrorPolicy::Stop + }); + let mut run = Run::new(editor, grammar, options.on_error, transaction.is_some()); + let opening = match transaction { + Some(dump) => run.open(dump, &mut file.units, cancel).await, + None => Opening { + opened: true, + began: false, + first: None, + }, + }; + if opening.opened { + run.units(&mut file, opening.first, progress, cancel).await; + } + let bytes_read = file.finish().await; + // Stop pressed as the last statement finished can lose the race with + // the end of the file, and must not commit. + if cancel.is_cancelled() && matches!(run.report.end, SqlImportEnd::Completed) { + run.report.end = SqlImportEnd::Cancelled; + } + run.report.transaction = run.close(transaction, opening.began, dialect).await; + run.report.elapsed = started.elapsed(); + progress(SqlImportProgress { + bytes_read, + statements: run.report.statements, + failed: run.report.failed, + }); + run.report +} + +type FileUnit = Result; + +/// The file's units as a blocking thread reads them, a few ahead of the +/// statement running. +struct FileUnits { + units: mpsc::Receiver, + consumed: Arc, + stop: Arc, + reading: tokio::task::JoinHandle<()>, +} + +impl FileUnits { + fn start(reader: SqlFileReader) -> Self { + let consumed = reader.consumed(); + let stop = reader.stop(); + let (sender, units) = mpsc::channel(UNITS_AHEAD); + let reading = tokio::task::spawn_blocking(move || { + for unit in reader { + if sender.blocking_send(unit).is_err() { + return; + } + } + }); + Self { + units, + consumed, + stop, + reading, + } + } + + fn progress(&self, report: &SqlImportReport) -> SqlImportProgress { + SqlImportProgress { + bytes_read: self.consumed.load(Ordering::Relaxed), + statements: report.statements, + failed: report.failed, + } + } + + /// Stop the reading thread, wait for it, and give the bytes it read. + async fn finish(self) -> u64 { + let Self { + units, + consumed, + stop, + reading, + } = self; + stop.store(true, Ordering::Relaxed); + drop(units); + let _ = reading.await; + consumed.load(Ordering::Relaxed) + } +} + +/// Whether the import's transaction opened, and a unit read to decide +/// that, still to run. +struct Opening { + opened: bool, + began: bool, + first: Option, +} + +struct Run<'a> { + editor: &'a dyn EditorSession, + grammar: SqlGrammar, + policy: SqlImportErrorPolicy, + wrapped: bool, + /// The file's own code closed the import's transaction, so what ran + /// before is kept. + ended_by_file: bool, + /// A SQL Server batch that ran a procedure or a string failed: it may + /// have committed before the error rolled back what followed. + uncertain: Option, + report: SqlImportReport, +} + +enum Fault { + Statement(DriverError), + Cancelled, + Session(DriverError), +} + +impl<'a> Run<'a> { + fn new(editor: &'a dyn EditorSession, grammar: SqlGrammar, policy: SqlImportErrorPolicy, wrapped: bool) -> Self { + Self { + editor, + grammar, + policy, + wrapped, + ended_by_file: false, + uncertain: None, + report: SqlImportReport { + end: SqlImportEnd::Completed, + statements: 0, + skipped: 0, + failures: Vec::new(), + failed: 0, + transaction: TransactionOutcome::AsWritten, + elapsed: Duration::ZERO, + }, + } + } + + /// Open the import's transaction. SQLite takes `foreign_keys` only + /// outside a transaction, so a file that opens with it off, as a dump + /// does, has it run first; another first unit is handed back. + async fn open( + &mut self, + dump: &dyn DumpDialect, + units: &mut mpsc::Receiver, + cancel: &CancellationToken, + ) -> Opening { + let mut first = None; + if self.grammar == SqlGrammar::Sqlite { + first = units.recv().await; + if let Some(Ok(unit)) = &first + && foreign_keys_off(&unit.text) + { + match self.execute(&unit.text, cancel).await { + Ok(()) => { + self.report.statements += 1; + first = None; + } + Err(fault) => { + self.fail_on_setup(&unit.text, fault); + return Opening { + opened: false, + began: false, + first: None, + }; + } + } + } + } + let mut began = false; + for statement in std::iter::once(dump.begin()).chain(dump.after_begin().iter().copied()) { + if let Err(fault) = self.execute(statement, cancel).await { + self.fail_on_setup(statement, fault); + return Opening { + opened: false, + began, + first, + }; + } + began = true; + } + Opening { + opened: true, + began, + first, + } + } + + /// Run the file's units until it ends, a unit stops the import or + /// Stop is pressed. + async fn units( + &mut self, + file: &mut FileUnits, + mut first: Option, + progress: &mut (dyn FnMut(SqlImportProgress) + Send), + cancel: &CancellationToken, + ) { + while matches!(self.report.end, SqlImportEnd::Completed) { + let next = match first.take() { + Some(unit) => Some(unit), + None => tokio::select! { + () = cancel.cancelled() => { + self.report.end = SqlImportEnd::Cancelled; + break; + } + next = file.units.recv() => next, + }, + }; + let Some(next) = next else { + break; + }; + match next { + Ok(unit) => self.unit(unit, cancel).await, + Err(error) => self.report.end = SqlImportEnd::File(error), + } + progress(file.progress(&self.report)); + } + } + + /// Commit or roll back the import's transaction, as the run ended, or + /// roll back one the file left open. + async fn close( + &mut self, + transaction: Option<&dyn DumpDialect>, + began: bool, + dialect: &dyn SqlDialect, + ) -> TransactionOutcome { + match transaction { + Some(_) if self.ended_by_file => TransactionOutcome::AsWritten, + Some(dump) if began => match self.report.end { + SqlImportEnd::Completed => match self.finish(dump.commit(), TransactionOutcome::Committed).await { + // A SQL Server file that opened a transaction of its own + // and never committed it leaves one open past the + // import's COMMIT, which closing the session would roll + // back. + TransactionOutcome::Committed if self.open_state().await == Some(true) => { + self.report.end = SqlImportEnd::Stopped; + self.finish(dump.rollback(), TransactionOutcome::OpenRolledBack).await + } + outcome => outcome, + }, + // SQL Server's XACT_ABORT has already rolled back after an + // error, and a ROLLBACK then would fail for want of one. + _ => match self.open_state().await { + Some(false) => match self.uncertain.take() { + Some(error) => TransactionOutcome::Unknown(error), + None => TransactionOutcome::RolledBack, + }, + Some(true) | None => self.finish(dump.rollback(), TransactionOutcome::RolledBack).await, + }, + }, + Some(_) => TransactionOutcome::AsWritten, + None => match self.open_state().await == Some(true) { + true => { + let rollback = dialect.dump().map_or("ROLLBACK", |dump| dump.rollback()); + match self.finish(rollback, TransactionOutcome::OpenRolledBack).await { + TransactionOutcome::OpenRolledBack if matches!(self.report.end, SqlImportEnd::Completed) => { + self.report.end = SqlImportEnd::Stopped; + TransactionOutcome::OpenRolledBack + } + outcome => outcome, + } + } + false => TransactionOutcome::AsWritten, + }, + } + } + + async fn unit(&mut self, unit: ScriptUnit, cancel: &CancellationToken) { + let refused = match &unit.kind { + ScriptUnitKind::ClientOnly => { + self.report.skipped += 1; + return; + } + ScriptUnitKind::Refused(refused) => Some(refused.clone()), + ScriptUnitKind::Run => None, + }; + if let Some(refused) = refused { + self.failure(&unit, SqlImportFault::Refused(refused)); + return; + } + if self.wrapped + && matches!( + self.grammar, + SqlGrammar::PostgreSql | SqlGrammar::Sqlite | SqlGrammar::DuckDb + ) + { + match transaction_control(&unit.text, self.grammar) { + Some(TransactionControl::Begin | TransactionControl::Commit) => { + self.report.skipped += 1; + return; + } + Some(TransactionControl::Rollback) => { + self.failure(&unit, SqlImportFault::FileEndsTransaction); + self.report.end = SqlImportEnd::Stopped; + return; + } + None => {} + } + } + // A SQL Server batch can hold a COMMIT or ROLLBACK among other + // statements, or run one from a string; either would end the + // import's transaction and leave what came before it kept. A + // PostgreSQL PREPARE TRANSACTION hands it to two-phase commit. + if self.wrapped && (ends_transaction(&unit.text, self.grammar) || two_phase(&unit.text, self.grammar)) { + self.failure(&unit, SqlImportFault::FileEndsTransaction); + self.report.end = SqlImportEnd::Stopped; + return; + } + for _ in 0..unit.repeat.max(1) { + match self.execute(&unit.text, cancel).await { + Ok(()) => self.report.statements += 1, + Err(Fault::Statement(error)) => { + if self.wrapped && runs_other_code(&unit.text, self.grammar) { + self.uncertain = Some(error.clone()); + } + self.failure(&unit, SqlImportFault::Statement(error)); + return; + } + Err(Fault::Cancelled) => { + self.report.end = SqlImportEnd::Cancelled; + return; + } + Err(Fault::Session(error)) => { + self.report.end = SqlImportEnd::Session(error); + return; + } + } + } + // A procedure or a string the batch runs can COMMIT where the + // batch's text does not show it. The session says whether the + // import's transaction is still there. + if self.wrapped && runs_other_code(&unit.text, self.grammar) && self.open_state().await == Some(false) { + self.ended_by_file = true; + self.failure(&unit, SqlImportFault::FileEndsTransaction); + self.report.end = SqlImportEnd::Stopped; + } + } + + fn failure(&mut self, unit: &ScriptUnit, fault: SqlImportFault) { + self.report.failed += 1; + if self.report.failures.len() < MAX_KEPT_FAILURES { + self.report.failures.push(SqlImportFailure { + line: unit.line, + statement: unit.text.chars().take(KEPT_STATEMENT_CHARS).collect(), + fault, + }); + } + if self.policy == SqlImportErrorPolicy::Stop { + self.report.end = SqlImportEnd::Stopped; + } + } + + fn fail_on_setup(&mut self, statement: &str, fault: Fault) { + self.report.end = match fault { + Fault::Statement(error) => { + self.report.failed += 1; + self.report.failures.push(SqlImportFailure { + line: 0, + statement: statement.to_owned(), + fault: SqlImportFault::Statement(error), + }); + SqlImportEnd::Stopped + } + Fault::Cancelled => SqlImportEnd::Cancelled, + Fault::Session(error) => SqlImportEnd::Session(error), + }; + } + + async fn execute(&self, sql: &str, cancel: &CancellationToken) -> Result<(), Fault> { + if cancel.is_cancelled() { + return Err(Fault::Cancelled); + } + // The rows go nowhere, but a limit would stop a statement early on + // an engine that reads through a cursor, and with it what it does. + let options = RunOptions::new(cancel.child_token()).with_row_limit(RowLimit::LARGEST); + match self.editor.run(UserSql::new(sql), options, Box::new(Discard)).await { + Ok(summary) => match summary.end { + RunEnd::Completed => Ok(()), + RunEnd::Failed(error) => Err(Fault::Statement(error)), + RunEnd::Cancelled => Err(Fault::Cancelled), + RunEnd::TimedOut { server_cancelled } => Err(Fault::Statement(DriverError::Timeout { + phase: TimeoutPhase::Statement, + server_cancelled, + })), + }, + Err(error) => Err(Fault::Session(error)), + } + } + + /// Run the commit or rollback that closes the import, past a stop. + async fn finish(&mut self, sql: &str, done: TransactionOutcome) -> TransactionOutcome { + let cancel = CancellationToken::new(); + let watchdog = cancel.clone(); + let deadline = tokio::time::sleep(FINISH_DEADLINE); + let outcome = tokio::select! { + outcome = self.execute(sql, &cancel) => outcome, + () = deadline => { + watchdog.cancel(); + Err(Fault::Cancelled) + } + }; + match outcome { + Ok(()) => done, + Err(Fault::Statement(error) | Fault::Session(error)) => { + if matches!(self.report.end, SqlImportEnd::Completed) { + self.report.end = SqlImportEnd::Stopped; + } + TransactionOutcome::Unknown(error) + } + Err(Fault::Cancelled) => { + if matches!(self.report.end, SqlImportEnd::Completed) { + self.report.end = SqlImportEnd::Stopped; + } + TransactionOutcome::Unknown(DriverError::Cancelled) + } + } + } + + /// Whether a transaction is open on the session, where the engine + /// can tell. + async fn open_state(&self) -> Option { + let options = CallOptions::new( + Some(tokio::time::Instant::now() + FINISH_DEADLINE), + CancellationToken::new(), + ); + match self.editor.transaction_state(options).await { + Ok(TransactionState::InTransaction | TransactionState::Failed) => Some(true), + Ok(TransactionState::Idle) => Some(false), + Ok(TransactionState::Unknown) | Err(_) => None, + } + } +} + +/// How the session reads a backslash in a string as the import starts. +/// The server's configuration decides it, and a file need not set it: +/// MySQL's `NO_BACKSLASH_ESCAPES` and PostgreSQL's +/// `standard_conforming_strings`. The default when the read fails. +async fn session_settings( + editor: &dyn EditorSession, + grammar: SqlGrammar, + cancel: &CancellationToken, +) -> LexicalSettings { + let default = LexicalSettings::default_for(grammar); + let (query, escapes): (&str, fn(&str) -> bool) = match grammar { + SqlGrammar::MySql => ("SELECT @@SESSION.sql_mode", |mode| { + !mode.to_ascii_uppercase().contains("NO_BACKSLASH_ESCAPES") + }), + SqlGrammar::PostgreSql => ("SHOW standard_conforming_strings", |value| { + value.trim().eq_ignore_ascii_case("off") + }), + _ => return default, + }; + let seen = Arc::new(Mutex::new(None)); + let sink = FirstText { + seen: Arc::clone(&seen), + }; + let options = RunOptions::new(cancel.child_token()); + let ran = editor.run(UserSql::new(query), options, Box::new(sink)).await; + if !matches!(ran.map(|summary| summary.end), Ok(RunEnd::Completed)) { + return default; + } + let value = seen.lock().ok().and_then(|mut seen| seen.take()); + match value { + Some(value) => LexicalSettings { + backslash_escapes: escapes(&value), + }, + None => default, + } +} + +/// The first value of the first row, as text. +struct FirstText { + seen: Arc>>, +} + +#[async_trait] +impl ResultSink for FirstText { + async fn accept(&mut self, event: ResultEvent) -> SinkFlow { + if let ResultEvent::Rows(rows) = event { + let text = rows + .into_iter() + .next() + .and_then(|row| row.into_iter().next()) + .and_then(|value| match value { + Value::Text(text) => Some(text), + Value::Bytes(bytes) => String::from_utf8(bytes).ok(), + _ => None, + }); + if let (Some(text), Ok(mut seen)) = (text, self.seen.lock()) + && seen.is_none() + { + *seen = Some(text); + } + } + SinkFlow::Continue + } +} + +/// `PRAGMA foreign_keys = OFF`, however it is spelled. +fn foreign_keys_off(text: &str) -> bool { + let compact: String = text + .chars() + .filter(|character| !character.is_whitespace() && *character != ';') + .collect::() + .to_ascii_lowercase(); + matches!( + compact.as_str(), + "pragmaforeign_keys=off" | "pragmaforeign_keys=0" | "pragmaforeign_keys=false" | "pragmaforeign_keys=no" + ) +} + +/// Whether a SQL Server batch runs a COMMIT or ROLLBACK. A batch that +/// makes a procedure, function or trigger only stores its body. One that +/// builds the statement in a string and runs it is not seen here. +/// PostgreSQL, SQLite and DuckDB send a statement at a time, each told +/// apart by `transaction_control` before it runs. +fn ends_transaction(text: &str, grammar: SqlGrammar) -> bool { + if grammar != SqlGrammar::MsSql { + return false; + } + let plan = ScriptPlan::build(text, grammar, LexicalSettings::default_for(grammar)); + let words: Vec = plan + .tokens(0..text.len()) + .filter(|token| token.kind == ScriptTokenKind::Word) + .filter_map(|token| text.get(token.range).map(str::to_ascii_uppercase)) + .collect(); + let module = words.first().is_some_and(|word| word == "CREATE" || word == "ALTER") + && words + .iter() + .skip(1) + .take(3) + .any(|word| matches!(word.as_str(), "PROC" | "PROCEDURE" | "FUNCTION" | "TRIGGER")); + !module && words.iter().any(|word| word == "COMMIT" || word == "ROLLBACK") +} + +/// Whether a SQL Server batch runs a procedure or a string, whose code +/// the batch's text does not hold. +fn runs_other_code(text: &str, grammar: SqlGrammar) -> bool { + if grammar != SqlGrammar::MsSql { + return false; + } + let plan = ScriptPlan::build(text, grammar, LexicalSettings::default_for(grammar)); + plan.tokens(0..text.len()) + .filter(|token| token.kind == ScriptTokenKind::Word) + .filter_map(|token| text.get(token.range)) + .any(|word| { + ["EXEC", "EXECUTE", "sp_executesql"] + .iter() + .any(|wanted| word.eq_ignore_ascii_case(wanted)) + }) +} + +/// PostgreSQL's PREPARE TRANSACTION, COMMIT PREPARED or ROLLBACK +/// PREPARED, which take a transaction out of the session. +fn two_phase(text: &str, grammar: SqlGrammar) -> bool { + if grammar != SqlGrammar::PostgreSql { + return false; + } + let plan = ScriptPlan::build(text, grammar, LexicalSettings::default_for(grammar)); + let words: Vec = plan + .tokens(0..text.len()) + .filter(|token| !token.kind.is_trivia()) + .take(2) + .filter_map(|token| text.get(token.range).map(str::to_ascii_uppercase)) + .collect(); + matches!( + words.iter().map(String::as_str).collect::>().as_slice(), + ["PREPARE", "TRANSACTION"] | ["COMMIT" | "ROLLBACK", "PREPARED"] + ) +} + +/// The import keeps no rows: a dump's SELECTs, such as `setval`, are run +/// for what they do. +struct Discard; + +#[async_trait] +impl ResultSink for Discard { + async fn accept(&mut self, _event: ResultEvent) -> SinkFlow { + SinkFlow::Continue + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_dump_turning_foreign_keys_off_is_told_from_other_pragmas() { + assert!(foreign_keys_off("PRAGMA foreign_keys=OFF")); + assert!(foreign_keys_off("pragma foreign_keys = 0;")); + assert!(!foreign_keys_off("PRAGMA foreign_keys=ON")); + assert!(!foreign_keys_off("PRAGMA defer_foreign_keys=OFF")); + } + + #[test] + fn a_batch_that_runs_a_procedure_or_a_string_is_found() { + let ms = SqlGrammar::MsSql; + + assert!(runs_other_code( + "INSERT INTO t VALUES (1); EXEC(N'COMMIT TRANSACTION')", + ms + )); + assert!(runs_other_code("execute app.load_rows", ms)); + assert!(runs_other_code("EXEC sys.sp_executesql N'COMMIT'", ms)); + assert!(!runs_other_code("INSERT INTO t VALUES (N'EXEC')", ms)); + assert!(!runs_other_code("EXEC p", SqlGrammar::PostgreSql)); + } + + #[test] + fn a_two_phase_command_is_told_from_a_prepared_statement() { + let pg = SqlGrammar::PostgreSql; + + assert!(two_phase("PREPARE TRANSACTION 'load'", pg)); + assert!(two_phase("commit prepared 'load'", pg)); + assert!(two_phase("ROLLBACK PREPARED 'load'", pg)); + assert!(!two_phase("PREPARE fetch_one AS SELECT 1", pg)); + assert!(!two_phase("COMMIT", pg)); + assert!(!two_phase("PREPARE TRANSACTION 'load'", SqlGrammar::MySql)); + } + + #[test] + fn a_sql_server_batch_that_ends_a_transaction_is_found_before_it_runs() { + let ms = SqlGrammar::MsSql; + + assert!(ends_transaction("DELETE FROM t; COMMIT; BEGIN TRANSACTION;", ms)); + assert!(ends_transaction("rollback tran", ms)); + assert!(!ends_transaction("INSERT INTO t VALUES (N'commitment', N'COMMIT')", ms)); + assert!(!ends_transaction("CREATE PROCEDURE p AS BEGIN COMMIT END", ms)); + assert!(!ends_transaction( + "CREATE OR ALTER TRIGGER t ON x AFTER INSERT AS ROLLBACK", + ms + )); + assert!(!ends_transaction("COMMIT", SqlGrammar::PostgreSql)); + } +} diff --git a/core/crates/core/src/lib.rs b/core/crates/core/src/lib.rs index 8eabc981e7..b19846cc17 100644 --- a/core/crates/core/src/lib.rs +++ b/core/crates/core/src/lib.rs @@ -20,6 +20,7 @@ pub mod dialect; pub mod dml; pub mod document; mod driver; +pub mod dump; pub mod edit; mod editor_session; mod engine_category; diff --git a/core/crates/core/src/sql_syntax/script/copy_rows.rs b/core/crates/core/src/sql_syntax/script/copy_rows.rs new file mode 100644 index 0000000000..50b1be9210 --- /dev/null +++ b/core/crates/core/src/sql_syntax/script/copy_rows.rs @@ -0,0 +1,256 @@ +use std::collections::VecDeque; + +use crate::sql_syntax::script::{ScriptToken, ScriptTokenKind}; + +use super::script_reader::{RefusedUnit, ScriptUnit, ScriptUnitKind}; + +/// Rows one INSERT made from COPY data carries, as a dump's INSERTs do. +const ROWS_PER_INSERT: usize = 1_000; +const BYTES_PER_INSERT: usize = 1_046_528; + +/// The rows of a `COPY table (columns) FROM stdin` block as INSERTs. +/// +/// A driver sends no COPY data, so the rows go in as statements instead: +/// each field of the text format as a string the server casts to its +/// column, as it casts COPY's own text. pg_dump writes this form. +#[derive(Debug)] +pub(super) struct CopyRows { + start: String, + refused: Option, + tuples: String, + rows: usize, + first_line: u64, + line: u64, +} + +impl CopyRows { + /// The block a statement opens, when it is a COPY from stdin. + pub(super) fn starting(text: &str, tokens: &[ScriptToken], overriding: Option<&str>) -> Option { + let code: Vec<&ScriptToken> = tokens + .iter() + .filter(|token| !token.kind.is_trivia() && token.kind != ScriptTokenKind::Semicolon) + .collect(); + let word = |token: &ScriptToken, wanted: &str| { + token.kind == ScriptTokenKind::Word + && text + .get(token.range.clone()) + .is_some_and(|word| word.eq_ignore_ascii_case(wanted)) + }; + let copy = code.first()?; + if !word(copy, "COPY") { + return None; + } + let mut depth = 0i32; + let mut from = None; + for (index, token) in code.iter().enumerate() { + match token.kind { + ScriptTokenKind::OpenParen => depth += 1, + ScriptTokenKind::CloseParen => depth -= 1, + _ if depth == 0 && word(token, "FROM") => { + from = Some(index); + break; + } + _ => {} + } + } + let from = from?; + let source = code.get(from + 1)?; + if !word(source, "STDIN") { + return None; + } + let target = text.get(copy.range.end..code[from].range.start)?.trim(); + let mut start = format!("INSERT INTO {target} "); + if let Some(clause) = overriding { + start.push_str(clause); + start.push(' '); + } + start.push_str("VALUES\n"); + let refused = (code.len() > from + 2).then_some(RefusedUnit::CopyFormat); + Some(Self { + start, + refused, + tuples: String::new(), + rows: 0, + first_line: 0, + line: 0, + }) + } + + pub(super) fn refusal(&self) -> Option { + self.refused.clone() + } + + /// The data's first line. + pub(super) fn at_line(mut self, line: u64) -> Self { + self.line = line; + self.first_line = line; + self + } + + pub(super) fn row(&mut self, line: &str, units: &mut VecDeque) { + let number = self.line; + self.line += 1; + if self.refused.is_some() { + return; + } + let Some(tuple) = tuple(line) else { + self.flush(units); + units.push_back(ScriptUnit { + text: line.to_owned(), + line: number, + repeat: 1, + kind: ScriptUnitKind::Refused(RefusedUnit::CopyNotText), + }); + return; + }; + if self.rows > 0 && self.tuples.len() + tuple.len() + self.start.len() > BYTES_PER_INSERT { + self.flush(units); + } + if self.rows == 0 { + self.first_line = number; + } else { + self.tuples.push_str(",\n"); + } + self.tuples.push_str(&tuple); + self.rows += 1; + if self.rows >= ROWS_PER_INSERT { + self.flush(units); + } + } + + pub(super) fn finish(&mut self, units: &mut VecDeque) { + self.flush(units); + } + + fn flush(&mut self, units: &mut VecDeque) { + if self.rows == 0 { + return; + } + let mut text = String::with_capacity(self.start.len() + self.tuples.len()); + text.push_str(&self.start); + text.push_str(&self.tuples); + units.push_back(ScriptUnit { + text, + line: self.first_line, + repeat: 1, + kind: ScriptUnitKind::Run, + }); + self.tuples.clear(); + self.rows = 0; + } +} + +/// One line of COPY's text format as a parenthesised list of literals: +/// fields split at tabs, `\N` as NULL, backslash escapes read. `None` when +/// the escapes make bytes that are not UTF-8. +fn tuple(line: &str) -> Option { + let mut out = String::from("("); + for (index, field) in line.split('\t').enumerate() { + if index > 0 { + out.push_str(", "); + } + if field == "\\N" { + out.push_str("NULL"); + continue; + } + let value = String::from_utf8(unescape(field)).ok()?; + // An E'' string reads the same whatever standard_conforming_strings is. + out.push_str("E'"); + for character in value.chars() { + match character { + '\'' => out.push_str("''"), + '\\' => out.push_str("\\\\"), + other => out.push(other), + } + } + out.push('\''); + } + out.push(')'); + Some(out) +} + +/// COPY's backslash sequences: `\b \f \n \r \t \v`, up to three octal +/// digits, `\x` and up to two hex digits, and a backslash before any other +/// character standing for that character. +fn unescape(field: &str) -> Vec { + let bytes = field.as_bytes(); + let mut out = Vec::with_capacity(bytes.len()); + let mut index = 0; + while let Some(&byte) = bytes.get(index) { + if byte != b'\\' { + out.push(byte); + index += 1; + continue; + } + let Some(&next) = bytes.get(index + 1) else { + out.push(b'\\'); + break; + }; + index += 2; + match next { + b'b' => out.push(0x08), + b'f' => out.push(0x0c), + b'n' => out.push(b'\n'), + b'r' => out.push(b'\r'), + b't' => out.push(b'\t'), + b'v' => out.push(0x0b), + b'0'..=b'7' => { + let mut value = u32::from(next - b'0'); + for _ in 0..2 { + match bytes.get(index) { + Some(digit @ b'0'..=b'7') => { + value = value * 8 + u32::from(digit - b'0'); + index += 1; + } + _ => break, + } + } + out.push((value & 0xff) as u8); + } + b'x' => { + let mut value = 0u32; + let mut digits = 0; + while digits < 2 { + match bytes.get(index).and_then(|digit| char::from(*digit).to_digit(16)) { + Some(digit) => { + value = value * 16 + digit; + index += 1; + digits += 1; + } + None => break, + } + } + match digits { + 0 => out.push(b'x'), + _ => out.push((value & 0xff) as u8), + } + } + other => out.push(other), + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_text_format_line_becomes_literals() { + assert_eq!( + tuple("1\tit's\t\\N\ta\\tb\\nc\\\\d\t\\\\x00ff").as_deref(), + Some("(E'1', E'it''s', NULL, E'a\tb\nc\\\\d', E'\\\\x00ff')") + ); + } + + #[test] + fn octal_and_hex_escapes_read_as_bytes() { + assert_eq!(unescape("\\101\\x42\\x4"), b"AB\x04"); + assert_eq!(tuple("\\377"), None, "a lone 0xFF byte is not UTF-8"); + } + + #[test] + fn an_empty_field_is_an_empty_string() { + assert_eq!(tuple("").as_deref(), Some("(E'')")); + } +} diff --git a/core/crates/core/src/sql_syntax/script/lexical_change.rs b/core/crates/core/src/sql_syntax/script/lexical_change.rs new file mode 100644 index 0000000000..1f40e004f8 --- /dev/null +++ b/core/crates/core/src/sql_syntax/script/lexical_change.rs @@ -0,0 +1,246 @@ +use std::collections::HashMap; + +use crate::sql_syntax::SqlGrammar; + +use super::LexicalSettings; + +/// How a session setting decides whether a backslash escapes in a string, +/// followed through a script as its statements set it: PostgreSQL's +/// `standard_conforming_strings`, and MySQL's `sql_mode` with or without +/// `NO_BACKSLASH_ESCAPES`, saved in a user variable and set back as +/// mysqldump does. +#[derive(Debug, Default)] +pub(crate) struct LexicalTracker { + /// The escape setting each user variable holds a saved `sql_mode` of. + saved: HashMap, +} + +impl LexicalTracker { + /// The settings after `statement` runs, when it changes them. + pub(crate) fn after( + &mut self, + statement: &str, + grammar: SqlGrammar, + current: LexicalSettings, + ) -> Option { + let assignments = set_assignments(statement)?; + let mut escapes = current.backslash_escapes; + for (target, value) in assignments { + let target = target.to_ascii_lowercase(); + match grammar { + SqlGrammar::PostgreSql if setting_name(&target) == "standard_conforming_strings" => { + match unquoted(&value).to_ascii_lowercase().as_str() { + "on" | "true" | "1" => escapes = false, + "off" | "false" | "0" => escapes = true, + _ => {} + } + } + SqlGrammar::MySql if setting_name(&target) == "sql_mode" => { + if let Some(variable) = value.strip_prefix('@').filter(|value| !value.starts_with('@')) { + if let Some(saved) = self.saved.get(&variable.to_ascii_lowercase()) { + escapes = *saved; + } + } else if value.starts_with('\'') || value.starts_with('"') { + escapes = !value.to_ascii_uppercase().contains("NO_BACKSLASH_ESCAPES"); + } + } + SqlGrammar::MySql + if target.starts_with('@') + && !target.starts_with("@@") + && value.starts_with("@@") + && setting_name(&value.to_ascii_lowercase()) == "sql_mode" => + { + self.saved.insert(target.trim_start_matches('@').to_owned(), escapes); + } + _ => {} + } + } + (escapes != current.backslash_escapes).then_some(LexicalSettings { + backslash_escapes: escapes, + }) + } +} + +/// `sql_mode` from `@@session.sql_mode`, `SESSION sql_mode` and the like. +fn setting_name(target: &str) -> &str { + let name = target + .rsplit(|character: char| character == '.' || character.is_whitespace()) + .next() + .unwrap_or(target); + name.trim_start_matches('@') +} + +fn unquoted(value: &str) -> &str { + value.trim_matches(|character| character == '\'' || character == '"') +} + +/// The `target = value` pairs of a SET statement, MySQL's executable +/// comment markers taken off. `None` for any other statement. +fn set_assignments(statement: &str) -> Option> { + let code = without_executable_markers(statement); + let code = code.trim().trim_end_matches(';').trim(); + let head = code.get(..3).filter(|head| head.eq_ignore_ascii_case("SET"))?; + let body = code.get(head.len()..)?; + if !body.starts_with(char::is_whitespace) { + return None; + } + let mut pairs = Vec::new(); + for part in split_outside_quotes(body, ',') { + if let Some((target, value)) = part.split_once(":=").or_else(|| part.split_once('=')) { + pairs.push((target.trim().to_owned(), value.trim().to_owned())); + continue; + } + let words: Vec<&str> = part.split_whitespace().collect(); + if let Some(at) = words.iter().position(|word| word.eq_ignore_ascii_case("TO")) + && at > 0 + { + pairs.push((words[..at].join(" "), words[at + 1..].join(" "))); + } + } + Some(pairs) +} + +/// The statement with `/*!` and MariaDB's `/*M!` openers, their version +/// digits and the closing `*/` taken off. +fn without_executable_markers(statement: &str) -> String { + let mut out = String::with_capacity(statement.len()); + let mut rest = statement; + while let Some(at) = rest.find("/*") { + let marker = ["/*!", "/*M!"] + .into_iter() + .find(|marker| rest[at..].starts_with(marker)); + let Some(marker) = marker else { + out.push_str(&rest[..at + 2]); + rest = &rest[at + 2..]; + continue; + }; + out.push_str(&rest[..at]); + let after = &rest[at + marker.len()..]; + let digits = after.len() - after.trim_start_matches(|c: char| c.is_ascii_digit()).len(); + rest = &after[digits..]; + } + out.push_str(rest); + out.replace("*/", " ") +} + +fn split_outside_quotes(text: &str, separator: char) -> Vec<&str> { + let mut parts = Vec::new(); + let mut quote = None; + let mut start = 0; + let mut escaped = false; + for (index, character) in text.char_indices() { + match quote { + Some(open) => { + if escaped { + escaped = false; + } else if character == '\\' { + escaped = true; + } else if character == open { + quote = None; + } + } + None if character == '\'' || character == '"' || character == '`' => quote = Some(character), + None if character == separator => { + parts.push(&text[start..index]); + start = index + character.len_utf8(); + } + None => {} + } + } + parts.push(&text[start..]); + parts +} + +#[cfg(test)] +mod tests { + use super::*; + + const ESCAPES: LexicalSettings = LexicalSettings { + backslash_escapes: true, + }; + const NO_ESCAPES: LexicalSettings = LexicalSettings { + backslash_escapes: false, + }; + + #[test] + fn mysqldump_saves_sets_and_restores_the_mode() { + let mut tracker = LexicalTracker::default(); + let my = SqlGrammar::MySql; + + assert_eq!( + tracker.after("/*!50003 SET @saved_sql_mode = @@sql_mode */", my, ESCAPES), + None + ); + let set = tracker.after( + "/*!50003 SET sql_mode = 'NO_BACKSLASH_ESCAPES,STRICT_TRANS_TABLES' */", + my, + ESCAPES, + ); + assert_eq!(set, Some(NO_ESCAPES)); + assert_eq!( + tracker.after("/*!50003 SET sql_mode = @saved_sql_mode */", my, NO_ESCAPES), + Some(ESCAPES) + ); + } + + #[test] + fn a_mariadb_executable_comment_sets_the_mode_too() { + let mut tracker = LexicalTracker::default(); + + assert_eq!( + tracker.after( + "/*M!100100 SET sql_mode='NO_BACKSLASH_ESCAPES' */", + SqlGrammar::MySql, + ESCAPES + ), + Some(NO_ESCAPES) + ); + } + + #[test] + fn the_dump_header_keeps_escapes_on() { + let mut tracker = LexicalTracker::default(); + + let header = "/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */"; + assert_eq!(tracker.after(header, SqlGrammar::MySql, ESCAPES), None); + } + + #[test] + fn postgresql_follows_standard_conforming_strings() { + let mut tracker = LexicalTracker::default(); + let pg = SqlGrammar::PostgreSql; + + assert_eq!( + tracker.after("SET standard_conforming_strings = off", pg, NO_ESCAPES), + Some(ESCAPES) + ); + assert_eq!( + tracker.after("SET SESSION standard_conforming_strings TO on", pg, ESCAPES), + Some(NO_ESCAPES) + ); + assert_eq!(tracker.after("SELECT 1", pg, NO_ESCAPES), None); + } + + #[test] + fn a_set_spread_over_lines_is_read() { + let mut tracker = LexicalTracker::default(); + + assert_eq!( + tracker.after( + "SET\n\tstandard_conforming_strings\nTO off", + SqlGrammar::PostgreSql, + NO_ESCAPES + ), + Some(ESCAPES) + ); + assert_eq!( + tracker.after( + "SET\nSESSION\tsql_mode = 'NO_BACKSLASH_ESCAPES'", + SqlGrammar::MySql, + ESCAPES + ), + Some(NO_ESCAPES) + ); + assert_eq!(tracker.after("SETTINGS = 1", SqlGrammar::MySql, ESCAPES), None); + } +} diff --git a/core/crates/core/src/sql_syntax/script/mod.rs b/core/crates/core/src/sql_syntax/script/mod.rs index f31a01de98..7423c01ab6 100644 --- a/core/crates/core/src/sql_syntax/script/mod.rs +++ b/core/crates/core/src/sql_syntax/script/mod.rs @@ -1,12 +1,15 @@ mod batch_error_policy; mod compound_tracker; +mod copy_rows; mod directive_scan; pub(crate) mod lexer; +mod lexical_change; mod lexical_settings; mod open_construct; mod script_batch; mod script_diagnostic; mod script_plan; +mod script_reader; pub(crate) mod script_rules; mod script_statement; mod script_token; @@ -20,6 +23,10 @@ pub use open_construct::OpenConstruct; pub use script_batch::ScriptBatch; pub use script_diagnostic::ScriptDiagnostic; pub use script_plan::ScriptPlan; +pub use script_reader::{ + MAX_UNIT_BYTES, RefusedUnit, ScriptReadError, ScriptReader, ScriptUnit, ScriptUnitKind, TransactionControl, + transaction_control, +}; pub use script_statement::ScriptStatement; pub use script_token::ScriptToken; pub use script_token_kind::ScriptTokenKind; diff --git a/core/crates/core/src/sql_syntax/script/script_plan.rs b/core/crates/core/src/sql_syntax/script/script_plan.rs index ee5aefae09..a0fbb0659b 100644 --- a/core/crates/core/src/sql_syntax/script/script_plan.rs +++ b/core/crates/core/src/sql_syntax/script/script_plan.rs @@ -33,12 +33,28 @@ impl ScriptPlan { reading: ServerReading, ) -> ScriptPlan { match rules_for(grammar) { - Some(rules) => Builder::new(text, rules, settings, reading).run(), + Some(rules) => Builder::new(text, rules, settings, reading, ";").run().0, None if grammar == SqlGrammar::Mongo => crate::sql_syntax::mongo::plan(text), None => crate::sql_syntax::redis::plan(text), } } + /// `text` read on from a point where `delimiter` is in effect, as a + /// client reads a file a piece at a time. Gives back where the last + /// statement, batch or directive line that is known to be whole ends, + /// and the delimiter in effect there. `None` for a grammar that is not + /// SQL. + pub(crate) fn build_continuing( + text: &str, + grammar: SqlGrammar, + settings: LexicalSettings, + delimiter: &str, + ) -> Option<(ScriptPlan, Settled)> { + let rules = rules_for(grammar)?; + let (plan, settled) = Builder::new(text, rules, settings, ServerReading::Client, delimiter).run(); + Some((plan, settled)) + } + pub(crate) fn from_parts( tokens: Vec, statements: Vec, @@ -117,6 +133,13 @@ impl ScriptPlan { } } +/// Where a plan of a piece of a script stops being sure of itself. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct Settled { + pub offset: usize, + pub delimiter: String, +} + struct Builder<'a> { text: &'a str, rules: &'static ScriptRules, @@ -129,26 +152,36 @@ struct Builder<'a> { statements: Vec, batches: Vec, diagnostics: Vec, + settled: usize, + settled_delimiter: &'a str, } impl<'a> Builder<'a> { - fn new(text: &'a str, rules: &'static ScriptRules, settings: LexicalSettings, reading: ServerReading) -> Self { + fn new( + text: &'a str, + rules: &'static ScriptRules, + settings: LexicalSettings, + reading: ServerReading, + delimiter: &'a str, + ) -> Self { Self { text, rules, lexer: Lexer::new(text, &rules.syntax, settings, reading), reads_directives: reading.reads_directives(), tracker: CompoundTracker::new(rules.compound, rules.parens_hold_semicolons), - delimiter: ";", + delimiter, open: None, tokens: Vec::new(), statements: Vec::new(), batches: Vec::new(), diagnostics: Vec::new(), + settled: 0, + settled_delimiter: delimiter, } } - fn run(mut self) -> ScriptPlan { + fn run(mut self) -> (ScriptPlan, Settled) { while self.lexer.position() < self.text.len() { if self.reads_directives && self.lexer.at_line_start() && self.accept_directive() { continue; @@ -167,15 +200,20 @@ impl<'a> Builder<'a> { start, }); } + let settled = Settled { + offset: self.settled, + delimiter: self.settled_delimiter.to_owned(), + }; self.finish_statement(None, NonZeroU32::MIN); self.extend_extents(); - ScriptPlan { + let plan = ScriptPlan { tokens: self.tokens, statements: self.statements, batches: self.batches, diagnostics: self.diagnostics, batch_error: self.rules.batch_error, - } + }; + (plan, settled) } fn accept_directive(&mut self) -> bool { @@ -187,6 +225,7 @@ impl<'a> Builder<'a> { }; self.push_directive(line_start, line.start, line.end); self.finish_statement(None, line.repeat); + self.settle(line.end); true } Boundary::DelimiterDirectives if self.open.is_none() => { @@ -197,6 +236,7 @@ impl<'a> Builder<'a> { if let Some(delimiter) = line.delimiter { self.delimiter = delimiter; } + self.settle(line.end); true } Boundary::DelimiterDirectives | Boundary::Semicolon => false, @@ -225,6 +265,7 @@ impl<'a> Builder<'a> { && self.tracker.semicolon_ends_statement(); if terminates { self.finish_statement(Some(range.end), NonZeroU32::MIN); + self.settle(range.end); } else { if kind != ScriptTokenKind::Semicolon { self.tracker.observe(kind, &self.text[range.clone()]); @@ -238,6 +279,14 @@ impl<'a> Builder<'a> { self.push_token(kind, range); } + /// Nothing open before `offset` can change with more text. + fn settle(&mut self, offset: usize) { + if self.open.is_none() { + self.settled = offset; + self.settled_delimiter = self.delimiter; + } + } + fn push_token(&mut self, kind: ScriptTokenKind, range: Range) { self.tokens.push(ScriptToken { kind, range }); } diff --git a/core/crates/core/src/sql_syntax/script/script_reader/mod.rs b/core/crates/core/src/sql_syntax/script/script_reader/mod.rs new file mode 100644 index 0000000000..c15f8d84cf --- /dev/null +++ b/core/crates/core/src/sql_syntax/script/script_reader/mod.rs @@ -0,0 +1,414 @@ +use std::collections::VecDeque; + +use crate::sql_syntax::SqlGrammar; +use crate::sql_syntax::script::{LexicalSettings, ScriptPlan, ScriptToken, ScriptTokenKind}; + +use super::copy_rows::CopyRows; +use super::directive_scan::delimiter_line; +use super::lexical_change::LexicalTracker; + +/// Text a reader holds before it plans again with nothing settled, so a +/// statement of many megabytes is lexed a few times, not once per piece. +const PLAN_STEP: usize = 64 * 1024; + +/// The longest statement or batch a reader holds. A dump's INSERTs are a +/// megabyte; a statement past this is not one a server takes either. +pub const MAX_UNIT_BYTES: usize = 128 * 1024 * 1024; + +/// One thing a file asks the server, as its engine's client would send +/// it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ScriptUnit { + pub text: String, + /// The line it starts on, from 1. + pub line: u64, + /// How many times it runs: a SQL Server `GO n` batch runs `n` times. + pub repeat: u32, + pub kind: ScriptUnitKind, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ScriptUnitKind { + /// Sent to the server. + Run, + /// A line only psql reads, such as pg_dump's `\restrict`, which keeps + /// psql from running meta-commands a server could plant in the data. + /// Nothing here runs meta-commands, so it is passed over. + ClientOnly, + /// A line nothing here can run. + Refused(RefusedUnit), +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum RefusedUnit { + /// A psql meta-command other than `\restrict` and `\unrestrict`. + ClientCommand { command: String }, + /// `COPY ... FROM stdin` in a form other than the text format pg_dump + /// writes, or with options. + CopyFormat, + /// `COPY` data that is not UTF-8 once its escapes are read. + CopyNotText, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +pub enum ScriptReadError { + #[error("the statement starting on line {line} is longer than the reader holds")] + TooLong { line: u64 }, +} + +/// Splits a script into the units its engine's client sends, read a +/// piece at a time so a file of any size is never held whole. +/// +/// It splits as `ScriptPlan` does: at `;`, at the delimiter a MySQL +/// `DELIMITER` line sets, at SQL Server `GO` lines. Only text up to the +/// last line break is planned until the end, since a directive is a whole +/// line. On PostgreSQL it also reads what psql reads in a pg_dump file: +/// `COPY ... FROM stdin` rows up to `\.`, which become INSERTs, and +/// backslash lines. A `SET` of `standard_conforming_strings` or of a +/// `sql_mode` with `NO_BACKSLASH_ESCAPES` changes how the rest reads, as +/// it does on the server. +#[derive(Debug)] +pub struct ScriptReader { + grammar: SqlGrammar, + settings: LexicalSettings, + pending: String, + /// The line `pending` starts on. + line: u64, + delimiter: String, + plan_at: usize, + copy: Option, + /// What lets an INSERT made from COPY rows set an identity column, as + /// COPY itself may. + copy_overriding: Option, + tracker: LexicalTracker, + units: VecDeque, +} + +impl ScriptReader { + pub fn new(grammar: SqlGrammar) -> Self { + Self { + grammar, + settings: LexicalSettings::default_for(grammar), + pending: String::new(), + line: 1, + delimiter: ";".to_owned(), + plan_at: 0, + copy: None, + copy_overriding: None, + tracker: LexicalTracker::default(), + units: VecDeque::new(), + } + } + + /// How the session reads a string before the script sets anything, + /// where the server's configuration can differ from the default. + pub fn with_settings(mut self, settings: LexicalSettings) -> Self { + self.settings = settings; + self + } + + /// The clause `SqlDialect::insert_overriding_clause` gives, for the + /// INSERTs COPY rows become. + pub fn with_copy_overriding(mut self, clause: Option<&str>) -> Self { + self.copy_overriding = clause.map(str::to_owned); + self + } + + /// Take the next piece of the script, and give back the units it + /// completed. + pub fn push(&mut self, text: &str) -> Result, ScriptReadError> { + self.pending.push_str(text); + self.process(false)?; + Ok(self.units.drain(..).collect()) + } + + /// The end of the script: what is left, whole or not, as the client + /// sends a last statement with no terminator. + pub fn finish(mut self) -> Result, ScriptReadError> { + self.process(true)?; + Ok(self.units.drain(..).collect()) + } + + fn process(&mut self, end: bool) -> Result<(), ScriptReadError> { + loop { + if self.copy.is_some() { + if self.copy_lines(end) { + continue; + } + break; + } + if self.grammar == SqlGrammar::PostgreSql && self.backslash_line(end) { + continue; + } + if !end && self.pending.len() < self.plan_at { + break; + } + // A line this long is no directive, and a file written on one + // line still has to be cut where its statements end. + let line_start = self.pending.rfind('\n').map_or(0, |at| at + 1); + let limit = match end || self.pending.len() - line_start >= PLAN_STEP { + true => self.pending.len(), + false if line_start > 0 => line_start, + false => break, + }; + match self.plan(limit, end) { + Planned::Interrupted => continue, + Planned::Settled => { + self.plan_at = self.pending.len() + PLAN_STEP; + break; + } + Planned::Nothing => { + if self.pending.len() > MAX_UNIT_BYTES { + return Err(ScriptReadError::TooLong { line: self.line }); + } + self.plan_at = (self.pending.len() * 2).max(self.pending.len() + PLAN_STEP); + break; + } + } + } + Ok(()) + } + + fn plan(&mut self, limit: usize, end: bool) -> Planned { + let Some(text) = self.pending.get(..limit) else { + return Planned::Nothing; + }; + let Some((plan, settled)) = ScriptPlan::build_continuing(text, self.grammar, self.settings, &self.delimiter) + else { + return Planned::Nothing; + }; + let whole_to = match end { + true => limit, + false => settled.offset, + }; + let mut found = Vec::new(); + let mut interrupted_at = None; + let mut delimiter_at_cut = None; + for batch in plan.batches() { + if batch.range.end > whole_to { + break; + } + let unit_text = text.get(batch.range.clone()).unwrap_or_default(); + if self.grammar == SqlGrammar::PostgreSql && unit_text.starts_with('\\') { + interrupted_at = Some((batch.range.start, None)); + break; + } + if self.grammar == SqlGrammar::PostgreSql { + let tokens: Vec = plan.tokens(batch.range.clone()).collect(); + if let Some(copy) = CopyRows::starting(text, &tokens, self.copy_overriding.as_deref()) { + let after_line = text + .get(batch.range.end..) + .and_then(|rest| rest.find('\n')) + .map_or(limit, |at| batch.range.end + at + 1); + // A COPY the reader turns into INSERTs is never sent; + // one it cannot read is reported and its rows passed over. + if let Some(refused) = copy.refusal() { + found.push((batch.range.clone(), 1, Some(refused))); + } + interrupted_at = Some((after_line, Some(copy))); + break; + } + } + found.push((batch.range.clone(), batch.repeat.get(), None)); + // A statement that changes how a backslash reads changes how + // the rest of the text splits, so it is planned again from here. + if let Some(settings) = self.tracker.after(unit_text, self.grammar, self.settings) { + self.settings = settings; + let cut = plan + .statements() + .get(batch.statements.start) + .map_or(batch.range.end, |statement| statement.extent.end.max(batch.range.end)); + delimiter_at_cut = Some(self.delimiter_at(&plan, text, cut)); + interrupted_at = Some((cut, None)); + break; + } + } + let lines_before = |offset: usize| text.get(..offset).unwrap_or_default().matches('\n').count() as u64; + for (range, repeat, refused) in &found { + let unit_text = text.get(range.clone()).unwrap_or_default().trim().to_owned(); + if unit_text.is_empty() { + continue; + } + self.units.push_back(ScriptUnit { + text: unit_text, + line: self.line + lines_before(range.start), + repeat: *repeat, + kind: match refused { + Some(refused) => ScriptUnitKind::Refused(refused.clone()), + None => ScriptUnitKind::Run, + }, + }); + } + match interrupted_at { + Some((cut, copy)) => { + self.cut(cut); + self.copy = copy.map(|copy| copy.at_line(self.line)); + self.delimiter = delimiter_at_cut.unwrap_or_else(|| self.delimiter.clone()); + Planned::Interrupted + } + None if end => { + self.cut(limit); + Planned::Settled + } + None if whole_to > 0 => { + self.cut(whole_to); + self.delimiter = settled.delimiter; + Planned::Settled + } + None => Planned::Nothing, + } + } + + /// The delimiter in effect at `offset` of `text`, from the `DELIMITER` + /// lines before it. + fn delimiter_at(&self, plan: &ScriptPlan, text: &str, offset: usize) -> String { + let mut delimiter = self.delimiter.clone(); + for token in plan.tokens(0..offset) { + if token.kind != ScriptTokenKind::Directive || token.range.end > offset { + continue; + } + if let Some(chosen) = delimiter_line(text, token.range.start).and_then(|line| line.delimiter) { + delimiter = chosen.to_owned(); + } + } + delimiter + } + + /// psql reads a line that starts with a backslash where no statement + /// is open as a meta-command, and only psql. pg_dump writes + /// `\restrict` and `\unrestrict` around a dump. + fn backslash_line(&mut self, end: bool) -> bool { + let mut offset = 0; + loop { + let rest = self.pending.get(offset..).unwrap_or_default(); + let trimmed = rest.trim_start(); + offset += rest.len() - trimmed.len(); + if trimmed.starts_with("--") { + match trimmed.find('\n') { + Some(at) => offset += at + 1, + None => return false, + } + continue; + } + if !trimmed.starts_with('\\') { + return false; + } + let line_end = match trimmed.find('\n') { + Some(at) => offset + at, + None if end => self.pending.len(), + None => return false, + }; + let line = self + .pending + .get(offset..line_end) + .unwrap_or_default() + .trim_end() + .to_owned(); + let starts_on = self.line + self.pending.get(..offset).unwrap_or_default().matches('\n').count() as u64; + let command = line.split_whitespace().next().unwrap_or_default().to_owned(); + let kind = match command.as_str() { + "\\restrict" | "\\unrestrict" => ScriptUnitKind::ClientOnly, + _ => ScriptUnitKind::Refused(RefusedUnit::ClientCommand { command }), + }; + self.units.push_back(ScriptUnit { + text: line, + line: starts_on, + repeat: 1, + kind, + }); + self.cut((line_end + 1).min(self.pending.len())); + return true; + } + } + + /// Read COPY rows a line at a time. True when the data ended and the + /// script goes on. + fn copy_lines(&mut self, end: bool) -> bool { + let Some(mut copy) = self.copy.take() else { + return false; + }; + let mut consumed = 0; + let mut finished = false; + while let Some(at) = self.pending.get(consumed..).and_then(|rest| rest.find('\n')) { + let line = self.pending.get(consumed..consumed + at).unwrap_or_default(); + let line = line.strip_suffix('\r').unwrap_or(line); + consumed += at + 1; + if line == "\\." { + finished = true; + break; + } + copy.row(line, &mut self.units); + } + if !finished && end { + let line = self.pending.get(consumed..).unwrap_or_default(); + let line = line.strip_suffix('\r').unwrap_or(line); + if !line.is_empty() && line != "\\." { + copy.row(line, &mut self.units); + } + consumed = self.pending.len(); + finished = true; + } + self.cut(consumed); + if finished { + copy.finish(&mut self.units); + self.plan_at = 0; + return true; + } + self.copy = Some(copy); + false + } + + fn cut(&mut self, offset: usize) { + let offset = offset.min(self.pending.len()); + self.line += self.pending.get(..offset).unwrap_or_default().matches('\n').count() as u64; + self.pending.drain(..offset); + } +} + +enum Planned { + /// A COPY or a backslash line stopped the plan; read on from there. + Interrupted, + Settled, + /// No unit is whole yet. + Nothing, +} + +/// Whether a statement starts or ends a transaction, for an import that +/// runs a file as one. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TransactionControl { + Begin, + Commit, + Rollback, +} + +/// The transaction statement `text` is, where it is one: BEGIN, START +/// TRANSACTION, COMMIT, END and ROLLBACK, not a ROLLBACK TO a savepoint or +/// a prepared transaction's COMMIT. +pub fn transaction_control(text: &str, grammar: SqlGrammar) -> Option { + let plan = ScriptPlan::build(text, grammar, LexicalSettings::default_for(grammar)); + let words: Vec = plan + .tokens(0..text.len()) + .filter(|token| !token.kind.is_trivia() && token.kind != ScriptTokenKind::Semicolon) + .take(3) + .map(|token| text.get(token.range).unwrap_or_default().to_ascii_uppercase()) + .collect(); + let first = words.first()?.as_str(); + let second = words.get(1).map(String::as_str); + match (first, second) { + ( + "BEGIN", + None | Some("TRANSACTION" | "WORK" | "DEFERRED" | "IMMEDIATE" | "EXCLUSIVE" | "ISOLATION" | "READ"), + ) => Some(TransactionControl::Begin), + ("START", Some("TRANSACTION")) => Some(TransactionControl::Begin), + ("COMMIT" | "END", None | Some("TRANSACTION" | "WORK" | "AND")) => Some(TransactionControl::Commit), + ("ROLLBACK" | "ABORT", None | Some("TRANSACTION" | "WORK" | "AND")) => match words.get(2).map(String::as_str) { + Some("TO") => None, + _ => Some(TransactionControl::Rollback), + }, + _ => None, + } +} + +#[cfg(test)] +mod tests; diff --git a/core/crates/core/src/sql_syntax/script/script_reader/tests.rs b/core/crates/core/src/sql_syntax/script/script_reader/tests.rs new file mode 100644 index 0000000000..036189ad74 --- /dev/null +++ b/core/crates/core/src/sql_syntax/script/script_reader/tests.rs @@ -0,0 +1,246 @@ +use super::*; + +fn read_in_pieces(grammar: SqlGrammar, text: &str, piece: usize) -> Vec { + let mut reader = ScriptReader::new(grammar).with_copy_overriding(Some("OVERRIDING SYSTEM VALUE")); + let mut units = Vec::new(); + let mut start = 0; + while start < text.len() { + let mut end = (start + piece).min(text.len()); + while !text.is_char_boundary(end) { + end += 1; + } + units.extend(reader.push(&text[start..end]).expect("a piece")); + start = end; + } + units.extend(reader.finish().expect("the end")); + units +} + +/// Every way of cutting the text into pieces reads the same units. +fn read_every_way(grammar: SqlGrammar, text: &str) -> Vec { + let whole = read_in_pieces(grammar, text, text.len().max(1)); + for piece in [1, 2, 3, 7, 64] { + assert_eq!(read_in_pieces(grammar, text, piece), whole, "pieces of {piece}"); + } + whole +} + +fn texts(units: &[ScriptUnit]) -> Vec<&str> { + units.iter().map(|unit| unit.text.as_str()).collect() +} + +#[test] +fn statements_split_at_semicolons_with_their_lines() { + let units = read_every_way( + SqlGrammar::PostgreSql, + "-- head\nCREATE TABLE t (a text);\nINSERT INTO t VALUES ('a;b'),\n('c');\n\nSELECT 1", + ); + + assert_eq!( + texts(&units), + [ + "CREATE TABLE t (a text)", + "INSERT INTO t VALUES ('a;b'),\n('c')", + "SELECT 1" + ] + ); + assert_eq!(units.iter().map(|unit| unit.line).collect::>(), [2, 3, 6]); +} + +#[test] +fn a_dollar_quoted_body_stays_one_statement() { + let units = read_every_way( + SqlGrammar::PostgreSql, + "CREATE FUNCTION f() RETURNS int AS $$ SELECT 1; $$ LANGUAGE sql;\nSELECT f();\n", + ); + + assert_eq!(units.len(), 2, "{units:?}"); + assert!(units[0].text.ends_with("LANGUAGE sql")); +} + +#[test] +fn a_mysql_delimiter_holds_across_pieces() { + let units = read_every_way( + SqlGrammar::MySql, + "/*!40101 SET NAMES utf8mb4 */;\nDELIMITER ;;\nCREATE PROCEDURE p()\nBEGIN\n SELECT 1;\n SELECT 2;\nEND ;;\nDELIMITER ;\nCALL p();\n", + ); + + assert_eq!( + texts(&units), + [ + "/*!40101 SET NAMES utf8mb4 */", + "CREATE PROCEDURE p()\nBEGIN\n SELECT 1;\n SELECT 2;\nEND", + "CALL p()" + ] + ); + assert_eq!(units[1].line, 3); +} + +#[test] +fn a_sql_server_batch_ends_at_go_and_keeps_its_count() { + let units = read_every_way( + SqlGrammar::MsSql, + "CREATE TABLE t (a int);\nINSERT INTO t VALUES (1);\nGO\nCREATE VIEW v AS SELECT a FROM t\nGO 2\nSELECT 1\n", + ); + + assert_eq!( + texts(&units), + [ + "CREATE TABLE t (a int);\nINSERT INTO t VALUES (1);", + "CREATE VIEW v AS SELECT a FROM t", + "SELECT 1" + ] + ); + assert_eq!(units.iter().map(|unit| unit.repeat).collect::>(), [1, 2, 1]); + assert_eq!(units[1].line, 4); +} + +#[test] +fn pg_dump_copy_rows_become_inserts_and_restrict_lines_are_passed_over() { + let dump = "--\n-- PostgreSQL database dump\n--\n\n\\restrict abc123\n\nSET client_encoding = 'UTF8';\n\nCOPY app.child (id, note) FROM stdin;\n1\tit's\n2\t\\N\n\\.\n\nSELECT pg_catalog.setval('app.child_id_seq', 5, true);\n\n\\unrestrict abc123\n"; + + let units = read_every_way(SqlGrammar::PostgreSql, dump); + + let kinds: Vec<&ScriptUnitKind> = units.iter().map(|unit| &unit.kind).collect(); + assert_eq!( + kinds, + [ + &ScriptUnitKind::ClientOnly, + &ScriptUnitKind::Run, + &ScriptUnitKind::Run, + &ScriptUnitKind::Run, + &ScriptUnitKind::ClientOnly + ], + "{units:?}" + ); + assert_eq!( + units[2].text, + "INSERT INTO app.child (id, note) OVERRIDING SYSTEM VALUE VALUES\n(E'1', E'it''s'),\n(E'2', NULL)" + ); + assert_eq!(units[2].line, 10); + assert_eq!(units[3].line, 14); +} + +#[test] +fn a_copy_with_options_is_refused_and_its_rows_skipped() { + let units = read_every_way( + SqlGrammar::PostgreSql, + "COPY t FROM stdin WITH (FORMAT csv);\n1,2\n\\.\nSELECT 1;\n", + ); + + assert_eq!(units.len(), 2, "{units:?}"); + assert_eq!(units[0].kind, ScriptUnitKind::Refused(RefusedUnit::CopyFormat)); + assert_eq!(units[1].text, "SELECT 1"); +} + +#[test] +fn another_psql_command_is_refused_with_its_name() { + let units = read_every_way(SqlGrammar::PostgreSql, "SELECT 1;\n\\connect other\nSELECT 2;\n"); + + assert_eq!(texts(&units), ["SELECT 1", "\\connect other", "SELECT 2"]); + assert_eq!( + units[1].kind, + ScriptUnitKind::Refused(RefusedUnit::ClientCommand { + command: "\\connect".to_owned() + }) + ); + assert_eq!(units[1].line, 2); +} + +#[test] +fn a_backslash_line_is_not_special_outside_postgresql() { + let units = read_every_way(SqlGrammar::MySql, "SELECT '\\\\';\n"); + + assert_eq!(texts(&units), ["SELECT '\\\\'"]); +} + +#[test] +fn transaction_statements_are_told_apart_from_savepoints() { + use TransactionControl::{Begin, Commit, Rollback}; + let cases = [ + ("BEGIN", Some(Begin)), + ("BEGIN TRANSACTION", Some(Begin)), + ("begin immediate", Some(Begin)), + ("START TRANSACTION", Some(Begin)), + ("COMMIT", Some(Commit)), + ("END", Some(Commit)), + ("ROLLBACK", Some(Rollback)), + ("ROLLBACK TO SAVEPOINT a", None), + ("ROLLBACK TRANSACTION TO a", None), + ("COMMIT PREPARED 'x'", None), + ("SELECT 1", None), + ]; + for (text, expected) in cases { + assert_eq!(transaction_control(text, SqlGrammar::PostgreSql), expected, "{text}"); + } +} + +#[test] +fn a_file_on_one_line_is_cut_where_its_statements_end() { + let line: String = (0..20_000).map(|n| format!("INSERT INTO t VALUES ({n});")).collect(); + let mut reader = ScriptReader::new(SqlGrammar::Sqlite); + + let early = reader.push(&line).expect("the line"); + + assert!( + early.len() > 19_000, + "only {} statements came before the end", + early.len() + ); + let rest = reader.finish().expect("the end"); + assert_eq!(early.len() + rest.len(), 20_000); +} + +#[test] +fn a_session_without_backslash_escapes_reads_a_file_that_sets_nothing() { + let mut reader = ScriptReader::new(SqlGrammar::MySql).with_settings(LexicalSettings { + backslash_escapes: false, + }); + + let mut units = reader + .push("INSERT INTO t VALUES ('a\\'); INSERT INTO t VALUES ('b;c');\n") + .expect("the file"); + units.extend(reader.finish().expect("the end")); + + assert_eq!( + texts(&units), + ["INSERT INTO t VALUES ('a\\')", "INSERT INTO t VALUES ('b;c')"] + ); +} + +#[test] +fn a_long_line_after_a_comment_line_is_cut_where_its_statements_end() { + let line: String = (0..20_000).map(|n| format!("INSERT INTO t VALUES ({n});")).collect(); + let mut reader = ScriptReader::new(SqlGrammar::Sqlite); + + let early = reader + .push(&format!("-- header\nINSERT INTO t VALUES\n(-1);{line}")) + .expect("the line"); + + assert!( + early.len() > 19_000, + "only {} statements came before the end", + early.len() + ); + let rest = reader.finish().expect("the end"); + assert_eq!(early.len() + rest.len(), 20_001); +} + +#[test] +fn a_routine_made_under_no_backslash_escapes_splits_as_the_server_reads_it() { + let units = read_every_way( + SqlGrammar::MySql, + "SET @saved_sql_mode = @@sql_mode;\nSET sql_mode = 'NO_BACKSLASH_ESCAPES';\nDELIMITER ;;\nCREATE FUNCTION f() RETURNS text RETURN 'a\\';;\nDELIMITER ;\nSET sql_mode = @saved_sql_mode;\nSELECT 'it\\'s';\n", + ); + + assert_eq!( + texts(&units), + [ + "SET @saved_sql_mode = @@sql_mode", + "SET sql_mode = 'NO_BACKSLASH_ESCAPES'", + "CREATE FUNCTION f() RETURNS text RETURN 'a\\'", + "SET sql_mode = @saved_sql_mode", + "SELECT 'it\\'s'" + ] + ); +} diff --git a/core/crates/core/src/unsupported_feature.rs b/core/crates/core/src/unsupported_feature.rs index 7c58083c7e..0b789d2472 100644 --- a/core/crates/core/src/unsupported_feature.rs +++ b/core/crates/core/src/unsupported_feature.rs @@ -61,6 +61,8 @@ pub enum UnsupportedFeature { EditorCommand { command: &'static str, }, + /// A whole database written as one SQL file. + SqlDump, } impl fmt::Display for UnsupportedFeature { @@ -92,6 +94,7 @@ impl fmt::Display for UnsupportedFeature { Self::ClrRoutine => f.write_str("the SQL of a CLR routine"), Self::EncryptedDefinition => f.write_str("the SQL of an encrypted module"), Self::EditorCommand { command } => write!(f, "{command} in the editor"), + Self::SqlDump => f.write_str("a SQL dump"), } } } diff --git a/core/crates/drivers/clickhouse/src/connection.rs b/core/crates/drivers/clickhouse/src/connection.rs index 59e06d3df8..214902cf86 100644 --- a/core/crates/drivers/clickhouse/src/connection.rs +++ b/core/crates/drivers/clickhouse/src/connection.rs @@ -7,6 +7,7 @@ use async_trait::async_trait; use tablepro_core::column::ColumnInfo; use tablepro_core::ddl::{InboundForeignKey, OutboundForeignKey, TableDefinition}; use tablepro_core::dialect::SqlDialect; +use tablepro_core::dump::{DumpCatalog, DumpScope}; use tablepro_core::meta::{ IndexInfo, ObjectDefinition, ObjectInfo, ObjectScope, RowIdentity, SchemaInfo, TableEstimate, TableRef, }; @@ -25,6 +26,7 @@ use crate::cancel::{ClickHouseStop, StopProgress}; use crate::catalog::{self, CatalogTable}; use crate::commit_resolution::resolve; use crate::dialect::ClickHouseDialect; +use crate::dump; use crate::editor_session::ClickHouseEditorSession; use crate::endpoint::ClickHouseEndpoint; use crate::http_call::HttpCall; @@ -306,6 +308,18 @@ impl Connection for ClickHouseConnection { .await } + async fn read_dump_catalog(&self, scope: &DumpScope, options: CallOptions) -> Result { + let database = match scope { + DumpScope::Database => self.default_database.clone(), + DumpScope::Schema(database) => database.clone(), + }; + let qualified = database != self.default_database; + self.workspace_read(options, move |calls| async move { + dump::read_dump_catalog(&calls, &database, qualified).await + }) + .await + } + async fn fetch_columns(&self, table: &TableRef, options: CallOptions) -> Result, DriverError> { let table = self.catalog_table(table); self.workspace_read(options, move |calls| async move { diff --git a/core/crates/drivers/clickhouse/src/dialect.rs b/core/crates/drivers/clickhouse/src/dialect.rs index 8d7aca130d..a326be4865 100644 --- a/core/crates/drivers/clickhouse/src/dialect.rs +++ b/core/crates/drivers/clickhouse/src/dialect.rs @@ -7,6 +7,7 @@ use tablepro_core::dialect::{ BindError, BindTarget, DatabaseLayout, DialectCapabilities, InsertBatchLimits, InsertConflict, KeysetDirection, LikeCase, LikeForm, LiteralError, Placeholder, SqlDialect, }; +use tablepro_core::dump::DumpDialect; use tablepro_core::edit::{CellExpression, MomentKind}; use tablepro_core::export::value_to_text; use tablepro_core::meta::{EngineRowId, EngineRowIdPart, TableRef}; @@ -177,6 +178,10 @@ impl SqlDialect for ClickHouseDialect { Some(self) } + fn dump(&self) -> Option<&dyn DumpDialect> { + Some(self) + } + /// The column alone: the FROM names the table as `db.table`, which a /// `table.column` sort would not match. fn sort_expression(&self, _table: &TableRef, column: &ColumnInfo) -> String { diff --git a/core/crates/drivers/clickhouse/src/dump/create_text.rs b/core/crates/drivers/clickhouse/src/dump/create_text.rs new file mode 100644 index 0000000000..c0fdf5ef04 --- /dev/null +++ b/core/crates/drivers/clickhouse/src/dump/create_text.rs @@ -0,0 +1,360 @@ +use std::ops::Range; + +use tablepro_core::sql_syntax::SqlGrammar; +use tablepro_core::sql_syntax::script::{LexicalSettings, ScriptPlan, ScriptToken, ScriptTokenKind}; + +/// `create` with every `database.` qualifier gone, so the file loads into +/// a database of any name. The server qualifies the names a view's query +/// reads, as well as the created name and a materialized view's target. +/// Where the query also uses the database's name as an alias, `name.` +/// there can be a column of that alias, so only the header's go. +pub(crate) fn without_database(create: &str, database: &str) -> String { + let tokens = tokens(create); + let aliased = tokens + .windows(2) + .any(|pair| is_word(create, &pair[0], "AS") && name_text(create, &pair[1]).as_deref() == Some(database)); + let considered = match aliased { + true => &tokens[..head(create, &tokens)], + false => &tokens[..], + }; + let mut cuts = Vec::new(); + for (at, window) in considered.windows(3).enumerate() { + let [name, dot, next] = window else { + continue; + }; + let follows_dot = at.checked_sub(1).is_some_and(|before| is_dot(create, &tokens[before])); + if !follows_dot && name_text(create, name).as_deref() == Some(database) && is_dot(create, dot) && is_name(next) + { + cuts.push(name.range.start..dot.range.end); + } + } + cut(create, &cuts) +} + +/// A view's text without its `DEFINER = user`, as mysqldump leaves the +/// definer out: the user may not exist where the file loads, and the +/// server makes whoever runs the CREATE the definer. +pub(crate) fn without_definer(create: &str) -> String { + let tokens = tokens(create); + let head = head(create, &tokens); + let mut cuts = Vec::new(); + for (at, window) in tokens[..head].windows(3).enumerate() { + let [definer, equals, user] = window else { + continue; + }; + if is_word(create, definer, "DEFINER") && token_text(create, equals) == "=" && is_name(user) { + let end = tokens.get(at + 3).map_or(user.range.end, |next| next.range.start); + cuts.push(definer.range.start..end); + } + } + cut(create, &cuts) +} + +/// Whether a materialized view keeps its rows in an inner table of its +/// own: one with no `TO` target and no refresh. That table's name changes +/// with every CREATE, so its rows cannot go in the file, and a POPULATE +/// would recompute them from the source rows still there, not the ones +/// the view kept. +pub(crate) fn keeps_own_rows(create: &str) -> bool { + let tokens = tokens(create); + let head = head(create, &tokens); + let Some(after_name) = after_created_name(create, &tokens) else { + return false; + }; + let mut depth = 0usize; + for token in &tokens[after_name.min(head)..] { + match token.kind { + ScriptTokenKind::OpenParen => depth += 1, + ScriptTokenKind::CloseParen => depth = depth.saturating_sub(1), + _ if depth > 0 => {} + _ if ["TO", "REFRESH"].iter().any(|word| is_word(create, token, word)) => return false, + _ if ["AS", "COMMENT", "DEFINER", "SQL"] + .iter() + .any(|word| is_word(create, token, word)) => + { + return true; + } + _ => {} + } + } + false +} + +/// Whether the server wrote `'[HIDDEN]'` in place of a secret: in an +/// engine's, a table function's or a dictionary source's arguments, or a +/// setting. The same string as a column's default or a comment is text. +pub(crate) fn hides_secret(create: &str) -> bool { + let tokens = tokens(create); + let mut depth = 0usize; + let mut seen_group = false; + let mut in_columns = false; + for (at, token) in tokens.iter().enumerate() { + match token.kind { + ScriptTokenKind::OpenParen => { + if depth == 0 && !seen_group { + seen_group = true; + let called = at + .checked_sub(2) + .and_then(|before| tokens.get(before)) + .is_some_and(|before| is_word(create, before, "AS") || token_text(create, before) == "="); + in_columns = !called; + } + depth += 1; + } + ScriptTokenKind::CloseParen => { + depth = depth.saturating_sub(1); + if depth == 0 { + in_columns = false; + } + } + ScriptTokenKind::StringLiteral if token_text(create, token) == "'[HIDDEN]'" => { + let comment = at + .checked_sub(1) + .is_some_and(|before| is_word(create, &tokens[before], "COMMENT")); + if !in_columns && !comment { + return true; + } + } + _ => {} + } + } + false +} + +/// Whether `create` makes a dictionary, rather than a table over one. +pub(crate) fn is_dictionary(create: &str) -> bool { + let tokens = tokens(create); + matches!(tokens.as_slice(), [first, second, ..] if is_word(create, first, "CREATE") && is_word(create, second, "DICTIONARY")) +} + +fn tokens(sql: &str) -> Vec { + let grammar = SqlGrammar::ClickHouse; + ScriptPlan::build(sql, grammar, LexicalSettings::default_for(grammar)) + .tokens(0..sql.len()) + .filter(|token| !token.kind.is_trivia()) + .collect() +} + +/// How many tokens come before the `AS` that starts a view's query. +fn head(sql: &str, tokens: &[ScriptToken]) -> usize { + let mut depth = 0usize; + for (at, token) in tokens.iter().enumerate() { + match token.kind { + ScriptTokenKind::OpenParen => depth += 1, + ScriptTokenKind::CloseParen => depth = depth.saturating_sub(1), + _ if depth == 0 && is_word(sql, token, "AS") => return at, + _ => {} + } + } + tokens.len() +} + +/// The position after `CREATE MATERIALIZED VIEW [IF NOT EXISTS] name`. +fn after_created_name(sql: &str, tokens: &[ScriptToken]) -> Option { + let mut at = 0; + for word in ["CREATE", "MATERIALIZED", "VIEW"] { + if !is_word(sql, tokens.get(at)?, word) { + return None; + } + at += 1; + } + if is_word(sql, tokens.get(at)?, "IF") { + at += 3; + } + if !is_name(tokens.get(at)?) { + return None; + } + at += 1; + while tokens.get(at).is_some_and(|token| is_dot(sql, token)) && tokens.get(at + 1).is_some_and(is_name) { + at += 2; + } + Some(at) +} + +fn cut(text: &str, ranges: &[Range]) -> String { + let mut kept = text.to_owned(); + for range in ranges.iter().rev() { + kept.replace_range(range.clone(), ""); + } + kept +} + +fn token_text<'a>(sql: &'a str, token: &ScriptToken) -> &'a str { + sql.get(token.range.clone()).unwrap_or_default() +} + +fn is_word(sql: &str, token: &ScriptToken, word: &str) -> bool { + token.kind == ScriptTokenKind::Word && token_text(sql, token).eq_ignore_ascii_case(word) +} + +fn is_dot(sql: &str, token: &ScriptToken) -> bool { + token.kind == ScriptTokenKind::Punct && token_text(sql, token) == "." +} + +fn is_name(token: &ScriptToken) -> bool { + matches!(token.kind, ScriptTokenKind::Word | ScriptTokenKind::QuotedIdentifier) +} + +/// The name a token spells, with a quoted one's escapes read as the +/// server reads them. +fn name_text(sql: &str, token: &ScriptToken) -> Option { + let text = token_text(sql, token); + match token.kind { + ScriptTokenKind::Word => Some(text.to_owned()), + ScriptTokenKind::QuotedIdentifier => unquoted(text), + _ => None, + } +} + +fn unquoted(text: &str) -> Option { + let quote = text.chars().next().filter(|first| matches!(first, '`' | '"'))?; + let inner = text.strip_prefix(quote)?.strip_suffix(quote)?; + let mut bytes = Vec::with_capacity(inner.len()); + let mut chars = inner.chars().peekable(); + let mut buffer = [0u8; 4]; + while let Some(character) = chars.next() { + let decoded = match character { + '\\' => match chars.next()? { + 'x' => { + let high = chars.next()?.to_digit(16)?; + let low = chars.next()?.to_digit(16)?; + bytes.push(u8::try_from(high * 16 + low).ok()?); + continue; + } + 'n' => '\n', + 't' => '\t', + 'r' => '\r', + 'b' => '\u{8}', + 'f' => '\u{c}', + 'a' => '\u{7}', + 'v' => '\u{b}', + '0' => '\0', + other => other, + }, + doubled if doubled == quote && chars.peek() == Some("e) => { + chars.next(); + quote + } + other => other, + }; + bytes.extend_from_slice(decoded.encode_utf8(&mut buffer).as_bytes()); + } + String::from_utf8(bytes).ok() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_secret_is_hidden_in_arguments_and_settings_only() { + assert!(hides_secret( + "CREATE TABLE app.r\n(\n `id` UInt64\n)\nENGINE = MySQL('h:3306', 'db', 't', 'u', '[HIDDEN]')" + )); + assert!(hides_secret( + "CREATE DICTIONARY app.d (`id` UInt64) PRIMARY KEY id SOURCE(CLICKHOUSE(HOST 'h' PASSWORD '[HIDDEN]'))" + )); + assert!(hides_secret( + "CREATE TABLE app.s AS s3('https://h/b/f.csv', 'key', '[HIDDEN]')" + )); + assert!(!hides_secret( + "CREATE TABLE app.t\n(\n `note` String DEFAULT '[HIDDEN]' COMMENT '[HIDDEN]'\n)\n\ + ENGINE = MergeTree\nORDER BY tuple()\nCOMMENT '[HIDDEN]'" + )); + } + + #[test] + fn the_database_qualifier_goes_from_every_name() { + let create = "CREATE VIEW app.v\n(\n `id` UInt64\n)\nAS SELECT app.parent.id\nFROM app.parent\nINNER JOIN `app`.`child` ON 1"; + assert_eq!( + without_database(create, "app"), + "CREATE VIEW v\n(\n `id` UInt64\n)\nAS SELECT parent.id\nFROM parent\nINNER JOIN `child` ON 1" + ); + } + + #[test] + fn only_the_dumped_database_is_stripped() { + let create = "CREATE MATERIALIZED VIEW app.mv TO app.counts AS SELECT * FROM other.t JOIN application.t2 ON 1"; + assert_eq!( + without_database(create, "app"), + "CREATE MATERIALIZED VIEW mv TO counts AS SELECT * FROM other.t JOIN application.t2 ON 1" + ); + } + + #[test] + fn strings_comments_and_inner_name_parts_are_left_alone() { + let create = "CREATE VIEW app.v AS SELECT 'app.x', x.app.y /* app.z */ FROM app.t"; + assert_eq!( + without_database(create, "app"), + "CREATE VIEW v AS SELECT 'app.x', x.app.y /* app.z */ FROM t" + ); + } + + #[test] + fn a_quoted_database_name_is_read_with_its_escapes() { + let create = "CREATE TABLE `my\\`db`.t (`a` UInt8) ENGINE = Memory"; + assert_eq!( + without_database(create, "my`db"), + "CREATE TABLE t (`a` UInt8) ENGINE = Memory" + ); + assert_eq!(unquoted("`a``b`").as_deref(), Some("a`b")); + assert_eq!(unquoted("\"\\x41\\n\"").as_deref(), Some("A\n")); + assert_eq!(unquoted("plain"), None); + } + + #[test] + fn the_definer_goes_and_the_security_stays() { + let create = "CREATE VIEW v\n(\n `id` UInt64\n)\nDEFINER = tablepro SQL SECURITY DEFINER\nAS SELECT definer = x FROM t"; + assert_eq!( + without_definer(create), + "CREATE VIEW v\n(\n `id` UInt64\n)\nSQL SECURITY DEFINER\nAS SELECT definer = x FROM t" + ); + assert_eq!( + without_definer("CREATE VIEW v DEFINER = `odd user` SQL SECURITY DEFINER AS SELECT 1"), + "CREATE VIEW v SQL SECURITY DEFINER AS SELECT 1" + ); + assert_eq!( + without_definer("CREATE VIEW v AS SELECT 1"), + "CREATE VIEW v AS SELECT 1" + ); + } + + #[test] + fn an_alias_named_like_the_database_keeps_its_columns() { + let create = "CREATE VIEW app.v AS SELECT app.id FROM app.events AS app"; + + assert_eq!( + without_database(create, "app"), + "CREATE VIEW v AS SELECT app.id FROM app.events AS app" + ); + } + + #[test] + fn an_inner_materialized_view_keeps_its_own_rows() { + assert!(keeps_own_rows( + "CREATE MATERIALIZED VIEW mv\n(\n `k` UInt64\n)\nENGINE = SummingMergeTree\nORDER BY k\nAS SELECT k FROM src" + )); + assert!(keeps_own_rows( + "CREATE MATERIALIZED VIEW mv (`k` UInt64) ENGINE = Memory SQL SECURITY DEFINER AS SELECT k FROM src" + )); + } + + #[test] + fn a_view_with_a_target_or_a_refresh_keeps_none() { + for create in [ + "CREATE MATERIALIZED VIEW mv TO counts (`k` UInt64) AS SELECT k FROM src", + "CREATE MATERIALIZED VIEW mv REFRESH EVERY 1 HOUR (`k` UInt64) ENGINE = Memory AS SELECT k FROM src", + "CREATE VIEW v", + ] { + assert!(!keeps_own_rows(create), "{create}"); + } + } + + #[test] + fn a_dictionary_is_told_from_a_table_over_one() { + assert!(is_dictionary( + "CREATE DICTIONARY d (`id` UInt64) PRIMARY KEY id SOURCE(NULL()) LIFETIME(0) LAYOUT(FLAT())" + )); + assert!(!is_dictionary("CREATE TABLE t (`id` UInt64) ENGINE = Dictionary(d)")); + } +} diff --git a/core/crates/drivers/clickhouse/src/dump/mod.rs b/core/crates/drivers/clickhouse/src/dump/mod.rs new file mode 100644 index 0000000000..53e3dbb20c --- /dev/null +++ b/core/crates/drivers/clickhouse/src/dump/mod.rs @@ -0,0 +1,297 @@ +//! A ClickHouse database as a SQL dump: each table, view and dictionary +//! as `SHOW CREATE TABLE` gives it, with the database's name taken out. +//! User-defined functions belong to the server, not to a database, so a +//! database's dump leaves them out. + +mod create_text; + +use tablepro_core::DriverError; +use tablepro_core::dump::{ + DumpCatalog, DumpDialect, DumpObject, DumpObjectKind, DumpSection, DumpSkip, DumpSkipReason, DumpStatement, + DumpTable, +}; +use tablepro_core::meta::TableRef; + +use crate::call_client::CallClient; +use clickhouse::sql::Identifier; + +use crate::catalog::CatalogTable; +use crate::dialect::ClickHouseDialect; +use crate::result_reader::{CatalogRow, read_table}; + +/// Statements end in `;`, and a CREATE does not roll back: ClickHouse +/// has no transaction a file could run in. +impl DumpDialect for ClickHouseDialect {} + +/// The objects of `database`, by name. `qualified` names the tables with +/// the database for reading their rows, where it is not the connection's. +pub(crate) async fn read_dump_catalog( + calls: &CallClient, + database: &str, + qualified: bool, +) -> Result { + // A materialized view's inner table is made again by its CREATE. + let sql = "SELECT name, engine FROM system.tables \ + WHERE database = {db:String} AND is_temporary = 0 AND NOT startsWith(name, '.inner') \ + ORDER BY name"; + let listed = read_table(calls.raw(sql)?.param("db", database), None).await?; + let mut catalog = DumpCatalog::default(); + for row in &listed.rows { + let row = CatalogRow(row); + let (name, engine) = (row.text(0)?, row.text(1)?); + let source = CatalogTable { + database: database.to_owned(), + name: name.clone(), + }; + let create = hidden_secrets_create(calls, &source).await?; + let table = match qualified { + true => TableRef::new(Some(database.to_owned()), name.clone()), + false => TableRef::unqualified(name.clone()), + }; + add_object(&mut catalog, table, &engine, create, database); + } + Ok(catalog) +} + +/// SHOW CREATE TABLE with any password or key the text names shown as +/// `[HIDDEN]`, which the dump then leaves out, even for a login allowed +/// to see secrets. +async fn hidden_secrets_create(calls: &CallClient, table: &CatalogTable) -> Result, DriverError> { + let query = calls + .template("SHOW CREATE TABLE ?.?")? + .bind(Identifier(&table.database)) + .bind(Identifier(&table.name)) + .with_setting("format_display_secrets_in_show_and_select", "0"); + let result = read_table(query, None).await?; + result.rows.first().map(|row| CatalogRow(row).text(0)).transpose() +} + +fn add_object(catalog: &mut DumpCatalog, table: TableRef, engine: &str, create: Option, database: &str) { + let placement = Placement::of(engine, create.as_deref().unwrap_or_default()); + let skip = |reason| DumpSkip { + kind: placement.kind(), + schema: None, + name: table.name.clone(), + reason, + }; + let Some(create) = create.filter(|create| !create.trim().is_empty()) else { + catalog.skipped.push(skip(DumpSkipReason::NoDefinition)); + return; + }; + if create_text::hides_secret(&create) { + catalog.skipped.push(skip(DumpSkipReason::HiddenSecret)); + return; + } + let create = create_text::without_database(&create, database); + match placement { + Placement::Table { rows } => { + let mut dumped = DumpTable::new(table.clone()); + dumped.rows = rows; + catalog + .objects + .push(DumpObject::table(table, vec![DumpStatement::new(create)])); + catalog.tables.push(dumped); + } + Placement::Program(kind) => { + let create = match kind { + DumpObjectKind::View | DumpObjectKind::MaterializedView => create_text::without_definer(&create), + _ => create, + }; + if kind == DumpObjectKind::MaterializedView && create_text::keeps_own_rows(&create) { + catalog.skipped.push(DumpSkip { + kind, + schema: None, + name: table.name.clone(), + reason: DumpSkipReason::RowsNotCarried, + }); + } + catalog.objects.push(DumpObject::new( + kind, + DumpSection::Programs, + None, + table.name, + vec![DumpStatement::new(create)], + )); + } + } +} + +/// Where an object of `system.tables` goes in the file. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Placement { + /// Its CREATE goes before the rows. Only an engine that keeps rows + /// of its own has them read: reading a Kafka table takes its + /// messages, and a Distributed or Merge table reads other tables. + Table { rows: bool }, + /// After the rows, so a materialized view does not see them load + /// and write them into its target a second time. + Program(DumpObjectKind), +} + +impl Placement { + fn of(engine: &str, create: &str) -> Self { + match engine { + "View" | "LiveView" | "WindowView" => Self::Program(DumpObjectKind::View), + "MaterializedView" => Self::Program(DumpObjectKind::MaterializedView), + "Dictionary" if create_text::is_dictionary(create) => Self::Program(DumpObjectKind::Dictionary), + _ => Self::Table { + rows: holds_rows(engine), + }, + } + } + + fn kind(self) -> DumpObjectKind { + match self { + Self::Table { .. } => DumpObjectKind::Table, + Self::Program(kind) => kind, + } + } +} + +fn holds_rows(engine: &str) -> bool { + engine.ends_with("MergeTree") + || matches!( + engine, + "Log" | "TinyLog" | "StripeLog" | "Memory" | "EmbeddedRocksDB" | "KeeperMap" | "Join" + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn catalog_of(objects: &[(&str, &str, Option<&str>)]) -> DumpCatalog { + let mut catalog = DumpCatalog::default(); + for (name, engine, create) in objects { + add_object( + &mut catalog, + TableRef::unqualified(*name), + engine, + create.map(str::to_owned), + "app", + ); + } + catalog + } + + #[test] + fn tables_come_before_the_rows_and_views_after() { + let catalog = catalog_of(&[ + ( + "counts", + "MergeTree", + Some("CREATE TABLE app.counts (`k` UInt64) ENGINE = MergeTree ORDER BY k"), + ), + ( + "mv", + "MaterializedView", + Some( + "CREATE MATERIALIZED VIEW app.mv TO app.counts (`k` UInt64) DEFINER = me SQL SECURITY DEFINER AS SELECT k FROM app.src", + ), + ), + ( + "v", + "View", + Some("CREATE VIEW app.v (`k` UInt64) AS SELECT k FROM app.counts"), + ), + ]); + + let statements: Vec<(DumpSection, &str)> = catalog + .objects + .iter() + .map(|object| (object.section, object.statements[0].sql.as_str())) + .collect(); + assert_eq!( + statements, + [ + ( + DumpSection::BeforeData, + "CREATE TABLE counts (`k` UInt64) ENGINE = MergeTree ORDER BY k" + ), + ( + DumpSection::Programs, + "CREATE MATERIALIZED VIEW mv TO counts (`k` UInt64) SQL SECURITY DEFINER AS SELECT k FROM src" + ), + ( + DumpSection::Programs, + "CREATE VIEW v (`k` UInt64) AS SELECT k FROM counts" + ), + ] + ); + assert_eq!(catalog.objects[0].table, Some(TableRef::unqualified("counts"))); + assert_eq!(catalog.tables.len(), 1); + assert!(catalog.tables[0].rows); + assert!(catalog.objects.iter().all(|object| object.schema.is_none())); + } + + #[test] + fn only_an_engine_that_keeps_rows_has_them_read() { + let catalog = catalog_of(&[ + ( + "events", + "Kafka", + Some("CREATE TABLE app.events (`m` String) ENGINE = Kafka"), + ), + ( + "log", + "ReplicatedReplacingMergeTree", + Some("CREATE TABLE app.log (`m` String) ENGINE = ReplicatedReplacingMergeTree ORDER BY m"), + ), + ( + "over", + "Dictionary", + Some("CREATE TABLE app.over (`id` UInt64) ENGINE = Dictionary(d)"), + ), + ]); + + let rows: Vec<(&str, bool)> = catalog + .tables + .iter() + .map(|table| (table.table.name.as_str(), table.rows)) + .collect(); + assert_eq!(rows, [("events", false), ("log", true), ("over", false)]); + } + + #[test] + fn a_dictionary_is_a_program_unless_it_hides_a_secret() { + let catalog = catalog_of(&[ + ( + "d", + "Dictionary", + Some( + "CREATE DICTIONARY app.d (`id` UInt64) PRIMARY KEY id SOURCE(CLICKHOUSE(TABLE 'src')) LIFETIME(0) LAYOUT(FLAT())", + ), + ), + ( + "remote", + "Dictionary", + Some( + "CREATE DICTIONARY app.remote (`id` UInt64) PRIMARY KEY id SOURCE(CLICKHOUSE(HOST 'h' PASSWORD '[HIDDEN]' TABLE 'src')) LIFETIME(0) LAYOUT(FLAT())", + ), + ), + ("gone", "MergeTree", None), + ]); + + assert_eq!(catalog.objects.len(), 1); + assert_eq!(catalog.objects[0].kind, DumpObjectKind::Dictionary); + assert_eq!(catalog.objects[0].section, DumpSection::Programs); + assert_eq!( + catalog.skipped, + [ + DumpSkip { + kind: DumpObjectKind::Dictionary, + schema: None, + name: "remote".to_owned(), + reason: DumpSkipReason::HiddenSecret, + }, + DumpSkip { + kind: DumpObjectKind::Table, + schema: None, + name: "gone".to_owned(), + reason: DumpSkipReason::NoDefinition, + }, + ] + ); + assert!(catalog.tables.is_empty()); + } +} diff --git a/core/crates/drivers/clickhouse/src/lib.rs b/core/crates/drivers/clickhouse/src/lib.rs index 5cf6138897..769cd1b56e 100644 --- a/core/crates/drivers/clickhouse/src/lib.rs +++ b/core/crates/drivers/clickhouse/src/lib.rs @@ -19,6 +19,7 @@ mod connection; mod ddl; mod dialect; mod driver; +mod dump; mod editor_run; mod editor_session; mod endpoint; diff --git a/core/crates/drivers/clickhouse/tests/integration/dump.rs b/core/crates/drivers/clickhouse/tests/integration/dump.rs new file mode 100644 index 0000000000..4b5a0b37f4 --- /dev/null +++ b/core/crates/drivers/clickhouse/tests/integration/dump.rs @@ -0,0 +1,79 @@ +//! The SQL dump round trip on ClickHouse. + +use async_trait::async_trait; +use tablepro_core::{ConnectOptions, ConnectTarget, DatabaseDriver, WorkspaceSettings, WriteKind}; +use tablepro_driver_clickhouse::ClickhouseDriver; +use tablepro_test_support::conformance::{ConformanceEngine, ConformanceFailure, DumpEngine, test_liveness}; +use tokio_util::sync::CancellationToken; + +use crate::conformance_engine::ClickHouseEngine; +use crate::support::{apply, quoted}; + +#[async_trait] +impl DumpEngine for ClickHouseEngine { + async fn empty_database(&self, name: &str) -> Result { + let mut options = self.connect_options(WorkspaceSettings::workspace(false), test_liveness()); + let connection = ClickhouseDriver + .connect(options.clone(), CancellationToken::new()) + .await?; + let database = quoted(name); + let drop = format!("DROP DATABASE IF EXISTS {database} SYNC"); + let create = format!("CREATE DATABASE {database}"); + let made = apply(connection.as_ref(), WriteKind::SchemaChanges, &[&drop, &create]).await; + connection.close().await?; + made?; + match &mut options.target { + ConnectTarget::Network(target) => target.database = Some(name.to_owned()), + _ => return Err(ConformanceFailure::setup("the fixture connects over the network")), + } + Ok(options) + } + + /// ClickHouse has no foreign keys, sequences, routines or triggers. + /// The materialized view stands in for a trigger: it is made after + /// the rows load, or its target would hold their counts twice. + fn dump_fixture(&self) -> Vec { + [ + "CREATE TABLE parent (\ + id UInt64, \ + name String, \ + note Nullable(String), \ + tags Array(String), \ + attrs Map(String, UInt8), \ + kind LowCardinality(String) DEFAULT 'plain', \ + payload String, \ + amount Decimal(18, 4), \ + created DateTime64(3, 'UTC'), \ + name_length UInt64 MATERIALIZED length(name)\ + ) ENGINE = MergeTree ORDER BY id", + "CREATE TABLE child (id UInt64, parent_id UInt64, label String, version UInt32) \ + ENGINE = ReplacingMergeTree(version) ORDER BY id", + "CREATE TABLE child_counts (parent_id UInt64, n UInt64) ENGINE = MergeTree ORDER BY (parent_id, n)", + "CREATE MATERIALIZED VIEW mv_child_counts TO child_counts \ + AS SELECT parent_id, count() AS n FROM child GROUP BY parent_id", + "CREATE VIEW v_top AS SELECT id, name FROM parent WHERE id > 1", + "CREATE VIEW v_over AS SELECT id FROM v_top WHERE name != ''", + "INSERT INTO parent (id, name, note, tags, attrs, kind, payload, amount, created) VALUES \ + (1, 'it\\'s; a \"quote\" \\\\ back\\nline two, Grüße 東京', NULL, ['x', 'y'], {'k': 1}, 'rare', \ + unhex('00FF27'), 12.5, '2024-01-02 03:04:05.678'), \ + (2, 'plain', 'a note', [], {}, DEFAULT, '', -0.0001, '1970-01-01 00:00:00.000'), \ + (3, '', 'ünïcode', ['only'], {'a': 0, 'b': 255}, 'plain', unhex('FF'), 0, '2038-01-19 03:14:08.000')", + "INSERT INTO child (id, parent_id, label, version) VALUES \ + (1, 1, 'first', 1), (2, 1, 'second', 1), (3, 2, 'old', 1)", + "INSERT INTO child (id, parent_id, label, version) VALUES (3, 1, 'replaced', 2)", + "OPTIMIZE TABLE child FINAL", + ] + .into_iter() + .map(str::to_owned) + .collect() + } +} + +mod suite { + use super::ClickHouseEngine; + + tablepro_test_support::dump_suite! { + engine: ClickHouseEngine, + docker: true, + } +} diff --git a/core/crates/drivers/clickhouse/tests/integration/main.rs b/core/crates/drivers/clickhouse/tests/integration/main.rs index 3ec640703f..725152b4b1 100644 --- a/core/crates/drivers/clickhouse/tests/integration/main.rs +++ b/core/crates/drivers/clickhouse/tests/integration/main.rs @@ -9,6 +9,7 @@ mod conformance_engine; mod csv_import; mod ddl; +mod dump; mod keys_and_decimals; mod metadata; mod round_trip; diff --git a/core/crates/drivers/duckdb/src/connection.rs b/core/crates/drivers/duckdb/src/connection.rs index 37bf79f855..09b85a593c 100644 --- a/core/crates/drivers/duckdb/src/connection.rs +++ b/core/crates/drivers/duckdb/src/connection.rs @@ -4,6 +4,7 @@ use std::sync::{Arc, Mutex, PoisonError}; use async_trait::async_trait; use tablepro_core::ddl::{InboundForeignKey, OutboundForeignKey, TableDefinition}; use tablepro_core::dialect::SqlDialect; +use tablepro_core::dump::{DumpCatalog, DumpScope}; use tablepro_core::meta::{IndexInfo, ObjectInfo, ObjectScope, RowIdentity, SchemaInfo, TableEstimate, TableRef}; use tablepro_core::statement::Statement; use tablepro_core::{ @@ -342,6 +343,19 @@ impl Connection for DuckDbConnection { .await } + async fn read_dump_catalog(&self, scope: &DumpScope, options: CallOptions) -> Result { + let scope = scope.clone(); + // One transaction, so the parts come from the same moment. + self.read(options, move |link, catalog| { + batch(link, "BEGIN TRANSACTION READ ONLY")?; + let read = crate::dump::read_dump_catalog(link, catalog, &scope); + link.hold_stops(); + batch(link, "ROLLBACK")?; + read + }) + .await + } + async fn row_identity(&self, table: &TableRef, options: CallOptions) -> Result { let table = table.clone(); self.read(options, move |link, catalog| { diff --git a/core/crates/drivers/duckdb/src/dialect.rs b/core/crates/drivers/duckdb/src/dialect.rs index 6da6dc45bb..74ee7a6858 100644 --- a/core/crates/drivers/duckdb/src/dialect.rs +++ b/core/crates/drivers/duckdb/src/dialect.rs @@ -3,6 +3,7 @@ use tablepro_core::ddl::DdlDialect; use tablepro_core::dialect::{ BindError, BindTarget, DialectCapabilities, InsertConflict, KeysetDirection, LiteralError, Placeholder, SqlDialect, }; +use tablepro_core::dump::DumpDialect; use tablepro_core::edit::{CellExpression, MomentKind}; use tablepro_core::meta::{EngineRowId, EngineRowIdPart, TableRef}; use tablepro_core::sql_syntax::{ @@ -236,6 +237,10 @@ impl SqlDialect for DuckDbDialect { Some(self) } + fn dump(&self) -> Option<&dyn DumpDialect> { + Some(self) + } + /// A value Arrow cannot carry whole is read as DuckDB's own text. fn projection(&self, column: &ColumnInfo) -> String { let name = self.quote_identifier(&column.name); diff --git a/core/crates/drivers/duckdb/src/dump.rs b/core/crates/drivers/duckdb/src/dump.rs new file mode 100644 index 0000000000..00df54ed7d --- /dev/null +++ b/core/crates/drivers/duckdb/src/dump.rs @@ -0,0 +1,641 @@ +//! A DuckDB file as one SQL dump, in the order EXPORT DATABASE writes +//! its schema.sql: schemas, sequences, enum types, tables with their +//! foreign keys inline, the rows, indexes, macros, views, and the +//! triggers EXPORT leaves out. Names carry their schema and never the +//! catalog, which is the file's own name. + +use duckdb::types::Value as DuckValue; +use tablepro_core::DriverError; +use tablepro_core::dialect::SqlDialect; +use tablepro_core::dump::{ + DumpCatalog, DumpDialect, DumpObject, DumpObjectKind, DumpScope, DumpSection, DumpSkip, DumpSkipReason, + DumpStatement, DumpTable, +}; +use tablepro_core::meta::TableRef; +use tablepro_core::sql_syntax::SqlGrammar; +use tablepro_core::sql_syntax::script::{LexicalSettings, ScriptPlan, ScriptToken, ScriptTokenKind}; + +use crate::catalog::DEFAULT_SCHEMA; +use crate::dialect::DuckDbDialect; +use crate::errors::map_error; +use crate::instance::Link; + +impl DumpDialect for DuckDbDialect { + fn header(&self) -> String { + "BEGIN TRANSACTION;\n".to_owned() + } + + fn footer(&self) -> String { + "\nCOMMIT;\n".to_owned() + } + + fn transactional_ddl(&self) -> bool { + true + } + + fn begin(&self) -> &'static str { + "BEGIN TRANSACTION" + } +} + +// Each query takes the catalog, then the schema twice, which is NULL for +// the whole database. +const SCHEMAS_SQL: &str = "SELECT schema_name FROM duckdb_schemas() \ + WHERE database_name = ? AND (? IS NULL OR schema_name = ?) AND NOT internal \ + ORDER BY schema_name"; + +const SEQUENCES_SQL: &str = "SELECT schema_name, sequence_name, increment_by, min_value, max_value, cycle, sql \ + FROM duckdb_sequences() \ + WHERE database_name = ? AND (? IS NULL OR schema_name = ?) AND NOT temporary \ + ORDER BY schema_name, sequence_name"; + +/// A type is listed once per constructor, and only its first row has +/// the oid. +const ENUMS_SQL: &str = "SELECT schema_name, type_name, labels FROM duckdb_types() \ + WHERE database_name = ? AND (? IS NULL OR schema_name = ?) AND NOT internal \ + AND logical_type = 'ENUM' AND type_oid IS NOT NULL \ + ORDER BY schema_name, type_name"; + +const TABLES_SQL: &str = "SELECT schema_name, table_name, sql FROM duckdb_tables() \ + WHERE database_name = ? AND (? IS NULL OR schema_name = ?) AND NOT internal AND NOT temporary \ + ORDER BY schema_name, table_name"; + +/// DuckDB keeps a foreign key's parent in the child's schema. +const FOREIGN_KEYS_SQL: &str = "SELECT schema_name, table_name, referenced_table FROM duckdb_constraints() \ + WHERE database_name = ? AND (? IS NULL OR schema_name = ?) AND constraint_type = 'FOREIGN KEY' \ + ORDER BY schema_name, table_name, constraint_index"; + +/// The index behind a PRIMARY KEY or UNIQUE has no text: its table's +/// CREATE makes it. +const INDEXES_SQL: &str = "SELECT schema_name, index_name, sql FROM duckdb_indexes() \ + WHERE database_name = ? AND (? IS NULL OR schema_name = ?) AND sql IS NOT NULL \ + ORDER BY schema_name, index_name"; + +/// A macro's overloads come one row each, in the order they were +/// declared, which an ORDER BY would not keep. +const MACROS_SQL: &str = "SELECT schema_name, function_name, function_type, parameters, parameter_types, \ + macro_definition \ + FROM duckdb_functions() \ + WHERE database_name = ? AND (? IS NULL OR schema_name = ?) AND NOT internal \ + AND function_type IN ('macro', 'table_macro')"; + +const VIEWS_SQL: &str = "SELECT schema_name, view_name, sql FROM duckdb_views() \ + WHERE database_name = ? AND (? IS NULL OR schema_name = ?) AND NOT internal AND NOT temporary \ + ORDER BY schema_name, view_name"; + +const TRIGGERS_SQL: &str = "SELECT schema_name, trigger_name, sql FROM duckdb_triggers() \ + WHERE database_name = ? AND (? IS NULL OR schema_name = ?) AND NOT temporary \ + ORDER BY schema_name, trigger_name"; + +struct Reader<'a> { + link: &'a Link, + catalog: &'a str, + schema: Option<&'a str>, +} + +impl Reader<'_> { + fn rows( + &self, + sql: &str, + read: impl FnMut(&duckdb::Row<'_>) -> duckdb::Result, + ) -> Result, DriverError> { + let mut statement = self.link.connection.prepare(sql).map_err(map_error)?; + let rows = statement + .query_map(duckdb::params![self.catalog, self.schema, self.schema], read) + .map_err(map_error)?; + rows.collect::, _>>().map_err(map_error) + } + + fn named(&self, sql: &str) -> Result, DriverError> { + self.rows(sql, |row| { + Ok(Named { + schema: row.get(0)?, + name: row.get(1)?, + sql: row + .get::<_, Option>(2)? + .map(|sql| without_catalog(&sql, self.catalog)), + }) + }) + } +} + +struct Named { + schema: String, + name: String, + sql: Option, +} + +impl Named { + fn text(&self) -> Option<&str> { + self.sql.as_deref().filter(|sql| !sql.trim().is_empty()) + } + + fn skip(&self, kind: DumpObjectKind) -> DumpSkip { + DumpSkip { + kind, + schema: Some(self.schema.clone()), + name: self.name.clone(), + reason: DumpSkipReason::NoDefinition, + } + } +} + +struct Sequence { + schema: String, + name: String, + increment: i64, + min: i64, + max: i64, + cycle: bool, + sql: String, +} + +struct MacroOverload { + schema: String, + name: String, + table: bool, + parameters: Vec<(String, Option)>, + definition: Option, +} + +pub(crate) fn read_dump_catalog(link: &Link, catalog: &str, scope: &DumpScope) -> Result { + let reader = Reader { + link, + catalog, + schema: match scope { + DumpScope::Database => None, + DumpScope::Schema(schema) => Some(schema.as_str()), + }, + }; + let dialect = DuckDbDialect::new(); + let mut dump = DumpCatalog::default(); + for schema in reader.rows(SCHEMAS_SQL, |row| row.get::<_, String>(0))? { + if schema != DEFAULT_SCHEMA { + let create = format!("CREATE SCHEMA IF NOT EXISTS {}", dialect.quote_identifier(&schema)); + dump.objects.push(DumpObject::new( + DumpObjectKind::Schema, + DumpSection::BeforeData, + None, + schema, + vec![DumpStatement::new(create)], + )); + } + } + for sequence in reader.rows(SEQUENCES_SQL, read_sequence)? { + dump.objects.push(sequence_object(&dialect, &sequence)?); + } + add_enums(&reader, &dialect, &mut dump)?; + add_tables(&reader, &mut dump)?; + place( + &mut dump, + reader.named(INDEXES_SQL)?, + DumpObjectKind::Index, + DumpSection::AfterData, + ); + add_macros(&reader, &dialect, &mut dump)?; + place( + &mut dump, + reader.named(VIEWS_SQL)?, + DumpObjectKind::View, + DumpSection::Programs, + ); + place( + &mut dump, + reader.named(TRIGGERS_SQL)?, + DumpObjectKind::Trigger, + DumpSection::Triggers, + ); + Ok(dump) +} + +/// DuckDB's text ends with a `;`, which the writer takes off. +fn place(dump: &mut DumpCatalog, objects: Vec, kind: DumpObjectKind, section: DumpSection) { + for object in objects { + match object.text() { + Some(sql) => dump.objects.push(DumpObject::new( + kind, + section, + Some(object.schema.clone()), + object.name.clone(), + vec![DumpStatement::new(sql)], + )), + None => dump.skipped.push(object.skip(kind)), + } + } +} + +fn read_sequence(row: &duckdb::Row<'_>) -> duckdb::Result { + Ok(Sequence { + schema: row.get(0)?, + name: row.get(1)?, + increment: row.get(2)?, + min: row.get(3)?, + max: row.get(4)?, + cycle: row.get(5)?, + sql: row.get(6)?, + }) +} + +/// The sequence made again at the value its next call would give, which +/// DuckDB's own text has as its START and lists nowhere else: a call to +/// `setval` moves it without a last value to count from. +fn sequence_object(dialect: &DuckDbDialect, sequence: &Sequence) -> Result { + let next = next_value(&sequence.sql) + .ok_or_else(|| DriverError::Protocol(format!("DuckDB gave sequence {} no START", sequence.name)))?; + let name = dialect.portable_name(&TableRef::new(Some(sequence.schema.clone()), sequence.name.clone())); + let cycle = match sequence.cycle { + true => "CYCLE", + false => "NO CYCLE", + }; + let create = format!( + "CREATE SEQUENCE {name} INCREMENT BY {} MINVALUE {} MAXVALUE {} START {next} {cycle}", + sequence.increment, sequence.min, sequence.max + ); + Ok(DumpObject::new( + DumpObjectKind::Sequence, + DumpSection::BeforeData, + Some(sequence.schema.clone()), + sequence.name.clone(), + vec![DumpStatement::new(create)], + )) +} + +/// The number after the last ` START `, since the sequence's name comes +/// first and can hold anything. +fn next_value(sql: &str) -> Option { + let (_, after) = sql.rsplit_once(" START ")?; + after.split_whitespace().next()?.parse().ok() +} + +/// A table names an enum by its labels, so a table loads without the +/// type, but a cast or a macro may name it. +fn add_enums(reader: &Reader<'_>, dialect: &DuckDbDialect, dump: &mut DumpCatalog) -> Result<(), DriverError> { + let enums = reader.rows(ENUMS_SQL, |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?, texts(row.get(2)?))) + })?; + for (schema, name, labels) in enums { + let Some(labels) = labels.into_iter().collect::>>() else { + dump.skipped.push(DumpSkip { + kind: DumpObjectKind::Type, + schema: Some(schema), + name, + reason: DumpSkipReason::NoDefinition, + }); + continue; + }; + let quoted: Vec = labels.iter().map(|label| dialect.string_literal(label)).collect(); + let create = format!( + "CREATE TYPE {} AS ENUM ({})", + dialect.portable_name(&TableRef::new(Some(schema.clone()), name.clone())), + quoted.join(", ") + ); + dump.objects.push(DumpObject::new( + DumpObjectKind::Type, + DumpSection::BeforeData, + Some(schema), + name, + vec![DumpStatement::new(create)], + )); + } + Ok(()) +} + +/// The items of a list of text, `None` for a NULL item. A value that is +/// no list has one `None`, so it reads as incomplete. +fn texts(value: DuckValue) -> Vec> { + match value { + DuckValue::List(items) => items + .into_iter() + .map(|item| match item { + DuckValue::Text(text) => Some(text), + _ => None, + }) + .collect(), + _ => vec![None], + } +} + +fn add_tables(reader: &Reader<'_>, dump: &mut DumpCatalog) -> Result<(), DriverError> { + let tables = reader.named(TABLES_SQL)?; + let keys = reader.rows(FOREIGN_KEYS_SQL, |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, Option>(2)?, + )) + })?; + let listed: Vec<&Named> = tables.iter().filter(|table| table.text().is_some()).collect(); + dump.tables.extend(dump_tables(&listed, &keys)); + for table in &tables { + match table.text() { + Some(sql) => dump.objects.push(DumpObject::table( + TableRef::new(Some(table.schema.clone()), table.name.clone()), + vec![DumpStatement::new(sql)], + )), + None => dump.skipped.push(table.skip(DumpObjectKind::Table)), + } + } + Ok(()) +} + +/// DuckDB checks a foreign key as each statement ends, and has no way +/// to add one after the rows, so a table's rows follow its parents', and +/// a table whose key points at itself takes a row per statement. +fn dump_tables(tables: &[&Named], keys: &[(String, String, Option)]) -> Vec { + tables + .iter() + .map(|table| { + let mut dumped = DumpTable::new(TableRef::new(Some(table.schema.clone()), table.name.clone())); + let parents = keys + .iter() + .filter(|(schema, child, _)| *schema == table.schema && *child == table.name) + .filter_map(|(_, _, parent)| parent.as_deref()); + for parent in parents { + // A key keeps its parent as the CREATE spelled it, and + // DuckDB folds the case of names. + let declared = tables + .iter() + .find(|other| other.schema == table.schema && other.name.eq_ignore_ascii_case(parent)) + .map_or(parent, |other| other.name.as_str()); + if declared == table.name { + dumped.one_row_per_insert = true; + continue; + } + let reference = TableRef::new(Some(table.schema.clone()), declared.to_owned()); + if !dumped.references.contains(&reference) { + dumped.references.push(reference); + } + } + dumped + }) + .collect() +} + +fn add_macros(reader: &Reader<'_>, dialect: &DuckDbDialect, dump: &mut DumpCatalog) -> Result<(), DriverError> { + let mut overloads = reader.rows(MACROS_SQL, |row| { + let names = texts(row.get(3)?); + let types = texts(row.get(4)?); + Ok(MacroOverload { + schema: row.get(0)?, + name: row.get(1)?, + table: row.get::<_, String>(2)? == "table_macro", + parameters: names + .into_iter() + .enumerate() + .filter_map(|(position, name)| Some((name?, types.get(position).cloned().flatten()))) + .collect(), + definition: row + .get::<_, Option>(5)? + .map(|body| without_catalog(&body, reader.catalog)), + }) + })?; + // Stable, so each macro's overloads keep their order. + overloads.sort_by(|left, right| (&left.schema, &left.name).cmp(&(&right.schema, &right.name))); + for group in overloads.chunk_by(|left, right| left.schema == right.schema && left.name == right.name) { + let Some(first) = group.first() else { + continue; + }; + let name = dialect.portable_name(&TableRef::new(Some(first.schema.clone()), first.name.clone())); + // A generated column or a default can call a scalar macro, and + // DuckDB keeps the call in the table's text; a table macro reads + // tables, which have to exist first. + let section = match first.table { + true => DumpSection::Programs, + false => DumpSection::BeforeData, + }; + match macro_sql(dialect, &name, group) { + Some(create) => dump.objects.push(DumpObject::new( + DumpObjectKind::Function, + section, + Some(first.schema.clone()), + first.name.clone(), + vec![DumpStatement::new(create)], + )), + None => dump.skipped.push(DumpSkip { + kind: DumpObjectKind::Function, + schema: Some(first.schema.clone()), + name: first.name.clone(), + reason: DumpSkipReason::NoDefinition, + }), + } + } + Ok(()) +} + +/// One CREATE MACRO for all of a macro's overloads, written as DuckDB +/// writes one. The catalogue lists no parameter's default, so a +/// parameter declared with one comes back without it. +fn macro_sql(dialect: &DuckDbDialect, name: &str, overloads: &[MacroOverload]) -> Option { + let definitions = overloads + .iter() + .map(|overload| { + let body = overload.definition.as_deref().filter(|body| !body.trim().is_empty())?; + let parameters: Vec = overload + .parameters + .iter() + .map(|(parameter, type_name)| match type_name { + Some(type_name) => format!("{} {type_name}", dialect.quote_identifier(parameter)), + None => dialect.quote_identifier(parameter), + }) + .collect(); + let parameters = parameters.join(", "); + Some(match overload.table { + true => format!("({parameters}) AS TABLE ({body})"), + false => format!("({parameters}) AS ({body})"), + }) + }) + .collect::>>()?; + Some(format!("CREATE MACRO {name} {}", definitions.join(", "))) +} + +/// `sql` without the catalog in front of `catalog.schema.name`: DuckDB +/// keeps a definition's names as they were written, and the catalog is +/// the source file's name, which the file the dump loads into does not +/// have. A text that names the catalog on its own, as an alias or a +/// table, can mean that by `name.schema.`, so it is kept. +fn without_catalog(sql: &str, catalog: &str) -> String { + let plan = ScriptPlan::build( + sql, + SqlGrammar::DuckDb, + LexicalSettings::default_for(SqlGrammar::DuckDb), + ); + let tokens: Vec = plan + .tokens(0..sql.len()) + .filter(|token| !token.kind.is_trivia()) + .collect(); + let text = |at: usize| tokens.get(at).and_then(|token| sql.get(token.range.clone())); + let dot = |at: usize| text(at) == Some("."); + let is_catalog = |at: usize| name_of(sql, &tokens[at]).is_some_and(|name| name.eq_ignore_ascii_case(catalog)); + if (0..tokens.len()).any(|at| is_catalog(at) && !dot(at + 1)) { + return sql.to_owned(); + } + let mut kept = String::with_capacity(sql.len()); + let mut from = 0; + for at in 0..tokens.len() { + let qualifies = is_catalog(at) && dot(at + 1) && dot(at + 3) && !at.checked_sub(1).is_some_and(dot); + if qualifies { + kept.push_str(&sql[from..tokens[at].range.start]); + from = tokens[at + 1].range.end; + } + } + kept.push_str(&sql[from..]); + kept +} + +fn name_of(sql: &str, token: &ScriptToken) -> Option { + let text = sql.get(token.range.clone())?; + match token.kind { + ScriptTokenKind::Word => Some(text.to_owned()), + ScriptTokenKind::QuotedIdentifier => Some(text.strip_prefix('"')?.strip_suffix('"')?.replace("\"\"", "\"")), + _ => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn named(schema: &str, name: &str) -> Named { + Named { + schema: schema.to_owned(), + name: name.to_owned(), + sql: Some(format!("CREATE TABLE {name}(id INTEGER);")), + } + } + + fn key(schema: &str, child: &str, parent: &str) -> (String, String, Option) { + (schema.to_owned(), child.to_owned(), Some(parent.to_owned())) + } + + #[test] + fn the_next_value_is_the_last_start_in_the_text() { + assert_eq!( + next_value("CREATE SEQUENCE s INCREMENT BY 1 MINVALUE 1 MAXVALUE 9223372036854775807 START 6 NO CYCLE;"), + Some(6) + ); + assert_eq!( + next_value("CREATE SEQUENCE \"a START 9 b\" INCREMENT BY -2 MINVALUE -10 MAXVALUE 0 START -4 CYCLE;"), + Some(-4) + ); + assert_eq!(next_value("CREATE SEQUENCE s;"), None); + } + + #[test] + fn a_sequence_starts_where_it_stopped() { + let sequence = Sequence { + schema: "app".to_owned(), + name: "ids".to_owned(), + increment: 1, + min: 1, + max: 100, + cycle: false, + sql: "CREATE SEQUENCE ids INCREMENT BY 1 MINVALUE 1 MAXVALUE 100 START 5 NO CYCLE;".to_owned(), + }; + + let object = sequence_object(&DuckDbDialect::new(), &sequence).unwrap(); + + assert_eq!( + object.statements[0].sql, + "CREATE SEQUENCE \"app\".\"ids\" INCREMENT BY 1 MINVALUE 1 MAXVALUE 100 START 5 NO CYCLE" + ); + } + + #[test] + fn a_table_follows_its_parents_and_a_self_reference_loads_a_row_at_a_time() { + let (parent, child, other) = (named("app", "parent"), named("app", "child"), named("main", "parent")); + let tables = [&child, &parent, &other]; + let keys = [ + key("app", "child", "Parent"), + key("app", "child", "parent"), + key("app", "child", "child"), + ]; + + let dumped = dump_tables(&tables, &keys); + + assert_eq!( + dumped[0].references, + [TableRef::new(Some("app".to_owned()), "parent".to_owned())] + ); + assert!(dumped[0].one_row_per_insert); + assert!(dumped[1].references.is_empty() && !dumped[1].one_row_per_insert); + } + + #[test] + fn a_macro_is_written_with_every_overload() { + let overload = |parameters: &[(&str, Option<&str>)], body: &str, table: bool| MacroOverload { + schema: "app".to_owned(), + name: "m".to_owned(), + table, + parameters: parameters + .iter() + .map(|(name, type_name)| ((*name).to_owned(), type_name.map(str::to_owned))) + .collect(), + definition: Some(body.to_owned()), + }; + let dialect = DuckDbDialect::new(); + + assert_eq!( + macro_sql( + &dialect, + "\"app\".\"m\"", + &[ + overload(&[("x", None)], "(x * 2)", false), + overload(&[("x", None), ("y", Some("INTEGER"))], "(x + y)", false) + ] + ) + .as_deref(), + Some("CREATE MACRO \"app\".\"m\" (\"x\") AS ((x * 2)), (\"x\", \"y\" INTEGER) AS ((x + y))") + ); + assert_eq!( + macro_sql(&dialect, "\"app\".\"t\"", &[overload(&[], "SELECT 1", true)]).as_deref(), + Some("CREATE MACRO \"app\".\"t\" () AS TABLE (SELECT 1)") + ); + let mut missing = overload(&[], "", false); + missing.definition = None; + assert_eq!(macro_sql(&dialect, "\"app\".\"m\"", &[missing]), None); + } + + #[test] + fn the_source_catalog_comes_off_three_part_names() { + assert_eq!( + without_catalog( + "CREATE VIEW v AS SELECT * FROM source.main.t JOIN \"Source\".app.u USING (id) JOIN other.main.t USING (id);", + "source" + ), + "CREATE VIEW v AS SELECT * FROM main.t JOIN app.u USING (id) JOIN other.main.t USING (id);" + ); + assert_eq!( + without_catalog( + "CREATE VIEW v AS SELECT 'source.main.t' AS s, x.source.main AS m FROM x;", + "source" + ), + "CREATE VIEW v AS SELECT 'source.main.t' AS s, x.source.main AS m FROM x;" + ); + } + + #[test] + fn a_text_that_names_the_catalog_alone_keeps_it() { + let view = "CREATE VIEW v AS SELECT source.payload.id FROM main.events AS source;"; + + assert_eq!(without_catalog(view, "source"), view); + } + + #[test] + fn a_list_with_a_null_or_no_list_reads_as_incomplete() { + assert_eq!( + texts(DuckValue::List(vec![DuckValue::Text("a".to_owned()), DuckValue::Null])), + [Some("a".to_owned()), None] + ); + assert_eq!(texts(DuckValue::Null), [None]); + } + + #[test] + fn the_file_is_one_transaction() { + let dialect = DuckDbDialect::new(); + + assert_eq!(dialect.header(), "BEGIN TRANSACTION;\n"); + assert_eq!(dialect.footer(), "\nCOMMIT;\n"); + assert_eq!(dialect.begin(), "BEGIN TRANSACTION"); + assert!(dialect.transactional_ddl()); + } +} diff --git a/core/crates/drivers/duckdb/src/lib.rs b/core/crates/drivers/duckdb/src/lib.rs index 73ae16e90a..c5b11a72f4 100644 --- a/core/crates/drivers/duckdb/src/lib.rs +++ b/core/crates/drivers/duckdb/src/lib.rs @@ -11,6 +11,7 @@ mod decode; mod dialect; mod driver; mod duck_type; +mod dump; mod editor_session; mod encode; mod errors; diff --git a/core/crates/drivers/duckdb/tests/integration/dump.rs b/core/crates/drivers/duckdb/tests/integration/dump.rs new file mode 100644 index 0000000000..79adecdfc2 --- /dev/null +++ b/core/crates/drivers/duckdb/tests/integration/dump.rs @@ -0,0 +1,94 @@ +//! The SQL dump round trip on DuckDB. + +use async_trait::async_trait; +use tablepro_core::{ConnectOptions, ConnectTarget, FileEndpoint, WorkspaceSettings}; +use tablepro_test_support::conformance::{ConformanceEngine, ConformanceFailure, DumpEngine, test_liveness}; + +use crate::duckdb_engine::{DuckDbEngine, create_database}; + +#[async_trait] +impl DumpEngine for DuckDbEngine { + /// A file of that name beside the engine's own, made by DuckDB with + /// its default storage version, which takes triggers. + async fn empty_database(&self, name: &str) -> Result { + let mut options = self.connect_options(WorkspaceSettings::workspace(false), test_liveness()); + let ConnectTarget::File(endpoint) = &options.target else { + return Err(ConformanceFailure::setup("the fixture opens a file")); + }; + let path = endpoint.path().with_file_name(format!("{name}.duckdb")); + for suffix in ["", ".wal"] { + let mut leftover = path.clone().into_os_string(); + leftover.push(suffix); + match std::fs::remove_file(&leftover) { + Err(error) if error.kind() != std::io::ErrorKind::NotFound => { + return Err(ConformanceFailure::setup(error)); + } + _ => {} + } + } + create_database(&path).map_err(ConformanceFailure::setup)?; + options.target = ConnectTarget::File(FileEndpoint::open_existing(path).map_err(ConformanceFailure::setup)?); + Ok(options) + } + + /// DuckDB's triggers fire only after a row and may not write to + /// their own table, so the trigger writes to another one: rows there + /// beyond the one it made would show it firing during the load. Its + /// table has no generated column, which a row trigger refuses. A + /// table whose key points at itself takes its rows one at a time. + fn dump_fixture(&self) -> Vec { + [ + "CREATE SCHEMA app", + "CREATE TYPE app.mood AS ENUM ('sad', 'ok', 'happy')", + "CREATE SEQUENCE app.parent_id_seq", + "CREATE SEQUENCE app.child_id_seq", + "CREATE TABLE app.parent (\ + id INTEGER DEFAULT nextval('app.parent_id_seq') PRIMARY KEY, \ + name VARCHAR NOT NULL UNIQUE, \ + mood app.mood, \ + score DOUBLE CHECK (score >= 0)\ + )", + "CREATE TABLE app.child (\ + id INTEGER DEFAULT nextval('app.child_id_seq') PRIMARY KEY, \ + parent_id INTEGER NOT NULL, \ + manager_id INTEGER, \ + note VARCHAR, \ + payload BLOB, \ + note_length BIGINT GENERATED ALWAYS AS (length(note)) VIRTUAL, \ + FOREIGN KEY (parent_id) REFERENCES app.parent (id), \ + FOREIGN KEY (manager_id) REFERENCES app.child (id)\ + )", + "CREATE TABLE app.audit (parent_id INTEGER NOT NULL, name VARCHAR NOT NULL)", + "CREATE INDEX child_note_idx ON app.child (note)", + "INSERT INTO app.parent (name, mood, score) VALUES \ + ('first', 'happy', 1.5), ('second', 'ok', 2), ('gone', NULL, 0)", + "INSERT INTO app.child (parent_id, manager_id, note, payload) \ + VALUES (1, NULL, 'it''s; a \"quote\" \\ back', '\\x00\\xFF\\x27'::BLOB)", + "INSERT INTO app.child (parent_id, manager_id, note, payload) \ + VALUES (1, 1, 'line one' || chr(10) || 'line two;', NULL)", + "INSERT INTO app.child (parent_id, manager_id, note, payload) \ + VALUES (2, 2, 'Grüße, 東京 ✓', '\\xFF'::BLOB)", + "INSERT INTO app.child (parent_id, manager_id, note, payload) VALUES (3, NULL, 'gone', NULL)", + "DELETE FROM app.child WHERE id = 4", + "DELETE FROM app.parent WHERE id = 3", + "CREATE MACRO app.name_length(s) AS length(s)", + "CREATE VIEW app.parent_names AS SELECT id, name FROM app.parent", + "CREATE VIEW app.parent_name_lengths AS SELECT id, app.name_length(name) AS n FROM app.parent_names", + "CREATE TRIGGER parent_audit AFTER INSERT ON app.parent FOR EACH ROW \ + INSERT INTO app.audit VALUES (NEW.id, NEW.name)", + "INSERT INTO app.parent (name, mood, score) VALUES ('after the trigger', 'sad', 4)", + ] + .into_iter() + .map(str::to_owned) + .collect() + } +} + +mod suite { + use super::DuckDbEngine; + + tablepro_test_support::dump_suite! { + engine: DuckDbEngine, + docker: false, + } +} diff --git a/core/crates/drivers/duckdb/tests/integration/main.rs b/core/crates/drivers/duckdb/tests/integration/main.rs index ed176da7d3..ad4d92dc03 100644 --- a/core/crates/drivers/duckdb/tests/integration/main.rs +++ b/core/crates/drivers/duckdb/tests/integration/main.rs @@ -15,6 +15,8 @@ mod decode; #[cfg(test)] mod duckdb_engine; #[cfg(test)] +mod dump; +#[cfg(test)] mod filter; #[cfg(test)] mod open_semantics; diff --git a/core/crates/drivers/mssql/src/catalog/mod.rs b/core/crates/drivers/mssql/src/catalog/mod.rs index 1d1110d388..c27331c116 100644 --- a/core/crates/drivers/mssql/src/catalog/mod.rs +++ b/core/crates/drivers/mssql/src/catalog/mod.rs @@ -19,7 +19,7 @@ use tiberius::{Column, ColumnType as TdsType}; pub(crate) use cells::Cells; pub(crate) use mssql_column_row::{COLUMNS_SQL, MssqlColumnRow}; -pub(crate) use sql_server_type::SqlServerType; +pub(crate) use sql_server_type::{CatalogFacts, SqlServerType}; pub(crate) use table_definition::{INBOUND_FOREIGN_KEYS_SQL, PRIMARY_KEY_SQL, inbound_foreign_keys, primary_key}; /// A column as browse and the insert form see it. @@ -71,6 +71,7 @@ pub(crate) fn column_definition(row: &MssqlColumnRow) -> ColumnDefinition { true => IdentityStyle::MssqlIdentity, false => IdentityStyle::None, }, + identity_seed: row.identity_seed.clone(), default_constraint_name: row.default_constraint.clone(), ..ColumnAttributes::default() }; diff --git a/core/crates/drivers/mssql/src/catalog/mssql_column_row.rs b/core/crates/drivers/mssql/src/catalog/mssql_column_row.rs index bcf4f2780b..8d532ec644 100644 --- a/core/crates/drivers/mssql/src/catalog/mssql_column_row.rs +++ b/core/crates/drivers/mssql/src/catalog/mssql_column_row.rs @@ -1,4 +1,5 @@ use tablepro_core::DriverError; +use tablepro_core::ddl::IdentitySeed; use tablepro_core::value::Value; use super::cells::Cells; @@ -11,8 +12,9 @@ use super::sql_server_type::{CatalogFacts, SqlServerType}; /// name. A collation is reported only where it differs from the /// database's, which is the default a column takes, and the column after /// it says whether its code page differs too. The last is the function -/// of the column's data mask, if it has one, and the one after whether -/// the server writes it as a system-versioned table's period column. +/// of the column's data mask, if it has one, the one after whether the +/// server writes it as a system-versioned table's period column, and the +/// last two an identity's seed and increment. pub(crate) const COLUMNS_SQL: &str = "SELECT \ c.name, \ t.name, \ @@ -42,11 +44,14 @@ pub(crate) const COLUMNS_SQL: &str = "SELECT \ <> COLLATIONPROPERTY(CAST(DATABASEPROPERTYEX(DB_NAME(), 'Collation') AS nvarchar(128)), 'CodePage') \ THEN 1 ELSE 0 END AS int), \ mc.masking_function, \ - CAST(CASE WHEN c.generated_always_type <> 0 THEN 1 ELSE 0 END AS int) \ + CAST(CASE WHEN c.generated_always_type <> 0 THEN 1 ELSE 0 END AS int), \ + CAST(ic.seed_value AS nvarchar(64)), \ + CAST(ic.increment_value AS nvarchar(64)) \ FROM sys.columns c \ JOIN sys.types t ON t.user_type_id = c.user_type_id \ LEFT JOIN sys.default_constraints dc ON dc.object_id = c.default_object_id \ LEFT JOIN sys.computed_columns cc ON cc.object_id = c.object_id AND cc.column_id = c.column_id \ + LEFT JOIN sys.identity_columns ic ON ic.object_id = c.object_id AND ic.column_id = c.column_id \ LEFT JOIN sys.masked_columns mc ON mc.object_id = c.object_id AND mc.column_id = c.column_id \ AND mc.is_masked = 1 \ LEFT JOIN ( \ @@ -80,6 +85,7 @@ pub(crate) struct MssqlColumnRow { pub(crate) masking_function: Option, /// `GENERATED ALWAYS AS ROW START` or `END`, or a ledger column. pub(crate) server_generated: bool, + pub(crate) identity_seed: Option, } impl MssqlColumnRow { @@ -116,6 +122,10 @@ impl MssqlColumnRow { persisted: cells.flag(20), masking_function: cells.text(22), server_generated: cells.flag(23), + identity_seed: cells + .text(24) + .zip(cells.text(25)) + .and_then(|(seed, increment)| IdentitySeed::new(&seed, &increment)), }) } } diff --git a/core/crates/drivers/mssql/src/connection.rs b/core/crates/drivers/mssql/src/connection.rs index aeb32f8d6f..f940b062a0 100644 --- a/core/crates/drivers/mssql/src/connection.rs +++ b/core/crates/drivers/mssql/src/connection.rs @@ -7,6 +7,7 @@ use deadpool::managed::{PoolError, TimeoutType}; use tablepro_core::column::ColumnInfo; use tablepro_core::ddl::{InboundForeignKey, OutboundForeignKey, TableDefinition}; use tablepro_core::dialect::SqlDialect; +use tablepro_core::dump::{DumpCatalog, DumpScope}; use tablepro_core::meta::{ IndexInfo, ObjectDefinition, ObjectInfo, ObjectScope, RowIdentity, SchemaInfo, TableEstimate, TableRef, }; @@ -22,6 +23,7 @@ use tokio_util::sync::{CancellationToken, WaitForCancellationFutureOwned}; use crate::cancel::{MssqlStop, StopProgress}; use crate::dialect::MssqlDialect; +use crate::dump; use crate::editor_session::MssqlEditorSession; use crate::endpoint::MssqlEndpoint; use crate::errors::map_pool_error; @@ -289,6 +291,18 @@ impl Connection for MssqlConnection { .await } + async fn read_dump_catalog(&self, scope: &DumpScope, options: CallOptions) -> Result { + let scope = scope.clone(); + let dialect = *self.dialect; + self.read( + options, + read_op(move |client, watch| { + Box::pin(async move { dump::read_dump_catalog(client, watch, &dialect, &scope).await }) + }), + ) + .await + } + async fn fetch_columns(&self, table: &TableRef, options: CallOptions) -> Result, DriverError> { let relation = self.relation(table); self.read( diff --git a/core/crates/drivers/mssql/src/ddl.rs b/core/crates/drivers/mssql/src/ddl.rs index 9d7f485e22..95a8f7c7e3 100644 --- a/core/crates/drivers/mssql/src/ddl.rs +++ b/core/crates/drivers/mssql/src/ddl.rs @@ -2,14 +2,14 @@ use tablepro_core::column::{ColumnDefault, ColumnKind}; use tablepro_core::ddl::ddl_sql::{column_list, create_index_sql, default_sql, foreign_key_sql, parenthesized}; use tablepro_core::ddl::{ BuildDdlError, ColumnAlteration, ColumnAttributes, ColumnCapabilities, ColumnDefinition, DdlCapabilities, - DdlDialect, DdlOperation, DdlPlan, DraftColumn, DraftType, GeneratedStorage, IdentityStyle, Permission, - ReferentialAction, RefusalReason, ResolvedType, TableChanges, TableDefinition, TableDraft, + DdlDialect, DdlOperation, DdlPlan, DraftColumn, DraftType, GeneratedStorage, IdentitySeed, IdentityStyle, + Permission, ReferentialAction, RefusalReason, ResolvedType, TableChanges, TableDefinition, TableDraft, }; use tablepro_core::dialect::SqlDialect; use tablepro_core::meta::{IndexInfo, TableRef}; use crate::catalog::SqlServerType; -use crate::dialect::{MssqlDialect, national_literal}; +use crate::dialect::{MssqlDialect, national_literal, quote}; use crate::index_statements::disabling; static CAPABILITIES: DdlCapabilities = DdlCapabilities { @@ -124,39 +124,15 @@ impl DdlDialect for MssqlDialect { } fn plan_create_table(&self, draft: &TableDraft) -> Result { - self.create_table(draft, "") + Ok(DdlPlan::from_sql( + self.create_table(draft, &CreateTableFacts::default(), false)? + .into_statements(), + )) } fn create_table_script(&self, definition: &TableDefinition) -> Result { - let mut draft = TableDraft::from_definition(definition); - let (stated, written): (Vec, Vec) = - draft.indexes.into_iter().partition(|index| index.definition.is_some()); - draft.indexes = written; - let stated: Vec<(&str, &str)> = stated - .iter() - .filter_map(|index| Some((index.name.as_str(), index.definition.as_deref()?))) - .collect(); - // A table has one clustered index, so a stated one leaves the key nonclustered. - let clustered = stated - .iter() - .any(|(_, sql)| sql.starts_with("CREATE CLUSTERED ") || sql.starts_with("CREATE UNIQUE CLUSTERED ")); - let mut plan = self.create_table(&draft, if clustered { " NONCLUSTERED" } else { "" })?; - let relation = self.qualify(&definition.table); - let (mut created, mut disabled) = (Vec::new(), Vec::new()); - // A disabled clustered index refuses every index made after it. - for (name, sql) in stated { - let disable = disabling(name, &relation); - match sql.strip_suffix(&format!(";\n\n{disable}")) { - Some(create) => { - created.push(create.to_owned()); - disabled.push(disable); - } - None => created.push(sql.to_owned()), - } - } - plan.statements - .extend(DdlPlan::from_sql(created.into_iter().chain(disabled)).statements); - Ok(plan) + let script = self.table_script(definition, &CreateTableFacts::default())?; + Ok(DdlPlan::from_sql(script.into_statements())) } fn plan_alter_table(&self, original: &TableDefinition, changes: &TableChanges) -> Result { @@ -203,7 +179,7 @@ impl DdlDialect for MssqlDialect { self.alter_column(&table, alteration, &mut statements)?; } for column in &changes.added_columns { - statements.push(alter(format!("ADD {}", self.column_definition(column)?))); + statements.push(alter(format!("ADD {}", self.column_definition(column, None)?))); if let Some(comment) = column.comment_text() { statements.push(column_comment(&table, &column.name, false, Some(comment))); } @@ -231,39 +207,153 @@ impl DdlDialect for MssqlDialect { } } +/// What a dump's CREATE TABLE states that the structure model does not +/// carry. Left at its default, the script is the model's. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub(crate) struct CreateTableFacts { + pub(crate) key: Option, + pub(crate) identity: Option, +} + +/// The primary key as its constraint and index have it. The model keeps +/// the key's columns in column order and no name. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct KeyClause { + pub(crate) name: String, + pub(crate) clustered: bool, + /// The keys in key order, each with its direction. + pub(crate) keys: String, +} + +impl KeyClause { + fn sql(&self) -> String { + let kind = match self.clustered { + true => "CLUSTERED", + false => "NONCLUSTERED", + }; + format!("CONSTRAINT {} PRIMARY KEY {kind} ({})", quote(&self.name), self.keys) + } +} + +fn identity_clause(seed: &IdentitySeed) -> String { + format!("IDENTITY({},{})", seed.seed(), seed.increment()) +} + +/// An integer as the catalogue prints one, the only text a counter's +/// value is spliced into a statement as. +pub(crate) fn whole_number(text: &str) -> bool { + let digits = text.strip_prefix('-').unwrap_or(text); + !digits.is_empty() && digits.bytes().all(|byte| byte.is_ascii_digit()) +} + +/// A table's script in the parts a dump places apart. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct TableScript { + pub(crate) create: String, + pub(crate) indexes: Vec, + pub(crate) comments: Vec, + /// The indexes the server's catalogue states, then the ones disabled. + pub(crate) stated: Vec, +} + +impl TableScript { + fn into_statements(self) -> Vec { + let mut statements = vec![self.create]; + statements.extend(self.indexes); + statements.extend(self.comments); + statements.extend(self.stated); + statements + } +} + impl MssqlDialect { - // `key_kind` is " NONCLUSTERED" where another index of the table is the clustered one. - fn create_table(&self, draft: &TableDraft, key_kind: &str) -> Result { + pub(crate) fn table_script( + &self, + definition: &TableDefinition, + facts: &CreateTableFacts, + ) -> Result { + let mut draft = TableDraft::from_definition(definition); + let (stated, written): (Vec, Vec) = + draft.indexes.into_iter().partition(|index| index.definition.is_some()); + draft.indexes = written; + let stated: Vec<(&str, &str)> = stated + .iter() + .filter_map(|index| Some((index.name.as_str(), index.definition.as_deref()?))) + .collect(); + // A table has one clustered index, so a stated one leaves the key nonclustered. + let clustered = stated + .iter() + .any(|(_, sql)| sql.starts_with("CREATE CLUSTERED ") || sql.starts_with("CREATE UNIQUE CLUSTERED ")); + let mut script = self.create_table(&draft, facts, clustered)?; + let relation = self.qualify(&definition.table); + let mut disabled = Vec::new(); + // A disabled clustered index refuses every index made after it. + for (name, sql) in stated { + let disable = disabling(name, &relation); + match sql.strip_suffix(&format!(";\n\n{disable}")) { + Some(create) => { + script.stated.push(create.to_owned()); + disabled.push(disable); + } + None => script.stated.push(sql.to_owned()), + } + } + script.stated.extend(disabled); + Ok(script) + } + + fn create_table( + &self, + draft: &TableDraft, + facts: &CreateTableFacts, + nonclustered_key: bool, + ) -> Result { draft.check_create(self)?; let table = TableRef::new(draft.table.schema.clone(), draft.table.name.trim()); let mut definitions = draft .columns .iter() - .map(|column| self.column_definition(column)) + .map(|column| self.column_definition(column, facts.identity.as_ref())) .collect::, _>>()?; let key: Vec = draft.primary_key().into_iter().map(str::to_owned).collect(); - if !key.is_empty() { - definitions.push(format!("PRIMARY KEY{key_kind} ({})", column_list(self, &key))); + match &facts.key { + Some(clause) => definitions.push(clause.sql()), + None if !key.is_empty() => { + let kind = if nonclustered_key { " NONCLUSTERED" } else { "" }; + definitions.push(format!("PRIMARY KEY{kind} ({})", column_list(self, &key))); + } + None => {} } definitions.extend(draft.foreign_keys.iter().map(|key| foreign_key_sql(self, &table, key))); - let mut statements = vec![format!( + let create = format!( "CREATE TABLE {} (\n {}\n)", self.qualify(&table), definitions.join(",\n ") - )]; - statements.extend( - draft - .indexes - .iter() - .filter(|index| !index.primary) - .map(|index| create_index_sql(self, &table, index)), ); - for column in &draft.columns { - if let Some(comment) = column.comment_text() { - statements.push(column_comment(&table, &column.name, false, Some(comment))); - } - } - Ok(DdlPlan::from_sql(statements)) + let indexes = draft + .indexes + .iter() + .filter(|index| !index.primary) + .map(|index| create_index_sql(self, &table, index)) + .collect(); + let comments = draft + .columns + .iter() + .filter_map(|column| { + Some(column_comment( + &table, + &column.name, + false, + Some(column.comment_text()?), + )) + }) + .collect(); + Ok(TableScript { + create, + indexes, + comments, + stated: Vec::new(), + }) } /// A column bound to a default cannot change type, so its default @@ -359,10 +449,11 @@ impl MssqlDialect { Ok(()) } - /// `name type [COLLATE c] [SPARSE] [MASKED WITH (..)] [IDENTITY(1,1)] + /// `name type [COLLATE c] [SPARSE] [MASKED WITH (..)] [IDENTITY(s,i)] /// NULL|NOT NULL [DEFAULT (..)]`, or a computed column read from the - /// catalogue as `name AS (..)`. - fn column_definition(&self, column: &DraftColumn) -> Result { + /// catalogue as `name AS (..)`. A new column with no seed of its own + /// is `IDENTITY(1,1)`. + fn column_definition(&self, column: &DraftColumn, seed: Option<&IdentitySeed>) -> Result { let name = self.quote_identifier(&column.name); let attributes = column.original.as_ref().map(|original| &original.attributes); if let Some(generated) = attributes.and_then(|attributes| attributes.generated.as_ref()) { @@ -392,7 +483,8 @@ impl MssqlDialect { let identity = column.auto_increment || attributes.is_some_and(|attributes| attributes.identity == IdentityStyle::MssqlIdentity); if identity { - parts.push("IDENTITY(1,1)".to_owned()); + let seed = seed.or_else(|| attributes.and_then(|attributes| attributes.identity_seed.as_ref())); + parts.push(seed.map_or_else(|| "IDENTITY(1,1)".to_owned(), identity_clause)); } // SQL Server refuses an identity on a nullable column (error 8147). parts.push( @@ -711,6 +803,47 @@ mod tests { ); } + #[test] + fn a_dumped_table_states_its_seed_and_its_key_as_the_catalogue_has_them() { + let mut table = orders(); + table.columns[0].attributes.identity = IdentityStyle::MssqlIdentity; + table.columns[0].info.is_auto_increment = true; + table.columns[2].info.comment = Some("why".to_owned()); + let facts = CreateTableFacts { + key: Some(KeyClause { + name: "PK_orders".to_owned(), + clustered: false, + keys: "[id] DESC".to_owned(), + }), + identity: IdentitySeed::new("1000", "-5"), + }; + + let script = dialect().table_script(&table, &facts).expect("the script"); + + assert_eq!( + script.create, + "CREATE TABLE [dbo].[orders] (\n [id] int IDENTITY(1000,-5) NOT NULL,\n [status] int NULL DEFAULT ((0)),\n \ + [note] nvarchar(20) NULL,\n CONSTRAINT [PK_orders] PRIMARY KEY NONCLUSTERED ([id] DESC)\n)" + ); + assert_eq!(script.comments.len(), 1); + assert!(script.indexes.is_empty() && script.stated.is_empty()); + let shown = sql(dialect().create_table_script(&table)); + assert!(shown[0].contains("[id] int IDENTITY(1,1) NOT NULL"), "{}", shown[0]); + assert!(shown[0].ends_with("PRIMARY KEY ([id])\n)"), "{}", shown[0]); + } + + #[test] + fn show_create_table_keeps_the_seed_the_catalogue_read() { + let mut table = orders(); + table.columns[0].attributes.identity = IdentityStyle::MssqlIdentity; + table.columns[0].attributes.identity_seed = IdentitySeed::new("100", "5"); + table.columns[0].info.is_auto_increment = true; + + let shown = sql(dialect().create_table_script(&table)); + + assert!(shown[0].contains("[id] int IDENTITY(100,5) NOT NULL"), "{}", shown[0]); + } + #[test] fn plan_create_table_composite_key_index_and_foreign_key() { let mut draft = TableDraft::new(TableRef::new(Some("dbo".to_owned()), "lines")); diff --git a/core/crates/drivers/mssql/src/dialect.rs b/core/crates/drivers/mssql/src/dialect.rs index 358b8b29c7..83a545d121 100644 --- a/core/crates/drivers/mssql/src/dialect.rs +++ b/core/crates/drivers/mssql/src/dialect.rs @@ -7,6 +7,7 @@ use tablepro_core::dialect::{ BindError, BindTarget, DatabaseLayout, DialectCapabilities, InsertBatchLimits, InsertConflict, KeysetDirection, LikeCase, LikeForm, LiteralError, Placeholder, SqlDialect, }; +use tablepro_core::dump::DumpDialect; use tablepro_core::edit::{CellExpression, MomentKind}; use tablepro_core::meta::{EngineRowId, EngineRowIdPart}; use tablepro_core::sql_syntax::{ @@ -17,6 +18,7 @@ use tablepro_core::statement::{BoundParam, ParamType}; use tablepro_core::value::{Temporal, Value}; use crate::catalog::SqlServerType; +use crate::dump::MssqlDump; use crate::encode::MAX_NUMERIC_DIGITS; /// A string literal in the national character set. Without the `N`, the @@ -259,6 +261,10 @@ impl SqlDialect for MssqlDialect { Some(self) } + fn dump(&self) -> Option<&dyn DumpDialect> { + Some(&MssqlDump) + } + /// A type the driver cannot decode is read as the server's text, and /// money as a decimal of its own scale: tiberius reads MONEY as a /// float, which would round it. diff --git a/core/crates/drivers/mssql/src/dump/dump_statements.rs b/core/crates/drivers/mssql/src/dump/dump_statements.rs new file mode 100644 index 0000000000..3193d1b9c7 --- /dev/null +++ b/core/crates/drivers/mssql/src/dump/dump_statements.rs @@ -0,0 +1,595 @@ +use tablepro_core::ForeignKeyInfo; +use tablepro_core::ddl::ddl_sql::{foreign_key_sql, index_keys_sql, parenthesized}; +use tablepro_core::dialect::SqlDialect; +use tablepro_core::dump::DumpStatement; +use tablepro_core::meta::{IndexInfo, TableRef}; +use tablepro_core::sql_syntax::script::{LexicalSettings, ScriptPlan, ScriptToken, ScriptTokenKind}; +use tablepro_core::sql_syntax::{SqlGrammar, qualify_created_name}; + +use crate::catalog::{CatalogFacts, SqlServerType}; +use crate::ddl::whole_number; +use crate::dialect::{MssqlDialect, national_literal, quote}; + +/// CREATE SCHEMA must open a batch, so it runs as dynamic SQL behind the +/// check that keeps a load into a database that has the schema going. +pub(super) fn create_schema(name: &str) -> String { + let create = format!("CREATE SCHEMA {}", quote(name)); + format!( + "IF SCHEMA_ID({}) IS NULL EXEC({})", + national_literal(name), + national_literal(&create) + ) +} + +/// An alias type's base as `sys.types` gives it, with the length in +/// bytes. +pub(super) struct AliasBase<'a> { + pub(super) system_type: &'a str, + pub(super) max_length: i64, + pub(super) precision: i64, + pub(super) scale: i64, + pub(super) nullable: bool, +} + +pub(super) fn create_alias_type(qualified: &str, base: &AliasBase<'_>) -> String { + let declaration = SqlServerType::from_catalog(CatalogFacts { + type_name: base.system_type, + base_type: base.system_type, + type_schema: None, + user_defined: false, + alias: false, + user_clr: false, + max_length: base.max_length, + precision: base.precision, + scale: base.scale, + foreign_code_page: false, + }) + .declaration(); + let null = if base.nullable { "" } else { " NOT NULL" }; + format!("CREATE TYPE {qualified} FROM {declaration}{null}") +} + +/// Where a sequence stands, in the text the catalogue gives its numbers. +pub(super) struct SequencePosition<'a> { + pub(super) start: &'a str, + pub(super) current: &'a str, + pub(super) increment: &'a str, + pub(super) minimum: &'a str, + pub(super) maximum: &'a str, + pub(super) cycles: bool, + /// Whether NEXT VALUE FOR ever ran: before, `current` is the start, + /// which is still to come. + pub(super) used: bool, +} + +/// The value the sequence hands out next, which a new sequence takes as +/// its START WITH. +pub(super) fn next_unused(position: &SequencePosition<'_>) -> String { + if !position.used { + return position.start.to_owned(); + } + let number = |text: &str| text.parse::().ok(); + let (Some(current), Some(increment), Some(minimum), Some(maximum)) = ( + number(position.current), + number(position.increment), + number(position.minimum), + number(position.maximum), + ) else { + return position.current.to_owned(); + }; + match current.checked_add(increment) { + Some(next) if (minimum..=maximum).contains(&next) => next.to_string(), + _ if position.cycles && increment > 0 => minimum.to_string(), + _ if position.cycles => maximum.to_string(), + // An exhausted sequence has no next value; a new one cannot start + // past its own bounds. + _ => current.to_string(), + } +} + +/// The identity's next number as it was on the server the dump was read +/// from. A table that took rows since it was made hands out the reseeded +/// number plus the increment, and one that took none the reseeded number +/// itself, so an empty table, as after a load of no rows, is reseeded one +/// increment further. +pub(super) fn reseed(relation: &str, last_value: &str, increment: &str) -> Option { + if !whole_number(last_value) || !whole_number(increment) { + return None; + } + let next = last_value + .parse::() + .ok()? + .checked_add(increment.parse::().ok()?)?; + let name = national_literal(relation); + Some(format!( + "IF EXISTS (SELECT 1 FROM {relation}) DBCC CHECKIDENT ({name}, RESEED, {last_value}) WITH NO_INFOMSGS \ + ELSE DBCC CHECKIDENT ({name}, RESEED, {next}) WITH NO_INFOMSGS" + )) +} + +/// Whether the server checks a constraint, as SMO restates it: an +/// untrusted one is added without checking the rows, and a disabled one +/// is turned off after. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) struct Enforcement { + pub(super) trusted: bool, + pub(super) disabled: bool, +} + +impl Enforcement { + fn with(self) -> &'static str { + match self.trusted { + true => "WITH CHECK", + false => "WITH NOCHECK", + } + } + + fn after(self, relation: &str, name: &str) -> Option { + self.disabled + .then(|| format!("ALTER TABLE {relation} NOCHECK CONSTRAINT {}", quote(name))) + } +} + +pub(super) fn check_constraint(relation: &str, name: &str, definition: &str, enforcement: Enforcement) -> Vec { + let mut statements = vec![format!( + "ALTER TABLE {relation} {} ADD CONSTRAINT {} CHECK {}", + enforcement.with(), + quote(name), + parenthesized(definition) + )]; + statements.extend(enforcement.after(relation, name)); + statements +} + +pub(super) fn foreign_key( + dialect: &MssqlDialect, + table: &TableRef, + key: &ForeignKeyInfo, + enforcement: Enforcement, +) -> Vec { + let relation = dialect.qualify(table); + let mut statements = vec![format!( + "ALTER TABLE {relation} {} ADD {}", + enforcement.with(), + foreign_key_sql(dialect, table, key) + )]; + statements.extend(enforcement.after(&relation, &key.name)); + statements +} + +/// A UNIQUE constraint as its index has it. The structure model keeps it +/// as a unique index, which a load would make an index, not a constraint. +pub(super) fn unique_constraint(dialect: &MssqlDialect, relation: &str, index: &IndexInfo, clustered: bool) -> String { + let kind = match clustered { + true => "CLUSTERED", + false => "NONCLUSTERED", + }; + format!( + "ALTER TABLE {relation} ADD CONSTRAINT {} UNIQUE {kind} ({})", + quote(&index.name), + index_keys_sql(dialect, index) + ) +} + +/// How a module was made: the SET options its body reads under. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) struct ModuleSettings { + pub(super) ansi_nulls: bool, + pub(super) quoted_identifier: bool, +} + +/// A view, routine or trigger as the server keeps its text, named in its +/// schema, between the SET options it was made under where they are not +/// the file's. The text starts at its first token: the server keeps +/// whatever comment opened the batch, and a load would keep the dump's +/// own comment line as part of the module. +pub(super) fn module_statements(schema: &str, name: &str, text: &str, settings: ModuleSettings) -> Vec { + let quoted = quote(schema); + let created = as_create(from_first_token(text)); + let created = named_as(&created, schema, name).unwrap_or(created); + let body = qualified_view(&qualify_created_name(&created, SqlGrammar::MsSql, "ed), "ed); + let mut set = Vec::new(); + if !settings.ansi_nulls { + set.push("ANSI_NULLS"); + } + if !settings.quoted_identifier { + set.push("QUOTED_IDENTIFIER"); + } + let mut statements: Vec = set + .iter() + .map(|option| DumpStatement::new(format!("SET {option} OFF"))) + .collect(); + statements.push(DumpStatement::body(body)); + statements.extend( + set.iter() + .rev() + .map(|option| DumpStatement::new(format!("SET {option} ON"))), + ); + statements +} + +/// `qualify_created_name` leaves a view's name as it is, and the server +/// keeps a view's CREATE as it was written, so one made without its +/// schema would land in the loading session's default schema. +fn qualified_view(text: &str, quoted_schema: &str) -> String { + let grammar = SqlGrammar::MsSql; + let plan = ScriptPlan::build(text, grammar, LexicalSettings::default_for(grammar)); + let tokens: Vec = plan + .tokens(0..text.len()) + .filter(|token| !token.kind.is_trivia()) + .collect(); + let spelled = |at: usize, words: &[&str]| { + tokens.get(at).is_some_and(|token| { + token.kind == ScriptTokenKind::Word + && text + .get(token.range.clone()) + .is_some_and(|word| words.iter().any(|wanted| word.eq_ignore_ascii_case(wanted))) + }) + }; + let mut at = 1; + if spelled(at, &["OR"]) { + at += 2; + } + let named = spelled(0, &["CREATE", "ALTER"]) && spelled(at, &["VIEW"]); + let name = tokens + .get(at + 1) + .filter(|token| matches!(token.kind, ScriptTokenKind::Word | ScriptTokenKind::QuotedIdentifier)); + let dotted = tokens + .get(at + 2) + .is_some_and(|token| token.kind == ScriptTokenKind::Punct && text.get(token.range.clone()) == Some(".")); + match (named, name, dotted) { + (true, Some(name), false) => { + let (head, tail) = text.split_at_checked(name.range.start).unwrap_or((text, "")); + format!("{head}{quoted_schema}.{tail}") + } + _ => text.to_owned(), + } +} + +/// The server keeps the text of the last CREATE or ALTER, so a module +/// changed with ALTER would be altered on load, where nothing exists yet. +fn as_create(text: &str) -> String { + let grammar = SqlGrammar::MsSql; + let plan = ScriptPlan::build(text, grammar, LexicalSettings::default_for(grammar)); + let mut words = plan + .tokens(0..text.len()) + .filter(|token| !token.kind.is_trivia()) + .take(2) + .map(|token| (token.kind, token.range.clone())); + let first = words.next(); + let second = words.next(); + let word = |token: &Option<(ScriptTokenKind, std::ops::Range)>, wanted: &[&str]| { + token.as_ref().is_some_and(|(kind, range)| { + *kind == ScriptTokenKind::Word + && text + .get(range.clone()) + .is_some_and(|word| wanted.iter().any(|candidate| word.eq_ignore_ascii_case(candidate))) + }) + }; + match first { + Some((_, range)) + if word(&first, &["ALTER"]) && word(&second, &["VIEW", "PROC", "PROCEDURE", "FUNCTION", "TRIGGER"]) => + { + format!("CREATE{}", text.get(range.end..).unwrap_or_default()) + } + _ => text.to_owned(), + } +} + +/// `text` with the name its CREATE states made the module's current one. +/// The server keeps a module's text as it was written, and `sp_rename` +/// or `ALTER SCHEMA ... TRANSFER` leaves the old name in it. +fn named_as(text: &str, schema: &str, name: &str) -> Option { + let grammar = SqlGrammar::MsSql; + let plan = ScriptPlan::build(text, grammar, LexicalSettings::default_for(grammar)); + let tokens: Vec = plan + .tokens(0..text.len()) + .filter(|token| !token.kind.is_trivia()) + .collect(); + let word = |at: usize, wanted: &[&str]| { + tokens.get(at).is_some_and(|token| { + token.kind == ScriptTokenKind::Word + && text + .get(token.range.clone()) + .is_some_and(|word| wanted.iter().any(|candidate| word.eq_ignore_ascii_case(candidate))) + }) + }; + let is_name = |at: usize| { + tokens + .get(at) + .is_some_and(|token| matches!(token.kind, ScriptTokenKind::Word | ScriptTokenKind::QuotedIdentifier)) + }; + let is_dot = |at: usize| { + tokens + .get(at) + .is_some_and(|token| text.get(token.range.clone()) == Some(".")) + }; + let mut at = 1; + if !word(0, &["CREATE"]) { + return None; + } + if word(at, &["OR"]) && word(at + 1, &["ALTER"]) { + at += 2; + } + if !word(at, &["VIEW", "PROC", "PROCEDURE", "FUNCTION", "TRIGGER"]) || !is_name(at + 1) { + return None; + } + let first = at + 1; + let mut last = first; + while is_dot(last + 1) && is_name(last + 2) { + last += 2; + } + let start = tokens[first].range.start; + let end = tokens[last].range.end; + Some(format!( + "{}{}.{}{}", + text.get(..start)?, + quote(schema), + quote(name), + text.get(end..)? + )) +} + +fn from_first_token(text: &str) -> &str { + let grammar = SqlGrammar::MsSql; + let plan = ScriptPlan::build(text, grammar, LexicalSettings::default_for(grammar)); + let start = plan + .tokens(0..text.len()) + .find(|token| !token.kind.is_trivia()) + .map_or(text.len(), |token| token.range.start); + text.get(start..).unwrap_or(text) +} + +pub(super) fn disable_trigger(schema: &str, name: &str, relation: &str) -> String { + format!("DISABLE TRIGGER {}.{} ON {relation}", quote(schema), quote(name)) +} + +#[cfg(test)] +mod tests { + use tablepro_core::ddl::ReferentialAction; + + #[test] + fn a_module_last_changed_with_alter_is_made_with_create() { + assert_eq!( + super::as_create("ALTER PROCEDURE p AS SELECT 1"), + "CREATE PROCEDURE p AS SELECT 1" + ); + assert_eq!( + super::as_create("alter view v AS SELECT 1"), + "CREATE view v AS SELECT 1" + ); + assert_eq!( + super::as_create("CREATE OR ALTER VIEW v AS SELECT 1"), + "CREATE OR ALTER VIEW v AS SELECT 1" + ); + assert_eq!(super::as_create("ALTER TABLE t ADD c int"), "ALTER TABLE t ADD c int"); + } + use tablepro_core::meta::IndexKey; + + use super::*; + + fn position<'a>(current: &'a str, used: bool) -> SequencePosition<'a> { + SequencePosition { + start: "100", + current, + increment: "5", + minimum: "1", + maximum: "120", + cycles: false, + used, + } + } + + const ENFORCED: Enforcement = Enforcement { + trusted: true, + disabled: false, + }; + + #[test] + fn a_schema_is_made_once_through_dynamic_sql() { + assert_eq!( + create_schema("o'k]s"), + "IF SCHEMA_ID(N'o''k]s') IS NULL EXEC(N'CREATE SCHEMA [o''k]]s]')" + ); + } + + #[test] + fn an_alias_type_restates_its_base_in_characters() { + let base = |system_type, max_length, nullable| AliasBase { + system_type, + max_length, + precision: 0, + scale: 0, + nullable, + }; + + assert_eq!( + create_alias_type("[app].[label]", &base("nvarchar", 80, false)), + "CREATE TYPE [app].[label] FROM nvarchar(40) NOT NULL" + ); + assert_eq!( + create_alias_type("[app].[blob]", &base("varbinary", -1, true)), + "CREATE TYPE [app].[blob] FROM varbinary(max)" + ); + let money = AliasBase { + system_type: "decimal", + max_length: 9, + precision: 12, + scale: 2, + nullable: true, + }; + assert_eq!( + create_alias_type("[app].[amount]", &money), + "CREATE TYPE [app].[amount] FROM decimal(12,2)" + ); + } + + #[test] + fn a_sequence_starts_where_the_old_one_would_hand_out_next() { + assert_eq!(next_unused(&position("100", false)), "100", "never used"); + assert_eq!(next_unused(&position("110", true)), "115"); + assert_eq!(next_unused(&position("120", true)), "120", "exhausted"); + let cycling = SequencePosition { + cycles: true, + ..position("120", true) + }; + assert_eq!(next_unused(&cycling), "1"); + let down = SequencePosition { + increment: "-5", + cycles: true, + ..position("3", true) + }; + assert_eq!(next_unused(&down), "120"); + let wide = SequencePosition { + increment: "1", + maximum: "99999999999999999999999999999999999999", + ..position("99999999999999999999999999999999999990", true) + }; + assert_eq!(next_unused(&wide), "99999999999999999999999999999999999991"); + } + + #[test] + fn only_a_whole_number_reseeds_and_an_empty_table_one_step_on() { + assert_eq!( + reseed("[app].[o'rders]", "20", "5").as_deref(), + Some( + "IF EXISTS (SELECT 1 FROM [app].[o'rders]) DBCC CHECKIDENT (N'[app].[o''rders]', RESEED, 20) \ + WITH NO_INFOMSGS ELSE DBCC CHECKIDENT (N'[app].[o''rders]', RESEED, 25) WITH NO_INFOMSGS" + ) + ); + assert!(reseed("[t]", "-3", "1").is_some()); + assert_eq!(reseed("[t]", "1; DROP TABLE x", "1"), None); + assert_eq!(reseed("[t]", "", "1"), None); + } + + #[test] + fn a_constraint_keeps_whether_the_server_checks_it() { + assert_eq!( + check_constraint("[app].[t]", "ck", "([price]>=(0))", ENFORCED), + ["ALTER TABLE [app].[t] WITH CHECK ADD CONSTRAINT [ck] CHECK ([price]>=(0))"] + ); + let off = Enforcement { + trusted: false, + disabled: true, + }; + assert_eq!( + check_constraint("[app].[t]", "ck", "[a]>(0)", off), + [ + "ALTER TABLE [app].[t] WITH NOCHECK ADD CONSTRAINT [ck] CHECK ([a]>(0))", + "ALTER TABLE [app].[t] NOCHECK CONSTRAINT [ck]", + ] + ); + } + + #[test] + fn a_foreign_key_is_added_to_its_table_after_the_rows() { + let key = ForeignKeyInfo { + name: "fk_child_parent".to_owned(), + columns: vec!["parent_id".to_owned()], + ref_schema: Some("app".to_owned()), + ref_table: "parent".to_owned(), + ref_columns: vec!["id".to_owned()], + on_delete: Some(ReferentialAction::Cascade), + on_update: None, + }; + let table = TableRef::new(Some("app".to_owned()), "child"); + + assert_eq!( + foreign_key(&MssqlDialect::new(16), &table, &key, ENFORCED), + [ + "ALTER TABLE [app].[child] WITH CHECK ADD CONSTRAINT [fk_child_parent] FOREIGN KEY ([parent_id]) \ + REFERENCES [app].[parent] ([id]) ON DELETE CASCADE" + ] + ); + } + + #[test] + fn a_unique_constraint_stays_a_constraint() { + let index = IndexInfo { + name: "uq".to_owned(), + keys: vec![ + IndexKey::Column { + name: "a".to_owned(), + descending: false, + }, + IndexKey::Column { + name: "b".to_owned(), + descending: true, + }, + ], + unique: true, + primary: false, + predicate: None, + definition: None, + }; + + assert_eq!( + unique_constraint(&MssqlDialect::new(16), "[app].[t]", &index, false), + "ALTER TABLE [app].[t] ADD CONSTRAINT [uq] UNIQUE NONCLUSTERED ([a], [b] DESC)" + ); + } + + #[test] + fn a_module_starts_at_its_create_named_in_its_schema() { + let settings = ModuleSettings { + ansi_nulls: true, + quoted_identifier: true, + }; + let text = "\n-- View: app.v\n/* note */ CREATE VIEW v AS SELECT 1 AS one; -- kept\n"; + + assert_eq!( + module_statements("app", "v", text, settings), + [DumpStatement::body( + "CREATE VIEW [app].[v] AS SELECT 1 AS one; -- kept\n" + )] + ); + assert_eq!( + module_statements("app", "v", "CREATE VIEW app.v AS SELECT 1", settings), + [DumpStatement::body("CREATE VIEW [app].[v] AS SELECT 1")] + ); + assert_eq!( + module_statements("app", "v w", "create or alter view [v w] (a) AS SELECT 1", settings), + [DumpStatement::body("create or alter view [app].[v w] (a) AS SELECT 1")] + ); + assert_eq!( + module_statements("app", "p", "CREATE PROCEDURE p AS SELECT 1", settings), + [DumpStatement::body("CREATE PROCEDURE [app].[p] AS SELECT 1")] + ); + } + + #[test] + fn a_renamed_or_moved_module_is_made_under_its_current_name() { + let settings = ModuleSettings { + ansi_nulls: true, + quoted_identifier: true, + }; + + assert_eq!( + module_statements("sales", "renamed", "ALTER PROCEDURE dbo.old_name AS SELECT 1", settings), + [DumpStatement::body("CREATE PROCEDURE [sales].[renamed] AS SELECT 1")] + ); + } + + #[test] + fn a_module_made_under_other_settings_is_framed_in_them() { + let settings = ModuleSettings { + ansi_nulls: false, + quoted_identifier: false, + }; + + let statements = module_statements("dbo", "p", "CREATE PROCEDURE p AS SELECT \"x\"", settings); + + let sql: Vec<&str> = statements.iter().map(|statement| statement.sql.as_str()).collect(); + assert_eq!( + sql, + [ + "SET ANSI_NULLS OFF", + "SET QUOTED_IDENTIFIER OFF", + "CREATE PROCEDURE [dbo].[p] AS SELECT \"x\"", + "SET QUOTED_IDENTIFIER ON", + "SET ANSI_NULLS ON", + ] + ); + assert!(statements[2].body && !statements[0].body); + } +} diff --git a/core/crates/drivers/mssql/src/dump/mod.rs b/core/crates/drivers/mssql/src/dump/mod.rs new file mode 100644 index 0000000000..15e55a6a03 --- /dev/null +++ b/core/crates/drivers/mssql/src/dump/mod.rs @@ -0,0 +1,972 @@ +//! A SQL Server database or schema as a SQL dump, in the order SMO +//! scripts one: schemas, alias types, sequences and tables before the +//! rows; indexes, identity positions, checks, foreign keys and comments +//! after them; then views and routines, then triggers. + +mod dump_statements; +mod mssql_dump; + +use std::collections::{HashMap, HashSet}; + +use tablepro_core::ddl::ddl_sql::index_keys_sql; +use tablepro_core::ddl::{CollationName, IdentitySeed}; +use tablepro_core::dialect::SqlDialect; +use tablepro_core::dump::{ + DumpCatalog, DumpObject, DumpObjectKind, DumpScope, DumpSection, DumpSkip, DumpSkipReason, DumpStatement, DumpTable, +}; +use tablepro_core::meta::{IndexInfo, TableRef}; +use tablepro_core::sql_syntax::{SqlGrammar, holds_client_directive}; +use tablepro_core::{DriverError, ForeignKeyInfo, Value}; +use tiberius::ColumnData; + +use crate::catalog::Cells; +use crate::ddl::{CreateTableFacts, KeyClause}; +use crate::dialect::MssqlDialect; +use crate::in_band_request::{Interrupt, Watch}; +use crate::metadata::{self, CLR_TYPES, SEQUENCE_COLUMNS, SequenceRow, referential_action, sequence_statement}; +use crate::physical::MssqlClient; +use crate::workspace_call::{catalogue_rows, text_param}; + +use dump_statements::{ + AliasBase, Enforcement, ModuleSettings, SequencePosition, check_constraint, create_alias_type, create_schema, + disable_trigger, foreign_key, module_statements, next_unused, reseed, unique_constraint, +}; +pub(crate) use mssql_dump::MssqlDump; + +/// `{scope}` in each query below is the condition on its schema id. +const SCHEMAS_SQL: &str = "SELECT s.name FROM sys.schemas s WHERE s.name <> N'dbo' AND {scope} ORDER BY s.name"; + +/// Alias types over a system type. Table types and CLR types take syntax +/// this does not write. +const ALIAS_TYPES_SQL: &str = "SELECT SCHEMA_NAME(t.schema_id), t.name, TYPE_NAME(t.system_type_id), \ + CAST(t.max_length AS int), CAST(t.precision AS int), CAST(t.scale AS int), CAST(t.is_nullable AS int) \ + FROM sys.types t \ + WHERE t.is_user_defined = 1 AND t.is_assembly_type = 0 AND t.is_table_type = 0 AND {scope} \ + ORDER BY 1, 2"; + +const TABLES_SQL: &str = "SELECT SCHEMA_NAME(t.schema_id), t.name FROM sys.tables t \ + WHERE t.is_ms_shipped = 0 AND {scope} ORDER BY 1, 2"; + +/// Primary keys and UNIQUE constraints, and whether each one's index is +/// the table's clustered one. +const KEYS_SQL: &str = "SELECT SCHEMA_NAME(t.schema_id), t.name, kc.name, RTRIM(kc.type), \ + CAST(CASE WHEN i.type = 1 THEN 1 ELSE 0 END AS int) \ + FROM sys.key_constraints kc \ + JOIN sys.tables t ON t.object_id = kc.parent_object_id \ + JOIN sys.indexes i ON i.object_id = kc.parent_object_id AND i.index_id = kc.unique_index_id \ + WHERE t.is_ms_shipped = 0 AND {scope} \ + ORDER BY 1, 2, 3"; + +/// Every stored character column's collation. The structure model keeps +/// one only where it differs from the database's default, and the +/// database the file loads into can have another. A column of an alias +/// type takes the database's. +const COLLATIONS_SQL: &str = "SELECT SCHEMA_NAME(t.schema_id), t.name, c.name, c.collation_name \ + FROM sys.columns c JOIN sys.tables t ON t.object_id = c.object_id \ + JOIN sys.types ty ON ty.user_type_id = c.user_type_id \ + WHERE c.collation_name IS NOT NULL AND c.is_computed = 0 AND ty.is_user_defined = 0 \ + AND t.is_ms_shipped = 0 AND {scope} \ + ORDER BY 1, 2, 3"; + +/// Tables an enabled security policy filters, whose rows a read sees +/// only some of. +const FILTERED_SQL: &str = "SELECT DISTINCT SCHEMA_NAME(t.schema_id), t.name \ + FROM sys.security_predicates p \ + JOIN sys.security_policies s ON s.object_id = p.object_id \ + JOIN sys.tables t ON t.object_id = p.target_object_id \ + WHERE s.is_enabled = 1 AND p.predicate_type = 0 AND t.is_ms_shipped = 0 AND {scope} \ + ORDER BY 1, 2"; + +/// The columns of a system-time period, which the server fills. +const PERIODS_SQL: &str = "SELECT SCHEMA_NAME(t.schema_id), t.name, c.name \ + FROM sys.columns c JOIN sys.tables t ON t.object_id = c.object_id \ + WHERE c.generated_always_type IN (1, 2) AND t.is_ms_shipped = 0 AND {scope} \ + ORDER BY 1, 2, 3"; + +/// `last_value` is NULL until a row takes a number. +const IDENTITIES_SQL: &str = "SELECT SCHEMA_NAME(t.schema_id), t.name, CAST(ic.seed_value AS nvarchar(64)), \ + CAST(ic.increment_value AS nvarchar(64)), CAST(ic.last_value AS nvarchar(64)) \ + FROM sys.identity_columns ic JOIN sys.tables t ON t.object_id = ic.object_id \ + WHERE t.is_ms_shipped = 0 AND {scope} \ + ORDER BY 1, 2"; + +const CHECKS_SQL: &str = "SELECT SCHEMA_NAME(t.schema_id), t.name, cc.name, cc.definition, \ + CAST(cc.is_not_trusted AS int), CAST(cc.is_disabled AS int) \ + FROM sys.check_constraints cc JOIN sys.tables t ON t.object_id = cc.parent_object_id \ + WHERE t.is_ms_shipped = 0 AND {scope} \ + ORDER BY 1, 2, 3"; + +const FOREIGN_KEYS_SQL: &str = "SELECT SCHEMA_NAME(t.schema_id), t.name, fk.name, pc.name, \ + SCHEMA_NAME(rt.schema_id), rt.name, rc.name, fk.delete_referential_action_desc, \ + fk.update_referential_action_desc, CAST(fk.is_not_trusted AS int), CAST(fk.is_disabled AS int) \ + FROM sys.foreign_keys fk \ + JOIN sys.tables t ON t.object_id = fk.parent_object_id \ + JOIN sys.foreign_key_columns fkc ON fkc.constraint_object_id = fk.object_id \ + JOIN sys.columns pc ON pc.object_id = fkc.parent_object_id AND pc.column_id = fkc.parent_column_id \ + JOIN sys.objects rt ON rt.object_id = fk.referenced_object_id \ + JOIN sys.columns rc ON rc.object_id = fkc.referenced_object_id AND rc.column_id = fkc.referenced_column_id \ + WHERE t.is_ms_shipped = 0 AND {scope} \ + ORDER BY 1, 2, 3, fkc.constraint_column_id"; + +/// Views, functions and procedures, with what tells why one has no text +/// and the SET options it was made under. A CLR module has no +/// `sys.sql_modules` row. +const MODULES_SQL: &str = "SELECT SCHEMA_NAME(o.schema_id), o.name, RTRIM(o.type), OBJECT_DEFINITION(o.object_id), \ + CAST(OBJECTPROPERTY(o.object_id, 'IsEncrypted') AS int), \ + CAST(m.uses_ansi_nulls AS int), CAST(m.uses_quoted_identifier AS int), \ + CAST(CASE WHEN EXISTS (SELECT 1 FROM sys.sql_expression_dependencies d \ + JOIN sys.objects r ON r.object_id = d.referenced_id \ + WHERE d.referencing_id = o.object_id AND r.type IN ('U', 'V')) THEN 1 ELSE 0 END AS int), \ + CAST(ISNULL(m.is_schema_bound, 0) AS int) \ + FROM sys.objects o LEFT JOIN sys.sql_modules m ON m.object_id = o.object_id \ + WHERE o.type IN ('V', 'FN', 'IF', 'TF', 'FS', 'FT', 'P', 'PC') AND o.is_ms_shipped = 0 AND {scope} \ + ORDER BY 1, 2"; + +/// The indexes of indexed views, which the dump does not make. +const VIEW_INDEXES_SQL: &str = "SELECT SCHEMA_NAME(v.schema_id), i.name \ + FROM sys.indexes i JOIN sys.views v ON v.object_id = i.object_id \ + WHERE i.index_id > 0 AND v.is_ms_shipped = 0 AND {scope} \ + ORDER BY 1, 2"; + +/// DML triggers of tables and views; `parent_class` 1 leaves out the +/// database's DDL triggers. +const TRIGGERS_SQL: &str = "SELECT SCHEMA_NAME(p.schema_id), tr.name, p.name, RTRIM(tr.type), \ + OBJECT_DEFINITION(tr.object_id), CAST(OBJECTPROPERTY(tr.object_id, 'IsEncrypted') AS int), \ + CAST(m.uses_ansi_nulls AS int), CAST(m.uses_quoted_identifier AS int), CAST(tr.is_disabled AS int) \ + FROM sys.triggers tr JOIN sys.objects p ON p.object_id = tr.parent_id \ + LEFT JOIN sys.sql_modules m ON m.object_id = tr.object_id \ + WHERE tr.parent_class = 1 AND tr.is_ms_shipped = 0 AND {scope} \ + ORDER BY 1, 3, 2"; + +/// The sequences, with where each stands. `last_used_value`, which says +/// whether one was ever used, came with SQL Server 2017; before it, a +/// sequence still at its start reads as unused. +fn sequences_sql(major_version: u32) -> String { + let used = match major_version >= 14 { + true => "CAST(CASE WHEN s.last_used_value IS NULL THEN 0 ELSE 1 END AS int)", + false => "CAST(CASE WHEN s.current_value = s.start_value THEN 0 ELSE 1 END AS int)", + }; + format!( + "SELECT SCHEMA_NAME(s.schema_id), s.name, {SEQUENCE_COLUMNS}, CAST(s.current_value AS nvarchar(64)), {used} \ + FROM sys.sequences s JOIN sys.types t ON t.user_type_id = s.user_type_id \ + WHERE {{scope}} ORDER BY 1, 2" + ) +} + +/// The condition that keeps a query to the scope's schemas: every one of +/// the database's but the server's own and the fixed roles' (ids from +/// 16384), or the one named in `@P1`. +fn scope_condition(scope: &DumpScope, schema_id: &str) -> String { + match scope { + DumpScope::Database => { + format!("{schema_id} < 16384 AND SCHEMA_NAME({schema_id}) NOT IN (N'sys', N'INFORMATION_SCHEMA', N'guest')") + } + DumpScope::Schema(_) => format!("{schema_id} = SCHEMA_ID(@P1)"), + } +} + +fn scope_params(scope: &DumpScope) -> Vec> { + match scope { + DumpScope::Database => Vec::new(), + DumpScope::Schema(schema) => vec![text_param(schema)], + } +} + +/// What goes after the rows, kept apart by kind so the file has each +/// kind together in the order a load needs. +#[derive(Default)] +struct AfterData { + indexes: Vec, + positions: Vec, + checks: Vec, + foreign_keys: Vec, + comments: Vec, +} + +impl AfterData { + fn into_objects(self) -> impl Iterator { + self.indexes + .into_iter() + .chain(self.positions) + .chain(self.checks) + .chain(self.foreign_keys) + .chain(self.comments) + } +} + +fn after(kind: DumpObjectKind, schema: Option, name: &str, statements: Vec) -> DumpObject { + DumpObject::new( + kind, + DumpSection::AfterData, + schema, + name, + statements.into_iter().map(DumpStatement::new).collect(), + ) +} + +pub(crate) async fn read_dump_catalog( + client: &mut MssqlClient, + watch: &Watch, + dialect: &MssqlDialect, + scope: &DumpScope, +) -> Result { + let mut read = CatalogRead { + client, + watch, + dialect, + scope, + catalog: DumpCatalog::default(), + after_data: AfterData::default(), + }; + read.before_data().await?; + read.tables().await?; + read.constraints().await?; + read.modules().await?; + let CatalogRead { + mut catalog, + after_data, + .. + } = read; + catalog.objects.extend(after_data.into_objects()); + Ok(catalog) +} + +/// One dump's catalogue read, on one connection. +struct CatalogRead<'a> { + client: &'a mut MssqlClient, + watch: &'a Watch, + dialect: &'a MssqlDialect, + scope: &'a DumpScope, + catalog: DumpCatalog, + after_data: AfterData, +} + +impl CatalogRead<'_> { + /// `sql`'s rows with its `{scope}` the condition on `schema_id`. + async fn rows(&mut self, sql: &str, schema_id: &str) -> Result>, Interrupt> { + let sql = sql.replace("{scope}", &scope_condition(self.scope, schema_id)); + catalogue_rows(self.client, self.watch, &sql, &scope_params(self.scope)).await + } + + async fn before_data(&mut self) -> Result<(), Interrupt> { + for row in self.rows(SCHEMAS_SQL, "s.schema_id").await? { + let name = Cells::new(&row, "sys.schemas").required_text(0)?; + let create = DumpStatement::new(create_schema(&name)); + self.catalog.objects.push(DumpObject::new( + DumpObjectKind::Schema, + DumpSection::BeforeData, + None, + name, + vec![create], + )); + } + for row in self.rows(ALIAS_TYPES_SQL, "t.schema_id").await? { + self.catalog.objects.push(alias_type(self.dialect, &row)?); + } + let sequences = sequences_sql(self.dialect.major_version()); + for row in self.rows(&sequences, "s.schema_id").await? { + self.catalog.objects.push(sequence(self.dialect, &row)?); + } + Ok(()) + } + + async fn tables(&mut self) -> Result<(), Interrupt> { + let keys = key_rows(&self.rows(KEYS_SQL, "t.schema_id").await?)?; + let identities = identity_rows(&self.rows(IDENTITIES_SQL, "t.schema_id").await?)?; + let periods = period_rows(&self.rows(PERIODS_SQL, "t.schema_id").await?)?; + let collations = collation_rows(&self.rows(COLLATIONS_SQL, "t.schema_id").await?)?; + let mut filtered = HashSet::new(); + for row in self.rows(FILTERED_SQL, "t.schema_id").await? { + let cells = Cells::new(&row, "sys.security_predicates"); + filtered.insert(TableRef::new(cells.text(0), cells.required_text(1)?)); + } + for row in self.rows(TABLES_SQL, "t.schema_id").await? { + let cells = Cells::new(&row, "sys.tables"); + let table = TableRef::new(cells.text(0), cells.required_text(1)?); + let parts = TableParts { + keys: keys.get(&table).map(Vec::as_slice).unwrap_or_default(), + identity: identities.get(&table), + periods: periods.get(&table).map(Vec::as_slice).unwrap_or_default(), + collations: collations.get(&table), + filtered: filtered.contains(&table), + }; + self.table(&table, &parts).await?; + } + Ok(()) + } + + /// The table's CREATE without its foreign keys, which go after every + /// table's rows, and what follows its rows: indexes, UNIQUE + /// constraints, the identity's position and column comments. + async fn table(&mut self, table: &TableRef, parts: &TableParts<'_>) -> Result<(), Interrupt> { + let dialect = self.dialect; + let relation = dialect.qualify(table); + let mut definition = + metadata::fetch_table_definition(self.client, self.watch, table, &relation, dialect.major_version()) + .await?; + definition.foreign_keys.clear(); + if let Some(collations) = parts.collations { + for column in &mut definition.columns { + if column.attributes.collation.is_none() + && let Some(collation) = collations.get(&column.info.name) + { + column.attributes.collation = Some(CollationName::new(None, collation)); + } + } + } + let unique = |index: &IndexInfo| parts.keys.iter().any(|key| !key.primary && key.name == index.name); + let (uniques, indexes): (Vec, Vec) = + std::mem::take(&mut definition.indexes).into_iter().partition(unique); + definition.indexes = indexes; + let facts = CreateTableFacts { + key: key_clause(dialect, &definition.indexes, parts.keys), + identity: parts + .identity + .and_then(|identity| IdentitySeed::new(&identity.seed, &identity.increment)), + }; + let script = dialect + .table_script(&definition, &facts) + .map_err(|error| DriverError::Decode { + column: format!("table {relation}"), + detail: error.to_string(), + })?; + + self.catalog.objects.push(DumpObject::table( + table.clone(), + vec![DumpStatement::new(script.create)], + )); + // The structure model has no period or SYSTEM_VERSIONING, so the + // table loads as an ordinary one, with the times its rows hold. + if !parts.periods.is_empty() { + self.catalog.skipped.push(DumpSkip { + kind: DumpObjectKind::Table, + schema: table.schema.clone(), + name: table.name.clone(), + reason: DumpSkipReason::VersioningNotCarried, + }); + } + // As pg_dump does under row-level security, no rows rather than + // the ones this login sees. + if parts.filtered { + self.catalog.skipped.push(DumpSkip { + kind: DumpObjectKind::Table, + schema: table.schema.clone(), + name: table.name.clone(), + reason: DumpSkipReason::RowsFiltered, + }); + } + self.catalog.tables.push(DumpTable { + plain_columns: parts.periods.to_vec(), + rows: !parts.filtered, + ..DumpTable::new(table.clone()) + }); + let schema = || table.schema.clone(); + let indexes: Vec = script.indexes.into_iter().chain(script.stated).collect(); + if !indexes.is_empty() { + self.after_data + .indexes + .push(after(DumpObjectKind::Index, schema(), &table.name, indexes)); + } + for index in &uniques { + let clustered = parts.keys.iter().any(|key| key.name == index.name && key.clustered); + let statement = unique_constraint(dialect, &relation, index, clustered); + self.after_data.indexes.push(after( + DumpObjectKind::Constraint, + schema(), + &index.name, + vec![statement], + )); + } + if let Some(identity) = parts.identity + && let Some(last_value) = identity.last_value.as_deref() + { + let statements = reseed(&relation, last_value, &identity.increment).into_iter().collect(); + self.after_data + .positions + .push(after(DumpObjectKind::SequenceValue, schema(), &table.name, statements)); + } + if !script.comments.is_empty() { + self.after_data + .comments + .push(after(DumpObjectKind::Comment, schema(), &table.name, script.comments)); + } + Ok(()) + } + + /// CHECK constraints and foreign keys, which the structure model + /// either leaves out or puts inside the CREATE. + async fn constraints(&mut self) -> Result<(), Interrupt> { + for row in self.rows(CHECKS_SQL, "t.schema_id").await? { + let cells = Cells::new(&row, "sys.check_constraints"); + let table = TableRef::new(cells.text(0), cells.required_text(1)?); + let name = cells.required_text(2)?; + let Some(definition) = cells.text(3) else { + self.catalog.skipped.push(DumpSkip { + kind: DumpObjectKind::Constraint, + schema: table.schema.clone(), + name, + reason: DumpSkipReason::NoDefinition, + }); + continue; + }; + let enforcement = Enforcement { + trusted: !cells.flag(4), + disabled: cells.flag(5), + }; + let statements = check_constraint(&self.dialect.qualify(&table), &name, &definition, enforcement); + self.after_data + .checks + .push(after(DumpObjectKind::Constraint, table.schema, &name, statements)); + } + let rows = self.rows(FOREIGN_KEYS_SQL, "t.schema_id").await?; + for found in foreign_key_rows(&rows)? { + let statements = foreign_key(self.dialect, &found.table, &found.key, found.enforcement); + self.after_data.foreign_keys.push(after( + DumpObjectKind::ForeignKey, + found.table.schema, + &found.key.name, + statements, + )); + } + Ok(()) + } + + async fn modules(&mut self) -> Result<(), Interrupt> { + for row in self.rows(MODULES_SQL, "o.schema_id").await? { + let cells = Cells::new(&row, "sys.sql_modules"); + let module = Module { + schema: cells.required_text(0)?, + name: cells.required_text(1)?, + code: cells.required_text(2)?, + text: cells.text(3), + encrypted: cells.flag(4), + settings: settings(&cells, 5), + }; + let section = module_section(&module.code, cells.flag(7), cells.flag(8)); + place_module(&mut self.catalog, module, section, None); + } + for row in self.rows(VIEW_INDEXES_SQL, "v.schema_id").await? { + let cells = Cells::new(&row, "sys.indexes"); + self.catalog.skipped.push(DumpSkip { + kind: DumpObjectKind::Index, + schema: cells.text(0), + name: cells.required_text(1)?, + reason: DumpSkipReason::NotSupported, + }); + } + for row in self.rows(TRIGGERS_SQL, "p.schema_id").await? { + let cells = Cells::new(&row, "sys.triggers"); + let schema = cells.required_text(0)?; + let table = TableRef::new(Some(schema.clone()), cells.required_text(2)?); + let module = Module { + schema, + name: cells.required_text(1)?, + code: cells.required_text(3)?, + text: cells.text(4), + encrypted: cells.flag(5), + settings: settings(&cells, 6), + }; + let disabled = cells.flag(8).then(|| self.dialect.qualify(&table)); + place_module(&mut self.catalog, module, DumpSection::Triggers, disabled); + } + Ok(()) + } +} + +/// The primary key as its constraint and index are, where both are read. +fn key_clause(dialect: &MssqlDialect, indexes: &[IndexInfo], keys: &[KeyRow]) -> Option { + let key = keys.iter().find(|key| key.primary)?; + let index = indexes.iter().find(|index| index.primary && index.name == key.name)?; + Some(KeyClause { + name: key.name.clone(), + clustered: key.clustered, + keys: index_keys_sql(dialect, index), + }) +} + +fn alias_type(dialect: &MssqlDialect, row: &[Value]) -> Result { + let cells = Cells::new(row, "sys.types"); + let schema = cells.text(0); + let name = cells.required_text(1)?; + let system_type = cells.required_text(2)?; + let base = AliasBase { + system_type: &system_type, + max_length: cells.int(3).unwrap_or(0), + precision: cells.int(4).unwrap_or(0), + scale: cells.int(5).unwrap_or(0), + nullable: cells.flag(6), + }; + let qualified = dialect.qualify(&TableRef::new(schema.clone(), name.clone())); + let create = DumpStatement::new(create_alias_type(&qualified, &base)); + Ok(DumpObject::new( + DumpObjectKind::Type, + DumpSection::BeforeData, + schema, + name, + vec![create], + )) +} + +/// The sequence made again where the old one would hand out its next +/// value, so a load hands out none twice. +fn sequence(dialect: &MssqlDialect, row: &[Value]) -> Result { + let cells = Cells::new(row, "sys.sequences"); + let schema = cells.text(0); + let name = cells.required_text(1)?; + let settings = row.get(2..11).unwrap_or_default(); + let mut sequence = SequenceRow::from_cells(&Cells::new(settings, "sys.sequences"))?; + let current = cells.required_text(11)?; + sequence.start = next_unused(&SequencePosition { + start: &sequence.start, + current: ¤t, + increment: &sequence.increment, + minimum: &sequence.minimum, + maximum: &sequence.maximum, + cycles: sequence.cycles, + used: cells.flag(12), + }); + let qualified = dialect.qualify(&TableRef::new(schema.clone(), name.clone())); + let create = DumpStatement::new(sequence_statement(&qualified, &sequence)); + Ok(DumpObject::new( + DumpObjectKind::Sequence, + DumpSection::BeforeData, + schema, + name, + vec![create], + )) +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct KeyRow { + name: String, + primary: bool, + clustered: bool, +} + +fn key_rows(rows: &[Vec]) -> Result>, DriverError> { + let mut keys: HashMap> = HashMap::new(); + for row in rows { + let cells = Cells::new(row, "sys.key_constraints"); + let table = TableRef::new(cells.text(0), cells.required_text(1)?); + keys.entry(table).or_default().push(KeyRow { + name: cells.required_text(2)?, + primary: cells.text(3).as_deref() == Some("PK"), + clustered: cells.flag(4), + }); + } + Ok(keys) +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct IdentityRow { + seed: String, + increment: String, + last_value: Option, +} + +fn identity_rows(rows: &[Vec]) -> Result, DriverError> { + let mut identities = HashMap::new(); + for row in rows { + let cells = Cells::new(row, "sys.identity_columns"); + let table = TableRef::new(cells.text(0), cells.required_text(1)?); + identities.insert( + table, + IdentityRow { + seed: cells.required_text(2)?, + increment: cells.required_text(3)?, + last_value: cells.text(4), + }, + ); + } + Ok(identities) +} + +/// What the scope-wide reads say about one table. +struct TableParts<'a> { + keys: &'a [KeyRow], + identity: Option<&'a IdentityRow>, + periods: &'a [String], + /// Each character column's collation, by column name. + collations: Option<&'a HashMap>, + filtered: bool, +} + +fn collation_rows(rows: &[Vec]) -> Result>, DriverError> { + let mut collations: HashMap> = HashMap::new(); + for row in rows { + let cells = Cells::new(row, "sys.columns"); + let table = TableRef::new(cells.text(0), cells.required_text(1)?); + collations + .entry(table) + .or_default() + .insert(cells.required_text(2)?, cells.required_text(3)?); + } + Ok(collations) +} + +fn period_rows(rows: &[Vec]) -> Result>, DriverError> { + let mut periods: HashMap> = HashMap::new(); + for row in rows { + let cells = Cells::new(row, "sys.columns"); + let table = TableRef::new(cells.text(0), cells.required_text(1)?); + periods.entry(table).or_default().push(cells.required_text(2)?); + } + Ok(periods) +} + +struct ForeignKeyRow { + table: TableRef, + key: ForeignKeyInfo, + enforcement: Enforcement, +} + +/// One row per column pair, in key order, grouped into keys. +fn foreign_key_rows(rows: &[Vec]) -> Result, DriverError> { + let mut keys: Vec = Vec::new(); + for row in rows { + let cells = Cells::new(row, "sys.foreign_keys"); + let table = TableRef::new(cells.text(0), cells.required_text(1)?); + let name = cells.required_text(2)?; + let column = cells.required_text(3)?; + let referenced = cells.required_text(6)?; + match keys.last_mut() { + Some(last) if last.table == table && last.key.name == name => { + last.key.columns.push(column); + last.key.ref_columns.push(referenced); + } + _ => keys.push(ForeignKeyRow { + table, + key: ForeignKeyInfo { + name, + columns: vec![column], + ref_schema: cells.text(4), + ref_table: cells.required_text(5)?, + ref_columns: vec![referenced], + on_delete: cells.text(7).as_deref().and_then(referential_action), + on_update: cells.text(8).as_deref().and_then(referential_action), + }, + enforcement: Enforcement { + trusted: !cells.flag(9), + disabled: cells.flag(10), + }, + }), + } + } + Ok(keys) +} + +/// A view, routine or trigger as the catalogue gives it. +struct Module { + schema: String, + name: String, + code: String, + text: Option, + encrypted: bool, + settings: ModuleSettings, +} + +/// Where a module goes. A table's DEFAULT, CHECK or computed column can +/// call a scalar or multi-statement function, so those come before the +/// tables: the server finds the tables their bodies name when they run. +/// A schema-bound one that names a table or view needs it to exist, as +/// inline functions and views do, so those wait for the tables. +fn module_section(code: &str, names_relations: bool, schema_bound: bool) -> DumpSection { + match code { + "FN" | "TF" if !(names_relations && schema_bound) => DumpSection::BeforeData, + _ => DumpSection::Programs, + } +} + +/// A module with no `sys.sql_modules` row reads as made under the +/// defaults. +fn settings(cells: &Cells<'_>, first: usize) -> ModuleSettings { + let on = |index: usize| cells.int(index).is_none_or(|value| value != 0); + ModuleSettings { + ansi_nulls: on(first), + quoted_identifier: on(first + 1), + } +} + +fn module_kind(code: &str) -> DumpObjectKind { + match code { + "V" => DumpObjectKind::View, + "P" | "PC" => DumpObjectKind::Procedure, + "TR" | "TA" => DumpObjectKind::Trigger, + _ => DumpObjectKind::Function, + } +} + +/// The module in `section`, or in the skipped list with why it has no +/// text. `disabled_on` names the table of a trigger that is turned off. +fn place_module(catalog: &mut DumpCatalog, module: Module, section: DumpSection, disabled_on: Option) { + let kind = module_kind(&module.code); + let Some(text) = module.text else { + let reason = if CLR_TYPES.contains(&module.code.as_str()) { + DumpSkipReason::ClrRoutine + } else if module.encrypted { + DumpSkipReason::Encrypted + } else { + DumpSkipReason::NoDefinition + }; + catalog.skipped.push(DumpSkip { + kind, + schema: Some(module.schema), + name: module.name, + reason, + }); + return; + }; + // A line that is only GO inside the text would end the batch there on + // load, and make a different module. + if holds_client_directive(&text, SqlGrammar::MsSql) { + catalog.skipped.push(DumpSkip { + kind, + schema: Some(module.schema), + name: module.name, + reason: DumpSkipReason::SplitsApart, + }); + return; + } + let mut statements = module_statements(&module.schema, &module.name, &text, module.settings); + if let Some(table) = disabled_on { + statements.push(DumpStatement::new(disable_trigger( + &module.schema, + &module.name, + &table, + ))); + } + catalog.objects.push(DumpObject::new( + kind, + section, + Some(module.schema), + module.name, + statements, + )); +} + +#[cfg(test)] +mod tests { + use tablepro_core::ddl::ReferentialAction; + + use super::*; + + fn text(value: &str) -> Value { + Value::Text(value.to_owned()) + } + + fn module(code: &str, text: Option<&str>, encrypted: bool) -> Module { + Module { + schema: "app".to_owned(), + name: "m".to_owned(), + code: code.to_owned(), + text: text.map(str::to_owned), + encrypted, + settings: ModuleSettings { + ansi_nulls: true, + quoted_identifier: true, + }, + } + } + + #[test] + fn a_scope_is_the_users_schemas_or_the_one_named() { + assert_eq!( + scope_condition(&DumpScope::Database, "o.schema_id"), + "o.schema_id < 16384 AND SCHEMA_NAME(o.schema_id) NOT IN (N'sys', N'INFORMATION_SCHEMA', N'guest')" + ); + assert_eq!( + scope_condition(&DumpScope::Schema("app".to_owned()), "t.schema_id"), + "t.schema_id = SCHEMA_ID(@P1)" + ); + assert!(scope_params(&DumpScope::Database).is_empty()); + assert_eq!(scope_params(&DumpScope::Schema("app".to_owned())).len(), 1); + } + + #[test] + fn every_query_is_ordered_and_scoped() { + for sql in [ + SCHEMAS_SQL, + ALIAS_TYPES_SQL, + TABLES_SQL, + KEYS_SQL, + IDENTITIES_SQL, + PERIODS_SQL, + COLLATIONS_SQL, + FILTERED_SQL, + CHECKS_SQL, + FOREIGN_KEYS_SQL, + MODULES_SQL, + VIEW_INDEXES_SQL, + TRIGGERS_SQL, + &sequences_sql(16), + &sequences_sql(13), + ] { + assert!(sql.contains("{scope}") && sql.contains(" ORDER BY "), "{sql}"); + } + assert!(sequences_sql(14).contains("last_used_value")); + assert!(!sequences_sql(13).contains("last_used_value")); + } + + #[test] + fn foreign_key_rows_group_by_table_and_constraint() { + let row = |table: &str, name: &str, column: &str, referenced: &str, trusted: i64| { + vec![ + text("app"), + text(table), + text(name), + text(column), + text("app"), + text("parent"), + text(referenced), + text("CASCADE"), + text("NO_ACTION"), + Value::Int(1 - trusted), + Value::Int(0), + ] + }; + let rows = vec![ + row("child", "fk", "tenant", "tenant", 1), + row("child", "fk", "parent_id", "id", 1), + row("other", "fk", "parent_id", "id", 0), + ]; + + let keys = foreign_key_rows(&rows).unwrap(); + + assert_eq!(keys.len(), 2); + assert_eq!(keys[0].key.columns, ["tenant", "parent_id"]); + assert_eq!(keys[0].key.ref_columns, ["tenant", "id"]); + assert_eq!(keys[0].key.on_delete, Some(ReferentialAction::Cascade)); + assert!(keys[0].enforcement.trusted); + assert_eq!(keys[1].table, TableRef::new(Some("app".to_owned()), "other")); + assert!(!keys[1].enforcement.trusted); + } + + #[test] + fn key_rows_tell_the_primary_key_from_a_unique_one() { + let rows = vec![ + vec![text("app"), text("t"), text("pk_t"), text("PK"), Value::Int(1)], + vec![text("app"), text("t"), text("uq_t"), text("UQ"), Value::Int(0)], + ]; + + let keys = key_rows(&rows).unwrap(); + + let table = TableRef::new(Some("app".to_owned()), "t"); + assert_eq!( + keys.get(&table).map(Vec::as_slice), + Some( + &[ + KeyRow { + name: "pk_t".to_owned(), + primary: true, + clustered: true, + }, + KeyRow { + name: "uq_t".to_owned(), + primary: false, + clustered: false, + }, + ][..] + ) + ); + } + + #[test] + fn a_sequence_is_made_again_at_its_next_value() { + let row = vec![ + text("app"), + text("ticket"), + text("int"), + text("100"), + text("5"), + text("-2147483648"), + text("2147483647"), + Value::Int(0), + Value::Int(1), + Value::Null, + Value::Int(10), + text("110"), + Value::Int(1), + ]; + + let object = sequence(&MssqlDialect::new(16), &row).unwrap(); + + assert_eq!(object.kind, DumpObjectKind::Sequence); + assert_eq!( + object.statements, + [DumpStatement::new( + "CREATE SEQUENCE [app].[ticket] AS int START WITH 115 INCREMENT BY 5 MINVALUE -2147483648 \ + MAXVALUE 2147483647 NO CYCLE CACHE;" + )] + ); + } + + #[test] + fn a_module_without_text_is_skipped_with_why() { + let mut catalog = DumpCatalog::default(); + + place_module(&mut catalog, module("PC", None, false), DumpSection::Programs, None); + place_module(&mut catalog, module("P", None, true), DumpSection::Programs, None); + place_module(&mut catalog, module("V", None, false), DumpSection::Programs, None); + + let reasons: Vec<(DumpObjectKind, DumpSkipReason)> = + catalog.skipped.iter().map(|skip| (skip.kind, skip.reason)).collect(); + assert_eq!( + reasons, + [ + (DumpObjectKind::Procedure, DumpSkipReason::ClrRoutine), + (DumpObjectKind::Procedure, DumpSkipReason::Encrypted), + (DumpObjectKind::View, DumpSkipReason::NoDefinition), + ] + ); + assert!(catalog.objects.is_empty()); + } + + #[test] + fn a_disabled_trigger_is_made_then_turned_off() { + let mut catalog = DumpCatalog::default(); + let trigger = module("TR", Some("CREATE TRIGGER tr ON t AFTER INSERT AS SELECT 1"), false); + + place_module( + &mut catalog, + trigger, + DumpSection::Triggers, + Some("[app].[t]".to_owned()), + ); + + let object = &catalog.objects[0]; + assert_eq!( + (object.kind, object.section), + (DumpObjectKind::Trigger, DumpSection::Triggers) + ); + let sql: Vec<&str> = object + .statements + .iter() + .map(|statement| statement.sql.as_str()) + .collect(); + assert_eq!( + sql, + [ + "CREATE TRIGGER [app].[m] ON [app].t AFTER INSERT AS SELECT 1", + "DISABLE TRIGGER [app].[m] ON [app].[t]", + ] + ); + } + + #[test] + fn a_module_that_a_go_line_would_split_is_left_out() { + let mut catalog = DumpCatalog::default(); + let split = module("P", Some("CREATE PROCEDURE p AS\nSELECT 1\nGO\nSELECT 2"), false); + + place_module(&mut catalog, split, DumpSection::Programs, None); + + assert!(catalog.objects.is_empty()); + assert_eq!( + catalog + .skipped + .iter() + .map(|skip| (skip.kind, skip.reason)) + .collect::>(), + [(DumpObjectKind::Procedure, DumpSkipReason::SplitsApart)] + ); + } +} diff --git a/core/crates/drivers/mssql/src/dump/mssql_dump.rs b/core/crates/drivers/mssql/src/dump/mssql_dump.rs new file mode 100644 index 0000000000..8ac4131ebc --- /dev/null +++ b/core/crates/drivers/mssql/src/dump/mssql_dump.rs @@ -0,0 +1,97 @@ +use tablepro_core::dump::{DumpDialect, DumpStatement}; + +/// How SQL Server writes a dump: every statement a batch of its own, +/// since CREATE VIEW, FUNCTION, PROCEDURE and TRIGGER must each open one, +/// and sqlcmd and SSMS send a file a `GO` at a time. +#[derive(Debug)] +pub(crate) struct MssqlDump; + +impl DumpDialect for MssqlDump { + /// The settings SMO scripts under; a module made under others is + /// framed in its own. + fn header(&self) -> String { + ["SET ANSI_NULLS ON", "SET QUOTED_IDENTIFIER ON"] + .into_iter() + .map(|sql| self.statement(&DumpStatement::new(sql))) + .collect() + } + + /// A batch of its own: SQL Server keeps a module's text from the + /// start of its batch, so a comment there would join the module. + fn comment(&self, text: &str) -> String { + format!("\n-- {text}\nGO\n") + } + + fn statement(&self, statement: &DumpStatement) -> String { + format!("{}\nGO\n", statement.sql) + } + + fn identity_insert(&self, table_sql: &str, on: bool) -> Option { + let state = if on { "ON" } else { "OFF" }; + Some(format!("SET IDENTITY_INSERT {table_sql} {state}")) + } + + fn transactional_ddl(&self) -> bool { + true + } + + fn begin(&self) -> &'static str { + "BEGIN TRANSACTION" + } + + fn commit(&self) -> &'static str { + "COMMIT TRANSACTION" + } + + fn rollback(&self) -> &'static str { + "ROLLBACK TRANSACTION" + } + + /// Without it an error can roll back only its own statement and leave + /// the transaction open, so the rest of the file would still commit. + fn after_begin(&self) -> &'static [&'static str] { + &["SET XACT_ABORT ON"] + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn every_statement_ends_its_batch() { + assert_eq!( + MssqlDump.header(), + "SET ANSI_NULLS ON\nGO\nSET QUOTED_IDENTIFIER ON\nGO\n" + ); + assert_eq!( + MssqlDump.statement(&DumpStatement::body("CREATE PROCEDURE p AS SELECT 1; -- done")), + "CREATE PROCEDURE p AS SELECT 1; -- done\nGO\n" + ); + assert_eq!( + MssqlDump.statement(&DumpStatement::new("INSERT INTO [t] ([a]) VALUES\n(1)")), + "INSERT INTO [t] ([a]) VALUES\n(1)\nGO\n" + ); + } + + #[test] + fn an_identity_insert_is_turned_on_and_off_by_name() { + assert_eq!( + MssqlDump.identity_insert("[app].[t]", true).as_deref(), + Some("SET IDENTITY_INSERT [app].[t] ON") + ); + assert_eq!( + MssqlDump.identity_insert("[app].[t]", false).as_deref(), + Some("SET IDENTITY_INSERT [app].[t] OFF") + ); + } + + #[test] + fn a_load_is_one_transaction_that_aborts_on_any_error() { + assert!(MssqlDump.transactional_ddl()); + assert_eq!(MssqlDump.begin(), "BEGIN TRANSACTION"); + assert_eq!(MssqlDump.commit(), "COMMIT TRANSACTION"); + assert_eq!(MssqlDump.rollback(), "ROLLBACK TRANSACTION"); + assert_eq!(MssqlDump.after_begin(), ["SET XACT_ABORT ON"]); + } +} diff --git a/core/crates/drivers/mssql/src/lib.rs b/core/crates/drivers/mssql/src/lib.rs index 0bd477a0ff..e72d3e0759 100644 --- a/core/crates/drivers/mssql/src/lib.rs +++ b/core/crates/drivers/mssql/src/lib.rs @@ -16,6 +16,7 @@ mod ddl; mod decode; mod dialect; mod driver; +mod dump; mod editor_run; mod editor_session; mod encode; diff --git a/core/crates/drivers/mssql/src/metadata.rs b/core/crates/drivers/mssql/src/metadata.rs index 065ce0d16a..a055129c55 100644 --- a/core/crates/drivers/mssql/src/metadata.rs +++ b/core/crates/drivers/mssql/src/metadata.rs @@ -71,20 +71,28 @@ const TRIGGER_DEFINITION_SQL: &str = "SELECT OBJECT_DEFINITION(t.object_id), RTR AND p.schema_id = SCHEMA_ID(COALESCE(@P1, SCHEMA_NAME()))"; /// The module types whose code is a .NET assembly, not SQL. -const CLR_TYPES: [&str; 4] = ["FS", "FT", "PC", "TA"]; - -/// The catalogue gives the numbers in the sequence's own type, which can -/// be wider than any integer here, so they come as text. -/// The type as declared: an alias type by its schema and name, which -/// `system_type_id` would give as the built-in type under it. -const SEQUENCE_SQL: &str = "SELECT CASE WHEN t.is_user_defined = 1 \ +pub(crate) const CLR_TYPES: [&str; 4] = ["FS", "FT", "PC", "TA"]; + +/// What `SequenceRow` reads, from `sys.sequences s` joined to the +/// `sys.types t` it is declared as. The catalogue gives the numbers in +/// the sequence's own type, which can be wider than any integer here, so +/// they come as text. The type as declared: an alias type by its schema +/// and name, which `system_type_id` would give as the built-in type +/// under it. +pub(crate) const SEQUENCE_COLUMNS: &str = "CASE WHEN t.is_user_defined = 1 \ THEN QUOTENAME(SCHEMA_NAME(t.schema_id)) + N'.' + QUOTENAME(t.name) ELSE t.name END, \ CAST(s.start_value AS nvarchar(64)), \ CAST(s.increment AS nvarchar(64)), CAST(s.minimum_value AS nvarchar(64)), \ CAST(s.maximum_value AS nvarchar(64)), CAST(s.is_cycling AS int), CAST(s.is_cached AS int), \ - CAST(s.cache_size AS bigint), CAST(s.precision AS int) \ - FROM sys.sequences s JOIN sys.types t ON t.user_type_id = s.user_type_id \ - WHERE s.schema_id = SCHEMA_ID(COALESCE(@P1, SCHEMA_NAME())) AND s.name = @P2"; + CAST(s.cache_size AS bigint), CAST(s.precision AS int)"; + +fn sequence_sql() -> String { + format!( + "SELECT {SEQUENCE_COLUMNS} \ + FROM sys.sequences s JOIN sys.types t ON t.user_type_id = s.user_type_id \ + WHERE s.schema_id = SCHEMA_ID(COALESCE(@P1, SCHEMA_NAME())) AND s.name = @P2" + ) +} const INDEXES_SQL: &str = "SELECT i.name, CAST(i.is_unique AS int), CAST(i.is_primary_key AS int), c.name, \ CAST(ic.is_descending_key AS int), i.filter_definition \ @@ -264,7 +272,7 @@ async fn sequence_definition( object: &ObjectInfo, ) -> Result, Interrupt> { let params = [optional_text_param(object.schema.as_deref()), text_param(&object.name)]; - let rows = catalogue_rows(client, watch, SEQUENCE_SQL, ¶ms).await?; + let rows = catalogue_rows(client, watch, &sequence_sql(), ¶ms).await?; let Some(row) = rows.first() else { return Ok(None); }; @@ -277,20 +285,20 @@ async fn sequence_definition( })) } -struct SequenceRow { +pub(crate) struct SequenceRow { type_name: String, - start: String, - increment: String, - minimum: String, - maximum: String, - cycles: bool, + pub(crate) start: String, + pub(crate) increment: String, + pub(crate) minimum: String, + pub(crate) maximum: String, + pub(crate) cycles: bool, cached: bool, cache_size: Option, precision: i64, } impl SequenceRow { - fn from_cells(cells: &Cells<'_>) -> Result { + pub(crate) fn from_cells(cells: &Cells<'_>) -> Result { Ok(Self { type_name: cells.required_text(0)?, start: cells.required_text(1)?, @@ -305,7 +313,7 @@ impl SequenceRow { } } -fn sequence_statement(qualified: &str, row: &SequenceRow) -> String { +pub(crate) fn sequence_statement(qualified: &str, row: &SequenceRow) -> String { let data_type = match row.type_name.as_str() { "decimal" | "numeric" => format!("{}({})", row.type_name, row.precision), other => other.to_owned(), @@ -431,7 +439,7 @@ fn foreign_keys(rows: &[Vec]) -> Result, DriverError> } /// `sys.foreign_keys` spells each action as its `_desc` text. -fn referential_action(description: &str) -> Option { +pub(crate) fn referential_action(description: &str) -> Option { match description { "NO_ACTION" => Some(ReferentialAction::NoAction), "CASCADE" => Some(ReferentialAction::Cascade), diff --git a/core/crates/drivers/mssql/tests/integration/dump.rs b/core/crates/drivers/mssql/tests/integration/dump.rs new file mode 100644 index 0000000000..528be14444 --- /dev/null +++ b/core/crates/drivers/mssql/tests/integration/dump.rs @@ -0,0 +1,146 @@ +//! The SQL dump round trip on SQL Server. + +use async_trait::async_trait; +use tablepro_core::dump::DumpSkipReason; +use tablepro_core::{ + ConnectOptions, ConnectTarget, DatabaseDriver, EditorSession, EditorSessionOptions, RunEnd, RunOptions, UserSql, + WorkspaceSettings, +}; +use tablepro_driver_mssql::MssqlDriver; +use tablepro_test_support::conformance::{ConformanceEngine, ConformanceFailure, DumpEngine, test_liveness}; +use tokio_util::sync::CancellationToken; + +use crate::conformance_engine::MssqlEngine; +use crate::support::{Events, quoted}; + +/// CREATE and DROP DATABASE run outside a transaction, as the editor +/// runs them. +async fn run(session: &dyn EditorSession, sql: &str) -> Result<(), ConformanceFailure> { + let summary = session + .run( + UserSql::new(sql), + RunOptions::new(CancellationToken::new()), + Box::new(Events::default()), + ) + .await?; + match summary.end { + RunEnd::Completed => Ok(()), + other => Err(ConformanceFailure::setup(format!("`{sql}` ended {other:?}"))), + } +} + +#[async_trait] +impl DumpEngine for MssqlEngine { + async fn empty_database(&self, name: &str) -> Result { + let mut options = self.connect_options(WorkspaceSettings::workspace(false), test_liveness()); + let connection = MssqlDriver.connect(options.clone(), CancellationToken::new()).await?; + let session = connection + .open_editor_session(EditorSessionOptions::default(), CancellationToken::new()) + .await?; + let database = quoted(name); + let literal = name.replace('\'', "''"); + let drop = format!( + "IF DB_ID(N'{literal}') IS NOT NULL BEGIN \ + ALTER DATABASE {database} SET SINGLE_USER WITH ROLLBACK IMMEDIATE; DROP DATABASE {database}; END" + ); + let mut made = run(session.as_ref(), &drop).await; + if made.is_ok() { + made = run(session.as_ref(), &format!("CREATE DATABASE {database}")).await; + } + session.close().await; + connection.close().await?; + made?; + match &mut options.target { + ConnectTarget::Network(target) => target.database = Some(name.to_owned()), + _ => return Err(ConformanceFailure::setup("the fixture connects over the network")), + } + Ok(options) + } + + /// Every object in a schema of its own, so the dump makes the schema + /// too. The procedure is made under QUOTED_IDENTIFIER OFF, which the + /// one session the sample runs in keeps until it is set back. The + /// trigger comes after the first rows and before the last, so a row + /// it touched is in the sample and a trigger firing during the load + /// would touch the others. + fn expected_skips(&self) -> Vec<(String, DumpSkipReason)> { + vec![("price_log".to_owned(), DumpSkipReason::VersioningNotCarried)] + } + + fn dump_fixture(&self) -> Vec { + [ + "CREATE SCHEMA app", + "CREATE TYPE app.label FROM nvarchar(40) NOT NULL", + "CREATE SEQUENCE app.ticket AS int START WITH 100 INCREMENT BY 5", + "CREATE TABLE app.parent (\ + id int IDENTITY(10, 5) NOT NULL, \ + name app.label, \ + note nvarchar(max) NULL, \ + payload varbinary(max) NULL, \ + price decimal(10, 2) NOT NULL CONSTRAINT df_parent_price DEFAULT (0), \ + ticket int NOT NULL CONSTRAINT df_parent_ticket DEFAULT (NEXT VALUE FOR app.ticket), \ + flag bit NULL, \ + ref uniqueidentifier NULL, \ + amount money NULL, \ + ratio float NULL, \ + created datetime2(3) NULL, \ + stamped datetimeoffset(2) NULL, \ + name_length AS LEN(name) PERSISTED, \ + touched nvarchar(20) NULL, \ + CONSTRAINT pk_parent PRIMARY KEY CLUSTERED (id), \ + CONSTRAINT uq_parent_ticket UNIQUE NONCLUSTERED (ticket), \ + CONSTRAINT ck_parent_price CHECK (price >= 0)\ + )", + "CREATE TABLE app.child (\ + id int IDENTITY(1, 1) NOT NULL CONSTRAINT pk_child PRIMARY KEY, \ + parent_id int NOT NULL CONSTRAINT fk_child_parent REFERENCES app.parent (id) ON DELETE CASCADE, \ + previous_id int NULL CONSTRAINT fk_child_previous REFERENCES app.child (id), \ + label nvarchar(50) NOT NULL\ + )", + "CREATE INDEX ix_child_label ON app.child (label DESC) INCLUDE (parent_id)", + "EXEC sys.sp_addextendedproperty @name = N'MS_Description', @value = N'what the row says', \ + @level0type = N'SCHEMA', @level0name = N'app', @level1type = N'TABLE', @level1name = N'parent', \ + @level2type = N'COLUMN', @level2name = N'note'", + "INSERT INTO app.parent (name, note, payload, price, flag, ref, amount, ratio, created, stamped) VALUES \ + (N'first', N'it''s; a \"quote\" \\ back' + NCHAR(10) + N'line two, Grüße 東京', 0x00FF27, 12.50, 1, \ + 'c3c9a0d4-4b7e-4f43-9a52-6a1f0c1d2e3f', 12.3456, 0.1, '2024-01-02T03:04:05.678', \ + '2024-01-02T03:04:05.67+07:00'), \ + (N'second', NULL, 0xFF, 0, 0, NULL, -1, -2.5E-10, NULL, NULL)", + "CREATE VIEW app.priced_parents AS SELECT id, name, price FROM app.parent WHERE price > 0", + "CREATE VIEW app.parent_names AS SELECT name FROM app.priced_parents", + "CREATE FUNCTION app.price_with_tax (@price decimal(10, 2)) RETURNS decimal(12, 2) \ + AS BEGIN RETURN @price * 1.1; END", + "SET QUOTED_IDENTIFIER OFF", + "CREATE PROCEDURE app.add_child @parent int, @label nvarchar(50) AS BEGIN SET NOCOUNT ON; \ + INSERT INTO app.child (parent_id, label) VALUES (@parent, @label); END", + "SET QUOTED_IDENTIFIER ON", + "CREATE TRIGGER app.parent_touch ON app.parent AFTER INSERT AS BEGIN SET NOCOUNT ON; \ + UPDATE p SET touched = N'trigger' FROM app.parent AS p JOIN inserted AS i ON i.id = p.id; END", + "INSERT INTO app.parent (name, price) VALUES (N'third', 1), (N'fourth', 2)", + "DELETE FROM app.parent WHERE name = N'fourth'", + "INSERT INTO app.child (parent_id, previous_id, label) VALUES (10, NULL, N'a')", + "INSERT INTO app.child (parent_id, previous_id, label) VALUES (15, 1, N'b')", + "CREATE TABLE app.price_log (\ + id int NOT NULL CONSTRAINT pk_price_log PRIMARY KEY, \ + price decimal(10, 2) NOT NULL, \ + valid_from datetime2 GENERATED ALWAYS AS ROW START NOT NULL, \ + valid_to datetime2 GENERATED ALWAYS AS ROW END NOT NULL, \ + PERIOD FOR SYSTEM_TIME (valid_from, valid_to)\ + ) WITH (SYSTEM_VERSIONING = ON (HISTORY_TABLE = app.price_log_history))", + "INSERT INTO app.price_log (id, price) VALUES (1, 5), (2, 7)", + "UPDATE app.price_log SET price = 6 WHERE id = 1", + ] + .into_iter() + .map(str::to_owned) + .collect() + } +} + +mod suite { + use super::MssqlEngine; + + tablepro_test_support::dump_suite! { + engine: MssqlEngine, + docker: true, + } +} diff --git a/core/crates/drivers/mssql/tests/integration/main.rs b/core/crates/drivers/mssql/tests/integration/main.rs index 6e26f190e3..e411e5b931 100644 --- a/core/crates/drivers/mssql/tests/integration/main.rs +++ b/core/crates/drivers/mssql/tests/integration/main.rs @@ -13,6 +13,7 @@ mod conformance_engine; mod csv_import; mod ddl; mod decode; +mod dump; mod filter; mod identity_insert; mod metadata; diff --git a/core/crates/drivers/mysql/src/connection.rs b/core/crates/drivers/mysql/src/connection.rs index 6c823e9ccc..d632289dee 100644 --- a/core/crates/drivers/mysql/src/connection.rs +++ b/core/crates/drivers/mysql/src/connection.rs @@ -7,6 +7,7 @@ use sqlx::Executor as _; use tablepro_core::column::ColumnInfo; use tablepro_core::ddl::{CharacterSet, InboundForeignKey, OutboundForeignKey, TableDefinition}; use tablepro_core::dialect::SqlDialect; +use tablepro_core::dump::{DumpCatalog, DumpScope}; use tablepro_core::meta::{ IndexInfo, ObjectDefinition, ObjectInfo, ObjectScope, RowIdentity, SchemaInfo, TableEstimate, TableRef, }; @@ -22,6 +23,7 @@ use tokio_util::sync::{CancellationToken, WaitForCancellationFutureOwned}; use crate::cancel::{ControlLink, MySqlStop, StopProgress}; use crate::dialect::MySqlDialect; +use crate::dump; use crate::editor_session::MysqlEditorSession; use crate::errors::{map_error, map_pool_error}; use crate::flavour::MySqlFlavour; @@ -119,6 +121,39 @@ impl MysqlConnection { self.settle(outcome, &stop).await } + /// Run one read on a login of its own, closed after it, so it never + /// holds a pool slot. + async fn read_alone( + &self, + options: CallOptions, + limits: SessionLimits, + op: ReadOp, + ) -> Result { + self.ensure_open()?; + let stop = Arc::new(MySqlStop::new(self.control.clone())); + let endpoint = self.endpoint.clone(); + let work_stop = stop.clone(); + let outcome = self + .calls + .run_read( + options, + Standalone::new(), + stop.clone(), + move |mut link, request| async move { + if let Err(error) = link.open(&endpoint, &limits, &request, &work_stop).await { + return (link, Err(error)); + } + let result = match link.connection() { + Ok((connection, flavour)) => op(connection, flavour).await, + Err(error) => Err(error), + }; + (link, result) + }, + ) + .await; + self.settle(outcome, &stop).await + } + /// Decide whether a failed call means the link itself is gone. /// /// sqlx has no keepalive on a direct route, so a dead peer shows up @@ -447,31 +482,33 @@ impl Connection for MysqlConnection { mut sink: Box, options: CallOptions, ) -> Result { - self.ensure_open()?; - let stop = Arc::new(MySqlStop::new(self.control.clone())); - let endpoint = self.endpoint.clone(); - let limits = self.stream_limits(); let read = read.clone(); - let work_stop = stop.clone(); - let outcome = self - .calls - .run_read( - options, - Standalone::new(), - stop.clone(), - move |mut link, request| async move { - if let Err(error) = link.open(&endpoint, &limits, &request, &work_stop).await { - return (link, Err(error)); - } - let result = match link.connection() { - Ok((connection, flavour)) => stream(connection, flavour, &read, sink.as_mut()).await, - Err(error) => Err(error), - }; - (link, result) - }, - ) - .await; - self.settle(outcome, &stop).await + self.read_alone( + options, + self.stream_limits(), + read_op(move |connection, flavour| { + Box::pin(async move { stream(connection, flavour, &read, sink.as_mut()).await }) + }), + ) + .await + } + + async fn read_dump_catalog(&self, scope: &DumpScope, options: CallOptions) -> Result { + let scope = scope.clone(); + let dialect = *self.dialect; + // The read sets the mode SHOW CREATE prints under, which no pooled session may keep. + let limits = SessionLimits { + read_only: true, + ..SessionLimits::workspace(&self.settings) + }; + self.read_alone( + options, + limits, + read_op(move |connection, _| { + Box::pin(async move { dump::read_catalog(connection, &dialect, &scope).await }) + }), + ) + .await } async fn write_batch(&self, batch: &WriteBatch, options: CallOptions) -> Result { diff --git a/core/crates/drivers/mysql/src/dialect.rs b/core/crates/drivers/mysql/src/dialect.rs index d27c77b792..4ef98a7b6a 100644 --- a/core/crates/drivers/mysql/src/dialect.rs +++ b/core/crates/drivers/mysql/src/dialect.rs @@ -8,6 +8,7 @@ use tablepro_core::dialect::{ BindError, BindTarget, DatabaseLayout, DialectCapabilities, InsertConflict, KeysetDirection, LikeCase, LikeForm, LiteralError, Placeholder, SqlDialect, }; +use tablepro_core::dump::DumpDialect; use tablepro_core::edit::{CellExpression, MomentKind}; use tablepro_core::export::value_to_text; use tablepro_core::meta::{EngineRowId, EngineRowIdPart}; @@ -233,6 +234,10 @@ impl SqlDialect for MySqlDialect { Some(self) } + fn dump(&self) -> Option<&dyn DumpDialect> { + Some(self) + } + fn insert_default_row(&self, table_sql: &str) -> Option { Some(format!("INSERT INTO {table_sql} () VALUES ()")) } diff --git a/core/crates/drivers/mysql/src/dump/create_text.rs b/core/crates/drivers/mysql/src/dump/create_text.rs new file mode 100644 index 0000000000..42f776a189 --- /dev/null +++ b/core/crates/drivers/mysql/src/dump/create_text.rs @@ -0,0 +1,315 @@ +use std::collections::HashSet; +use std::ops::Range; + +use tablepro_core::sql_syntax::SqlGrammar; +use tablepro_core::sql_syntax::script::{LexicalSettings, ScriptPlan, ScriptToken, ScriptTokenKind}; + +/// The words that end the part of a CREATE where `DEFINER=` can stand. +const OBJECT_WORDS: [&str; 7] = [ + "VIEW", + "FUNCTION", + "PROCEDURE", + "TRIGGER", + "EVENT", + "AGGREGATE", + "PACKAGE", +]; + +/// `create` without its `DEFINER=user@host`: the account may not exist +/// where the file loads, and without the clause the server makes +/// whoever runs the CREATE the definer. +pub(crate) fn without_definer(create: &str) -> String { + let tokens = significant(create); + let head = tokens + .iter() + .position(|token| OBJECT_WORDS.iter().any(|word| is_word(create, token, word))) + .unwrap_or(tokens.len()); + for (at, window) in tokens[..head].windows(3).enumerate() { + let [definer, equals, _user] = window else { + continue; + }; + if !is_word(create, definer, "DEFINER") || text(create, equals) != "=" { + continue; + } + let mut last = at + 2; + match tokens.get(last + 1).map(|token| text(create, token)) { + Some("@") => last += 2, + Some(host) if host.starts_with('@') => last += 1, + _ => {} + } + let end = tokens.get(last + 1).map_or(create.len(), |next| next.range.start); + return cut(create, std::slice::from_ref(&(definer.range.start..end))); + } + create.to_owned() +} + +/// `create` with `` `database`. `` gone from before the names of the +/// dump's own objects, so the file loads into a database of any name. +/// MySQL qualifies a stored function a view calls, and every name of a +/// view that reads another database too; MariaDB qualifies the sequence +/// a column's default takes from; a routine or trigger keeps its body as +/// written. `names` is in lower case. +/// +/// A text that names the database other than as `name.` keeps every +/// qualifier: there the name can be a table alias, which SHOW CREATE +/// VIEW writes without `AS`, a label, a variable or a table of another +/// database, and `name.` can mean that one. +pub(crate) fn without_database(create: &str, database: &str, names: &HashSet, reading: ModeReading) -> String { + let tokens = reading.tokens(create); + let name_of = |token: &ScriptToken| reading.name_of(create, token); + let alone = (0..tokens.len()).any(|at| { + let named = name_of(&tokens[at]).is_some_and(|name| name.eq_ignore_ascii_case(database)); + let qualifies = tokens.get(at + 1).is_some_and(|token| text(create, token) == "."); + named && !qualifies + }); + if alone { + return create.to_owned(); + } + let mut cuts = Vec::new(); + for (at, window) in tokens.windows(3).enumerate() { + let [qualifier, dot, name] = window else { + continue; + }; + let follows_dot = at + .checked_sub(1) + .is_some_and(|before| text(create, &tokens[before]) == "."); + let names_ours = name_of(name).is_some_and(|name| names.contains(&name.to_lowercase())); + if !follows_dot && name_of(qualifier).as_deref() == Some(database) && text(create, dot) == "." && names_ours { + cuts.push(qualifier.range.start..dot.range.end); + } + } + cut(create, &cuts) +} + +/// A table whose rows are another server's, through FEDERATED or +/// MariaDB's CONNECT: its CREATE would reach that server again on load, +/// password and all, and its rows would be written there. Read from the +/// table options, not from a comment or a default that spells one. +pub(crate) fn remote_table(create: &str) -> bool { + let tokens = significant(create); + tokens.windows(3).any(|window| { + let [option, equals, value] = window else { + return false; + }; + if text(create, equals) != "=" { + return false; + } + is_word(create, option, "CONNECTION") + || (is_word(create, option, "ENGINE") + && (is_word(create, value, "FEDERATED") || is_word(create, value, "CONNECT"))) + }) +} + +fn significant(sql: &str) -> Vec { + let plan = ScriptPlan::build(sql, SqlGrammar::MySql, LexicalSettings::default_for(SqlGrammar::MySql)); + plan.tokens(0..sql.len()) + .filter(|token| !token.kind.is_trivia()) + .collect() +} + +fn text<'a>(sql: &'a str, token: &ScriptToken) -> &'a str { + sql.get(token.range.clone()).unwrap_or_default() +} + +fn is_word(sql: &str, token: &ScriptToken, word: &str) -> bool { + token.kind == ScriptTokenKind::Word && text(sql, token).eq_ignore_ascii_case(word) +} + +/// How a text reads under the `sql_mode` a routine or trigger was made +/// in: `ANSI_QUOTES` makes `"name"` a name, and `NO_BACKSLASH_ESCAPES` +/// ends a string at a quote after a backslash. SHOW CREATE TABLE and +/// VIEW print under the session's mode, which is the default here. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct ModeReading { + ansi_quotes: bool, + backslash_escapes: bool, +} + +impl ModeReading { + pub(crate) fn of(sql_mode: Option<&str>) -> Self { + let mode = sql_mode.unwrap_or_default().to_ascii_uppercase(); + let has = |flag: &str| mode.split(',').any(|set| set.trim() == flag); + Self { + ansi_quotes: has("ANSI_QUOTES"), + backslash_escapes: !has("NO_BACKSLASH_ESCAPES"), + } + } + + fn tokens(self, sql: &str) -> Vec { + let settings = LexicalSettings { + backslash_escapes: self.backslash_escapes, + }; + ScriptPlan::build(sql, SqlGrammar::MySql, settings) + .tokens(0..sql.len()) + .filter(|token| !token.kind.is_trivia()) + .collect() + } + + /// The name a token spells, quoted or not. + fn name_of(self, sql: &str, token: &ScriptToken) -> Option { + let spelled = text(sql, token); + match token.kind { + ScriptTokenKind::Word => Some(spelled.to_owned()), + ScriptTokenKind::QuotedIdentifier => { + let inner = spelled.strip_prefix('`')?.strip_suffix('`')?; + Some(inner.replace("``", "`")) + } + ScriptTokenKind::StringLiteral if self.ansi_quotes => { + let inner = spelled.strip_prefix('"')?.strip_suffix('"')?; + Some(inner.replace("\"\"", "\"")) + } + _ => None, + } + } +} + +fn cut(sql: &str, cuts: &[Range]) -> String { + let mut kept = String::with_capacity(sql.len()); + let mut from = 0; + for range in cuts { + kept.push_str(sql.get(from..range.start).unwrap_or_default()); + from = range.end; + } + kept.push_str(sql.get(from..).unwrap_or_default()); + kept +} + +#[cfg(test)] +mod tests { + use super::*; + + fn names(list: &[&str]) -> HashSet { + list.iter().map(|name| (*name).to_owned()).collect() + } + + #[test] + fn the_definer_goes_and_the_security_stays() { + assert_eq!( + without_definer( + "CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`%` SQL SECURITY DEFINER VIEW `v` AS select 1 AS `1`" + ), + "CREATE ALGORITHM=UNDEFINED SQL SECURITY DEFINER VIEW `v` AS select 1 AS `1`" + ); + assert_eq!( + without_definer("CREATE DEFINER=`app``s`@`localhost` FUNCTION `f`() RETURNS int\nRETURN 1"), + "CREATE FUNCTION `f`() RETURNS int\nRETURN 1" + ); + assert_eq!( + without_definer( + "CREATE DEFINER=`root`@`%` TRIGGER `t` BEFORE INSERT ON `c` FOR EACH ROW SET NEW.definer = 'x'" + ), + "CREATE TRIGGER `t` BEFORE INSERT ON `c` FOR EACH ROW SET NEW.definer = 'x'" + ); + } + + #[test] + fn a_role_definer_has_no_host() { + assert_eq!( + without_definer("CREATE DEFINER=`reporting` PROCEDURE `p`()\nBEGIN SELECT 1; END"), + "CREATE PROCEDURE `p`()\nBEGIN SELECT 1; END" + ); + } + + #[test] + fn a_definer_in_the_body_is_not_the_clause() { + let create = "CREATE PROCEDURE `p`()\nBEGIN SET @x = 'DEFINER=`root`@`%`'; END"; + + assert_eq!(without_definer(create), create); + } + + #[test] + fn the_database_comes_off_names_of_the_dumps_own_objects() { + let ours = names(&["label_of", "parent", "invoice_no"]); + + assert_eq!( + without_database( + "CREATE VIEW `v` AS select `app`.`label_of`(`p`.`id`) AS `l` from `app`.`parent` `p`", + "app", + &ours, + ModeReading::of(None) + ), + "CREATE VIEW `v` AS select `label_of`(`p`.`id`) AS `l` from `parent` `p`" + ); + assert_eq!( + without_database( + "CREATE TABLE `invoice` (\n `no` bigint(20) NOT NULL DEFAULT nextval(`app`.`invoice_no`)\n)", + "app", + &ours, + ModeReading::of(None) + ), + "CREATE TABLE `invoice` (\n `no` bigint(20) NOT NULL DEFAULT nextval(`invoice_no`)\n)" + ); + } + + #[test] + fn other_databases_columns_and_literals_keep_their_qualifier() { + let ours = names(&["parent"]); + let create = "CREATE VIEW `v` AS select `app`.`id` AS `id`, '`app`.`parent`' AS `s` \ + from `other`.`parent` where `x`.`app`.`parent` = 1"; + + assert_eq!( + without_database(create, "app", &ours, ModeReading::of(None)), + "CREATE VIEW `v` AS select `app`.`id` AS `id`, '`app`.`parent`' AS `s` \ + from `other`.`parent` where `x`.`app`.`parent` = 1" + ); + } + + #[test] + fn a_view_with_an_alias_named_like_the_database_keeps_its_qualifiers() { + let ours = names(&["items", "id"]); + let create = "CREATE VIEW `v` AS select `shop`.`id` AS `id` \ + from (`shop`.`items` `shop` join `shop`.`id` `j` on(`j`.`id` = `shop`.`id`))"; + + assert_eq!(without_database(create, "shop", &ours, ModeReading::of(None)), create); + } + + #[test] + fn a_remote_table_is_read_from_its_options() { + assert!(remote_table( + "CREATE TABLE `r` (\n `id` int\n) ENGINE=FEDERATED DEFAULT CHARSET=utf8mb4 \ + CONNECTION='mysql://u:p@h/db/t'" + )); + assert!(remote_table( + "CREATE TABLE `r` (\n `id` int\n) ENGINE=CONNECT TABLE_TYPE=MYSQL" + )); + assert!(!remote_table( + "CREATE TABLE `l` (\n `note` varchar(40) DEFAULT 'CONNECTION=x'\n) ENGINE=InnoDB \ + COMMENT='was ENGINE=CONNECT'" + )); + } + + #[test] + fn a_routine_body_loses_the_qualifier_as_written() { + let ours = names(&["orders", "label_of"]); + let create = "CREATE PROCEDURE `p`()\nBEGIN\n SELECT shop.label_of(id) FROM shop.orders o;\n \ + SELECT * FROM `shop`.`orders` JOIN other.orders USING (id);\nEND"; + + assert_eq!( + without_database(create, "shop", &ours, ModeReading::of(None)), + "CREATE PROCEDURE `p`()\nBEGIN\n SELECT label_of(id) FROM orders o;\n \ + SELECT * FROM `orders` JOIN other.orders USING (id);\nEND" + ); + } + + #[test] + fn a_routine_made_under_ansi_quotes_loses_a_double_quoted_qualifier() { + let ours = names(&["t"]); + let create = "CREATE PROCEDURE `p`()\nBEGIN\n DELETE FROM \"shop\".\"t\" WHERE note = 'a\\';\nEND"; + let reading = ModeReading::of(Some("ANSI_QUOTES,NO_BACKSLASH_ESCAPES")); + + assert_eq!( + without_database(create, "shop", &ours, reading), + "CREATE PROCEDURE `p`()\nBEGIN\n DELETE FROM \"t\" WHERE note = 'a\\';\nEND" + ); + assert_eq!(without_database(create, "shop", &ours, ModeReading::of(None)), create); + } + + #[test] + fn a_body_that_names_the_database_alone_keeps_it() { + let ours = names(&["orders"]); + let create = "CREATE TRIGGER `t` AFTER INSERT ON `orders` FOR EACH ROW \ + UPDATE shop.orders SET total = 0 WHERE shop.id IN (SELECT id FROM orders shop)"; + + assert_eq!(without_database(create, "shop", &ours, ModeReading::of(None)), create); + } +} diff --git a/core/crates/drivers/mysql/src/dump/dump_dialect.rs b/core/crates/drivers/mysql/src/dump/dump_dialect.rs new file mode 100644 index 0000000000..600e96c360 --- /dev/null +++ b/core/crates/drivers/mysql/src/dump/dump_dialect.rs @@ -0,0 +1,120 @@ +use tablepro_core::dump::{DumpDialect, DumpStatement}; + +use crate::dialect::MySqlDialect; +use crate::flavour::MySqlFlavour; + +/// mysqldump 8.4's opening lines: the session's settings saved, then the +/// ones every load needs. Foreign keys go unchecked, so tables load in +/// any order, and `NO_AUTO_VALUE_ON_ZERO` keeps a row whose key is 0. +const HEADER: &str = "\ +/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; +/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; +/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; +/*!50503 SET NAMES utf8mb4 */; +/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; +/*!40103 SET TIME_ZONE='+00:00' */; +/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; +/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; +/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; +"; + +const FOOTER: &str = "\ +/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; + +/*!40101 SET SQL_MODE=@OLD_SQL_MODE */; +/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; +/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; +/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; +/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; +/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; +"; + +impl MySqlDialect { + /// mariadb-dump writes `NOTE_VERBOSITY`, which MariaDB 10.6.16 added, + /// in place of `SQL_NOTES`, in a comment only MariaDB runs. + fn notes(&self, saving: bool) -> &'static str { + match (self.flavour(), saving) { + (MySqlFlavour::MySql, true) => "/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;\n", + (MySqlFlavour::MySql, false) => "/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;\n", + (MySqlFlavour::MariaDb, true) => { + "/*M!100616 SET @OLD_NOTE_VERBOSITY=@@NOTE_VERBOSITY, NOTE_VERBOSITY=0 */;\n" + } + (MySqlFlavour::MariaDb, false) => "/*M!100616 SET NOTE_VERBOSITY=@OLD_NOTE_VERBOSITY */;\n", + } + } +} + +impl DumpDialect for MySqlDialect { + fn header(&self) -> String { + format!("\n{HEADER}{}", self.notes(true)) + } + + fn footer(&self) -> String { + format!("\n{FOOTER}{}", self.notes(false)) + } + + /// A body holds semicolons of its own, which the mysql client reads + /// as one statement only under another delimiter. + fn statement(&self, statement: &DumpStatement) -> String { + match statement.body { + true => format!("DELIMITER ;;\n{} ;;\nDELIMITER ;\n", statement.sql), + false => format!("{};\n", statement.sql), + } + } +} + +#[cfg(test)] +mod tests { + use tablepro_core::ServerInfo; + + use super::*; + + fn dialect(flavour: MySqlFlavour) -> MySqlDialect { + MySqlDialect::new(flavour, &ServerInfo::new(flavour.product(), "8.4.3", 8, 4, 3)) + } + + #[test] + fn mysql_saves_and_restores_its_notes_setting() { + let mysql = dialect(MySqlFlavour::MySql); + + assert!(mysql.header().starts_with("\n/*!40101 SET @OLD_CHARACTER_SET_CLIENT")); + assert!(mysql.header().ends_with( + "SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;\n/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;\n" + )); + assert!(mysql.footer().ends_with("/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;\n")); + assert!(!mysql.header().contains("NOTE_VERBOSITY")); + } + + #[test] + fn mariadb_saves_and_restores_its_note_verbosity_instead() { + let mariadb = dialect(MySqlFlavour::MariaDb); + + assert!( + mariadb + .header() + .ends_with("/*M!100616 SET @OLD_NOTE_VERBOSITY=@@NOTE_VERBOSITY, NOTE_VERBOSITY=0 */;\n") + ); + assert!( + mariadb + .footer() + .ends_with("/*M!100616 SET NOTE_VERBOSITY=@OLD_NOTE_VERBOSITY */;\n") + ); + assert!(!mariadb.header().contains("SQL_NOTES")); + assert!(!mariadb.header().contains("sandbox")); + } + + #[test] + fn a_body_goes_in_a_delimiter_block_and_the_rest_ends_in_a_semicolon() { + let mysql = dialect(MySqlFlavour::MySql); + + assert_eq!( + mysql.statement(&DumpStatement::body("CREATE PROCEDURE `p`()\nBEGIN SELECT 1; END")), + "DELIMITER ;;\nCREATE PROCEDURE `p`()\nBEGIN SELECT 1; END ;;\nDELIMITER ;\n" + ); + assert_eq!( + mysql.statement(&DumpStatement::new("SET sql_mode = @saved_sql_mode")), + "SET sql_mode = @saved_sql_mode;\n" + ); + assert!(!mysql.transactional_ddl()); + } +} diff --git a/core/crates/drivers/mysql/src/dump/mod.rs b/core/crates/drivers/mysql/src/dump/mod.rs new file mode 100644 index 0000000000..a99353dd67 --- /dev/null +++ b/core/crates/drivers/mysql/src/dump/mod.rs @@ -0,0 +1,425 @@ +//! A database as one SQL file, read the way mysqldump reads it: each +//! object's own `SHOW CREATE` text, on a login of its own that prints it +//! under an empty `sql_mode`. + +mod create_text; +mod dump_dialect; +mod program_settings; + +use std::collections::HashSet; + +use sqlx::mysql::{MySqlConnection, MySqlDatabaseError, MySqlRow}; +use sqlx::{AssertSqlSafe, Executor as _, Row as _}; +use tablepro_core::DriverError; +use tablepro_core::dialect::SqlDialect; +use tablepro_core::dump::{ + DumpCatalog, DumpObject, DumpObjectKind, DumpScope, DumpSection, DumpSkip, DumpSkipReason, DumpStatement, DumpTable, +}; +use tablepro_core::meta::TableRef; + +use crate::dialect::MySqlDialect; +use crate::errors::map_error; +use crate::metadata::{catalog_rows, not_there, text_column}; + +use create_text::{ModeReading, remote_table, without_database, without_definer}; +use program_settings::ProgramSettings; + +/// SHOW CREATE prints by the session's mode: `ANSI_QUOTES` quotes names +/// in `"`, and MariaDB's `NO_TABLE_OPTIONS` drops ENGINE and the counter. +/// mysqldump prints under an empty mode too. +const PRINT_SETTINGS_STATEMENT: &str = "SET SESSION sql_mode = '', SESSION sql_quote_show_create = 1"; + +/// MariaDB's Oracle-mode packages are left out. +const ROUTINES_SQL: &str = "\ +SELECT CONVERT(ROUTINE_NAME USING utf8mb4) COLLATE utf8mb4_bin, ROUTINE_TYPE +FROM information_schema.ROUTINES +WHERE ROUTINE_SCHEMA = ? AND ROUTINE_TYPE IN ('FUNCTION', 'PROCEDURE') +ORDER BY 2, 1"; + +/// In firing order within each table: a trigger made after another for +/// the same event fires after it again. +const TRIGGERS_SQL: &str = "\ +SELECT CONVERT(TRIGGER_NAME USING utf8mb4) COLLATE utf8mb4_bin +FROM information_schema.TRIGGERS +WHERE EVENT_OBJECT_SCHEMA = ? +ORDER BY CONVERT(EVENT_OBJECT_TABLE USING utf8mb4) COLLATE utf8mb4_bin, ACTION_ORDER, 1"; + +/// ER_TABLEACCESS_DENIED_ERROR, what SHOW CREATE answers for an object +/// the login may list but not read. +const DENIED: u16 = 1142; + +/// The scope's tables with their rows, sequences, routines, views and +/// triggers. Events are left out. +pub(crate) async fn read_catalog( + connection: &mut MySqlConnection, + dialect: &MySqlDialect, + scope: &DumpScope, +) -> Result { + let flavour = dialect.flavour(); + connection + .execute(PRINT_SETTINGS_STATEMENT) + .await + .map_err(|error| map_error(&error, flavour))?; + let qualifier = match scope { + DumpScope::Database => None, + DumpScope::Schema(name) => Some(name.clone()), + }; + let listed = list_tables(connection, dialect, qualifier.as_deref()).await?; + let database = match &qualifier { + Some(name) => name.clone(), + None => current_database(connection, dialect).await?, + }; + let binds = [Some(database.as_str())]; + let mut routines = Vec::new(); + for row in catalog_rows(connection, flavour, ROUTINES_SQL, "ROUTINES", &binds).await? { + let kind = match row.required_text(1)?.as_str() { + "FUNCTION" => DumpObjectKind::Function, + _ => DumpObjectKind::Procedure, + }; + routines.push((row.required_text(0)?, kind)); + } + let mut triggers = Vec::new(); + for row in catalog_rows(connection, flavour, TRIGGERS_SQL, "TRIGGERS", &binds).await? { + triggers.push(row.required_text(0)?); + } + let names = listed + .iter() + .map(|(name, _)| name) + .chain(routines.iter().map(|(name, _)| name)) + .map(|name| name.to_lowercase()) + .collect(); + let mut reader = Reader { + connection, + dialect, + qualifier, + database, + names, + catalog: DumpCatalog::default(), + }; + let mut views = Vec::new(); + for (name, kind) in listed { + match kind { + ListedKind::Table => reader.table(&name).await?, + ListedKind::Sequence => reader.sequence(&name).await?, + ListedKind::View => views.push(name), + } + } + for (name, kind) in routines { + reader.routine(&name, kind).await?; + } + for name in views { + reader.view(&name).await?; + } + for name in triggers { + reader.trigger(&name).await?; + } + Ok(reader.catalog) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ListedKind { + Table, + View, + Sequence, +} + +/// The scope's tables, views and sequences, by name. On a session with +/// no database the server's own error says so. +async fn list_tables( + connection: &mut MySqlConnection, + dialect: &MySqlDialect, + qualifier: Option<&str>, +) -> Result, DriverError> { + let sql = match qualifier { + Some(database) => format!("SHOW FULL TABLES FROM {}", dialect.quote_identifier(database)), + None => "SHOW FULL TABLES".to_owned(), + }; + let rows = sqlx::raw_sql(AssertSqlSafe(sql)) + .fetch_all(&mut *connection) + .await + .map_err(|error| map_error(&error, dialect.flavour()))?; + let mut listed = Vec::with_capacity(rows.len()); + for row in &rows { + let kind = match text_at(row, 1, "Table_type")?.as_deref() { + Some("BASE TABLE" | "SYSTEM VERSIONED") => ListedKind::Table, + Some("VIEW") => ListedKind::View, + Some("SEQUENCE") => ListedKind::Sequence, + _ => continue, + }; + if let Some(name) = text_at(row, 0, "Tables_in")? { + listed.push((name, kind)); + } + } + listed.sort_by(|left, right| left.0.cmp(&right.0)); + Ok(listed) +} + +async fn current_database(connection: &mut MySqlConnection, dialect: &MySqlDialect) -> Result { + let rows = catalog_rows(connection, dialect.flavour(), "SELECT DATABASE()", "DATABASE", &[]).await?; + rows.first() + .map(|row| row.required_text(0)) + .transpose()? + .ok_or_else(|| DriverError::Protocol("SELECT DATABASE() returned no row".to_owned())) +} + +fn text_at(row: &MySqlRow, index: usize, column: &str) -> Result, DriverError> { + let decode = |detail: String| DriverError::Decode { + column: column.to_owned(), + detail, + }; + let bytes: Option<&[u8]> = row + .try_get_unchecked(index) + .map_err(|error| decode(error.to_string()))?; + bytes + .map(|bytes| String::from_utf8(bytes.to_vec()).map_err(|error| decode(error.to_string()))) + .transpose() +} + +/// What a SHOW CREATE found. +enum Shown { + Row(MySqlRow), + /// Dropped since the list was read. + Gone, + /// The login may list it but not read its text. + Hidden, +} + +struct Reader<'a> { + connection: &'a mut MySqlConnection, + dialect: &'a MySqlDialect, + /// The database a schema scope names, which each read qualifies its + /// names with. The file itself never names it. + qualifier: Option, + database: String, + /// The scope's tables, views, sequences and routines, in lower case. + names: HashSet, + catalog: DumpCatalog, +} + +impl Reader<'_> { + fn reference(&self, name: &str) -> TableRef { + TableRef::new(self.qualifier.clone(), name) + } + + async fn show(&mut self, what: &str, name: &str) -> Result { + let sql = format!("SHOW CREATE {what} {}", self.dialect.qualify(&self.reference(name))); + match sqlx::raw_sql(AssertSqlSafe(sql)).fetch_one(&mut *self.connection).await { + Ok(row) => Ok(Shown::Row(row)), + Err(error) if not_there(&error) => Ok(Shown::Gone), + Err(error) if number(&error) == Some(DENIED) => Ok(Shown::Hidden), + Err(error) => Err(map_error(&error, self.dialect.flavour())), + } + } + + /// The object's text from `column`, or `None` with the object listed + /// as skipped where the login may not read it. + async fn create_text( + &mut self, + kind: DumpObjectKind, + name: &str, + column: &str, + ) -> Result, DriverError> { + let what = match kind { + DumpObjectKind::Function => "FUNCTION", + DumpObjectKind::Procedure => "PROCEDURE", + DumpObjectKind::Sequence => "SEQUENCE", + DumpObjectKind::View => "VIEW", + DumpObjectKind::Trigger => "TRIGGER", + _ => "TABLE", + }; + let row = match self.show(what, name).await? { + Shown::Row(row) => row, + Shown::Gone => return Ok(None), + Shown::Hidden => { + self.skip(kind, name); + return Ok(None); + } + }; + // NULL where the login may not read a routine's body. + match text_column(&row, column)? { + Some(text) => Ok(Some((text, row))), + None => { + self.skip(kind, name); + Ok(None) + } + } + } + + fn skip(&mut self, kind: DumpObjectKind, name: &str) { + self.catalog.skipped.push(DumpSkip { + kind, + schema: self.qualifier.clone(), + name: name.to_owned(), + reason: DumpSkipReason::NoDefinition, + }); + } + + fn push(&mut self, kind: DumpObjectKind, section: DumpSection, name: &str, statements: Vec) { + self.catalog + .objects + .push(DumpObject::new(kind, section, self.qualifier.clone(), name, statements)); + } + + /// SHOW CREATE TABLE keeps the foreign keys inline, which the + /// header's `FOREIGN_KEY_CHECKS=0` lets load in any order, and + /// `AUTO_INCREMENT=` with the counter past the highest key. + async fn table(&mut self, name: &str) -> Result<(), DriverError> { + let Some((create, _)) = self.create_text(DumpObjectKind::Table, name, "Create Table").await? else { + return Ok(()); + }; + if remote_table(&create) { + self.catalog.skipped.push(DumpSkip { + kind: DumpObjectKind::Table, + schema: self.qualifier.clone(), + name: name.to_owned(), + reason: DumpSkipReason::RemoteRows, + }); + return Ok(()); + } + let create = without_database(&create, &self.database, &self.names, ModeReading::of(None)); + let table = self.reference(name); + self.catalog + .objects + .push(DumpObject::table(table.clone(), vec![DumpStatement::new(create)])); + self.catalog.tables.push(DumpTable::new(table)); + Ok(()) + } + + /// The sequence as made, and after the rows the value it hands out + /// next, as mariadb-dump writes it. + async fn sequence(&mut self, name: &str) -> Result<(), DriverError> { + let Some((create, _)) = self.create_text(DumpObjectKind::Sequence, name, "Create Table").await? else { + return Ok(()); + }; + let next = self.next_value(name).await?; + self.push( + DumpObjectKind::Sequence, + DumpSection::BeforeData, + name, + vec![DumpStatement::new(create)], + ); + if let Some(next) = next { + let statement = set_value(&self.dialect.quote_identifier(name), &next)?; + self.push( + DumpObjectKind::SequenceValue, + DumpSection::AfterData, + name, + vec![statement], + ); + } + Ok(()) + } + + async fn next_value(&mut self, name: &str) -> Result, DriverError> { + let sql = format!( + "SELECT CAST(next_not_cached_value AS CHAR) AS next_value FROM {}", + self.dialect.qualify(&self.reference(name)) + ); + match sqlx::raw_sql(AssertSqlSafe(sql)) + .fetch_optional(&mut *self.connection) + .await + { + Ok(Some(row)) => text_column(&row, "next_value"), + Ok(None) => Ok(None), + Err(error) if not_there(&error) => Ok(None), + Err(error) => Err(map_error(&error, self.dialect.flavour())), + } + } + + async fn routine(&mut self, name: &str, kind: DumpObjectKind) -> Result<(), DriverError> { + let column = match kind { + DumpObjectKind::Function => "Create Function", + _ => "Create Procedure", + }; + let Some((create, row)) = self.create_text(kind, name, column).await? else { + return Ok(()); + }; + let settings = ProgramSettings { + sql_mode: text_column(&row, "sql_mode")?, + collation_connection: text_column(&row, "collation_connection")?, + }; + let reading = ModeReading::of(settings.sql_mode.as_deref()); + let create = without_database(&without_definer(&create), &self.database, &self.names, reading); + let statements = settings.around(DumpStatement::body(create)); + self.push(kind, DumpSection::Programs, name, statements); + Ok(()) + } + + async fn view(&mut self, name: &str) -> Result<(), DriverError> { + let Some((create, row)) = self.create_text(DumpObjectKind::View, name, "Create View").await? else { + return Ok(()); + }; + let settings = ProgramSettings { + sql_mode: None, + collation_connection: text_column(&row, "collation_connection")?, + }; + let create = without_database( + &without_definer(&create), + &self.database, + &self.names, + ModeReading::of(None), + ); + let statements = settings.around(DumpStatement::new(create)); + self.push(DumpObjectKind::View, DumpSection::Programs, name, statements); + Ok(()) + } + + async fn trigger(&mut self, name: &str) -> Result<(), DriverError> { + let Some((create, row)) = self + .create_text(DumpObjectKind::Trigger, name, "SQL Original Statement") + .await? + else { + return Ok(()); + }; + let settings = ProgramSettings { + sql_mode: text_column(&row, "sql_mode")?, + collation_connection: text_column(&row, "collation_connection")?, + }; + let reading = ModeReading::of(settings.sql_mode.as_deref()); + let create = without_database(&without_definer(&create), &self.database, &self.names, reading); + let statements = settings.around(DumpStatement::body(create)); + self.push(DumpObjectKind::Trigger, DumpSection::Triggers, name, statements); + Ok(()) + } +} + +fn number(error: &sqlx::Error) -> Option { + let sqlx::Error::Database(database) = error else { + return None; + }; + database + .try_downcast_ref::() + .map(MySqlDatabaseError::number) +} + +/// `DO SETVAL(seq, next, 0)`: the next NEXTVAL hands out `next`. The +/// value is the server's own number, checked to be one before it +/// becomes SQL. +fn set_value(sequence_sql: &str, next: &str) -> Result { + let value: i64 = next.trim().parse().map_err(|_| DriverError::Decode { + column: "next_not_cached_value".to_owned(), + detail: format!("{next:?} is not a number"), + })?; + Ok(DumpStatement::new(format!("DO SETVAL({sequence_sql}, {value}, 0)"))) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_sequence_restarts_at_the_value_it_hands_out_next() { + assert_eq!( + set_value("`invoice_no`", "2000") + .map(|statement| statement.sql) + .ok() + .as_deref(), + Some("DO SETVAL(`invoice_no`, 2000, 0)") + ); + assert_eq!( + set_value("`down`", "-5").map(|statement| statement.sql).ok().as_deref(), + Some("DO SETVAL(`down`, -5, 0)") + ); + assert!(set_value("`s`", "1; DROP TABLE t").is_err()); + } +} diff --git a/core/crates/drivers/mysql/src/dump/program_settings.rs b/core/crates/drivers/mysql/src/dump/program_settings.rs new file mode 100644 index 0000000000..c312ad164e --- /dev/null +++ b/core/crates/drivers/mysql/src/dump/program_settings.rs @@ -0,0 +1,115 @@ +use tablepro_core::dump::DumpStatement; + +/// What a view, routine or trigger was made under, which its CREATE runs +/// under again, as mysqldump writes it, so the server reads the text as +/// it did then. +/// +/// `character_set_client` is not switched: mysqldump writes the stored +/// bytes as they are (`character_set_results=binary`), while this file +/// is UTF-8, which only the header's `SET NAMES utf8mb4` reads right. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub(crate) struct ProgramSettings { + /// `None` for a view, which keeps no mode. + pub sql_mode: Option, + pub collation_connection: Option, +} + +impl ProgramSettings { + /// The statements that save the session's settings, set the + /// object's, run `create`, and set the session's back. + pub(crate) fn around(&self, create: DumpStatement) -> Vec { + let mut before = Vec::new(); + let mut after = Vec::new(); + if let Some(collation) = &self.collation_connection { + before.push("SET @saved_col_connection = @@collation_connection".to_owned()); + before.push(format!("SET collation_connection = {}", quoted(collation))); + after.push("SET collation_connection = @saved_col_connection".to_owned()); + } + if let Some(mode) = &self.sql_mode { + before.push("SET @saved_sql_mode = @@sql_mode".to_owned()); + before.push(format!("SET sql_mode = {}", quoted(mode))); + after.insert(0, "SET sql_mode = @saved_sql_mode".to_owned()); + } + before + .into_iter() + .map(DumpStatement::new) + .chain(std::iter::once(create)) + .chain(after.into_iter().map(DumpStatement::new)) + .collect() + } +} + +/// A setting's value as a string, which reads the same whatever mode +/// the session is in: neither a mode list nor a collation name holds a +/// backslash. +fn quoted(value: &str) -> String { + format!("'{}'", value.replace('\'', "''")) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn texts(statements: &[DumpStatement]) -> Vec<(&str, bool)> { + statements + .iter() + .map(|statement| (statement.sql.as_str(), statement.body)) + .collect() + } + + #[test] + fn a_routine_runs_under_its_own_mode_and_collation_then_the_sessions() { + let settings = ProgramSettings { + sql_mode: Some("STRICT_TRANS_TABLES,NO_ZERO_DATE".to_owned()), + collation_connection: Some("utf8mb4_0900_ai_ci".to_owned()), + }; + + let statements = settings.around(DumpStatement::body("CREATE PROCEDURE `p`()\nBEGIN SELECT 1; END")); + + assert_eq!( + texts(&statements), + [ + ("SET @saved_col_connection = @@collation_connection", false), + ("SET collation_connection = 'utf8mb4_0900_ai_ci'", false), + ("SET @saved_sql_mode = @@sql_mode", false), + ("SET sql_mode = 'STRICT_TRANS_TABLES,NO_ZERO_DATE'", false), + ("CREATE PROCEDURE `p`()\nBEGIN SELECT 1; END", true), + ("SET sql_mode = @saved_sql_mode", false), + ("SET collation_connection = @saved_col_connection", false), + ] + ); + } + + #[test] + fn a_view_keeps_the_sessions_mode_and_an_empty_mode_is_still_set() { + let view = ProgramSettings { + sql_mode: None, + collation_connection: Some("latin1_swedish_ci".to_owned()), + }; + let unmoded = ProgramSettings { + sql_mode: Some(String::new()), + collation_connection: None, + }; + + assert_eq!( + texts(&view.around(DumpStatement::new("CREATE VIEW `v` AS select 1 AS `1`"))), + [ + ("SET @saved_col_connection = @@collation_connection", false), + ("SET collation_connection = 'latin1_swedish_ci'", false), + ("CREATE VIEW `v` AS select 1 AS `1`", false), + ("SET collation_connection = @saved_col_connection", false), + ] + ); + assert_eq!( + texts(&unmoded.around(DumpStatement::body( + "CREATE TRIGGER `t` BEFORE INSERT ON `c` FOR EACH ROW SET @n = 1" + ))), + [ + ("SET @saved_sql_mode = @@sql_mode", false), + ("SET sql_mode = ''", false), + ("CREATE TRIGGER `t` BEFORE INSERT ON `c` FOR EACH ROW SET @n = 1", true), + ("SET sql_mode = @saved_sql_mode", false), + ] + ); + } +} diff --git a/core/crates/drivers/mysql/src/lib.rs b/core/crates/drivers/mysql/src/lib.rs index fbfdb4ddc9..e774985393 100644 --- a/core/crates/drivers/mysql/src/lib.rs +++ b/core/crates/drivers/mysql/src/lib.rs @@ -14,6 +14,7 @@ mod ddl; mod decode; mod dialect; mod driver; +mod dump; mod editor_run; mod editor_session; mod encode; diff --git a/core/crates/drivers/mysql/src/metadata.rs b/core/crates/drivers/mysql/src/metadata.rs index c718bc126e..38631e631f 100644 --- a/core/crates/drivers/mysql/src/metadata.rs +++ b/core/crates/drivers/mysql/src/metadata.rs @@ -283,7 +283,7 @@ pub(crate) async fn fetch_object_definition( })) } -fn text_column(row: &sqlx::mysql::MySqlRow, column: &str) -> Result, DriverError> { +pub(crate) fn text_column(row: &sqlx::mysql::MySqlRow, column: &str) -> Result, DriverError> { let decode = |detail: String| DriverError::Decode { column: column.to_owned(), detail, @@ -296,7 +296,7 @@ fn text_column(row: &sqlx::mysql::MySqlRow, column: &str) -> Result bool { +pub(crate) fn not_there(error: &sqlx::Error) -> bool { let sqlx::Error::Database(database) = error else { return false; }; diff --git a/core/crates/drivers/mysql/tests/integration/conformance_engine.rs b/core/crates/drivers/mysql/tests/integration/conformance_engine.rs index 57df619e2e..08185a2266 100644 --- a/core/crates/drivers/mysql/tests/integration/conformance_engine.rs +++ b/core/crates/drivers/mysql/tests/integration/conformance_engine.rs @@ -42,6 +42,12 @@ pub struct MySqlEngine { server: PhantomData, } +impl MySqlEngine { + pub fn fixture(&self) -> &MysqlFixture { + &self.fixture + } +} + fn quoted(name: &str) -> String { format!("`{}`", name.replace('`', "``")) } diff --git a/core/crates/drivers/mysql/tests/integration/dump.rs b/core/crates/drivers/mysql/tests/integration/dump.rs new file mode 100644 index 0000000000..acced80d9b --- /dev/null +++ b/core/crates/drivers/mysql/tests/integration/dump.rs @@ -0,0 +1,202 @@ +//! The SQL dump round trip on MySQL and MariaDB. + +use async_trait::async_trait; +use tablepro_core::{ConnectOptions, DatabaseDriver, TlsConfig, WorkspaceSettings, WriteKind}; +use tablepro_driver_mysql::MysqlDriver; +use tablepro_test_fixtures::MysqlFlavour; +use tablepro_test_support::conformance::{ConformanceFailure, DumpEngine, NativeDump, test_liveness}; +use tokio_util::sync::CancellationToken; + +use crate::conformance_engine::{MySqlEngine, Server}; +use crate::support::{apply, direct, options_over, password}; + +fn quoted(name: &str) -> String { + format!("`{}`", name.replace('`', "``")) +} + +#[async_trait] +impl DumpEngine for MySqlEngine { + /// Made and reached as root: the password user may create no + /// database, and under MySQL's binary log only an account with SUPER + /// may create a trigger. + async fn empty_database(&self, name: &str) -> Result { + let fixture = self.fixture(); + let options = |database: &str| { + let liveness = test_liveness(); + options_over( + direct(fixture.host(), fixture.port(), &liveness), + password(&fixture.root_credentials()), + database, + TlsConfig::disabled(), + WorkspaceSettings::workspace(false), + liveness, + ) + }; + let connection = MysqlDriver + .connect(options(fixture.database()), CancellationToken::new()) + .await?; + let database = quoted(name); + let drop = format!("DROP DATABASE IF EXISTS {database}"); + let create = format!("CREATE DATABASE {database}"); + let made = apply(connection.as_ref(), WriteKind::SchemaChanges, &[&drop, &create]).await; + connection.close().await?; + made?; + Ok(options(name)) + } + + /// The parent row with key 0 loads only under the header's + /// `NO_AUTO_VALUE_ON_ZERO`. The view that calls a function is printed + /// with the database's name by MySQL, and MariaDB names it in the + /// sequence a default takes from. + fn dump_fixture(&self) -> Vec { + let mariadb = S::FLAVOUR == MysqlFlavour::Mariadb; + let mut statements: Vec<&str> = Vec::new(); + if mariadb { + statements.push("CREATE SEQUENCE invoice_no START WITH 1000 INCREMENT BY 1 CACHE 10"); + } + statements.extend([ + "CREATE TABLE parent (\ + id int NOT NULL AUTO_INCREMENT, \ + name varchar(50) NOT NULL, \ + PRIMARY KEY (id), \ + CONSTRAINT parent_name_present CHECK (name <> '')\ + )", + "CREATE TABLE child (\ + id int NOT NULL AUTO_INCREMENT, \ + parent_id int NOT NULL, \ + next_id int NULL, \ + note text, \ + payload varbinary(16), \ + amount decimal(10,2) NOT NULL DEFAULT 0, \ + doubled decimal(12,2) GENERATED ALWAYS AS (amount * 2) STORED, \ + PRIMARY KEY (id), \ + CONSTRAINT child_parent FOREIGN KEY (parent_id) REFERENCES parent (id), \ + CONSTRAINT child_next FOREIGN KEY (next_id) REFERENCES child (id), \ + CONSTRAINT child_amount_positive CHECK (amount >= 0)\ + )", + ]); + if mariadb { + statements.push( + "CREATE TABLE invoice (\ + no bigint NOT NULL DEFAULT NEXTVAL(invoice_no), \ + note varchar(20), \ + PRIMARY KEY (no)\ + )", + ); + } + let typed = typed_table(S::FLAVOUR); + statements.push(&typed.0); + statements.extend([ + "CREATE FUNCTION label_of(p int) RETURNS varchar(80) DETERMINISTIC READS SQL DATA\n\ + BEGIN\n\ + DECLARE found varchar(50);\n\ + SELECT name INTO found FROM parent WHERE id = p;\n\ + RETURN CONCAT('#', p, ' ', COALESCE(found, '?'));\n\ + END", + "CREATE PROCEDURE add_parent(IN n varchar(50))\n\ + BEGIN\n\ + INSERT INTO parent (name) VALUES (n);\n\ + SELECT LAST_INSERT_ID() AS id;\n\ + END", + "CREATE VIEW parent_names AS SELECT id, name FROM parent", + "CREATE VIEW parent_labels AS SELECT id, label_of(id) AS label FROM parent_names", + "CREATE TRIGGER child_touch BEFORE INSERT ON child FOR EACH ROW \ + SET NEW.note = CONCAT(COALESCE(NEW.note, ''), ';')", + "INSERT INTO parent (id, name) VALUES (1, 'a'), (2, 'b'), (3, 'c'), (4, 'zero'), (5, 'gone')", + "UPDATE parent SET id = 0 WHERE id = 4", + "DELETE FROM parent WHERE id = 5", + "INSERT INTO child (id, parent_id, next_id, note, payload, amount) VALUES \ + (1, 1, NULL, 'it''s \"quoted\" back\\\\slash; line\\nbreak naïve 日本', X'00FF7F', 10.50), \ + (2, 0, 1, NULL, NULL, 0), \ + (3, 2, 2, 'plain', X'FF', 3.25), \ + (4, 3, NULL, 'gone', NULL, 1)", + "UPDATE child SET next_id = 3 WHERE id = 1", + "DELETE FROM child WHERE id = 4", + ]); + if mariadb { + statements.push("INSERT INTO invoice (note) VALUES ('first'), ('second')"); + } + statements.push(&typed.1); + statements.push("INSERT INTO typed (id) VALUES (2)"); + statements.into_iter().map(str::to_owned).collect() + } + + /// Each server's own tool, with `--hex-blob --routines`: without + /// `--hex-blob` the file holds raw bytes and is not UTF-8. + fn native_dump(&self) -> Option { + let file = match S::FLAVOUR { + MysqlFlavour::Mysql => include_str!("fixtures/mysqldump-8.4-hex-blob.sql"), + MysqlFlavour::Mariadb => include_str!("fixtures/mariadb-dump-11.4-hex-blob.sql"), + }; + let checks = [ + ("SELECT COUNT(*) FROM child", "4"), + ("SELECT id FROM parent WHERE name = 'zero'", "0"), + ("SELECT note FROM child WHERE id = 2", "line1\nline2;"), + ("SELECT HEX(payload) FROM child WHERE id = 1", "00FF27"), + ("SELECT COUNT(*) FROM v2", "3"), + ("SELECT f_double(21)", "42"), + ]; + Some(NativeDump { + file, + checks: checks + .into_iter() + .map(|(sql, value)| (sql.to_owned(), value.to_owned())) + .collect(), + }) + } +} + +/// A column of each kind the value corpus covers, one row of values at +/// their edges and one of NULLs. +fn typed_table(flavour: MysqlFlavour) -> (String, String) { + let (columns, values) = match flavour { + MysqlFlavour::Mysql => ( + "doc json, packed_uuid binary(16)", + "'{\"b\": 1, \"a\": [1, 2.50, \"x\"], \"ü\": null}', \ + UUID_TO_BIN('123e4567-e89b-12d3-a456-426614174000')", + ), + MysqlFlavour::Mariadb => ( + "doc json, uid uuid, address inet6", + "'{\"b\": 1, \"a\": [1, 2.50, \"x\"]}', '123e4567-e89b-12d3-a456-426614174000', '::ffff:192.0.2.1'", + ), + }; + let create = format!( + "CREATE TABLE typed (\ + id int NOT NULL PRIMARY KEY, \ + fine datetime(6), at datetime, stamp timestamp(3) NULL, day date, span time(6), yr year, \ + wide decimal(65, 30), money decimal(10, 2), huge bigint unsigned, low bigint, small tinyint unsigned, \ + single float, twice double, word varchar(100), latin varchar(20) CHARACTER SET latin1, \ + fixed char(5), body text, bytes varbinary(8), blob_value blob, flags bit(10), \ + mood enum('happy', 'sad'), tags set('a', 'b', 'c'), {columns}\ + )" + ); + let insert = format!( + "INSERT INTO typed VALUES (1, \ + '9999-12-31 23:59:59.999999', '2024-03-10 02:30:00', '2038-01-19 03:14:07.999', '2024-02-29', \ + '-838:59:59', 2155, \ + 12345678901234567890123456789012345.123456789012345678901234567890, -12345678.9, \ + 18446744073709551615, -9223372036854775808, 255, \ + 1.5, -2.5e-5, '🦀 emoji 👩‍💻', 'café', \ + 'ab', 'tab\\there', X'DEADBEEF', X'00FF', b'1010101010', \ + 'sad', 'a,c', {values})" + ); + (create, insert) +} + +mod mysql_suite { + use crate::conformance_engine::{MySqlEngine, MysqlServer}; + + tablepro_test_support::dump_suite! { + engine: MySqlEngine, + docker: true, + } +} + +mod mariadb_suite { + use crate::conformance_engine::{MariadbServer, MySqlEngine}; + + tablepro_test_support::dump_suite! { + engine: MySqlEngine, + docker: true, + } +} diff --git a/core/crates/drivers/mysql/tests/integration/fixtures/mariadb-dump-11.4-hex-blob.sql b/core/crates/drivers/mysql/tests/integration/fixtures/mariadb-dump-11.4-hex-blob.sql new file mode 100644 index 0000000000..dc51f66ce3 --- /dev/null +++ b/core/crates/drivers/mysql/tests/integration/fixtures/mariadb-dump-11.4-hex-blob.sql @@ -0,0 +1,221 @@ +/*M!999999\- enable the sandbox mode */ +-- MariaDB dump 10.19-11.4.13-MariaDB, for debian-linux-gnu (aarch64) +-- +-- Host: localhost Database: app +-- ------------------------------------------------------ +-- Server version 11.4.13-MariaDB-ubu2404 + +/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; +/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; +/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; +/*!40101 SET NAMES utf8mb4 */; +/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; +/*!40103 SET TIME_ZONE='+00:00' */; +/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; +/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; +/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; +/*M!100616 SET @OLD_NOTE_VERBOSITY=@@NOTE_VERBOSITY, NOTE_VERBOSITY=0 */; + +-- +-- Table structure for table `child` +-- + +DROP TABLE IF EXISTS `child`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8mb4 */; +CREATE TABLE `child` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `parent_id` int(11) NOT NULL, + `manager_id` bigint(20) DEFAULT NULL, + `note` text DEFAULT NULL, + `payload` blob DEFAULT NULL, + `created` datetime(3) DEFAULT current_timestamp(3), + PRIMARY KEY (`id`), + KEY `child_parent_fk` (`parent_id`), + KEY `child_manager_fk` (`manager_id`), + KEY `child_note_idx` (`note`(20)), + CONSTRAINT `child_manager_fk` FOREIGN KEY (`manager_id`) REFERENCES `child` (`id`), + CONSTRAINT `child_parent_fk` FOREIGN KEY (`parent_id`) REFERENCES `parent` (`id`) ON DELETE CASCADE +) ENGINE=InnoDB AUTO_INCREMENT=6 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `child` +-- + +LOCK TABLES `child` WRITE; +/*!40000 ALTER TABLE `child` DISABLE KEYS */; +INSERT INTO `child` VALUES +(1,1,3,'it\'s; a \"quote\" \\ back;',0x00FF27,'2026-01-02 03:04:05.678'), +(2,1,NULL,'line1\nline2;',NULL,'2026-01-02 03:04:05.678'), +(3,2,1,'ünïcødé 😀 DELIMITER ;;;',NULL,'2026-01-02 03:04:05.678'), +(4,0,NULL,';',NULL,'2026-01-02 03:04:05.678'); +/*!40000 ALTER TABLE `child` ENABLE KEYS */; +UNLOCK TABLES; +/*!50003 SET @saved_cs_client = @@character_set_client */ ; +/*!50003 SET @saved_cs_results = @@character_set_results */ ; +/*!50003 SET @saved_col_connection = @@collation_connection */ ; +/*!50003 SET character_set_client = utf8mb4 */ ; +/*!50003 SET character_set_results = utf8mb4 */ ; +/*!50003 SET collation_connection = utf8mb4_uca1400_ai_ci */ ; +/*!50003 SET @saved_sql_mode = @@sql_mode */ ; +/*!50003 SET sql_mode = 'NO_AUTO_VALUE_ON_ZERO,STRICT_TRANS_TABLES,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION' */ ; +DELIMITER ;; +/*!50003 CREATE*/ /*!50017 DEFINER=`root`@`localhost`*/ /*!50003 TRIGGER child_touch BEFORE INSERT ON child FOR EACH ROW SET NEW.note = CONCAT(COALESCE(NEW.note, ''), ';') +*/;; +DELIMITER ; +/*!50003 SET sql_mode = @saved_sql_mode */ ; +/*!50003 SET character_set_client = @saved_cs_client */ ; +/*!50003 SET character_set_results = @saved_cs_results */ ; +/*!50003 SET collation_connection = @saved_col_connection */ ; + +-- +-- Table structure for table `parent` +-- + +DROP TABLE IF EXISTS `parent`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8mb4 */; +CREATE TABLE `parent` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `name` varchar(50) NOT NULL, + `mood` enum('sad','ok','happy') DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `name` (`name`) +) ENGINE=InnoDB AUTO_INCREMENT=6 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `parent` +-- + +LOCK TABLES `parent` WRITE; +/*!40000 ALTER TABLE `parent` DISABLE KEYS */; +INSERT INTO `parent` VALUES +(0,'zero','ok'), +(1,'a','ok'), +(2,'b','sad'), +(3,'c','happy'); +/*!40000 ALTER TABLE `parent` ENABLE KEYS */; +UNLOCK TABLES; + +-- +-- Temporary table structure for view `v1` +-- + +DROP TABLE IF EXISTS `v1`; +/*!50001 DROP VIEW IF EXISTS `v1`*/; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8mb4; +/*!50001 CREATE VIEW `v1` AS SELECT + NULL AS `id`, + NULL AS `name`, + NULL AS `n` */; +SET character_set_client = @saved_cs_client; + +-- +-- Temporary table structure for view `v2` +-- + +DROP TABLE IF EXISTS `v2`; +/*!50001 DROP VIEW IF EXISTS `v2`*/; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8mb4; +/*!50001 CREATE VIEW `v2` AS SELECT + NULL AS `id`, + NULL AS `name`, + NULL AS `n` */; +SET character_set_client = @saved_cs_client; + +-- +-- Dumping routines for database 'app' +-- +/*!50003 SET @saved_sql_mode = @@sql_mode */ ; +/*!50003 SET sql_mode = 'NO_AUTO_VALUE_ON_ZERO,STRICT_TRANS_TABLES,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION' */ ; +/*!50003 DROP FUNCTION IF EXISTS `f_double` */; +/*!50003 SET @saved_cs_client = @@character_set_client */ ; +/*!50003 SET @saved_cs_results = @@character_set_results */ ; +/*!50003 SET @saved_col_connection = @@collation_connection */ ; +/*!50003 SET character_set_client = utf8mb4 */ ; +/*!50003 SET character_set_results = utf8mb4 */ ; +/*!50003 SET collation_connection = utf8mb4_uca1400_ai_ci */ ; +DELIMITER ;; +CREATE DEFINER=`root`@`localhost` FUNCTION `f_double`(x INT) RETURNS int(11) + DETERMINISTIC +BEGIN + RETURN x * 2; +END +;; +DELIMITER ; +/*!50003 SET sql_mode = @saved_sql_mode */ ; +/*!50003 SET character_set_client = @saved_cs_client */ ; +/*!50003 SET character_set_results = @saved_cs_results */ ; +/*!50003 SET collation_connection = @saved_col_connection */ ; +/*!50003 SET @saved_sql_mode = @@sql_mode */ ; +/*!50003 SET sql_mode = 'NO_AUTO_VALUE_ON_ZERO,STRICT_TRANS_TABLES,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION' */ ; +/*!50003 DROP PROCEDURE IF EXISTS `add_parent` */; +/*!50003 SET @saved_cs_client = @@character_set_client */ ; +/*!50003 SET @saved_cs_results = @@character_set_results */ ; +/*!50003 SET @saved_col_connection = @@collation_connection */ ; +/*!50003 SET character_set_client = utf8mb4 */ ; +/*!50003 SET character_set_results = utf8mb4 */ ; +/*!50003 SET collation_connection = utf8mb4_uca1400_ai_ci */ ; +DELIMITER ;; +CREATE DEFINER=`root`@`localhost` PROCEDURE `add_parent`(IN n VARCHAR(50)) +BEGIN + INSERT INTO parent (name) VALUES (n); + SELECT LAST_INSERT_ID() AS id; +END +;; +DELIMITER ; +/*!50003 SET sql_mode = @saved_sql_mode */ ; +/*!50003 SET character_set_client = @saved_cs_client */ ; +/*!50003 SET character_set_results = @saved_cs_results */ ; +/*!50003 SET collation_connection = @saved_col_connection */ ; + +-- +-- Final view structure for view `v1` +-- + +/*!50001 DROP VIEW IF EXISTS `v1`*/; +/*!50001 SET @saved_cs_client = @@character_set_client */; +/*!50001 SET @saved_cs_results = @@character_set_results */; +/*!50001 SET @saved_col_connection = @@collation_connection */; +/*!50001 SET character_set_client = utf8mb4 */; +/*!50001 SET character_set_results = utf8mb4 */; +/*!50001 SET collation_connection = utf8mb4_uca1400_ai_ci */; +/*!50001 CREATE ALGORITHM=UNDEFINED */ +/*!50013 DEFINER=`root`@`localhost` SQL SECURITY DEFINER */ +/*!50001 VIEW `v1` AS select `p`.`id` AS `id`,`p`.`name` AS `name`,count(`c`.`id`) AS `n` from (`parent` `p` left join `child` `c` on(`c`.`parent_id` = `p`.`id`)) group by `p`.`id`,`p`.`name` */; +/*!50001 SET character_set_client = @saved_cs_client */; +/*!50001 SET character_set_results = @saved_cs_results */; +/*!50001 SET collation_connection = @saved_col_connection */; + +-- +-- Final view structure for view `v2` +-- + +/*!50001 DROP VIEW IF EXISTS `v2`*/; +/*!50001 SET @saved_cs_client = @@character_set_client */; +/*!50001 SET @saved_cs_results = @@character_set_results */; +/*!50001 SET @saved_col_connection = @@collation_connection */; +/*!50001 SET character_set_client = utf8mb4 */; +/*!50001 SET character_set_results = utf8mb4 */; +/*!50001 SET collation_connection = utf8mb4_uca1400_ai_ci */; +/*!50001 CREATE ALGORITHM=UNDEFINED */ +/*!50013 DEFINER=`root`@`localhost` SQL SECURITY DEFINER */ +/*!50001 VIEW `v2` AS select `v1`.`id` AS `id`,`v1`.`name` AS `name`,`v1`.`n` AS `n` from `v1` where `v1`.`n` > 0 */; +/*!50001 SET character_set_client = @saved_cs_client */; +/*!50001 SET character_set_results = @saved_cs_results */; +/*!50001 SET collation_connection = @saved_col_connection */; +/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; + +/*!40101 SET SQL_MODE=@OLD_SQL_MODE */; +/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; +/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; +/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; +/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; +/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; +/*M!100616 SET NOTE_VERBOSITY=@OLD_NOTE_VERBOSITY */; + +-- Dump completed on 2026-11-05 5:39:09 diff --git a/core/crates/drivers/mysql/tests/integration/fixtures/mysqldump-8.4-hex-blob.sql b/core/crates/drivers/mysql/tests/integration/fixtures/mysqldump-8.4-hex-blob.sql new file mode 100644 index 0000000000..23ba24099d --- /dev/null +++ b/core/crates/drivers/mysql/tests/integration/fixtures/mysqldump-8.4-hex-blob.sql @@ -0,0 +1,209 @@ +-- MySQL dump 10.13 Distrib 8.4.13, for Linux (aarch64) +-- +-- Host: localhost Database: app +-- ------------------------------------------------------ +-- Server version 8.4.13 + +/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; +/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; +/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; +/*!50503 SET NAMES utf8mb4 */; +/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; +/*!40103 SET TIME_ZONE='+00:00' */; +/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; +/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; +/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; +/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; + +-- +-- Table structure for table `child` +-- + +DROP TABLE IF EXISTS `child`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!50503 SET character_set_client = utf8mb4 */; +CREATE TABLE `child` ( + `id` bigint NOT NULL AUTO_INCREMENT, + `parent_id` int NOT NULL, + `manager_id` bigint DEFAULT NULL, + `note` text, + `payload` blob, + `created` datetime(3) DEFAULT CURRENT_TIMESTAMP(3), + PRIMARY KEY (`id`), + KEY `child_parent_fk` (`parent_id`), + KEY `child_manager_fk` (`manager_id`), + KEY `child_note_idx` (`note`(20)), + CONSTRAINT `child_manager_fk` FOREIGN KEY (`manager_id`) REFERENCES `child` (`id`), + CONSTRAINT `child_parent_fk` FOREIGN KEY (`parent_id`) REFERENCES `parent` (`id`) ON DELETE CASCADE +) ENGINE=InnoDB AUTO_INCREMENT=6 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `child` +-- + +LOCK TABLES `child` WRITE; +/*!40000 ALTER TABLE `child` DISABLE KEYS */; +INSERT INTO `child` VALUES (1,1,3,'it\'s; a \"quote\" \\ back;',0x00FF27,'2026-01-02 03:04:05.678'),(2,1,NULL,'line1\nline2;',NULL,'2026-01-02 03:04:05.678'),(3,2,1,'ünïcødé 😀 DELIMITER ;;;',NULL,'2026-01-02 03:04:05.678'),(4,0,NULL,';',NULL,'2026-01-02 03:04:05.678'); +/*!40000 ALTER TABLE `child` ENABLE KEYS */; +UNLOCK TABLES; +/*!50003 SET @saved_cs_client = @@character_set_client */ ; +/*!50003 SET @saved_cs_results = @@character_set_results */ ; +/*!50003 SET @saved_col_connection = @@collation_connection */ ; +/*!50003 SET character_set_client = latin1 */ ; +/*!50003 SET character_set_results = latin1 */ ; +/*!50003 SET collation_connection = latin1_swedish_ci */ ; +/*!50003 SET @saved_sql_mode = @@sql_mode */ ; +/*!50003 SET sql_mode = 'ONLY_FULL_GROUP_BY,NO_AUTO_VALUE_ON_ZERO,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION' */ ; +DELIMITER ;; +/*!50003 CREATE*/ /*!50017 DEFINER=`root`@`localhost`*/ /*!50003 TRIGGER `child_touch` BEFORE INSERT ON `child` FOR EACH ROW SET NEW.note = CONCAT(COALESCE(NEW.note, ''), ';') */;; +DELIMITER ; +/*!50003 SET sql_mode = @saved_sql_mode */ ; +/*!50003 SET character_set_client = @saved_cs_client */ ; +/*!50003 SET character_set_results = @saved_cs_results */ ; +/*!50003 SET collation_connection = @saved_col_connection */ ; + +-- +-- Table structure for table `parent` +-- + +DROP TABLE IF EXISTS `parent`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!50503 SET character_set_client = utf8mb4 */; +CREATE TABLE `parent` ( + `id` int NOT NULL AUTO_INCREMENT, + `name` varchar(50) NOT NULL, + `mood` enum('sad','ok','happy') DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `name` (`name`) +) ENGINE=InnoDB AUTO_INCREMENT=6 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `parent` +-- + +LOCK TABLES `parent` WRITE; +/*!40000 ALTER TABLE `parent` DISABLE KEYS */; +INSERT INTO `parent` VALUES (0,'zero','ok'),(1,'a','ok'),(2,'b','sad'),(3,'c','happy'); +/*!40000 ALTER TABLE `parent` ENABLE KEYS */; +UNLOCK TABLES; + +-- +-- Temporary view structure for view `v1` +-- + +DROP TABLE IF EXISTS `v1`; +/*!50001 DROP VIEW IF EXISTS `v1`*/; +SET @saved_cs_client = @@character_set_client; +/*!50503 SET character_set_client = utf8mb4 */; +/*!50001 CREATE VIEW `v1` AS SELECT + 1 AS `id`, + 1 AS `name`, + 1 AS `n`*/; +SET character_set_client = @saved_cs_client; + +-- +-- Temporary view structure for view `v2` +-- + +DROP TABLE IF EXISTS `v2`; +/*!50001 DROP VIEW IF EXISTS `v2`*/; +SET @saved_cs_client = @@character_set_client; +/*!50503 SET character_set_client = utf8mb4 */; +/*!50001 CREATE VIEW `v2` AS SELECT + 1 AS `id`, + 1 AS `name`, + 1 AS `n`*/; +SET character_set_client = @saved_cs_client; + +-- +-- Dumping routines for database 'app' +-- +/*!50003 DROP FUNCTION IF EXISTS `f_double` */; +/*!50003 SET @saved_cs_client = @@character_set_client */ ; +/*!50003 SET @saved_cs_results = @@character_set_results */ ; +/*!50003 SET @saved_col_connection = @@collation_connection */ ; +/*!50003 SET character_set_client = latin1 */ ; +/*!50003 SET character_set_results = latin1 */ ; +/*!50003 SET collation_connection = latin1_swedish_ci */ ; +/*!50003 SET @saved_sql_mode = @@sql_mode */ ; +/*!50003 SET sql_mode = 'ONLY_FULL_GROUP_BY,NO_AUTO_VALUE_ON_ZERO,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION' */ ; +DELIMITER ;; +CREATE DEFINER=`root`@`localhost` FUNCTION `f_double`(x INT) RETURNS int + DETERMINISTIC +BEGIN + RETURN x * 2; +END ;; +DELIMITER ; +/*!50003 SET sql_mode = @saved_sql_mode */ ; +/*!50003 SET character_set_client = @saved_cs_client */ ; +/*!50003 SET character_set_results = @saved_cs_results */ ; +/*!50003 SET collation_connection = @saved_col_connection */ ; +/*!50003 DROP PROCEDURE IF EXISTS `add_parent` */; +/*!50003 SET @saved_cs_client = @@character_set_client */ ; +/*!50003 SET @saved_cs_results = @@character_set_results */ ; +/*!50003 SET @saved_col_connection = @@collation_connection */ ; +/*!50003 SET character_set_client = latin1 */ ; +/*!50003 SET character_set_results = latin1 */ ; +/*!50003 SET collation_connection = latin1_swedish_ci */ ; +/*!50003 SET @saved_sql_mode = @@sql_mode */ ; +/*!50003 SET sql_mode = 'ONLY_FULL_GROUP_BY,NO_AUTO_VALUE_ON_ZERO,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION' */ ; +DELIMITER ;; +CREATE DEFINER=`root`@`localhost` PROCEDURE `add_parent`(IN n VARCHAR(50)) +BEGIN + INSERT INTO parent (name) VALUES (n); + SELECT LAST_INSERT_ID() AS id; +END ;; +DELIMITER ; +/*!50003 SET sql_mode = @saved_sql_mode */ ; +/*!50003 SET character_set_client = @saved_cs_client */ ; +/*!50003 SET character_set_results = @saved_cs_results */ ; +/*!50003 SET collation_connection = @saved_col_connection */ ; + +-- +-- Final view structure for view `v1` +-- + +/*!50001 DROP VIEW IF EXISTS `v1`*/; +/*!50001 SET @saved_cs_client = @@character_set_client */; +/*!50001 SET @saved_cs_results = @@character_set_results */; +/*!50001 SET @saved_col_connection = @@collation_connection */; +/*!50001 SET character_set_client = latin1 */; +/*!50001 SET character_set_results = latin1 */; +/*!50001 SET collation_connection = latin1_swedish_ci */; +/*!50001 CREATE ALGORITHM=UNDEFINED */ +/*!50013 DEFINER=`root`@`localhost` SQL SECURITY DEFINER */ +/*!50001 VIEW `v1` AS select `p`.`id` AS `id`,`p`.`name` AS `name`,count(`c`.`id`) AS `n` from (`parent` `p` left join `child` `c` on((`c`.`parent_id` = `p`.`id`))) group by `p`.`id`,`p`.`name` */; +/*!50001 SET character_set_client = @saved_cs_client */; +/*!50001 SET character_set_results = @saved_cs_results */; +/*!50001 SET collation_connection = @saved_col_connection */; + +-- +-- Final view structure for view `v2` +-- + +/*!50001 DROP VIEW IF EXISTS `v2`*/; +/*!50001 SET @saved_cs_client = @@character_set_client */; +/*!50001 SET @saved_cs_results = @@character_set_results */; +/*!50001 SET @saved_col_connection = @@collation_connection */; +/*!50001 SET character_set_client = latin1 */; +/*!50001 SET character_set_results = latin1 */; +/*!50001 SET collation_connection = latin1_swedish_ci */; +/*!50001 CREATE ALGORITHM=UNDEFINED */ +/*!50013 DEFINER=`root`@`localhost` SQL SECURITY DEFINER */ +/*!50001 VIEW `v2` AS select `v1`.`id` AS `id`,`v1`.`name` AS `name`,`v1`.`n` AS `n` from `v1` where (`v1`.`n` > 0) */; +/*!50001 SET character_set_client = @saved_cs_client */; +/*!50001 SET character_set_results = @saved_cs_results */; +/*!50001 SET collation_connection = @saved_col_connection */; +/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; + +/*!40101 SET SQL_MODE=@OLD_SQL_MODE */; +/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; +/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; +/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; +/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; +/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; +/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; + +-- Dump completed on 2026-11-05 5:20:36 diff --git a/core/crates/drivers/mysql/tests/integration/main.rs b/core/crates/drivers/mysql/tests/integration/main.rs index b916b3de6f..90331f91bf 100644 --- a/core/crates/drivers/mysql/tests/integration/main.rs +++ b/core/crates/drivers/mysql/tests/integration/main.rs @@ -10,6 +10,7 @@ mod conformance_engine; mod csv_import; mod ddl; mod decode; +mod dump; mod editor_session; mod flavour_suite; mod lenient_server; diff --git a/core/crates/drivers/postgres/src/connection.rs b/core/crates/drivers/postgres/src/connection.rs index 374e3e59d5..ecab805d37 100644 --- a/core/crates/drivers/postgres/src/connection.rs +++ b/core/crates/drivers/postgres/src/connection.rs @@ -6,6 +6,7 @@ use deadpool_postgres::{Pool, PoolError, TimeoutType, Timeouts}; use tablepro_core::column::ColumnInfo; use tablepro_core::ddl::{InboundForeignKey, OutboundForeignKey, TableDefinition}; use tablepro_core::dialect::SqlDialect; +use tablepro_core::dump::{DumpCatalog, DumpScope}; use tablepro_core::meta::{ IndexInfo, ObjectDefinition, ObjectInfo, ObjectScope, RowIdentity, SchemaInfo, TableEstimate, TableRef, }; @@ -24,6 +25,7 @@ use crate::cancel::{PgStop, StopProgress}; use crate::catalog; use crate::commit_resolution::resolve_op; use crate::dialect::PgDialect; +use crate::dump; use crate::editor::PgEditorSession; use crate::errors::{map_error, map_pool_error}; use crate::metadata; @@ -338,6 +340,20 @@ impl Connection for PgConnection { .await } + /// Every read runs in one transaction that first empties the search + /// path, under which the server names every object with its schema. + async fn read_dump_catalog(&self, scope: &DumpScope, options: CallOptions) -> Result { + let scope = scope.clone(); + let sql_bodies = self.server.at_least(14, 0, 0); + let begin = self.read_preamble(&options).with_dependent_queries(); + self.read_in( + options, + begin, + read_op(move |client| Box::pin(async move { dump::read_dump_catalog(client, &scope, sql_bodies).await })), + ) + .await + } + async fn fetch_columns(&self, table: &TableRef, options: CallOptions) -> Result, DriverError> { let relation = self.relation(table); let columns_sql = self.columns_sql(); diff --git a/core/crates/drivers/postgres/src/dialect.rs b/core/crates/drivers/postgres/src/dialect.rs index 826e3d898b..bf3882bc7b 100644 --- a/core/crates/drivers/postgres/src/dialect.rs +++ b/core/crates/drivers/postgres/src/dialect.rs @@ -9,6 +9,7 @@ use tablepro_core::dialect::{ BindError, BindTarget, DatabaseLayout, DialectCapabilities, InsertConflict, KeysetDirection, LiteralError, Placeholder, SqlDialect, }; +use tablepro_core::dump::DumpDialect; use tablepro_core::edit::{CellExpression, MomentKind}; use tablepro_core::export::value_to_text; use tablepro_core::meta::{EngineRowId, EngineRowIdPart}; @@ -19,6 +20,8 @@ use tablepro_core::sql_syntax::{ use tablepro_core::statement::{BoundParam, ParamType}; use tablepro_core::value::{Temporal, Value}; +use crate::dump::PgDump; + /// The type a bit string is cast to. A cast to the column's own `bit` /// would silently truncate to one bit, while `varbit` keeps every bit and /// leaves the length check to the column's assignment cast. @@ -183,6 +186,10 @@ impl SqlDialect for PgDialect { Some(self) } + fn dump(&self) -> Option<&dyn DumpDialect> { + Some(&PgDump) + } + /// A column without a binary decoder is read as the server's text, /// and money as the numeric it is, so no row ever carries a type the /// driver would have to look up by name. @@ -417,7 +424,7 @@ fn array_element(text: &str) -> String { /// A string literal that reads the same whatever /// `standard_conforming_strings` is. -fn quoted(text: &str) -> String { +pub(crate) fn quoted(text: &str) -> String { escape_literal(text).trim_start().to_owned() } diff --git a/core/crates/drivers/postgres/src/dump/catalog_rows.rs b/core/crates/drivers/postgres/src/dump/catalog_rows.rs new file mode 100644 index 0000000000..2cce3d9ae3 --- /dev/null +++ b/core/crates/drivers/postgres/src/dump/catalog_rows.rs @@ -0,0 +1,407 @@ +use tablepro_core::DriverError; +use tokio_postgres::Row; + +use crate::codec::TextArray; +use crate::metadata::{decode_error, in_tablespace, without_only}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct Extension { + pub(super) name: String, + pub(super) schema: String, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct Collation { + pub(super) schema: String, + pub(super) name: String, +} + +/// A collation a user made, with each release's locale columns. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct UserCollation { + pub(super) schema: String, + pub(super) name: String, + /// `c` for libc, `i` for ICU, `b` for builtin. + pub(super) provider: String, + pub(super) deterministic: bool, + pub(super) collate: Option, + pub(super) ctype: Option, + pub(super) locale: Option, + pub(super) rules: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct Domain { + pub(super) base: String, + pub(super) not_null: bool, + pub(super) default: Option, + pub(super) collation: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) enum TypeShape { + Enum(Vec), + Domain(Domain), + Composite, + /// A range or a base type. + Other, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct UserType { + pub(super) schema: String, + pub(super) name: String, + pub(super) shape: TypeShape, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct DomainCheck { + pub(super) schema: String, + pub(super) domain: String, + pub(super) name: String, + pub(super) definition: String, + pub(super) validated: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct Routine { + pub(super) schema: String, + pub(super) name: String, + /// `pg_proc.prokind`. + pub(super) kind: String, + pub(super) definition: Option, + /// The body is standard SQL, which the server parses when the routine + /// is created, so what it names must exist first. + pub(super) sql_body: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct SequenceOwner { + pub(super) identity: bool, + pub(super) schema: String, + pub(super) table: String, + pub(super) column: String, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct Sequence { + pub(super) schema: String, + pub(super) name: String, + pub(super) data_type: String, + pub(super) start: i64, + pub(super) increment: i64, + pub(super) min: i64, + pub(super) max: i64, + pub(super) cache: i64, + pub(super) cycle: bool, + pub(super) unlogged: bool, + pub(super) last_value: Option, + /// Whether `last_value` was handed out, so the next is the one after. + pub(super) is_called: Option, + pub(super) owner: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct Partition { + pub(super) parent_schema: String, + pub(super) parent: String, + pub(super) bound: String, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct Table { + pub(super) schema: String, + pub(super) name: String, + /// `pg_class.relkind`: `r`, `p` or `f`. + pub(super) kind: String, + pub(super) unlogged: bool, + pub(super) partition_key: Option, + pub(super) partition: Option, + pub(super) inherits: bool, + pub(super) options: Vec, + pub(super) access_method: Option, + pub(super) tablespace: Option, + pub(super) comment: Option, + /// Row-level security filters what this login reads of it. + pub(super) filtered: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct Column { + pub(super) schema: String, + pub(super) table: String, + pub(super) name: String, + pub(super) type_sql: String, + pub(super) collation: Option, + /// The DEFAULT, or a generated column's expression. + pub(super) default: Option, + /// `pg_attribute.attidentity`, empty when the column has none. + pub(super) identity: String, + /// `pg_attribute.attgenerated`, empty when the column is not. + pub(super) generated: String, + pub(super) not_null: bool, + pub(super) comment: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct Constraint { + pub(super) schema: String, + pub(super) table: String, + pub(super) partitioned: bool, + pub(super) name: String, + /// `pg_constraint.contype`. + pub(super) kind: String, + pub(super) definition: String, + pub(super) validated: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct Index { + pub(super) schema: String, + /// The table or materialized view it is on. + pub(super) table: String, + pub(super) name: String, + pub(super) statement: String, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct View { + pub(super) schema: String, + pub(super) name: String, + pub(super) materialized: bool, + pub(super) definition: String, + pub(super) options: Vec, + pub(super) populated: bool, + pub(super) access_method: Option, + pub(super) tablespace: Option, + pub(super) comment: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct Trigger { + pub(super) schema: String, + pub(super) table: String, + pub(super) name: String, + pub(super) definition: String, + /// `pg_trigger.tgenabled`. + pub(super) enabled: String, +} + +fn collation(schema: Option, name: Option) -> Option { + Some(Collation { + schema: schema?, + name: name?, + }) +} + +fn options(array: Option) -> Vec { + array.map(|array| array.0).unwrap_or_default() +} + +pub(super) fn schema_name(row: &Row) -> Result { + row.try_get(0).map_err(decode_error("pg_namespace")) +} + +pub(super) fn extension(row: &Row) -> Result { + let field = decode_error("pg_extension"); + Ok(Extension { + name: row.try_get(0).map_err(&field)?, + schema: row.try_get(1).map_err(&field)?, + }) +} + +pub(super) fn user_type(row: &Row) -> Result { + let field = decode_error("pg_type"); + let typtype: String = row.try_get(2).map_err(&field)?; + let shape = match typtype.as_str() { + "e" => TypeShape::Enum(row.try_get::<_, TextArray>(3).map_err(&field)?.0), + "d" => TypeShape::Domain(Domain { + base: row.try_get(4).map_err(&field)?, + not_null: row.try_get(5).map_err(&field)?, + default: row.try_get(6).map_err(&field)?, + collation: collation(row.try_get(7).map_err(&field)?, row.try_get(8).map_err(&field)?), + }), + "c" => TypeShape::Composite, + _ => TypeShape::Other, + }; + Ok(UserType { + schema: row.try_get(0).map_err(&field)?, + name: row.try_get(1).map_err(&field)?, + shape, + }) +} + +pub(super) fn user_collation(row: &Row) -> Result { + let field = decode_error("pg_collation"); + Ok(UserCollation { + schema: row.try_get(0).map_err(&field)?, + name: row.try_get(1).map_err(&field)?, + provider: row.try_get(2).map_err(&field)?, + deterministic: row.try_get(3).map_err(&field)?, + collate: row.try_get(4).map_err(&field)?, + ctype: row.try_get(5).map_err(&field)?, + locale: row.try_get(6).map_err(&field)?, + rules: row.try_get(7).map_err(&field)?, + }) +} + +pub(super) fn domain_check(row: &Row) -> Result { + let field = decode_error("pg_constraint"); + Ok(DomainCheck { + schema: row.try_get(0).map_err(&field)?, + domain: row.try_get(1).map_err(&field)?, + name: row.try_get(2).map_err(&field)?, + definition: row.try_get(3).map_err(&field)?, + validated: row.try_get(4).map_err(&field)?, + }) +} + +pub(super) fn routine(row: &Row) -> Result { + let field = decode_error("pg_proc"); + Ok(Routine { + schema: row.try_get(0).map_err(&field)?, + name: row.try_get(1).map_err(&field)?, + kind: row.try_get(2).map_err(&field)?, + definition: row.try_get(3).map_err(&field)?, + sql_body: row.try_get(4).map_err(&field)?, + }) +} + +pub(super) fn sequence(row: &Row) -> Result { + let field = decode_error("pg_sequences"); + let persistence: String = row.try_get(9).map_err(&field)?; + let dependency: Option = row.try_get(11).map_err(&field)?; + let owner_schema: Option = row.try_get(12).map_err(&field)?; + let owner_table: Option = row.try_get(13).map_err(&field)?; + let owner_column: Option = row.try_get(14).map_err(&field)?; + let owner = match (dependency, owner_schema, owner_table, owner_column) { + (Some(dependency), Some(schema), Some(table), Some(column)) => Some(SequenceOwner { + identity: dependency == "i", + schema, + table, + column, + }), + _ => None, + }; + Ok(Sequence { + schema: row.try_get(0).map_err(&field)?, + name: row.try_get(1).map_err(&field)?, + data_type: row.try_get(2).map_err(&field)?, + start: row.try_get(3).map_err(&field)?, + increment: row.try_get(4).map_err(&field)?, + min: row.try_get(5).map_err(&field)?, + max: row.try_get(6).map_err(&field)?, + cache: row.try_get(7).map_err(&field)?, + cycle: row.try_get(8).map_err(&field)?, + unlogged: persistence == "u", + last_value: row.try_get(10).map_err(&field)?, + is_called: row.try_get(15).map_err(&field)?, + owner, + }) +} + +pub(super) fn table(row: &Row) -> Result { + let field = decode_error("pg_class"); + let persistence: String = row.try_get(3).map_err(&field)?; + let bound: Option = row.try_get(5).map_err(&field)?; + let parent_schema: Option = row.try_get(6).map_err(&field)?; + let parent: Option = row.try_get(7).map_err(&field)?; + let partition = match (bound, parent_schema, parent) { + (Some(bound), Some(parent_schema), Some(parent)) => Some(Partition { + parent_schema, + parent, + bound, + }), + _ => None, + }; + Ok(Table { + schema: row.try_get(0).map_err(&field)?, + name: row.try_get(1).map_err(&field)?, + kind: row.try_get(2).map_err(&field)?, + unlogged: persistence == "u", + partition_key: row.try_get(4).map_err(&field)?, + partition, + inherits: row.try_get(8).map_err(&field)?, + options: options(row.try_get(9).map_err(&field)?), + access_method: row.try_get(10).map_err(&field)?, + tablespace: row.try_get(11).map_err(&field)?, + comment: row.try_get(12).map_err(&field)?, + filtered: row.try_get(13).map_err(&field)?, + }) +} + +pub(super) fn column(row: &Row) -> Result { + let field = decode_error("pg_attribute"); + Ok(Column { + schema: row.try_get(0).map_err(&field)?, + table: row.try_get(1).map_err(&field)?, + name: row.try_get(2).map_err(&field)?, + type_sql: row.try_get(3).map_err(&field)?, + collation: collation(row.try_get(4).map_err(&field)?, row.try_get(5).map_err(&field)?), + default: row.try_get(6).map_err(&field)?, + identity: row.try_get(7).map_err(&field)?, + generated: row.try_get(8).map_err(&field)?, + not_null: row.try_get(9).map_err(&field)?, + comment: row.try_get(10).map_err(&field)?, + }) +} + +pub(super) fn constraint(row: &Row) -> Result { + let field = decode_error("pg_constraint"); + let relkind: String = row.try_get(2).map_err(&field)?; + Ok(Constraint { + schema: row.try_get(0).map_err(&field)?, + table: row.try_get(1).map_err(&field)?, + partitioned: relkind == "p", + name: row.try_get(3).map_err(&field)?, + kind: row.try_get(4).map_err(&field)?, + definition: row.try_get(5).map_err(&field)?, + validated: row.try_get(6).map_err(&field)?, + }) +} + +pub(super) fn index(row: &Row) -> Result { + let field = decode_error("pg_index"); + let mut statement: String = row.try_get(3).map_err(&field)?; + if row.try_get(4).map_err(&field)? { + let quoted_name: String = row.try_get(6).map_err(&field)?; + statement = without_only(statement, row.try_get(5).map_err(&field)?, "ed_name); + } + if let Some(tablespace) = row.try_get::<_, Option>(7).map_err(&field)? { + let predicate: Option = row.try_get(8).map_err(&field)?; + statement = in_tablespace(statement, &tablespace, predicate.as_deref()); + } + Ok(Index { + schema: row.try_get(0).map_err(&field)?, + table: row.try_get(1).map_err(&field)?, + name: row.try_get(2).map_err(&field)?, + statement, + }) +} + +pub(super) fn view(row: &Row) -> Result { + let field = decode_error("pg_class"); + let relkind: String = row.try_get(2).map_err(&field)?; + Ok(View { + schema: row.try_get(0).map_err(&field)?, + name: row.try_get(1).map_err(&field)?, + materialized: relkind == "m", + definition: row.try_get(3).map_err(&field)?, + options: options(row.try_get(4).map_err(&field)?), + populated: row.try_get(5).map_err(&field)?, + access_method: row.try_get(6).map_err(&field)?, + tablespace: row.try_get(7).map_err(&field)?, + comment: row.try_get(8).map_err(&field)?, + }) +} + +pub(super) fn trigger(row: &Row) -> Result { + let field = decode_error("pg_trigger"); + Ok(Trigger { + schema: row.try_get(0).map_err(&field)?, + table: row.try_get(1).map_err(&field)?, + name: row.try_get(2).map_err(&field)?, + definition: row.try_get(3).map_err(&field)?, + enabled: row.try_get(4).map_err(&field)?, + }) +} diff --git a/core/crates/drivers/postgres/src/dump/catalog_sql.rs b/core/crates/drivers/postgres/src/dump/catalog_sql.rs new file mode 100644 index 0000000000..9822dc66d7 --- /dev/null +++ b/core/crates/drivers/postgres/src/dump/catalog_sql.rs @@ -0,0 +1,332 @@ +//! The catalog reads behind a dump. Each takes the scope as `$1` (`all` +//! or `schema`) and `$2` (the schema's name), and orders its rows by +//! schema and name so two dumps of the same database read alike. + +/// The schemas a dump covers, the server's own left out. +macro_rules! in_scope { + () => { + "n.nspname <> 'information_schema' AND NOT pg_catalog.starts_with(n.nspname, 'pg_') + AND ($1 = 'all' OR n.nspname = $2)" + }; +} + +/// An extension makes its own objects again when it is created. +macro_rules! not_extension_member { + ($catalog:literal, $oid:literal) => { + concat!( + "NOT EXISTS (SELECT 1 FROM pg_catalog.pg_depend x + WHERE x.classid = 'pg_catalog.", + $catalog, + "'::pg_catalog.regclass::pg_catalog.oid AND x.objid = ", + $oid, + " AND x.deptype = 'e')" + ) + }; +} + +/// `public` is in every new database already. +pub(crate) const SCHEMAS_SQL: &str = concat!( + "SELECT n.nspname::text +FROM pg_catalog.pg_namespace n +WHERE ", + in_scope!(), + " + AND n.nspname <> 'public' + AND ", + not_extension_member!("pg_namespace", "n.oid"), + " +ORDER BY n.nspname" +); + +/// plpgsql is in every new database already. +pub(crate) const EXTENSIONS_SQL: &str = concat!( + "SELECT e.extname::text, n.nspname::text +FROM pg_catalog.pg_extension e +JOIN pg_catalog.pg_namespace n ON n.oid = e.extnamespace +WHERE ", + in_scope!(), + " + AND e.extname <> 'plpgsql' +ORDER BY e.extname" +); + +/// Collations users made. The column that names an ICU or builtin +/// locale moved between releases (`colliculocale` in 15 and 16, +/// `colllocale` from 17), so the row is read as JSON. +pub(crate) const COLLATIONS_SQL: &str = concat!( + "SELECT n.nspname::text, c.collname::text, c.collprovider::text, + COALESCE((pg_catalog.to_jsonb(c) ->> 'collisdeterministic')::boolean, true), + pg_catalog.to_jsonb(c) ->> 'collcollate', pg_catalog.to_jsonb(c) ->> 'collctype', + COALESCE(pg_catalog.to_jsonb(c) ->> 'colllocale', pg_catalog.to_jsonb(c) ->> 'colliculocale'), + pg_catalog.to_jsonb(c) ->> 'collicurules' +FROM pg_catalog.pg_collation c +JOIN pg_catalog.pg_namespace n ON n.oid = c.collnamespace +WHERE ", + in_scope!(), + " + AND ", + not_extension_member!("pg_collation", "c.oid"), + " +ORDER BY n.nspname, c.collname" +); + +/// Enums, domains and composite types, then range and base types, which +/// the dump lists as left out. An array type comes with its element type +/// and a table's row type with its table. +pub(crate) const TYPES_SQL: &str = concat!( + "SELECT n.nspname::text, t.typname::text, t.typtype::text, + ARRAY(SELECT e.enumlabel::text FROM pg_catalog.pg_enum e + WHERE e.enumtypid = t.oid ORDER BY e.enumsortorder), + pg_catalog.format_type(t.typbasetype, t.typtypmod)::text, + t.typnotnull, + pg_catalog.pg_get_expr(t.typdefaultbin, 0::pg_catalog.oid)::text, + cn.nspname::text, + co.collname::text +FROM pg_catalog.pg_type t +JOIN pg_catalog.pg_namespace n ON n.oid = t.typnamespace +LEFT JOIN pg_catalog.pg_type b ON b.oid = t.typbasetype +LEFT JOIN pg_catalog.pg_collation co ON co.oid = t.typcollation AND t.typcollation <> b.typcollation +LEFT JOIN pg_catalog.pg_namespace cn ON cn.oid = co.collnamespace +WHERE ", + in_scope!(), + " + AND t.typtype IN ('b', 'c', 'd', 'e', 'r') + AND NOT (t.typtype = 'b' AND t.typelem <> 0::pg_catalog.oid AND t.typlen = -1) + AND NOT EXISTS (SELECT 1 FROM pg_catalog.pg_class r WHERE r.oid = t.typrelid AND r.relkind <> 'c') + AND ", + not_extension_member!("pg_type", "t.oid"), + " +ORDER BY CASE t.typtype WHEN 'e' THEN 0 WHEN 'd' THEN 1 ELSE 2 END, n.nspname, t.typname" +); + +pub(crate) const DOMAIN_CHECKS_SQL: &str = concat!( + "SELECT n.nspname::text, t.typname::text, k.conname::text, + pg_catalog.pg_get_constraintdef(k.oid)::text, k.convalidated +FROM pg_catalog.pg_constraint k +JOIN pg_catalog.pg_type t ON t.oid = k.contypid +JOIN pg_catalog.pg_namespace n ON n.oid = t.typnamespace +WHERE k.contype = 'c' AND ", + in_scope!(), + " +ORDER BY n.nspname, t.typname, k.conname" +); + +/// `pg_get_functiondef` refuses an aggregate, so one has no text. +macro_rules! functions_sql { + ($sql_body:expr) => { + concat!( + "SELECT n.nspname::text, p.proname::text, p.prokind::text, + CASE WHEN p.prokind <> 'a' THEN pg_catalog.pg_get_functiondef(p.oid)::text END, + ", + $sql_body, + " +FROM pg_catalog.pg_proc p +JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace +WHERE ", + in_scope!(), + " + AND ", + not_extension_member!("pg_proc", "p.oid"), + " +ORDER BY n.nspname, p.proname, pg_catalog.pg_get_function_identity_arguments(p.oid)" + ) + }; +} + +/// Whether a routine waits for the tables: a body in standard SQL that +/// names a table or view, which the server resolves when it is made +/// (PostgreSQL 14 and later), or a signature that names a table's or +/// view's row type, which exists only once the relation does. Any other +/// routine goes first, where a table's DEFAULT or CHECK can call it. +macro_rules! after_tables { + ($sql_body:literal) => { + concat!( + "(", + $sql_body, + " OR EXISTS (SELECT 1 FROM pg_catalog.pg_type rt + JOIN pg_catalog.pg_class rc ON rc.oid = rt.typrelid + WHERE rc.relkind IN ('r', 'p', 'v', 'm', 'f') + AND (rt.oid = p.prorettype OR rt.typarray = p.prorettype + OR rt.oid = ANY (COALESCE(p.proallargtypes, p.proargtypes::pg_catalog.oid[])) + OR rt.typarray = ANY (COALESCE(p.proallargtypes, p.proargtypes::pg_catalog.oid[])))))" + ) + }; +} + +pub(crate) const FUNCTIONS_SQL: &str = functions_sql!(after_tables!( + "(p.prosqlbody IS NOT NULL AND EXISTS (SELECT 1 FROM pg_catalog.pg_depend dep + WHERE dep.classid = 'pg_catalog.pg_proc'::pg_catalog.regclass AND dep.objid = p.oid + AND dep.refclassid = 'pg_catalog.pg_class'::pg_catalog.regclass))" +)); +pub(crate) const FUNCTIONS_SQL_BEFORE_14: &str = functions_sql!(after_tables!("false")); + +/// Every sequence, with the column that owns it (`a`) or whose identity +/// it is (`i`), and its position read from the sequence itself: +/// `pg_sequences.last_value` is NULL until the next `nextval`, after a +/// RESTART as well as before the first use. NULL where the login may not +/// read the sequence. +pub(crate) const SEQUENCES_SQL: &str = concat!( + "SELECT n.nspname::text, c.relname::text, s.data_type::text, + s.start_value, s.increment_by, s.min_value, s.max_value, s.cache_size, s.cycle, + c.relpersistence::text, seq_state.last_value, + d.deptype::text, tn.nspname::text, t.relname::text, a.attname::text, seq_state.is_called +FROM pg_catalog.pg_class c +JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace +JOIN pg_catalog.pg_sequences s ON s.schemaname = n.nspname AND s.sequencename = c.relname +LEFT JOIN LATERAL ( + SELECT (pg_catalog.xpath('/row/last_value/text()', probe.state))[1]::text::bigint AS last_value, + (pg_catalog.xpath('/row/is_called/text()', probe.state))[1]::text::boolean AS is_called + FROM (SELECT CASE WHEN pg_catalog.has_sequence_privilege(c.oid, 'SELECT') + THEN pg_catalog.query_to_xml( + pg_catalog.format('SELECT last_value, is_called FROM %I.%I', n.nspname, c.relname), + false, true, '') + END AS state) probe +) seq_state ON true +LEFT JOIN pg_catalog.pg_depend d + ON d.classid = 'pg_catalog.pg_class'::pg_catalog.regclass::pg_catalog.oid AND d.objid = c.oid + AND d.refclassid = 'pg_catalog.pg_class'::pg_catalog.regclass::pg_catalog.oid + AND d.refobjsubid > 0 AND d.deptype IN ('a', 'i') +LEFT JOIN pg_catalog.pg_class t ON t.oid = d.refobjid +LEFT JOIN pg_catalog.pg_namespace tn ON tn.oid = t.relnamespace +LEFT JOIN pg_catalog.pg_attribute a ON a.attrelid = d.refobjid AND a.attnum::pg_catalog.int4 = d.refobjsubid +WHERE c.relkind = 'S' AND ", + in_scope!(), + " + AND ", + not_extension_member!("pg_class", "c.oid"), + " +ORDER BY n.nspname, c.relname" +); + +/// Tables, partitioned tables and foreign tables. A partition names its +/// parent; a child or parent by INHERITS is marked, since a read of the +/// parent returns its children's rows too. The last column says whether +/// row-level security shows this login only some of the table's rows. +pub(crate) const TABLES_SQL: &str = concat!( + "SELECT n.nspname::text, c.relname::text, c.relkind::text, c.relpersistence::text, + CASE WHEN c.relkind = 'p' THEN pg_catalog.pg_get_partkeydef(c.oid)::text END, + CASE WHEN c.relispartition THEN pg_catalog.pg_get_expr(c.relpartbound, c.oid)::text END, + pn.nspname::text, pc.relname::text, + NOT c.relispartition AND (EXISTS (SELECT 1 FROM pg_catalog.pg_inherits h WHERE h.inhrelid = c.oid) + OR EXISTS (SELECT 1 FROM pg_catalog.pg_inherits h + JOIN pg_catalog.pg_class ch ON ch.oid = h.inhrelid + WHERE h.inhparent = c.oid AND NOT ch.relispartition)), + c.reloptions::text[], am.amname::text, ts.spcname::text, + pg_catalog.obj_description(c.oid, 'pg_class')::text, + c.relrowsecurity + AND NOT EXISTS (SELECT 1 FROM pg_catalog.pg_roles r + WHERE r.rolname = CURRENT_USER AND (r.rolsuper OR r.rolbypassrls)) + AND (c.relforcerowsecurity OR NOT pg_catalog.pg_has_role(c.relowner, 'USAGE')) +FROM pg_catalog.pg_class c +JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace +LEFT JOIN pg_catalog.pg_inherits i ON i.inhrelid = c.oid AND c.relispartition +LEFT JOIN pg_catalog.pg_class pc ON pc.oid = i.inhparent +LEFT JOIN pg_catalog.pg_namespace pn ON pn.oid = pc.relnamespace +LEFT JOIN pg_catalog.pg_am am ON am.oid = c.relam AND am.amname <> 'heap' +LEFT JOIN pg_catalog.pg_tablespace ts ON ts.oid = c.reltablespace +WHERE c.relkind IN ('r', 'p', 'f') AND ", + in_scope!(), + " + AND ", + not_extension_member!("pg_class", "c.oid"), + " +ORDER BY n.nspname, c.relname" +); + +/// The columns of tables, views and composite types. The collation is +/// named only where it is not the type's own. +pub(crate) const COLUMNS_SQL: &str = concat!( + "SELECT n.nspname::text, c.relname::text, a.attname::text, + pg_catalog.format_type(a.atttypid, a.atttypmod)::text, + cn.nspname::text, co.collname::text, + pg_catalog.pg_get_expr(d.adbin, d.adrelid)::text, + a.attidentity::text, a.attgenerated::text, a.attnotnull, + pg_catalog.col_description(a.attrelid, a.attnum::pg_catalog.int4)::text +FROM pg_catalog.pg_attribute a +JOIN pg_catalog.pg_class c ON c.oid = a.attrelid +JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace +JOIN pg_catalog.pg_type t ON t.oid = a.atttypid +LEFT JOIN pg_catalog.pg_attrdef d ON d.adrelid = a.attrelid AND d.adnum = a.attnum +LEFT JOIN pg_catalog.pg_collation co ON co.oid = a.attcollation AND a.attcollation <> t.typcollation +LEFT JOIN pg_catalog.pg_namespace cn ON cn.oid = co.collnamespace +WHERE c.relkind IN ('r', 'p', 'v', 'm', 'c') AND a.attnum > 0 AND NOT a.attisdropped AND ", + in_scope!(), + " +ORDER BY n.nspname, c.relname, a.attnum" +); + +/// A table's own CHECK constraints, and the keys it declared: those a +/// partition took from its parent, or a key copied for each partition of +/// the table it references, come back with the declared one. +pub(crate) const CONSTRAINTS_SQL: &str = concat!( + "SELECT n.nspname::text, c.relname::text, c.relkind::text, k.conname::text, k.contype::text, + pg_catalog.pg_get_constraintdef(k.oid)::text, k.convalidated +FROM pg_catalog.pg_constraint k +JOIN pg_catalog.pg_class c ON c.oid = k.conrelid +JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace +WHERE c.relkind IN ('r', 'p') AND ", + in_scope!(), + " + AND ((k.contype = 'c' AND k.conislocal) + OR (k.contype IN ('f', 'p', 'u', 'x') AND k.conparentid = 0::pg_catalog.oid)) +ORDER BY n.nspname, c.relname, k.conname" +); + +/// Indexes no constraint owns. A partition's index attached to its +/// parent's is made by the parent's statement. An invalid index is a +/// build that failed, except on a partitioned table, where it waits for +/// its partitions. +pub(crate) const INDEXES_SQL: &str = concat!( + "SELECT n.nspname::text, c.relname::text, ic.relname::text, + pg_catalog.pg_get_indexdef(i.indexrelid)::text, ic.relkind = 'I', i.indisunique, + pg_catalog.quote_ident(ic.relname)::text, pg_catalog.quote_ident(ts.spcname)::text, + pg_catalog.pg_get_expr(i.indpred, i.indrelid)::text +FROM pg_catalog.pg_index i +JOIN pg_catalog.pg_class ic ON ic.oid = i.indexrelid +JOIN pg_catalog.pg_class c ON c.oid = i.indrelid +JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace +LEFT JOIN pg_catalog.pg_tablespace ts ON ts.oid = ic.reltablespace +WHERE c.relkind IN ('r', 'p', 'm') AND ", + in_scope!(), + " + AND i.indislive AND (i.indisvalid OR ic.relkind = 'I') + AND NOT EXISTS ( + SELECT 1 FROM pg_catalog.pg_constraint k + WHERE k.conindid = i.indexrelid AND k.conrelid = i.indrelid AND k.contype IN ('p', 'u', 'x')) + AND NOT EXISTS (SELECT 1 FROM pg_catalog.pg_inherits h WHERE h.inhrelid = i.indexrelid) +ORDER BY n.nspname, c.relname, ic.relname" +); + +pub(crate) const VIEWS_SQL: &str = concat!( + "SELECT n.nspname::text, c.relname::text, c.relkind::text, pg_catalog.pg_get_viewdef(c.oid)::text, + c.reloptions::text[], c.relispopulated, am.amname::text, ts.spcname::text, + pg_catalog.obj_description(c.oid, 'pg_class')::text +FROM pg_catalog.pg_class c +JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace +LEFT JOIN pg_catalog.pg_am am ON am.oid = c.relam AND am.amname <> 'heap' +LEFT JOIN pg_catalog.pg_tablespace ts ON ts.oid = c.reltablespace +WHERE c.relkind IN ('v', 'm') AND ", + in_scope!(), + " + AND ", + not_extension_member!("pg_class", "c.oid"), + " +ORDER BY n.nspname, c.relname" +); + +/// A partition's copy of its parent's trigger comes with the parent's. +/// Not pretty-printed, so the table is named with its schema. +pub(crate) const TRIGGERS_SQL: &str = concat!( + "SELECT n.nspname::text, c.relname::text, t.tgname::text, + pg_catalog.pg_get_triggerdef(t.oid, false)::text, t.tgenabled::text +FROM pg_catalog.pg_trigger t +JOIN pg_catalog.pg_class c ON c.oid = t.tgrelid +JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace +WHERE NOT t.tgisinternal AND t.tgparentid = 0::pg_catalog.oid AND c.relkind IN ('r', 'p', 'v') AND ", + in_scope!(), + " + AND ", + not_extension_member!("pg_class", "c.oid"), + " +ORDER BY n.nspname, c.relname, t.tgname" +); diff --git a/core/crates/drivers/postgres/src/dump/mod.rs b/core/crates/drivers/postgres/src/dump/mod.rs new file mode 100644 index 0000000000..e3baf6b037 --- /dev/null +++ b/core/crates/drivers/postgres/src/dump/mod.rs @@ -0,0 +1,1198 @@ +//! A PostgreSQL database or schema as a SQL dump, in pg_dump's plain +//! order: structure, rows, then sequence positions, keys and indexes, +//! then views, then triggers. Ownership, grants, policies, rules, +//! publications and event triggers are left out, as with +//! `pg_dump --no-owner --no-acl`. + +mod catalog_rows; +pub(crate) mod catalog_sql; +mod statements; + +use std::collections::{HashMap, HashSet}; + +use tablepro_core::DriverError; +use tablepro_core::dump::{ + DumpCatalog, DumpDialect, DumpObject, DumpObjectKind, DumpScope, DumpSection, DumpSkip, DumpSkipReason, + DumpStatement, DumpTable, +}; +use tablepro_core::meta::TableRef; +use tokio_postgres::{Client, Row}; + +use crate::errors::map_error; +use crate::metadata::text_rows; +use catalog_rows::{ + Column, Constraint, DomainCheck, Extension, Index, Routine, Sequence, Table, Trigger, TypeShape, UserCollation, + UserType, View, +}; +use catalog_sql::{ + COLLATIONS_SQL, COLUMNS_SQL, CONSTRAINTS_SQL, DOMAIN_CHECKS_SQL, EXTENSIONS_SQL, FUNCTIONS_SQL, + FUNCTIONS_SQL_BEFORE_14, INDEXES_SQL, SCHEMAS_SQL, SEQUENCES_SQL, TABLES_SQL, TRIGGERS_SQL, TYPES_SQL, VIEWS_SQL, +}; +use statements::qualified; + +/// pg_dump's session settings, less `SET transaction_timeout`, which a +/// server before 17 refuses. An empty `search_path` makes every name in +/// the file resolve as written. +const HEADER: &str = " +SET statement_timeout = 0; +SET lock_timeout = 0; +SET idle_in_transaction_session_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = on; +SELECT pg_catalog.set_config('search_path', '', false); +SET check_function_bodies = false; +SET xmloption = content; +SET client_min_messages = warning; +SET row_security = off; +"; + +/// Plain SQL, no COPY and no psql command, so the file loads through a +/// driver as well as through psql of any release. Apart from `PgDialect`, +/// whose `SqlDialect` has an `identity_insert` of its own. +#[derive(Debug)] +pub(crate) struct PgDump; + +impl DumpDialect for PgDump { + fn header(&self) -> String { + HEADER.to_owned() + } + + fn transactional_ddl(&self) -> bool { + true + } +} + +/// The catalog prints every name but pg_catalog's with its schema only +/// while the search path is empty. +const EMPTY_SEARCH_PATH: &str = "SELECT pg_catalog.set_config('search_path', '', true)"; + +/// Run inside the read's own transaction, which ends with it. +pub(crate) async fn read_dump_catalog( + client: &Client, + scope: &DumpScope, + sql_bodies: bool, +) -> Result { + client + .batch_execute(EMPTY_SEARCH_PATH) + .await + .map_err(|error| map_error(&error))?; + let params = match scope { + DumpScope::Database => ["all", ""], + DumpScope::Schema(schema) => ["schema", schema.as_str()], + }; + let functions_sql = match sql_bodies { + true => FUNCTIONS_SQL, + false => FUNCTIONS_SQL_BEFORE_14, + }; + let ( + schemas, + extensions, + collations, + types, + domain_checks, + routines, + sequences, + tables, + columns, + constraints, + indexes, + views, + triggers, + ) = futures_util::try_join!( + text_rows(client, SCHEMAS_SQL, ¶ms), + text_rows(client, EXTENSIONS_SQL, ¶ms), + text_rows(client, COLLATIONS_SQL, ¶ms), + text_rows(client, TYPES_SQL, ¶ms), + text_rows(client, DOMAIN_CHECKS_SQL, ¶ms), + text_rows(client, functions_sql, ¶ms), + text_rows(client, SEQUENCES_SQL, ¶ms), + text_rows(client, TABLES_SQL, ¶ms), + text_rows(client, COLUMNS_SQL, ¶ms), + text_rows(client, CONSTRAINTS_SQL, ¶ms), + text_rows(client, INDEXES_SQL, ¶ms), + text_rows(client, VIEWS_SQL, ¶ms), + text_rows(client, TRIGGERS_SQL, ¶ms), + )?; + let facts = CatalogFacts { + schemas: decoded(&schemas, catalog_rows::schema_name)?, + extensions: decoded(&extensions, catalog_rows::extension)?, + collations: decoded(&collations, catalog_rows::user_collation)?, + types: decoded(&types, catalog_rows::user_type)?, + domain_checks: decoded(&domain_checks, catalog_rows::domain_check)?, + routines: decoded(&routines, catalog_rows::routine)?, + sequences: decoded(&sequences, catalog_rows::sequence)?, + tables: decoded(&tables, catalog_rows::table)?, + columns: decoded(&columns, catalog_rows::column)?, + constraints: decoded(&constraints, catalog_rows::constraint)?, + indexes: decoded(&indexes, catalog_rows::index)?, + views: decoded(&views, catalog_rows::view)?, + triggers: decoded(&triggers, catalog_rows::trigger)?, + }; + Ok(assemble(&facts)) +} + +fn decoded(rows: &[Row], decode: fn(&Row) -> Result) -> Result, DriverError> { + rows.iter().map(decode).collect() +} + +/// What the catalog reads returned, each list in schema and name order. +#[derive(Debug, Default)] +struct CatalogFacts { + schemas: Vec, + extensions: Vec, + collations: Vec, + types: Vec, + domain_checks: Vec, + routines: Vec, + sequences: Vec, + tables: Vec, + columns: Vec, + constraints: Vec, + indexes: Vec, + views: Vec, + triggers: Vec, +} + +type Key<'a> = (&'a str, &'a str); + +/// Each item under the schema and name of the relation it belongs to, +/// in the order it came. +fn by_relation<'a, T>(items: &'a [T], key: impl Fn(&'a T) -> Key<'a>) -> HashMap, Vec<&'a T>> { + let mut grouped: HashMap, Vec<&'a T>> = HashMap::new(); + for item in items { + grouped.entry(key(item)).or_default().push(item); + } + grouped +} + +fn listed<'m, 'a, T>(grouped: &'m HashMap, Vec<&'a T>>, key: Key<'a>) -> &'m [&'a T] { + grouped.get(&key).map(Vec::as_slice).unwrap_or_default() +} + +fn plain(sql: impl IntoIterator) -> Vec { + sql.into_iter().map(DumpStatement::new).collect() +} + +fn skip(kind: DumpObjectKind, schema: &str, name: &str) -> DumpSkip { + DumpSkip { + kind, + schema: Some(schema.to_owned()), + name: name.to_owned(), + reason: DumpSkipReason::NotSupported, + } +} + +/// A table the dump writes: a regular or partitioned table or a +/// partition. A foreign table's rows live elsewhere, a child by INHERITS +/// has no statement that makes it again here, and its parent's rows +/// would carry the child's. +fn written(table: &Table) -> bool { + matches!(table.kind.as_str(), "r" | "p") && !table.inherits +} + +/// The catalog grouped by the relation or type each part belongs to. +struct Lookup<'a> { + columns: HashMap, Vec<&'a Column>>, + constraints: HashMap, Vec<&'a Constraint>>, + indexes: HashMap, Vec<&'a Index>>, + checks: HashMap, Vec<&'a DomainCheck>>, + written: HashSet>, + /// A written table's identity sequences, by table and column. + identities: HashMap<(&'a str, &'a str, &'a str), &'a Sequence>, + /// The sequences a written table's columns own. + serials: HashMap, Vec<&'a Sequence>>, +} + +impl<'a> Lookup<'a> { + fn new(facts: &'a CatalogFacts) -> Self { + let written: HashSet> = facts + .tables + .iter() + .filter(|table| written(table)) + .map(|table| (table.schema.as_str(), table.name.as_str())) + .collect(); + let mut identities = HashMap::new(); + let mut serials: HashMap, Vec<&'a Sequence>> = HashMap::new(); + for sequence in &facts.sequences { + let Some(owner) = &sequence.owner else { + continue; + }; + let table = (owner.schema.as_str(), owner.table.as_str()); + match (written.contains(&table), owner.identity) { + (false, _) => {} + (true, true) => { + identities.insert((table.0, table.1, owner.column.as_str()), sequence); + } + (true, false) => serials.entry(table).or_default().push(sequence), + } + } + Self { + columns: by_relation(&facts.columns, |column| (column.schema.as_str(), column.table.as_str())), + constraints: by_relation(&facts.constraints, |constraint| { + (constraint.schema.as_str(), constraint.table.as_str()) + }), + indexes: by_relation(&facts.indexes, |index| (index.schema.as_str(), index.table.as_str())), + checks: by_relation(&facts.domain_checks, |check| { + (check.schema.as_str(), check.domain.as_str()) + }), + written, + identities, + serials, + } + } + + /// An identity sequence goes with its table, so one whose table the + /// dump leaves out has no position to set. + fn makes(&self, sequence: &Sequence) -> bool { + match &sequence.owner { + Some(owner) if owner.identity => self.written.contains(&(owner.schema.as_str(), owner.table.as_str())), + _ => true, + } + } +} + +/// What follows the rows, in the order it runs. +#[derive(Default)] +struct AfterData { + values: Vec, + keys: Vec, + indexes: Vec, + comments: Vec, + foreign_keys: Vec, +} + +impl AfterData { + fn into_objects(self) -> impl Iterator { + self.values + .into_iter() + .chain(self.keys) + .chain(self.indexes) + .chain(self.comments) + .chain(self.foreign_keys) + } +} + +fn assemble(facts: &CatalogFacts) -> DumpCatalog { + let lookup = Lookup::new(facts); + let mut catalog = DumpCatalog::default(); + let mut before = namespaces(facts); + before.extend(collations(facts, &mut catalog.skipped)); + // A domain's DEFAULT can take from a sequence, which it names in a + // string the dump's order does not read. A sequence needs no type. + before.extend(sequences(facts)); + let (types, domain_checks) = types(facts, &lookup, &mut catalog.skipped); + before.extend(types); + let (routines_before, mut programs) = routines(facts, &mut catalog.skipped); + before.extend(routines_before); + before.extend(domain_checks); + let mut after = AfterData { + values: sequence_values(facts, &lookup, &mut catalog.skipped), + ..AfterData::default() + }; + for table in &facts.tables { + if !written(table) { + catalog + .skipped + .push(skip(DumpObjectKind::Table, &table.schema, &table.name)); + continue; + } + if let Some(partition) = &table.partition { + let parent = (partition.parent_schema.as_str(), partition.parent.as_str()); + let own = listed(&lookup.columns, (table.schema.as_str(), table.name.as_str())); + if generation_differs(own, listed(&lookup.columns, parent)) { + catalog.skipped.push(DumpSkip { + reason: DumpSkipReason::ExpressionNotCarried, + ..skip(DumpObjectKind::Table, &table.schema, &table.name) + }); + } + } + before.push(create_table(table, &lookup)); + table_after_data(table, &lookup, &mut after); + let mut dumped = dump_table(table); + // A read would give the policy's rows as if they were all, as + // pg_dump's `row_security = off` refuses to. + if table.filtered { + dumped.rows = false; + catalog.skipped.push(DumpSkip { + reason: DumpSkipReason::RowsFiltered, + ..skip(DumpObjectKind::Table, &table.schema, &table.name) + }); + } + catalog.tables.push(dumped); + } + programs.extend(views(facts, &lookup)); + let triggers = triggers(facts, &lookup); + catalog.objects = before + .into_iter() + .chain(after.into_objects()) + .chain(programs) + .chain(triggers) + .collect(); + catalog +} + +fn collations(facts: &CatalogFacts, skipped: &mut Vec) -> Vec { + let mut objects = Vec::new(); + for collation in &facts.collations { + match statements::create_collation(collation) { + Some(sql) => objects.push(DumpObject::new( + DumpObjectKind::Collation, + DumpSection::BeforeData, + Some(collation.schema.clone()), + collation.name.clone(), + plain([sql]), + )), + None => skipped.push(skip(DumpObjectKind::Collation, &collation.schema, &collation.name)), + } + } + objects +} + +fn namespaces(facts: &CatalogFacts) -> Vec { + let schemas = facts.schemas.iter().map(|schema| { + DumpObject::new( + DumpObjectKind::Schema, + DumpSection::BeforeData, + None, + schema.clone(), + plain([statements::create_schema(schema)]), + ) + }); + let extensions = facts.extensions.iter().map(|extension| { + DumpObject::new( + DumpObjectKind::Extension, + DumpSection::BeforeData, + None, + extension.name.clone(), + plain([statements::create_extension(extension)]), + ) + }); + schemas.chain(extensions).collect() +} + +/// The types, and the domains' checks, which wait for the routines. +fn types(facts: &CatalogFacts, lookup: &Lookup<'_>, skipped: &mut Vec) -> (Vec, Vec) { + let mut objects = Vec::new(); + let mut checks = Vec::new(); + for user_type in &facts.types { + let key = (user_type.schema.as_str(), user_type.name.as_str()); + let sql = match &user_type.shape { + TypeShape::Enum(labels) => vec![statements::create_enum(key.0, key.1, labels)], + TypeShape::Domain(domain) => { + let listed_checks = listed(&lookup.checks, key); + let mut sql = statements::create_domain(key.0, key.1, domain, listed_checks); + let added = sql.split_off(1); + // A check added NOT VALID holds for rows written after it + // and not for the ones already there, so it follows them. + let (held, pending): (Vec<_>, Vec<_>) = added + .into_iter() + .zip(listed_checks) + .partition(|(_, check)| check.validated); + for (section, group) in [(DumpSection::BeforeData, held), (DumpSection::AfterData, pending)] { + if !group.is_empty() { + checks.push(DumpObject::new( + DumpObjectKind::Constraint, + section, + Some(user_type.schema.clone()), + user_type.name.clone(), + plain(group.into_iter().map(|(statement, _)| statement)), + )); + } + } + sql + } + TypeShape::Composite => vec![statements::create_composite(key.0, key.1, listed(&lookup.columns, key))], + TypeShape::Other => { + skipped.push(skip(DumpObjectKind::Type, key.0, key.1)); + continue; + } + }; + objects.push(DumpObject::new( + DumpObjectKind::Type, + DumpSection::BeforeData, + Some(user_type.schema.clone()), + user_type.name.clone(), + plain(sql), + )); + } + (objects, checks) +} + +/// The routines made before the tables, and those made with the views. +/// A body in standard SQL names tables and views the server must find +/// when it is made; a quoted body is read only when it runs. +fn routines(facts: &CatalogFacts, skipped: &mut Vec) -> (Vec, Vec) { + let mut before = Vec::new(); + let mut programs = Vec::new(); + for routine in &facts.routines { + let kind = match routine.kind.as_str() { + "p" => DumpObjectKind::Procedure, + _ => DumpObjectKind::Function, + }; + let Some(definition) = routine.definition.as_ref().filter(|_| routine.kind != "a") else { + skipped.push(skip(kind, &routine.schema, &routine.name)); + continue; + }; + let (section, list) = match routine.sql_body { + true => (DumpSection::Programs, &mut programs), + false => (DumpSection::BeforeData, &mut before), + }; + list.push(DumpObject::new( + kind, + section, + Some(routine.schema.clone()), + routine.name.clone(), + vec![DumpStatement::body(definition.clone())], + )); + } + (before, programs) +} + +/// Every sequence but an identity's, which its column makes. +fn sequences(facts: &CatalogFacts) -> Vec { + facts + .sequences + .iter() + .filter(|sequence| !sequence.owner.as_ref().is_some_and(|owner| owner.identity)) + .map(|sequence| { + DumpObject::new( + DumpObjectKind::Sequence, + DumpSection::BeforeData, + Some(sequence.schema.clone()), + sequence.name.clone(), + plain([statements::create_sequence(sequence)]), + ) + }) + .collect() +} + +/// Each sequence's position, and the ones the login may not read, which +/// the report names: their numbers would start over on load. +fn sequence_values(facts: &CatalogFacts, lookup: &Lookup<'_>, skipped: &mut Vec) -> Vec { + for sequence in facts.sequences.iter().filter(|sequence| lookup.makes(sequence)) { + if sequence.last_value.is_none() { + skipped.push(DumpSkip { + reason: DumpSkipReason::NotReadable, + ..skip(DumpObjectKind::SequenceValue, &sequence.schema, &sequence.name) + }); + } + } + facts + .sequences + .iter() + .filter(|sequence| lookup.makes(sequence)) + .filter_map(|sequence| { + statements::set_value(sequence).map(|sql| { + DumpObject::new( + DumpObjectKind::SequenceValue, + DumpSection::AfterData, + Some(sequence.schema.clone()), + sequence.name.clone(), + plain([sql]), + ) + }) + }) + .collect() +} + +/// The table with its columns and checks, then the sequences its columns +/// own, which need the table to exist. +fn create_table(table: &Table, lookup: &Lookup<'_>) -> DumpObject { + let key = (table.schema.as_str(), table.name.as_str()); + let columns: Vec = listed(&lookup.columns, key) + .iter() + .map(|column| { + let identity = lookup.identities.get(&(key.0, key.1, column.name.as_str())).copied(); + statements::column_sql(column, identity) + }) + .collect(); + let checks: Vec<&Constraint> = listed(&lookup.constraints, key) + .iter() + .filter(|constraint| constraint.kind == "c" && constraint.validated) + .copied() + .collect(); + let mut sql = vec![statements::create_table(table, &columns, &checks)]; + if let Some(partition) = &table.partition { + let parent = (partition.parent_schema.as_str(), partition.parent.as_str()); + sql.extend(partition_overrides( + table, + listed(&lookup.columns, key), + listed(&lookup.columns, parent), + )); + } + for sequence in listed(&lookup.serials, key) { + if let Some(owner) = &sequence.owner { + sql.push(statements::owned_by(sequence, owner)); + } + } + DumpObject::table( + TableRef::new(Some(table.schema.clone()), table.name.clone()), + plain(sql), + ) +} + +/// What a partition sets on its columns apart from its parent: a +/// default of its own, or a NOT NULL the parent does not have. `PARTITION +/// OF` takes the parent's columns as they are. +fn partition_overrides(table: &Table, own: &[&Column], parent: &[&Column]) -> Vec { + let name = statements::qualified(&table.schema, &table.name); + let mut sql = Vec::new(); + for column in own + .iter() + .filter(|column| column.identity.is_empty() && column.generated.is_empty()) + { + let Some(inherited) = parent.iter().find(|candidate| candidate.name == column.name) else { + continue; + }; + let target = format!( + "ALTER TABLE ONLY {name} ALTER COLUMN {}", + statements::quote(&column.name) + ); + if column.default != inherited.default { + sql.push(match &column.default { + Some(default) => format!("{target} SET DEFAULT {default}"), + None => format!("{target} DROP DEFAULT"), + }); + } + if column.not_null && !inherited.not_null { + sql.push(format!("{target} SET NOT NULL")); + } + } + sql +} + +/// Whether a partition computes a generated column apart from its +/// parent, which `PARTITION OF` cannot say. +fn generation_differs(own: &[&Column], parent: &[&Column]) -> bool { + // A generated column keeps its expression where another keeps its + // DEFAULT. + own.iter().filter(|column| !column.generated.is_empty()).any(|column| { + parent + .iter() + .find(|candidate| candidate.name == column.name) + .is_some_and(|inherited| inherited.default != column.default) + }) +} + +/// A partitioned table's rows are its partitions', which come after it. +fn dump_table(table: &Table) -> DumpTable { + DumpTable { + references: table + .partition + .iter() + .map(|partition| TableRef::new(Some(partition.parent_schema.clone()), partition.parent.clone())) + .collect(), + rows: table.kind != "p", + ..DumpTable::new(TableRef::new(Some(table.schema.clone()), table.name.clone())) + } +} + +/// Keys, a check not yet held, indexes, comments and foreign keys. +fn table_after_data(table: &Table, lookup: &Lookup<'_>, after: &mut AfterData) { + let key = (table.schema.as_str(), table.name.as_str()); + let object = |kind, name: &str, sql: Vec| { + DumpObject::new( + kind, + DumpSection::AfterData, + Some(table.schema.clone()), + name.to_owned(), + plain(sql), + ) + }; + for constraint in listed(&lookup.constraints, key) { + let (kind, list) = match constraint.kind.as_str() { + "f" => (DumpObjectKind::ForeignKey, &mut after.foreign_keys), + "c" if constraint.validated => continue, + _ => (DumpObjectKind::Constraint, &mut after.keys), + }; + list.push(object( + kind, + &constraint.name, + vec![statements::add_constraint(constraint)], + )); + } + for index in listed(&lookup.indexes, key) { + after.indexes.push(object( + DumpObjectKind::Index, + &index.name, + vec![index.statement.clone()], + )); + } + let mut comments = Vec::new(); + if let Some(comment) = &table.comment { + comments.push(statements::comment_on("TABLE", &qualified(key.0, key.1), comment)); + } + comments.extend(column_comments(listed(&lookup.columns, key))); + if !comments.is_empty() { + after + .comments + .push(object(DumpObjectKind::Comment, &table.name, comments)); + } +} + +/// A view with its comments; a materialized view with its indexes too. +fn views(facts: &CatalogFacts, lookup: &Lookup<'_>) -> Vec { + let mut objects = Vec::new(); + for view in &facts.views { + let key = (view.schema.as_str(), view.name.as_str()); + let (kind, target) = match view.materialized { + true => (DumpObjectKind::MaterializedView, "MATERIALIZED VIEW"), + false => (DumpObjectKind::View, "VIEW"), + }; + let mut rest = Vec::new(); + if view.materialized { + rest.extend(listed(&lookup.indexes, key).iter().map(|index| index.statement.clone())); + } + if let Some(comment) = &view.comment { + rest.push(statements::comment_on(target, &qualified(key.0, key.1), comment)); + } + // pg_get_viewdef leaves out a default set on a view's column, which + // an INSERT through the view takes. + rest.extend(listed(&lookup.columns, key).iter().filter_map(|column| { + let default = column.default.as_ref().filter(|_| !view.materialized)?; + Some(format!( + "ALTER VIEW {} ALTER COLUMN {} SET DEFAULT {default}", + qualified(key.0, key.1), + statements::quote(&column.name) + )) + })); + rest.extend(column_comments(listed(&lookup.columns, key))); + let with_create = |create: String| plain(std::iter::once(create).chain(rest.iter().cloned())); + let mut object = DumpObject::new( + kind, + DumpSection::Programs, + Some(view.schema.clone()), + view.name.clone(), + with_create(statements::create_view(view)), + ); + // A dump without rows leaves a materialized view to be refreshed, + // as pg_dump's --schema-only does. + if view.materialized && view.populated { + let unpopulated = View { + populated: false, + ..view.clone() + }; + object.without_rows = Some(with_create(statements::create_view(&unpopulated))); + } + objects.push(object); + } + objects +} + +/// The triggers of the tables and views the dump writes. +fn triggers(facts: &CatalogFacts, lookup: &Lookup<'_>) -> Vec { + let views: HashSet> = facts + .views + .iter() + .filter(|view| !view.materialized) + .map(|view| (view.schema.as_str(), view.name.as_str())) + .collect(); + facts + .triggers + .iter() + .filter(|trigger| { + let target = (trigger.schema.as_str(), trigger.table.as_str()); + lookup.written.contains(&target) || views.contains(&target) + }) + .map(|trigger| { + let mut sql = vec![trigger.definition.clone()]; + sql.extend(statements::trigger_state(trigger)); + DumpObject::new( + DumpObjectKind::Trigger, + DumpSection::Triggers, + Some(trigger.schema.clone()), + trigger.name.clone(), + plain(sql), + ) + }) + .collect() +} + +fn column_comments(columns: &[&Column]) -> Vec { + columns + .iter() + .filter_map(|column| { + column + .comment + .as_ref() + .map(|comment| statements::comment_on_column(&column.schema, &column.table, &column.name, comment)) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use tablepro_core::dialect::SqlDialect; + + use super::catalog_rows::{Domain, Partition, SequenceOwner}; + use super::*; + use crate::dialect::PgDialect; + + fn table(name: &str, kind: &str) -> Table { + Table { + schema: "app".to_owned(), + name: name.to_owned(), + kind: kind.to_owned(), + unlogged: false, + partition_key: (kind == "p").then(|| "RANGE (k)".to_owned()), + partition: None, + inherits: false, + filtered: false, + options: Vec::new(), + access_method: None, + tablespace: None, + comment: None, + } + } + + fn column(table: &str, name: &str) -> Column { + Column { + schema: "app".to_owned(), + table: table.to_owned(), + name: name.to_owned(), + type_sql: "integer".to_owned(), + collation: None, + default: None, + identity: String::new(), + generated: String::new(), + not_null: true, + comment: None, + } + } + + #[test] + fn a_partition_computing_a_column_apart_from_its_parent_is_found() { + let generated = |table: &str, expression: &str| Column { + generated: "s".to_owned(), + default: Some(expression.to_owned()), + ..column(table, "doubled") + }; + let (parent, same, other) = ( + generated("p", "(id * 2)"), + generated("c", "(id * 2)"), + generated("c", "(id * 3)"), + ); + + assert!(!generation_differs(&[&same], &[&parent])); + assert!(generation_differs(&[&other], &[&parent])); + } + + fn sequence(name: &str, owner: Option<(&str, &str, bool)>) -> Sequence { + Sequence { + schema: "app".to_owned(), + name: name.to_owned(), + data_type: "integer".to_owned(), + start: 1, + increment: 1, + min: 1, + max: 2_147_483_647, + cache: 1, + cycle: false, + unlogged: false, + last_value: Some(4), + is_called: Some(true), + owner: owner.map(|(table, column, identity)| SequenceOwner { + identity, + schema: "app".to_owned(), + table: table.to_owned(), + column: column.to_owned(), + }), + } + } + + fn constraint(table: &str, name: &str, kind: &str, definition: &str) -> Constraint { + Constraint { + schema: "app".to_owned(), + table: table.to_owned(), + partitioned: false, + name: name.to_owned(), + kind: kind.to_owned(), + definition: definition.to_owned(), + validated: true, + } + } + + fn sections(catalog: &DumpCatalog) -> Vec<(DumpSection, DumpObjectKind, String)> { + catalog + .objects + .iter() + .map(|object| (object.section, object.kind, object.name.clone())) + .collect() + } + + fn sql_of<'a>(catalog: &'a DumpCatalog, name: &str) -> Vec<&'a str> { + catalog + .objects + .iter() + .filter(|object| object.name == name) + .flat_map(|object| object.statements.iter().map(|statement| statement.sql.as_str())) + .collect() + } + + #[test] + fn the_header_is_pg_dumps_without_transaction_timeout() { + let dialect = PgDialect::new(&tablepro_core::ServerInfo::new("PostgreSQL", "18.0", 18, 0, 0)); + let header = dialect.dump().map(|dump| dump.header()).unwrap_or_default(); + + assert_eq!(header.lines().filter(|line| !line.is_empty()).count(), 10); + assert!(header.contains("SELECT pg_catalog.set_config('search_path', '', false);\n")); + assert!(!header.contains("transaction_timeout")); + assert!(!header.contains('\\')); + } + + #[test] + fn serial_and_identity_sequences_are_made_where_their_columns_need_them() { + let mut id = column("child", "id"); + id.identity = "a".to_owned(); + let mut parent_id = column("parent", "id"); + parent_id.default = Some("nextval('app.parent_id_seq'::regclass)".to_owned()); + let facts = CatalogFacts { + sequences: vec![ + sequence("child_id_seq", Some(("child", "id", true))), + sequence("parent_id_seq", Some(("parent", "id", false))), + ], + tables: vec![table("child", "r"), table("parent", "r")], + columns: vec![id, parent_id], + ..CatalogFacts::default() + }; + + let catalog = assemble(&facts); + + assert_eq!( + sections(&catalog), + [ + ( + DumpSection::BeforeData, + DumpObjectKind::Sequence, + "parent_id_seq".to_owned() + ), + (DumpSection::BeforeData, DumpObjectKind::Table, "child".to_owned()), + (DumpSection::BeforeData, DumpObjectKind::Table, "parent".to_owned()), + ( + DumpSection::AfterData, + DumpObjectKind::SequenceValue, + "child_id_seq".to_owned() + ), + ( + DumpSection::AfterData, + DumpObjectKind::SequenceValue, + "parent_id_seq".to_owned() + ), + ] + ); + assert!(sql_of(&catalog, "child")[0].contains("SEQUENCE NAME \"app\".\"child_id_seq\"")); + assert_eq!( + sql_of(&catalog, "parent")[1], + "ALTER SEQUENCE \"app\".\"parent_id_seq\" OWNED BY \"app\".\"parent\".\"id\"" + ); + } + + #[test] + fn keys_and_checks_not_yet_held_follow_the_rows_and_foreign_keys_come_last() { + let mut pending = constraint("t", "small", "c", "CHECK ((id < 10)) NOT VALID"); + pending.validated = false; + let facts = CatalogFacts { + tables: vec![table("t", "r")], + columns: vec![column("t", "id")], + constraints: vec![ + constraint("t", "positive", "c", "CHECK ((id > 0))"), + pending, + constraint("t", "t_next_fkey", "f", "FOREIGN KEY (id) REFERENCES app.t(id)"), + constraint("t", "t_pkey", "p", "PRIMARY KEY (id)"), + ], + indexes: vec![Index { + schema: "app".to_owned(), + table: "t".to_owned(), + name: "t_id_idx".to_owned(), + statement: "CREATE INDEX t_id_idx ON app.t USING btree (id)".to_owned(), + }], + ..CatalogFacts::default() + }; + + let catalog = assemble(&facts); + + assert_eq!( + sections(&catalog) + .into_iter() + .filter(|(section, ..)| *section == DumpSection::AfterData) + .map(|(_, _, name)| name) + .collect::>(), + ["small", "t_pkey", "t_id_idx", "t_next_fkey"] + ); + assert_eq!( + sql_of(&catalog, "t"), + [ + "CREATE TABLE \"app\".\"t\" (\n \"id\" integer NOT NULL,\n CONSTRAINT \"positive\" CHECK ((id > 0))\n)" + ] + ); + } + + #[test] + fn a_partition_follows_its_parent_whose_rows_are_its_partitions() { + let mut partition = table("p_2025", "r"); + partition.partition = Some(Partition { + parent_schema: "app".to_owned(), + parent: "p".to_owned(), + bound: "FOR VALUES FROM (1) TO (10)".to_owned(), + }); + let facts = CatalogFacts { + tables: vec![table("p", "p"), partition], + columns: vec![column("p", "k"), column("p_2025", "k")], + ..CatalogFacts::default() + }; + + let catalog = assemble(&facts); + + let parent = TableRef::new(Some("app".to_owned()), "p"); + assert_eq!(catalog.tables.len(), 2); + assert!(!catalog.tables[0].rows); + assert!(catalog.tables[1].rows); + assert_eq!(catalog.tables[1].references, [parent]); + assert_eq!( + sql_of(&catalog, "p_2025"), + ["CREATE TABLE \"app\".\"p_2025\" PARTITION OF \"app\".\"p\" FOR VALUES FROM (1) TO (10)"] + ); + } + + #[test] + fn what_the_dump_cannot_make_again_is_listed_as_left_out() { + let mut child = table("child", "r"); + child.inherits = true; + let mut sum = Routine { + schema: "app".to_owned(), + name: "total".to_owned(), + kind: "a".to_owned(), + definition: None, + sql_body: false, + }; + let facts = CatalogFacts { + types: vec![UserType { + schema: "app".to_owned(), + name: "span".to_owned(), + shape: TypeShape::Other, + }], + routines: vec![sum.clone()], + tables: vec![table("remote", "f"), child], + triggers: vec![Trigger { + schema: "app".to_owned(), + table: "child".to_owned(), + name: "touch".to_owned(), + definition: "CREATE TRIGGER touch".to_owned(), + enabled: "O".to_owned(), + }], + ..CatalogFacts::default() + }; + + let catalog = assemble(&facts); + + let skipped: Vec<(DumpObjectKind, &str)> = catalog + .skipped + .iter() + .map(|skip| (skip.kind, skip.name.as_str())) + .collect(); + assert_eq!( + skipped, + [ + (DumpObjectKind::Type, "span"), + (DumpObjectKind::Function, "total"), + (DumpObjectKind::Table, "remote"), + (DumpObjectKind::Table, "child"), + ] + ); + assert!(catalog.objects.is_empty()); + assert!(catalog.tables.is_empty()); + + sum.kind = "f".to_owned(); + sum.definition = Some("CREATE OR REPLACE FUNCTION app.total()\n RETURNS integer\n".to_owned()); + sum.sql_body = true; + let catalog = assemble(&CatalogFacts { + routines: vec![sum], + ..CatalogFacts::default() + }); + assert_eq!(catalog.objects[0].section, DumpSection::Programs); + assert!(catalog.objects[0].statements[0].body); + } + + #[test] + fn a_view_carries_its_comments_and_a_materialized_view_its_indexes() { + let mut labelled = column("v", "id"); + labelled.comment = Some("the key".to_owned()); + labelled.default = Some("42".to_owned()); + let view = |name: &str, materialized: bool| View { + schema: "app".to_owned(), + name: name.to_owned(), + materialized, + definition: " SELECT 1 AS id;".to_owned(), + options: Vec::new(), + populated: true, + access_method: None, + tablespace: None, + comment: Some("shown".to_owned()), + }; + let facts = CatalogFacts { + columns: vec![labelled], + indexes: vec![Index { + schema: "app".to_owned(), + table: "m".to_owned(), + name: "m_id".to_owned(), + statement: "CREATE UNIQUE INDEX m_id ON app.m USING btree (id)".to_owned(), + }], + views: vec![view("m", true), view("v", false)], + ..CatalogFacts::default() + }; + + let catalog = assemble(&facts); + + assert_eq!( + sql_of(&catalog, "m")[1..], + [ + "CREATE UNIQUE INDEX m_id ON app.m USING btree (id)", + "COMMENT ON MATERIALIZED VIEW \"app\".\"m\" IS 'shown'" + ] + ); + assert_eq!( + sql_of(&catalog, "v")[1..], + [ + "COMMENT ON VIEW \"app\".\"v\" IS 'shown'", + "ALTER VIEW \"app\".\"v\" ALTER COLUMN \"id\" SET DEFAULT 42", + "COMMENT ON COLUMN \"app\".\"v\".\"id\" IS 'the key'" + ] + ); + let without_rows: Vec<&str> = catalog + .objects + .iter() + .filter_map(|object| object.without_rows.as_ref()) + .flatten() + .map(|statement| statement.sql.as_str()) + .collect(); + assert!(without_rows[0].ends_with("WITH NO DATA"), "{without_rows:?}"); + assert_eq!(without_rows[1..], sql_of(&catalog, "m")[1..]); + assert!( + catalog + .objects + .iter() + .filter(|object| object.name == "v") + .all(|object| object.without_rows.is_none()) + ); + assert!( + catalog + .objects + .iter() + .all(|object| object.section == DumpSection::Programs) + ); + } + + #[test] + fn a_domain_and_a_composite_type_are_made_from_their_parts() { + let mut attribute = column("pair", "a"); + attribute.not_null = false; + let facts = CatalogFacts { + types: vec![ + UserType { + schema: "app".to_owned(), + name: "positive".to_owned(), + shape: TypeShape::Domain(Domain { + base: "integer".to_owned(), + not_null: false, + default: None, + collation: None, + }), + }, + UserType { + schema: "app".to_owned(), + name: "pair".to_owned(), + shape: TypeShape::Composite, + }, + ], + domain_checks: vec![ + DomainCheck { + schema: "app".to_owned(), + domain: "positive".to_owned(), + name: "positive_check".to_owned(), + definition: "CHECK ((VALUE > 0))".to_owned(), + validated: true, + }, + DomainCheck { + schema: "app".to_owned(), + domain: "positive".to_owned(), + name: "small".to_owned(), + definition: "CHECK ((VALUE < 100)) NOT VALID".to_owned(), + validated: false, + }, + ], + columns: vec![attribute], + ..CatalogFacts::default() + }; + + let catalog = assemble(&facts); + + assert_eq!( + sql_of(&catalog, "positive"), + [ + "CREATE DOMAIN \"app\".\"positive\" AS integer", + "ALTER DOMAIN \"app\".\"positive\" ADD CONSTRAINT \"positive_check\" CHECK ((VALUE > 0))", + "ALTER DOMAIN \"app\".\"positive\" ADD CONSTRAINT \"small\" CHECK ((VALUE < 100)) NOT VALID" + ] + ); + assert_eq!( + sections(&catalog) + .into_iter() + .filter(|(_, kind, _)| *kind == DumpObjectKind::Constraint) + .map(|(section, ..)| section) + .collect::>(), + [DumpSection::BeforeData, DumpSection::AfterData] + ); + assert_eq!( + sql_of(&catalog, "pair"), + ["CREATE TYPE \"app\".\"pair\" AS (\n \"a\" integer\n)"] + ); + } + + #[test] + fn a_trigger_on_a_table_left_out_is_left_out_with_it() { + let trigger = |table: &str| Trigger { + schema: "app".to_owned(), + table: table.to_owned(), + name: format!("{table}_touch"), + definition: format!("CREATE TRIGGER {table}_touch BEFORE INSERT ON app.{table}"), + enabled: "D".to_owned(), + }; + let facts = CatalogFacts { + tables: vec![table("kept", "r"), table("remote", "f")], + triggers: vec![trigger("kept"), trigger("remote")], + ..CatalogFacts::default() + }; + + let catalog = assemble(&facts); + + assert_eq!( + sql_of(&catalog, "kept_touch"), + [ + "CREATE TRIGGER kept_touch BEFORE INSERT ON app.kept", + "ALTER TABLE \"app\".\"kept\" DISABLE TRIGGER \"kept_touch\"" + ] + ); + assert!(sql_of(&catalog, "remote_touch").is_empty()); + } + + #[test] + fn every_catalog_read_names_its_scope() { + for sql in [ + SCHEMAS_SQL, + EXTENSIONS_SQL, + TYPES_SQL, + DOMAIN_CHECKS_SQL, + FUNCTIONS_SQL, + FUNCTIONS_SQL_BEFORE_14, + SEQUENCES_SQL, + TABLES_SQL, + COLUMNS_SQL, + CONSTRAINTS_SQL, + INDEXES_SQL, + VIEWS_SQL, + TRIGGERS_SQL, + ] { + assert!(sql.contains("$1 = 'all' OR n.nspname = $2"), "{sql}"); + } + } +} diff --git a/core/crates/drivers/postgres/src/dump/statements.rs b/core/crates/drivers/postgres/src/dump/statements.rs new file mode 100644 index 0000000000..511cc851b0 --- /dev/null +++ b/core/crates/drivers/postgres/src/dump/statements.rs @@ -0,0 +1,673 @@ +//! The statements a dump writes for what the catalog holds. The file +//! empties `search_path`, so every name is quoted and schema-qualified. + +use postgres_protocol::escape::escape_identifier; + +use super::catalog_rows::{ + Collation, Column, Constraint, Domain, DomainCheck, Extension, Sequence, SequenceOwner, Table, Trigger, + UserCollation, View, +}; +use crate::dialect::quoted as literal; + +pub(super) fn quote(name: &str) -> String { + escape_identifier(name) +} + +pub(super) fn qualified(schema: &str, name: &str) -> String { + format!("{}.{}", quote(schema), quote(name)) +} + +fn collate(collation: &Collation) -> String { + format!(" COLLATE {}", qualified(&collation.schema, &collation.name)) +} + +/// As pg_dump writes one. `None` for a provider this does not know. +pub(super) fn create_collation(collation: &UserCollation) -> Option { + let provider = match collation.provider.as_str() { + "c" => "libc", + "i" => "icu", + "b" => "builtin", + _ => return None, + }; + let mut options = vec![format!("provider = {provider}")]; + if !collation.deterministic { + options.push("deterministic = false".to_owned()); + } + match (provider, collation.collate.as_deref(), collation.ctype.as_deref()) { + ("libc", Some(collate), Some(ctype)) if collate == ctype => { + options.push(format!("locale = {}", literal(collate))) + } + ("libc", Some(collate), Some(ctype)) => { + options.push(format!("lc_collate = {}", literal(collate))); + options.push(format!("lc_ctype = {}", literal(ctype))); + } + ("libc", ..) => return None, + // Before 15 an ICU collation kept its locale in collcollate. + _ => { + let locale = collation.locale.as_deref().or(collation.collate.as_deref())?; + options.push(format!("locale = {}", literal(locale))); + } + } + if let Some(rules) = &collation.rules { + options.push(format!("rules = {}", literal(rules))); + } + Some(format!( + "CREATE COLLATION {} ({})", + qualified(&collation.schema, &collation.name), + options.join(", ") + )) +} + +pub(super) fn create_schema(name: &str) -> String { + format!("CREATE SCHEMA IF NOT EXISTS {}", quote(name)) +} + +pub(super) fn create_extension(extension: &Extension) -> String { + format!( + "CREATE EXTENSION IF NOT EXISTS {} WITH SCHEMA {}", + quote(&extension.name), + quote(&extension.schema) + ) +} + +pub(super) fn create_enum(schema: &str, name: &str, labels: &[String]) -> String { + let labels: Vec = labels.iter().map(|label| format!(" {}", literal(label))).collect(); + format!( + "CREATE TYPE {} AS ENUM (\n{}\n)", + qualified(schema, name), + labels.join(",\n") + ) +} + +/// The CREATE first, then each check as an ALTER DOMAIN, which the dump +/// runs once the routines exist: a check can call one, and the server +/// resolves it when the check is added. One the values have not all been +/// held to stays NOT VALID. +pub(super) fn create_domain(schema: &str, name: &str, domain: &Domain, checks: &[&DomainCheck]) -> Vec { + let mut create = format!("CREATE DOMAIN {} AS {}", qualified(schema, name), domain.base); + if let Some(collation) = &domain.collation { + create.push_str(&collate(collation)); + } + if let Some(default) = &domain.default { + create.push_str(&format!(" DEFAULT {default}")); + } + if domain.not_null { + create.push_str(" NOT NULL"); + } + let mut statements = vec![create]; + statements.extend(checks.iter().map(|check| { + format!( + "ALTER DOMAIN {} ADD CONSTRAINT {} {}", + qualified(schema, name), + quote(&check.name), + check.definition + ) + })); + statements +} + +fn column_head(column: &Column) -> String { + let mut head = format!("{} {}", quote(&column.name), column.type_sql); + if let Some(collation) = &column.collation { + head.push_str(&collate(collation)); + } + head +} + +pub(super) fn create_composite(schema: &str, name: &str, attributes: &[&Column]) -> String { + let attributes: Vec = attributes + .iter() + .map(|attribute| format!(" {}", column_head(attribute))) + .collect(); + format!( + "CREATE TYPE {} AS (\n{}\n)", + qualified(schema, name), + attributes.join(",\n") + ) +} + +fn sequence_options(sequence: &Sequence) -> Vec { + let mut options = vec![ + format!("START WITH {}", sequence.start), + format!("INCREMENT BY {}", sequence.increment), + format!("MINVALUE {}", sequence.min), + format!("MAXVALUE {}", sequence.max), + format!("CACHE {}", sequence.cache), + ]; + if sequence.cycle { + options.push("CYCLE".to_owned()); + } + options +} + +/// Without OWNED BY: the column it names does not exist yet. +pub(super) fn create_sequence(sequence: &Sequence) -> String { + let kind = match sequence.unlogged { + true => "UNLOGGED SEQUENCE", + false => "SEQUENCE", + }; + let mut lines = vec![format!("AS {}", sequence.data_type)]; + lines.extend(sequence_options(sequence)); + format!( + "CREATE {kind} {}\n {}", + qualified(&sequence.schema, &sequence.name), + lines.join("\n ") + ) +} + +/// The identity's sequence keeps its name and settings, so its position +/// can be set by name after the rows. +fn identity(column: &Column, sequence: Option<&Sequence>) -> String { + let when = match column.identity.as_str() { + "a" => "ALWAYS", + _ => "BY DEFAULT", + }; + match sequence { + Some(sequence) => { + let mut options = vec![format!("SEQUENCE NAME {}", qualified(&sequence.schema, &sequence.name))]; + options.extend(sequence_options(sequence)); + format!("GENERATED {when} AS IDENTITY ({})", options.join(" ")) + } + None => format!("GENERATED {when} AS IDENTITY"), + } +} + +pub(super) fn column_sql(column: &Column, identity_sequence: Option<&Sequence>) -> String { + let mut sql = column_head(column); + match (column.generated.as_str(), column.identity.as_str(), &column.default) { + ("s", _, Some(expression)) => sql.push_str(&format!(" GENERATED ALWAYS AS ({expression}) STORED")), + ("v", _, Some(expression)) => sql.push_str(&format!(" GENERATED ALWAYS AS ({expression}) VIRTUAL")), + (_, "a" | "d", _) => { + sql.push(' '); + sql.push_str(&identity(column, identity_sequence)); + } + (_, _, Some(default)) => sql.push_str(&format!(" DEFAULT {default}")), + _ => {} + } + if column.not_null { + sql.push_str(" NOT NULL"); + } + sql +} + +/// `columns` are the column clauses; a partition takes its columns from +/// its parent and states only its own checks. +pub(super) fn create_table(table: &Table, columns: &[String], checks: &[&Constraint]) -> String { + let unlogged = match table.unlogged { + true => "UNLOGGED ", + false => "", + }; + let name = qualified(&table.schema, &table.name); + let checks: Vec = checks + .iter() + .map(|check| format!("CONSTRAINT {} {}", quote(&check.name), check.definition)) + .collect(); + let mut sql = match &table.partition { + Some(partition) => { + let mut sql = format!( + "CREATE {unlogged}TABLE {name} PARTITION OF {}", + qualified(&partition.parent_schema, &partition.parent) + ); + if !checks.is_empty() { + sql.push_str(&format!(" (\n {}\n)", checks.join(",\n "))); + } + sql.push(' '); + sql.push_str(&partition.bound); + sql + } + None => { + let mut body = columns.to_vec(); + body.extend(checks); + format!("CREATE {unlogged}TABLE {name} (\n {}\n)", body.join(",\n ")) + } + }; + if let Some(key) = &table.partition_key { + sql.push_str(&format!(" PARTITION BY {key}")); + } + if let Some(method) = &table.access_method { + sql.push_str(&format!(" USING {}", quote(method))); + } + if let Some(options) = storage_options(&table.options) { + sql.push_str(&format!(" WITH ({options})")); + } + if let Some(tablespace) = &table.tablespace { + sql.push_str(&format!(" TABLESPACE {}", quote(tablespace))); + } + sql +} + +/// `reloptions` as `name='value'`, as pg_dump writes them. +fn storage_options(options: &[String]) -> Option { + let written: Vec = options + .iter() + .map(|option| match option.split_once('=') { + Some((name, value)) => format!("{name}={}", literal(value)), + None => option.clone(), + }) + .collect(); + (!written.is_empty()).then(|| written.join(", ")) +} + +pub(super) fn owned_by(sequence: &Sequence, owner: &SequenceOwner) -> String { + format!( + "ALTER SEQUENCE {} OWNED BY {}.{}", + qualified(&sequence.schema, &sequence.name), + qualified(&owner.schema, &owner.table), + quote(&owner.column) + ) +} + +/// Where the sequence stands, used or not: a RESTART leaves the next +/// value away from START. `None` where the login could not read it. +pub(super) fn set_value(sequence: &Sequence) -> Option { + sequence.last_value.map(|value| { + format!( + "SELECT pg_catalog.setval({}, {value}, {})", + literal(&qualified(&sequence.schema, &sequence.name)), + sequence.is_called.unwrap_or(true) + ) + }) +} + +/// ONLY keeps the statement off a regular table's inheritance children; +/// a partitioned table refuses it, and its partitions take the +/// constraint from it. +pub(super) fn add_constraint(constraint: &Constraint) -> String { + let only = match constraint.partitioned { + true => "", + false => "ONLY ", + }; + format!( + "ALTER TABLE {only}{} ADD CONSTRAINT {} {}", + qualified(&constraint.schema, &constraint.table), + quote(&constraint.name), + constraint.definition + ) +} + +pub(super) fn comment_on(target: &str, name: &str, text: &str) -> String { + format!("COMMENT ON {target} {name} IS {}", literal(text)) +} + +pub(super) fn comment_on_column(schema: &str, table: &str, column: &str, text: &str) -> String { + comment_on( + "COLUMN", + &format!("{}.{}", qualified(schema, table), quote(column)), + text, + ) +} + +/// A materialized view is filled as it is made, after the rows it reads. +pub(super) fn create_view(view: &View) -> String { + let name = qualified(&view.schema, &view.name); + let definition = view.definition.trim_end().trim_end_matches(';').trim_end(); + if view.materialized { + let mut head = format!("CREATE MATERIALIZED VIEW {name}"); + if let Some(method) = &view.access_method { + head.push_str(&format!(" USING {}", quote(method))); + } + if let Some(options) = storage_options(&view.options) { + head.push_str(&format!(" WITH ({options})")); + } + if let Some(tablespace) = &view.tablespace { + head.push_str(&format!(" TABLESPACE {}", quote(tablespace))); + } + let data = match view.populated { + true => "WITH DATA", + false => "WITH NO DATA", + }; + return format!("{head} AS\n{definition}\n {data}"); + } + let (check, options): (Vec<&String>, Vec<&String>) = view + .options + .iter() + .partition(|option| option.starts_with("check_option=")); + let mut head = format!("CREATE VIEW {name}"); + let options: Vec = options.into_iter().cloned().collect(); + if let Some(options) = storage_options(&options) { + head.push_str(&format!(" WITH ({options})")); + } + let mut sql = format!("{head} AS\n{definition}"); + if let Some(level) = check + .first() + .and_then(|option| option.split_once('=')) + .map(|(_, level)| level) + { + sql.push_str(&format!("\n WITH {} CHECK OPTION", level.to_uppercase())); + } + sql +} + +/// A trigger is made enabled; one turned off or set to fire only on a +/// replica is set so again. +pub(super) fn trigger_state(trigger: &Trigger) -> Option { + let state = match trigger.enabled.as_str() { + "D" => "DISABLE", + "R" => "ENABLE REPLICA", + "A" => "ENABLE ALWAYS", + _ => return None, + }; + Some(format!( + "ALTER TABLE {} {state} TRIGGER {}", + qualified(&trigger.schema, &trigger.table), + quote(&trigger.name) + )) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::dump::catalog_rows::Partition; + + #[test] + fn a_collation_is_written_for_each_provider_and_release() { + let collation = + |provider: &str, collate: Option<&str>, ctype: Option<&str>, locale: Option<&str>| UserCollation { + schema: "app".to_owned(), + name: "sorted".to_owned(), + provider: provider.to_owned(), + deterministic: true, + collate: collate.map(str::to_owned), + ctype: ctype.map(str::to_owned), + locale: locale.map(str::to_owned), + rules: None, + }; + + assert_eq!( + create_collation(&collation("c", Some("C"), Some("C"), None)).as_deref(), + Some("CREATE COLLATION \"app\".\"sorted\" (provider = libc, locale = 'C')") + ); + assert_eq!( + create_collation(&collation("c", Some("C"), Some("en_US.utf8"), None)).as_deref(), + Some("CREATE COLLATION \"app\".\"sorted\" (provider = libc, lc_collate = 'C', lc_ctype = 'en_US.utf8')") + ); + let mut nocase = collation("i", None, None, Some("und-u-ks-level2")); + nocase.deterministic = false; + assert_eq!( + create_collation(&nocase).as_deref(), + Some( + "CREATE COLLATION \"app\".\"sorted\" (provider = icu, deterministic = false, locale = 'und-u-ks-level2')" + ) + ); + assert_eq!( + create_collation(&collation("i", Some("de-DE"), Some("de-DE"), None)).as_deref(), + Some("CREATE COLLATION \"app\".\"sorted\" (provider = icu, locale = 'de-DE')") + ); + assert_eq!(create_collation(&collation("d", None, None, None)), None); + } + + fn sequence(name: &str) -> Sequence { + Sequence { + schema: "app".to_owned(), + name: name.to_owned(), + data_type: "integer".to_owned(), + start: 1, + increment: 1, + min: 1, + max: 2_147_483_647, + cache: 1, + cycle: false, + unlogged: false, + last_value: Some(4), + is_called: Some(true), + owner: None, + } + } + + fn column(name: &str, type_sql: &str) -> Column { + Column { + schema: "app".to_owned(), + table: "t".to_owned(), + name: name.to_owned(), + type_sql: type_sql.to_owned(), + collation: None, + default: None, + identity: String::new(), + generated: String::new(), + not_null: false, + comment: None, + } + } + + fn table(name: &str) -> Table { + Table { + schema: "app".to_owned(), + name: name.to_owned(), + kind: "r".to_owned(), + unlogged: false, + partition_key: None, + partition: None, + inherits: false, + filtered: false, + options: Vec::new(), + access_method: None, + tablespace: None, + comment: None, + } + } + + fn check(name: &str, definition: &str) -> Constraint { + Constraint { + schema: "app".to_owned(), + table: "t".to_owned(), + partitioned: false, + name: name.to_owned(), + kind: "c".to_owned(), + definition: definition.to_owned(), + validated: true, + } + } + + #[test] + fn a_sequence_states_every_setting_and_no_owner() { + let mut cycling = sequence("Order Seq"); + cycling.cycle = true; + cycling.unlogged = true; + + assert_eq!( + create_sequence(&cycling), + "CREATE UNLOGGED SEQUENCE \"app\".\"Order Seq\"\n AS integer\n START WITH 1\n INCREMENT BY 1\n MINVALUE 1\n MAXVALUE 2147483647\n CACHE 1\n CYCLE" + ); + } + + #[test] + fn an_identity_keeps_its_sequence_name_and_settings() { + let mut id = column("id", "integer"); + id.identity = "a".to_owned(); + id.not_null = true; + + assert_eq!( + column_sql(&id, Some(&sequence("t_id_seq"))), + "\"id\" integer GENERATED ALWAYS AS IDENTITY (SEQUENCE NAME \"app\".\"t_id_seq\" START WITH 1 INCREMENT BY 1 MINVALUE 1 MAXVALUE 2147483647 CACHE 1) NOT NULL" + ); + id.identity = "d".to_owned(); + assert_eq!( + column_sql(&id, None), + "\"id\" integer GENERATED BY DEFAULT AS IDENTITY NOT NULL" + ); + } + + #[test] + fn a_column_states_its_collation_default_or_expression() { + let mut note = column("note", "text"); + note.collation = Some(Collation { + schema: "pg_catalog".to_owned(), + name: "C".to_owned(), + }); + note.default = Some("'none'::text".to_owned()); + assert_eq!( + column_sql(¬e, None), + "\"note\" text COLLATE \"pg_catalog\".\"C\" DEFAULT 'none'::text" + ); + + let mut total = column("total", "numeric"); + total.generated = "s".to_owned(); + total.default = Some("(qty * price)".to_owned()); + assert_eq!( + column_sql(&total, None), + "\"total\" numeric GENERATED ALWAYS AS ((qty * price)) STORED" + ); + } + + #[test] + fn a_table_states_its_columns_checks_and_options() { + let mut readings = table("readings"); + readings.unlogged = true; + readings.partition_key = Some("RANGE (taken)".to_owned()); + readings.options = vec!["fillfactor=70".to_owned()]; + + assert_eq!( + create_table( + &readings, + &["\"taken\" date NOT NULL".to_owned()], + &[&check("positive", "CHECK ((amount > 0))")] + ), + "CREATE UNLOGGED TABLE \"app\".\"readings\" (\n \"taken\" date NOT NULL,\n CONSTRAINT \"positive\" CHECK ((amount > 0))\n) PARTITION BY RANGE (taken) WITH (fillfactor='70')" + ); + } + + #[test] + fn a_partition_names_its_parent_and_bound_and_no_columns() { + let mut partition = table("readings_2025"); + partition.partition = Some(Partition { + parent_schema: "app".to_owned(), + parent: "readings".to_owned(), + bound: "FOR VALUES FROM ('2025-01-01') TO ('2026-01-01')".to_owned(), + }); + + assert_eq!( + create_table(&partition, &["\"taken\" date".to_owned()], &[]), + "CREATE TABLE \"app\".\"readings_2025\" PARTITION OF \"app\".\"readings\" FOR VALUES FROM ('2025-01-01') TO ('2026-01-01')" + ); + assert_eq!( + create_table(&partition, &[], &[&check("own", "CHECK ((amount < 10))")]), + "CREATE TABLE \"app\".\"readings_2025\" PARTITION OF \"app\".\"readings\" (\n CONSTRAINT \"own\" CHECK ((amount < 10))\n) FOR VALUES FROM ('2025-01-01') TO ('2026-01-01')" + ); + } + + #[test] + fn a_constraint_on_a_partitioned_table_reaches_its_partitions() { + let mut key = check("t_pkey", "PRIMARY KEY (id)"); + assert_eq!( + add_constraint(&key), + "ALTER TABLE ONLY \"app\".\"t\" ADD CONSTRAINT \"t_pkey\" PRIMARY KEY (id)" + ); + key.partitioned = true; + assert_eq!( + add_constraint(&key), + "ALTER TABLE \"app\".\"t\" ADD CONSTRAINT \"t_pkey\" PRIMARY KEY (id)" + ); + } + + #[test] + fn a_sequence_position_is_set_by_its_quoted_name() { + assert_eq!( + set_value(&sequence("it's")).as_deref(), + Some("SELECT pg_catalog.setval('\"app\".\"it''s\"', 4, true)") + ); + let mut unread = sequence("unread"); + unread.last_value = None; + assert_eq!(set_value(&unread), None); + let mut restarted = sequence("restarted"); + restarted.last_value = Some(1000); + restarted.is_called = Some(false); + assert_eq!( + set_value(&restarted).as_deref(), + Some("SELECT pg_catalog.setval('\"app\".\"restarted\"', 1000, false)") + ); + } + + #[test] + fn a_domain_adds_a_check_not_yet_held_after_it() { + let domain = Domain { + base: "integer".to_owned(), + not_null: true, + default: Some("1".to_owned()), + collation: None, + }; + let held = DomainCheck { + schema: "app".to_owned(), + domain: "positive".to_owned(), + name: "positive_check".to_owned(), + definition: "CHECK ((VALUE > 0))".to_owned(), + validated: true, + }; + let pending = DomainCheck { + name: "small".to_owned(), + definition: "CHECK ((VALUE < 100)) NOT VALID".to_owned(), + validated: false, + ..held.clone() + }; + + assert_eq!( + create_domain("app", "positive", &domain, &[&held, &pending]), + [ + "CREATE DOMAIN \"app\".\"positive\" AS integer DEFAULT 1 NOT NULL", + "ALTER DOMAIN \"app\".\"positive\" ADD CONSTRAINT \"positive_check\" CHECK ((VALUE > 0))", + "ALTER DOMAIN \"app\".\"positive\" ADD CONSTRAINT \"small\" CHECK ((VALUE < 100)) NOT VALID", + ] + ); + } + + #[test] + fn enum_labels_and_comments_are_literals() { + assert_eq!( + create_enum("app", "mood", &["sad".to_owned(), "it's".to_owned()]), + "CREATE TYPE \"app\".\"mood\" AS ENUM (\n 'sad',\n 'it''s'\n)" + ); + assert_eq!( + comment_on_column("app", "t", "note", "a \\ b"), + "COMMENT ON COLUMN \"app\".\"t\".\"note\" IS E'a \\\\ b'" + ); + } + + fn view(materialized: bool, options: &[&str]) -> View { + View { + schema: "app".to_owned(), + name: "v".to_owned(), + materialized, + definition: " SELECT t.id\n FROM app.t;".to_owned(), + options: options.iter().map(|option| (*option).to_owned()).collect(), + populated: true, + access_method: None, + tablespace: None, + comment: None, + } + } + + #[test] + fn a_view_keeps_its_options_and_check_option_and_drops_the_semicolon() { + assert_eq!( + create_view(&view(false, &["security_barrier=true", "check_option=local"])), + "CREATE VIEW \"app\".\"v\" WITH (security_barrier='true') AS\n SELECT t.id\n FROM app.t\n WITH LOCAL CHECK OPTION" + ); + } + + #[test] + fn a_materialized_view_fills_itself_only_where_it_was_filled() { + assert_eq!( + create_view(&view(true, &[])), + "CREATE MATERIALIZED VIEW \"app\".\"v\" AS\n SELECT t.id\n FROM app.t\n WITH DATA" + ); + let mut empty = view(true, &[]); + empty.populated = false; + assert!(create_view(&empty).ends_with("\n WITH NO DATA")); + } + + #[test] + fn only_a_trigger_not_plainly_enabled_gets_a_state() { + let mut trigger = Trigger { + schema: "app".to_owned(), + table: "t".to_owned(), + name: "touch".to_owned(), + definition: String::new(), + enabled: "O".to_owned(), + }; + assert_eq!(trigger_state(&trigger), None); + trigger.enabled = "D".to_owned(); + assert_eq!( + trigger_state(&trigger).as_deref(), + Some("ALTER TABLE \"app\".\"t\" DISABLE TRIGGER \"touch\"") + ); + } +} diff --git a/core/crates/drivers/postgres/src/lib.rs b/core/crates/drivers/postgres/src/lib.rs index c2ed03aa50..f58020fdea 100644 --- a/core/crates/drivers/postgres/src/lib.rs +++ b/core/crates/drivers/postgres/src/lib.rs @@ -15,6 +15,7 @@ mod connection; mod ddl; mod dialect; mod driver; +mod dump; mod editor; mod errors; mod metadata; diff --git a/core/crates/drivers/postgres/src/metadata/indexes.rs b/core/crates/drivers/postgres/src/metadata/indexes.rs index 1c46d30b9c..1cff852590 100644 --- a/core/crates/drivers/postgres/src/metadata/indexes.rs +++ b/core/crates/drivers/postgres/src/metadata/indexes.rs @@ -99,7 +99,7 @@ pub(super) async fn fetch_index_statements( // pg_get_indexdef prints ON ONLY for a partitioned table's index and has no form without it; // run as is, it builds the parent's index alone, invalid and on no partition. -fn without_only(statement: String, unique: bool, quoted_name: &str) -> String { +pub(crate) fn without_only(statement: String, unique: bool, quoted_name: &str) -> String { let head = format!("CREATE {}INDEX {quoted_name} ON ", if unique { "UNIQUE " } else { "" }); match statement .strip_prefix(&head) @@ -111,7 +111,7 @@ fn without_only(statement: String, unique: bool, quoted_name: &str) -> String { } // pg_get_indexdef leaves the tablespace out. It goes before the predicate, which ends the text. -fn in_tablespace(statement: String, tablespace: &str, predicate: Option<&str>) -> String { +pub(crate) fn in_tablespace(statement: String, tablespace: &str, predicate: Option<&str>) -> String { let clause = format!(" TABLESPACE {tablespace}"); let head = predicate.and_then(|predicate| statement.strip_suffix(&format!(" WHERE {predicate}"))); match (head, predicate) { diff --git a/core/crates/drivers/postgres/src/metadata/mod.rs b/core/crates/drivers/postgres/src/metadata/mod.rs index bb58057803..dc0432e6f1 100644 --- a/core/crates/drivers/postgres/src/metadata/mod.rs +++ b/core/crates/drivers/postgres/src/metadata/mod.rs @@ -13,7 +13,7 @@ pub(crate) use definitions::fetch_object_definition; pub(crate) use estimate::table_estimate; pub(crate) use foreign_keys::{fetch_foreign_keys, fetch_outbound_foreign_keys}; pub(crate) use identity::row_identity; -pub(crate) use indexes::fetch_indexes; +pub(crate) use indexes::{fetch_indexes, in_tablespace, without_only}; pub(crate) use objects::{list_databases, list_objects, list_schemas}; use futures_util::TryStreamExt; @@ -129,11 +129,12 @@ pub(crate) async fn fetch_table_definition( mod tests { use super::*; use crate::catalog::COLUMNS_SQL_BEFORE_14; + use crate::dump::catalog_sql as dump; /// Words that take a parenthesis without calling a function by name. - const SYNTAX: [&str; 14] = [ + const SYNTAX: [&str; 16] = [ "AS", "ARRAY", "COALESCE", "EXISTS", "FROM", "IN", "ANY", "NULLIF", "GREATEST", "ON", "AND", "OR", "NOT", - "OPERATOR", + "OPERATOR", "LATERAL", "SELECT", ]; /// Names called without `pg_catalog.` in front, which the session's @@ -192,6 +193,19 @@ mod tests { ("primary key", PRIMARY_KEY_NAME_SQL.to_owned()), ("columns", crate::catalog::COLUMNS_SQL.to_owned()), ("columns before 14", COLUMNS_SQL_BEFORE_14.to_owned()), + ("dump schemas", dump::SCHEMAS_SQL.to_owned()), + ("dump extensions", dump::EXTENSIONS_SQL.to_owned()), + ("dump types", dump::TYPES_SQL.to_owned()), + ("dump domain checks", dump::DOMAIN_CHECKS_SQL.to_owned()), + ("dump functions", dump::FUNCTIONS_SQL.to_owned()), + ("dump functions before 14", dump::FUNCTIONS_SQL_BEFORE_14.to_owned()), + ("dump sequences", dump::SEQUENCES_SQL.to_owned()), + ("dump tables", dump::TABLES_SQL.to_owned()), + ("dump columns", dump::COLUMNS_SQL.to_owned()), + ("dump constraints", dump::CONSTRAINTS_SQL.to_owned()), + ("dump indexes", dump::INDEXES_SQL.to_owned()), + ("dump views", dump::VIEWS_SQL.to_owned()), + ("dump triggers", dump::TRIGGERS_SQL.to_owned()), ] { assert_eq!(bare_calls(&sql), Vec::::new(), "{name}"); } diff --git a/core/crates/drivers/postgres/tests/integration/conformance_engine.rs b/core/crates/drivers/postgres/tests/integration/conformance_engine.rs index 298862ed00..22cc522df3 100644 --- a/core/crates/drivers/postgres/tests/integration/conformance_engine.rs +++ b/core/crates/drivers/postgres/tests/integration/conformance_engine.rs @@ -5,18 +5,20 @@ use std::time::Duration; use async_trait::async_trait; use tablepro_core::{ - ConnectOptions, DatabaseDriver, LivenessPolicy, NetworkEndpoint, TableRef, TlsConfig, WorkspaceSettings, + ConnectOptions, DatabaseDriver, EditorSessionOptions, LivenessPolicy, NetworkEndpoint, RunEnd, RunOptions, + TableRef, TlsConfig, UserSql, WorkspaceSettings, }; use tablepro_driver_postgres::{PgDriver, WORKSPACE_POOL_MAX_SIZE}; -use tablepro_test_fixtures::{HbaMode, PostgresFixture}; +use tablepro_test_fixtures::{HbaMode, PostgresFixture, PostgresVersion, ServerVersion}; use tablepro_test_support::FaultTransport; use tablepro_test_support::conformance::{ - BoundedDdlEngine, ConformanceDdlEngine, ConformanceEngine, ConformanceFailure, EditorTransactionEngine, - LargeTableEngine, NetworkConformanceEngine, PooledConformanceEngine, ServerStopEngine, TypedConformanceEngine, - UnsupportedTypeCase, + BoundedDdlEngine, ConformanceDdlEngine, ConformanceEngine, ConformanceFailure, DumpEngine, EditorTransactionEngine, + LargeTableEngine, NativeDump, NetworkConformanceEngine, PooledConformanceEngine, ServerStopEngine, + TypedConformanceEngine, UnsupportedTypeCase, test_liveness, }; +use tokio_util::sync::CancellationToken; -use crate::support::{direct, options_over, password}; +use crate::support::{EventSink, Events, direct, options_over, password}; pub struct PgEngine { fixture: PostgresFixture, @@ -190,6 +192,129 @@ impl ConformanceDdlEngine for PgEngine { } } +#[async_trait] +impl DumpEngine for PgEngine { + async fn empty_database(&self, name: &str) -> Result { + let liveness = test_liveness(); + let admin = Self::driver() + .connect( + self.connect_options(WorkspaceSettings::workspace(false), liveness), + CancellationToken::new(), + ) + .await?; + // CREATE DATABASE refuses a transaction, and the editor runs each + // statement outside one. + let session = admin + .open_editor_session(EditorSessionOptions::default(), CancellationToken::new()) + .await?; + for sql in [ + format!("DROP DATABASE IF EXISTS {} WITH (FORCE)", quoted(name)), + format!("CREATE DATABASE {}", quoted(name)), + ] { + let summary = session + .run( + UserSql::new(sql.as_str()), + RunOptions::new(CancellationToken::new()), + Box::new(EventSink::new(&Events::default())), + ) + .await?; + if summary.end != RunEnd::Completed { + return Err(ConformanceFailure::setup(format!("`{sql}` ended {:?}", summary.end))); + } + } + session.close().await; + admin.close().await?; + Ok(options_over( + direct(self.fixture.host(), self.fixture.port(), &liveness), + password(&self.fixture.password_credentials()), + name, + TlsConfig::disabled(), + WorkspaceSettings::workspace(false), + liveness, + )) + } + + /// The child's identity, the parent's serial and the invoice sequence + /// all stand past the highest key. The trigger appends `;` to every + /// note it sees, so a trigger made before the rows load shows twice. + fn dump_fixture(&self) -> Vec { + [ + "CREATE SCHEMA app", + "CREATE COLLATION app.bytes (locale = 'C')", + "CREATE TYPE app.mood AS ENUM ('sad', 'ok', 'happy')", + "CREATE DOMAIN app.quantity AS integer CONSTRAINT quantity_positive CHECK (VALUE > 0)", + "CREATE SEQUENCE app.invoice_no START WITH 1000", + "CREATE FUNCTION app.mark_note() RETURNS trigger LANGUAGE plpgsql AS $$ \ + BEGIN NEW.note := coalesce(NEW.note, '') || ';'; RETURN NEW; END $$", + "CREATE FUNCTION app.label(name text, mood app.mood) RETURNS text LANGUAGE sql IMMUTABLE AS $$ \ + SELECT name || ' (' || coalesce(mood::text, 'none') || ')' $$", + "CREATE TABLE app.parent (id serial PRIMARY KEY, name text NOT NULL UNIQUE, mood app.mood)", + "CREATE TABLE app.child (\ + id integer GENERATED ALWAYS AS IDENTITY PRIMARY KEY, \ + parent_id integer NOT NULL REFERENCES app.parent (id), \ + next_id integer REFERENCES app.child (id), \ + invoice integer NOT NULL DEFAULT nextval('app.invoice_no'), \ + qty app.quantity NOT NULL, \ + price numeric(10, 2) NOT NULL CHECK (price >= 0), \ + total numeric GENERATED ALWAYS AS (qty * price) STORED, \ + note text COLLATE \"C\", \ + code text COLLATE app.bytes, \ + payload bytea)", + "CREATE INDEX child_parent_idx ON app.child (parent_id)", + "CREATE PROCEDURE app.clear_note(target integer) LANGUAGE plpgsql AS $$ \ + BEGIN UPDATE app.child SET note = NULL WHERE id = target; END $$", + "CREATE TABLE app.reading (taken date NOT NULL, amount integer NOT NULL, PRIMARY KEY (taken, amount)) \ + PARTITION BY RANGE (taken)", + "CREATE TABLE app.reading_2025 PARTITION OF app.reading FOR VALUES FROM ('2025-01-01') TO ('2026-01-01')", + "CREATE TABLE app.reading_2026 PARTITION OF app.reading FOR VALUES FROM ('2026-01-01') TO ('2027-01-01')", + "CREATE TRIGGER child_mark BEFORE INSERT ON app.child FOR EACH ROW EXECUTE FUNCTION app.mark_note()", + "INSERT INTO app.parent (name, mood) VALUES ('ann', 'happy'), ('bob', 'sad'), ('cy', NULL), ('dee', 'ok')", + "DELETE FROM app.parent WHERE name = 'dee'", + r#"INSERT INTO app.child (parent_id, qty, price, note, payload) VALUES + (1, 2, 1.50, 'it''s "quoted" \ back; and +a new line: ünïcødé', '\x00ff'::bytea), + (1, 1, 0.25, NULL, NULL), + (2, 3, 9.99, 'plain', '\xdeadbeef'::bytea), + (3, 1, 100.00, 'last', NULL)"#, + "DELETE FROM app.child WHERE id = 4", + "UPDATE app.child SET next_id = 3 WHERE id = 1", + "UPDATE app.child SET next_id = 1 WHERE id = 2", + "INSERT INTO app.reading (taken, amount) VALUES ('2025-03-01', 1), ('2026-07-01', 2), ('2026-08-01', 3)", + "CREATE VIEW app.parent_labels AS SELECT id, app.label(name, mood) AS label FROM app.parent", + "CREATE VIEW app.all_labels AS SELECT upper(label) AS label FROM app.parent_labels", + "CREATE MATERIALIZED VIEW app.child_totals AS \ + SELECT parent_id, sum(total) AS total FROM app.child GROUP BY parent_id", + "CREATE UNIQUE INDEX child_totals_parent ON app.child_totals (parent_id)", + "COMMENT ON TABLE app.parent IS 'People; with ''quotes'''", + "COMMENT ON COLUMN app.child.note IS 'Free text'", + "COMMENT ON VIEW app.parent_labels IS 'Labels'", + ] + .map(str::to_owned) + .to_vec() + } + + /// pg_dump 18.6's plain output with its OWNER lines taken out, as + /// `--no-owner` writes it: `\restrict`, COPY blocks and all. + /// pg_dump 18 writes `SET transaction_timeout = 0`, which a server + /// before 17 refuses, as psql with `ON_ERROR_STOP` would show. + fn native_dump(&self) -> Option { + if PostgresVersion::selected().is_ok_and(|version| version.major() < 17) { + return None; + } + let check = |sql: &str, value: &str| (sql.to_owned(), value.to_owned()); + Some(NativeDump { + file: include_str!("fixtures/pg_dump-18-no-owner.sql"), + checks: vec![ + check("SELECT count(*) FROM app.child", "4"), + check("SELECT note FROM app.child WHERE id = 1", r#"it's; a "quote" \ back;"#), + check("SELECT nextval('app.child_id_seq')", "6"), + check("SELECT count(*) FROM app.v2", "3"), + check("SELECT encode(payload, 'hex') FROM app.child WHERE id = 1", "00ff"), + ], + }) + } +} + mod suite { use super::PgEngine; diff --git a/core/crates/drivers/postgres/tests/integration/dump.rs b/core/crates/drivers/postgres/tests/integration/dump.rs new file mode 100644 index 0000000000..daee0d8d54 --- /dev/null +++ b/core/crates/drivers/postgres/tests/integration/dump.rs @@ -0,0 +1,11 @@ +//! The dump round trip on PostgreSQL: the sample out as a SQL file, into +//! an empty database through the SQL file import, and out again the same. + +mod suite { + use crate::conformance_engine::PgEngine; + + tablepro_test_support::dump_suite! { + engine: PgEngine, + docker: true, + } +} diff --git a/core/crates/drivers/postgres/tests/integration/fixtures/pg_dump-18-no-owner.sql b/core/crates/drivers/postgres/tests/integration/fixtures/pg_dump-18-no-owner.sql new file mode 100644 index 0000000000..c2676f6a8c --- /dev/null +++ b/core/crates/drivers/postgres/tests/integration/fixtures/pg_dump-18-no-owner.sql @@ -0,0 +1,332 @@ +-- +-- PostgreSQL database dump +-- + +\restrict mWamWyqv07yVo3aCxQxVd5KrzbHohlp5dNMiGv13zLPzagfuMqNtgHkxSMT0nST + +-- Dumped from database version 18.6 (Debian 18.6-1.pgdg13+2) +-- Dumped by pg_dump version 18.6 (Debian 18.6-1.pgdg13+2) + +SET statement_timeout = 0; +SET lock_timeout = 0; +SET idle_in_transaction_session_timeout = 0; +SET transaction_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = on; +SELECT pg_catalog.set_config('search_path', '', false); +SET check_function_bodies = false; +SET xmloption = content; +SET client_min_messages = warning; +SET row_security = off; + +-- +-- Name: app; Type: SCHEMA; Schema: -; Owner: postgres +-- + +CREATE SCHEMA app; + + + +-- +-- Name: mood; Type: TYPE; Schema: app; Owner: postgres +-- + +CREATE TYPE app.mood AS ENUM ( + 'sad', + 'ok', + 'happy' +); + + + +-- +-- Name: add_parent(text); Type: PROCEDURE; Schema: app; Owner: postgres +-- + +CREATE PROCEDURE app.add_parent(IN n text) + LANGUAGE sql + AS $$ INSERT INTO app.parent (name) VALUES (n) $$; + + + +-- +-- Name: touch(); Type: FUNCTION; Schema: app; Owner: postgres +-- + +CREATE FUNCTION app.touch() RETURNS trigger + LANGUAGE plpgsql + AS $$ +BEGIN + NEW.note := coalesce(NEW.note, '') || ';'; + RETURN NEW; +END +$$; + + + +SET default_tablespace = ''; + +SET default_table_access_method = heap; + +-- +-- Name: always_ident; Type: TABLE; Schema: app; Owner: postgres +-- + +CREATE TABLE app.always_ident ( + id integer NOT NULL, + v text +); + + + +-- +-- Name: always_ident_id_seq; Type: SEQUENCE; Schema: app; Owner: postgres +-- + +ALTER TABLE app.always_ident ALTER COLUMN id ADD GENERATED ALWAYS AS IDENTITY ( + SEQUENCE NAME app.always_ident_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1 +); + + +-- +-- Name: invoice_no; Type: SEQUENCE; Schema: app; Owner: postgres +-- + +CREATE SEQUENCE app.invoice_no + START WITH 1000 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + + +-- +-- Name: child; Type: TABLE; Schema: app; Owner: postgres +-- + +CREATE TABLE app.child ( + id bigint NOT NULL, + parent_id integer NOT NULL, + manager_id bigint, + invoice integer DEFAULT nextval('app.invoice_no'::regclass), + note text, + payload bytea +); + + + +-- +-- Name: child_id_seq; Type: SEQUENCE; Schema: app; Owner: postgres +-- + +CREATE SEQUENCE app.child_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + + +-- +-- Name: child_id_seq; Type: SEQUENCE OWNED BY; Schema: app; Owner: postgres +-- + +ALTER SEQUENCE app.child_id_seq OWNED BY app.child.id; + + +-- +-- Name: parent; Type: TABLE; Schema: app; Owner: postgres +-- + +CREATE TABLE app.parent ( + id integer NOT NULL, + name text NOT NULL, + mood app.mood +); + + + +-- +-- Name: parent_id_seq; Type: SEQUENCE; Schema: app; Owner: postgres +-- + +ALTER TABLE app.parent ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY ( + SEQUENCE NAME app.parent_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1 +); + + +-- +-- Name: v1; Type: VIEW; Schema: app; Owner: postgres +-- + +CREATE VIEW app.v1 AS + SELECT p.id, + p.name, + count(c.id) AS n + FROM (app.parent p + LEFT JOIN app.child c ON ((c.parent_id = p.id))) + GROUP BY p.id, p.name; + + + +-- +-- Name: v2; Type: VIEW; Schema: app; Owner: postgres +-- + +CREATE VIEW app.v2 AS + SELECT id, + name, + n + FROM app.v1 + WHERE (n > 0); + + + +-- +-- Name: child id; Type: DEFAULT; Schema: app; Owner: postgres +-- + +ALTER TABLE ONLY app.child ALTER COLUMN id SET DEFAULT nextval('app.child_id_seq'::regclass); + + +-- +-- Data for Name: always_ident; Type: TABLE DATA; Schema: app; Owner: postgres +-- + +COPY app.always_ident (id, v) FROM stdin; +1 x +2 y +\. + + +-- +-- Data for Name: child; Type: TABLE DATA; Schema: app; Owner: postgres +-- + +COPY app.child (id, parent_id, manager_id, invoice, note, payload) FROM stdin; +2 1 \N 1001 line1\nline2; \N +4 3 \N 1003 ; \N +1 1 3 1000 it's; a "quote" \\ back; \\x00ff +3 2 1 1002 ünïcødé $$ dollar; \N +\. + + +-- +-- Data for Name: parent; Type: TABLE DATA; Schema: app; Owner: postgres +-- + +COPY app.parent (id, name, mood) FROM stdin; +1 a ok +2 b sad +3 c happy +\. + + +-- +-- Name: always_ident_id_seq; Type: SEQUENCE SET; Schema: app; Owner: postgres +-- + +SELECT pg_catalog.setval('app.always_ident_id_seq', 3, true); + + +-- +-- Name: child_id_seq; Type: SEQUENCE SET; Schema: app; Owner: postgres +-- + +SELECT pg_catalog.setval('app.child_id_seq', 5, true); + + +-- +-- Name: invoice_no; Type: SEQUENCE SET; Schema: app; Owner: postgres +-- + +SELECT pg_catalog.setval('app.invoice_no', 1004, true); + + +-- +-- Name: parent_id_seq; Type: SEQUENCE SET; Schema: app; Owner: postgres +-- + +SELECT pg_catalog.setval('app.parent_id_seq', 5, true); + + +-- +-- Name: always_ident always_ident_pkey; Type: CONSTRAINT; Schema: app; Owner: postgres +-- + +ALTER TABLE ONLY app.always_ident + ADD CONSTRAINT always_ident_pkey PRIMARY KEY (id); + + +-- +-- Name: child child_pkey; Type: CONSTRAINT; Schema: app; Owner: postgres +-- + +ALTER TABLE ONLY app.child + ADD CONSTRAINT child_pkey PRIMARY KEY (id); + + +-- +-- Name: parent parent_name_key; Type: CONSTRAINT; Schema: app; Owner: postgres +-- + +ALTER TABLE ONLY app.parent + ADD CONSTRAINT parent_name_key UNIQUE (name); + + +-- +-- Name: parent parent_pkey; Type: CONSTRAINT; Schema: app; Owner: postgres +-- + +ALTER TABLE ONLY app.parent + ADD CONSTRAINT parent_pkey PRIMARY KEY (id); + + +-- +-- Name: child_parent_idx; Type: INDEX; Schema: app; Owner: postgres +-- + +CREATE INDEX child_parent_idx ON app.child USING btree (parent_id); + + +-- +-- Name: child child_touch; Type: TRIGGER; Schema: app; Owner: postgres +-- + +CREATE TRIGGER child_touch BEFORE INSERT ON app.child FOR EACH ROW EXECUTE FUNCTION app.touch(); + + +-- +-- Name: child child_manager_id_fkey; Type: FK CONSTRAINT; Schema: app; Owner: postgres +-- + +ALTER TABLE ONLY app.child + ADD CONSTRAINT child_manager_id_fkey FOREIGN KEY (manager_id) REFERENCES app.child(id); + + +-- +-- Name: child child_parent_id_fkey; Type: FK CONSTRAINT; Schema: app; Owner: postgres +-- + +ALTER TABLE ONLY app.child + ADD CONSTRAINT child_parent_id_fkey FOREIGN KEY (parent_id) REFERENCES app.parent(id) ON DELETE CASCADE; + + +-- +-- PostgreSQL database dump complete +-- + +\unrestrict mWamWyqv07yVo3aCxQxVd5KrzbHohlp5dNMiGv13zLPzagfuMqNtgHkxSMT0nST + diff --git a/core/crates/drivers/postgres/tests/integration/main.rs b/core/crates/drivers/postgres/tests/integration/main.rs index 75e1453b3c..0880a52a92 100644 --- a/core/crates/drivers/postgres/tests/integration/main.rs +++ b/core/crates/drivers/postgres/tests/integration/main.rs @@ -11,6 +11,7 @@ mod conformance_engine; mod csv_import; mod ddl; mod decode; +mod dump; mod editor_session; mod filter; mod flight; diff --git a/core/crates/drivers/sqlite/src/connection.rs b/core/crates/drivers/sqlite/src/connection.rs index a0b6814523..743231865f 100644 --- a/core/crates/drivers/sqlite/src/connection.rs +++ b/core/crates/drivers/sqlite/src/connection.rs @@ -8,6 +8,7 @@ use sqlx::sqlite::{SqliteConnectOptions, SqliteConnection as SqlxConnection, Sql use sqlx::{Connection as _, Executor as _}; use tablepro_core::ddl::{InboundForeignKey, OutboundForeignKey, TableDefinition}; use tablepro_core::dialect::SqlDialect; +use tablepro_core::dump::{DumpCatalog, DumpScope}; use tablepro_core::meta::{ IndexInfo, ObjectDefinition, ObjectInfo, ObjectScope, RowIdentity, SchemaInfo, TableEstimate, TableRef, }; @@ -342,6 +343,14 @@ impl Connection for SqliteConnection { .await } + async fn read_dump_catalog(&self, scope: &DumpScope, options: CallOptions) -> Result { + let scope = scope.clone(); + self.read(options, move |connection| { + Box::pin(async move { crate::dump::read_dump_catalog(connection, &scope).await }) + }) + .await + } + async fn fetch_columns(&self, table: &TableRef, options: CallOptions) -> Result, DriverError> { let table = table.clone(); self.read(options, move |connection| { diff --git a/core/crates/drivers/sqlite/src/dialect.rs b/core/crates/drivers/sqlite/src/dialect.rs index d5e57a29bc..9b1e923004 100644 --- a/core/crates/drivers/sqlite/src/dialect.rs +++ b/core/crates/drivers/sqlite/src/dialect.rs @@ -4,6 +4,7 @@ use tablepro_core::dialect::{ BindError, BindTarget, DialectCapabilities, InsertConflict, KeysetDirection, LikeCase, LikeForm, LiteralError, Placeholder, SqlDialect, }; +use tablepro_core::dump::DumpDialect; use tablepro_core::edit::{CellExpression, MomentKind}; use tablepro_core::export::value_to_text; use tablepro_core::meta::{EngineRowId, EngineRowIdPart}; @@ -188,6 +189,10 @@ impl SqlDialect for SqliteDialect { fn ddl(&self) -> Option<&dyn DdlDialect> { Some(self) } + + fn dump(&self) -> Option<&dyn DumpDialect> { + Some(self) + } } /// SQLite ends SQL text at the first NUL, wherever it stands, so a NUL diff --git a/core/crates/drivers/sqlite/src/dump.rs b/core/crates/drivers/sqlite/src/dump.rs new file mode 100644 index 0000000000..fe1541c763 --- /dev/null +++ b/core/crates/drivers/sqlite/src/dump.rs @@ -0,0 +1,284 @@ +//! A SQLite file as a SQL dump in the order `sqlite3 .dump` writes it: +//! the stored CREATE text of `main`, its AUTOINCREMENT counters, and the +//! rows in between. Virtual tables are left out and named in the report, +//! and their shadow tables with them: their rows belong to a module the +//! loading side may not have. + +use sqlx::Row; +use sqlx::sqlite::{SqliteConnection as SqlxConnection, SqliteRow}; +use tablepro_core::dump::{ + DumpCatalog, DumpDialect, DumpObject, DumpObjectKind, DumpScope, DumpSection, DumpSkip, DumpSkipReason, + DumpStatement, DumpTable, +}; +use tablepro_core::meta::TableRef; +use tablepro_core::{DriverError, UnsupportedFeature}; + +use crate::catalog::{self, MAIN_SCHEMA}; +use crate::dialect::SqliteDialect; +use crate::errors::map_error; + +impl DumpDialect for SqliteDialect { + /// Foreign keys go off before the transaction, the only place the + /// pragma takes effect, so the file loads in any table order. + fn header(&self) -> String { + "PRAGMA foreign_keys=OFF;\nBEGIN TRANSACTION;\n".to_owned() + } + + fn footer(&self) -> String { + "\nCOMMIT;\n".to_owned() + } + + fn transactional_ddl(&self) -> bool { + true + } +} + +/// `sqlite_schema` in the order the objects were made, each table with +/// what `table_list` calls it, so a virtual table and its shadow tables +/// can be told from the rest. An automatic index has no text. +const OBJECTS_SQL: &str = "SELECT m.type AS type, m.name AS name, m.sql AS sql, coalesce(l.type, '') AS kind \ + FROM \"main\".sqlite_schema AS m \ + LEFT JOIN pragma_table_list AS l ON m.type = 'table' AND l.schema = ?1 AND l.name = m.name \ + WHERE m.sql IS NOT NULL AND m.name NOT LIKE 'sqlite!_%' ESCAPE '!' \ + ORDER BY m.rowid"; + +const SEQUENCE_SQL: &str = + "SELECT name, CAST(seq AS INTEGER) AS seq FROM \"main\".sqlite_sequence WHERE name IS NOT NULL ORDER BY rowid"; + +struct SchemaEntry { + kind: String, + name: String, + sql: String, + table_kind: String, +} + +/// The objects of `main`. An attached database's names would follow it +/// into the file, so only `main` is dumped. +pub(crate) async fn read_dump_catalog( + connection: &mut SqlxConnection, + scope: &DumpScope, +) -> Result { + if let DumpScope::Schema(schema) = scope + && schema != MAIN_SCHEMA + { + return Err(DriverError::Unsupported { + feature: UnsupportedFeature::SqlDump, + }); + } + let entries = sqlx::query(OBJECTS_SQL) + .bind(MAIN_SCHEMA) + .fetch_all(&mut *connection) + .await + .map_err(map_error)? + .iter() + .map(schema_entry) + .collect::, _>>()?; + let counters = match has_sequence_table(connection).await? { + true => sqlx::query(SEQUENCE_SQL) + .fetch_all(&mut *connection) + .await + .map_err(map_error)? + .iter() + .map(|row| Ok((text(row, "name")?, row.try_get::("seq").map_err(map_error)?))) + .collect::, DriverError>>()?, + false => Vec::new(), + }; + Ok(dump_catalog(entries, &counters)) +} + +async fn has_sequence_table(connection: &mut SqlxConnection) -> Result { + let found: i64 = sqlx::query_scalar(sqlx::AssertSqlSafe(catalog::has_table_sql("\"main\""))) + .bind("sqlite_sequence") + .fetch_one(&mut *connection) + .await + .map_err(map_error)?; + Ok(found > 0) +} + +fn schema_entry(row: &SqliteRow) -> Result { + Ok(SchemaEntry { + kind: text(row, "type")?, + name: text(row, "name")?, + sql: text(row, "sql")?, + table_kind: text(row, "kind")?, + }) +} + +fn text(row: &SqliteRow, column: &str) -> Result { + row.try_get(column).map_err(map_error) +} + +/// The counters go back after the rows, which move them, and before the +/// indexes, as `.dump` has them. +fn dump_catalog(entries: Vec, counters: &[(String, i64)]) -> DumpCatalog { + let mut catalog = DumpCatalog::default(); + let mut indexes = Vec::new(); + for entry in entries { + let object = + |kind, section, statement| DumpObject::new(kind, section, None, entry.name.clone(), vec![statement]); + match (entry.kind.as_str(), entry.table_kind.as_str()) { + ("table", "virtual") => catalog.skipped.push(DumpSkip { + kind: DumpObjectKind::Table, + schema: None, + name: entry.name.clone(), + reason: DumpSkipReason::NotSupported, + }), + ("table", "shadow") => {} + ("table", _) => { + let table = TableRef::unqualified(entry.name.clone()); + catalog + .objects + .push(DumpObject::table(table.clone(), vec![DumpStatement::new(entry.sql)])); + catalog.tables.push(DumpTable::new(table)); + } + ("index", _) => indexes.push(object( + DumpObjectKind::Index, + DumpSection::AfterData, + DumpStatement::new(entry.sql), + )), + ("view", _) => catalog.objects.push(object( + DumpObjectKind::View, + DumpSection::Programs, + DumpStatement::new(entry.sql), + )), + ("trigger", _) => catalog.objects.push(object( + DumpObjectKind::Trigger, + DumpSection::Triggers, + DumpStatement::body(entry.sql), + )), + _ => {} + } + } + if !counters.is_empty() { + catalog.objects.push(counters_object(counters)); + } + catalog.objects.extend(indexes); + catalog +} + +/// Loading the rows puts a counter at each table's highest key, which a +/// deleted last row had passed. +fn counters_object(counters: &[(String, i64)]) -> DumpObject { + let mut statements = vec![DumpStatement::new("DELETE FROM sqlite_sequence")]; + statements.extend(counters.iter().map(|(name, seq)| { + DumpStatement::new(format!( + "INSERT INTO sqlite_sequence VALUES('{}',{seq})", + name.replace('\'', "''") + )) + })); + DumpObject::new( + DumpObjectKind::SequenceValue, + DumpSection::AfterData, + None, + "sqlite_sequence", + statements, + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn entry(kind: &str, name: &str, sql: &str, table_kind: &str) -> SchemaEntry { + SchemaEntry { + kind: kind.to_owned(), + name: name.to_owned(), + sql: sql.to_owned(), + table_kind: table_kind.to_owned(), + } + } + + fn placed(catalog: &DumpCatalog) -> Vec<(DumpSection, &str)> { + catalog + .objects + .iter() + .map(|object| (object.section, object.name.as_str())) + .collect() + } + + #[test] + fn objects_keep_their_order_and_counters_come_before_the_indexes() { + let catalog = dump_catalog( + vec![ + entry( + "table", + "parent", + "CREATE TABLE parent (id INTEGER PRIMARY KEY AUTOINCREMENT)", + "table", + ), + entry("index", "parent_idx", "CREATE INDEX parent_idx ON parent (id)", ""), + entry("view", "v", "CREATE VIEW v AS SELECT 1", ""), + entry("table", "child", "CREATE TABLE child (id)", "table"), + entry( + "trigger", + "t", + "CREATE TRIGGER t AFTER INSERT ON child BEGIN SELECT 1; END", + "", + ), + ], + &[("parent".to_owned(), 7)], + ); + + assert_eq!( + placed(&catalog), + [ + (DumpSection::BeforeData, "parent"), + (DumpSection::Programs, "v"), + (DumpSection::BeforeData, "child"), + (DumpSection::Triggers, "t"), + (DumpSection::AfterData, "sqlite_sequence"), + (DumpSection::AfterData, "parent_idx"), + ] + ); + let tables: Vec<&str> = catalog.tables.iter().map(|table| table.table.name.as_str()).collect(); + assert_eq!(tables, ["parent", "child"]); + assert!(catalog.objects[3].statements[0].body); + } + + #[test] + fn a_virtual_table_and_its_shadow_tables_are_left_out() { + let catalog = dump_catalog( + vec![ + entry("table", "docs", "CREATE VIRTUAL TABLE docs USING fts5(body)", "virtual"), + entry( + "table", + "docs_data", + "CREATE TABLE 'docs_data'(id INTEGER PRIMARY KEY, block BLOB)", + "shadow", + ), + entry("table", "kept", "CREATE TABLE kept (a)", "table"), + ], + &[], + ); + + assert_eq!(placed(&catalog), [(DumpSection::BeforeData, "kept")]); + assert_eq!(catalog.tables.len(), 1); + } + + #[test] + fn counters_are_cleared_then_written_back() { + let object = counters_object(&[("it's".to_owned(), 5), ("child".to_owned(), 12)]); + + let sql: Vec<&str> = object + .statements + .iter() + .map(|statement| statement.sql.as_str()) + .collect(); + assert_eq!( + sql, + [ + "DELETE FROM sqlite_sequence", + "INSERT INTO sqlite_sequence VALUES('it''s',5)", + "INSERT INTO sqlite_sequence VALUES('child',12)", + ] + ); + } + + #[test] + fn the_file_turns_foreign_keys_off_outside_its_transaction() { + let dialect = SqliteDialect::new(crate::catalog::SqliteVersion::new(3, 46, 1)); + + assert_eq!(dialect.header(), "PRAGMA foreign_keys=OFF;\nBEGIN TRANSACTION;\n"); + assert!(dialect.transactional_ddl()); + } +} diff --git a/core/crates/drivers/sqlite/src/lib.rs b/core/crates/drivers/sqlite/src/lib.rs index 8450200cd6..6c25f2501e 100644 --- a/core/crates/drivers/sqlite/src/lib.rs +++ b/core/crates/drivers/sqlite/src/lib.rs @@ -12,6 +12,7 @@ mod deadline; mod decode; mod dialect; mod driver; +mod dump; mod editor_session; mod encode; mod errors; diff --git a/core/crates/drivers/sqlite/tests/integration/dump.rs b/core/crates/drivers/sqlite/tests/integration/dump.rs new file mode 100644 index 0000000000..d00f9ef772 --- /dev/null +++ b/core/crates/drivers/sqlite/tests/integration/dump.rs @@ -0,0 +1,105 @@ +//! The SQL dump round trip on SQLite. + +use async_trait::async_trait; +use tablepro_core::{ConnectOptions, ConnectTarget, FileEndpoint, WorkspaceSettings}; +use tablepro_test_support::conformance::{ + ConformanceEngine, ConformanceFailure, DumpEngine, NativeDump, test_liveness, +}; + +use crate::sqlite_engine::{SqliteEngine, create_database}; + +#[async_trait] +impl DumpEngine for SqliteEngine { + /// A file of that name beside the engine's own. + async fn empty_database(&self, name: &str) -> Result { + let mut options = self.connect_options(WorkspaceSettings::workspace(false), test_liveness()); + let ConnectTarget::File(endpoint) = &options.target else { + return Err(ConformanceFailure::setup("the fixture opens a file")); + }; + let path = endpoint.path().with_file_name(format!("{name}.sqlite3")); + for suffix in ["", "-journal", "-wal", "-shm"] { + let mut leftover = path.clone().into_os_string(); + leftover.push(suffix); + match std::fs::remove_file(&leftover) { + Err(error) if error.kind() != std::io::ErrorKind::NotFound => { + return Err(ConformanceFailure::setup(error)); + } + _ => {} + } + } + create_database(&path).await.map_err(ConformanceFailure::setup)?; + options.target = ConnectTarget::File(FileEndpoint::open_existing(path).map_err(ConformanceFailure::setup)?); + Ok(options) + } + + /// SQLite has no sequences or routines. Its counters are + /// `sqlite_sequence`, and the trigger counts its own firings in the + /// row it fires on. + fn dump_fixture(&self) -> Vec { + [ + "CREATE TABLE parent (\ + id INTEGER PRIMARY KEY AUTOINCREMENT, \ + name TEXT NOT NULL UNIQUE, \ + score REAL CHECK (score >= 0)\ + )", + "CREATE TABLE child (\ + id INTEGER PRIMARY KEY AUTOINCREMENT, \ + parent_id INTEGER NOT NULL REFERENCES parent (id), \ + manager_id INTEGER REFERENCES child (id), \ + note TEXT, \ + payload BLOB, \ + note_length INTEGER GENERATED ALWAYS AS (length(note)) VIRTUAL, \ + touched INTEGER NOT NULL DEFAULT 0\ + )", + "CREATE TABLE plain (k TEXT, v REAL)", + "CREATE INDEX child_parent_idx ON child (parent_id)", + "INSERT INTO parent (name, score) VALUES ('first', 1.5), ('second', 2), ('gone', 0)", + "INSERT INTO child (parent_id, manager_id, note, payload) VALUES \ + (1, NULL, 'it''s; a \"quote\" \\ back', X'00FF27'), \ + (1, 1, 'line one' || char(10) || 'line two;', NULL), \ + (2, 2, 'Grüße, 東京 ✓', X'FF'), \ + (3, NULL, 'gone', NULL)", + "DELETE FROM child WHERE id = 4", + "DELETE FROM parent WHERE id = 3", + "INSERT INTO plain (k, v) VALUES ('x', 1.5), ('y', NULL)", + "CREATE VIEW parent_names AS SELECT id, name FROM parent", + "CREATE VIEW parent_name_lengths AS SELECT id, length(name) AS n FROM parent_names", + "CREATE TRIGGER child_touch AFTER INSERT ON child BEGIN \ + UPDATE child SET touched = touched + 1 WHERE id = NEW.id; \ + END", + "INSERT INTO child (parent_id, manager_id, note) VALUES (2, 3, 'after the trigger')", + ] + .into_iter() + .map(str::to_owned) + .collect() + } + + /// `sqlite3 .dump` from 3.46.1, the shell Ubuntu 26.04 ships: its + /// own `PRAGMA foreign_keys=OFF; BEGIN TRANSACTION; ... COMMIT;`, a + /// child row that points at a later one, and `sqlite_sequence`. + fn native_dump(&self) -> Option { + let checks = [ + ("SELECT count(*) FROM child", "4"), + ("SELECT seq FROM sqlite_sequence WHERE name = 'child'", "5"), + ("SELECT note FROM child WHERE id = 2", "line1\nline2;"), + ("SELECT hex(payload) FROM child WHERE id = 1", "00FF27"), + ("SELECT count(*) FROM v2", "3"), + ]; + Some(NativeDump { + file: include_str!("fixtures/sqlite3-3.46-dump.sql"), + checks: checks + .into_iter() + .map(|(sql, value)| (sql.to_owned(), value.to_owned())) + .collect(), + }) + } +} + +mod suite { + use super::SqliteEngine; + + tablepro_test_support::dump_suite! { + engine: SqliteEngine, + docker: false, + } +} diff --git a/core/crates/drivers/sqlite/tests/integration/fixtures/sqlite3-3.46-dump.sql b/core/crates/drivers/sqlite/tests/integration/fixtures/sqlite3-3.46-dump.sql new file mode 100644 index 0000000000..3185ccf479 --- /dev/null +++ b/core/crates/drivers/sqlite/tests/integration/fixtures/sqlite3-3.46-dump.sql @@ -0,0 +1,33 @@ +PRAGMA foreign_keys=OFF; +BEGIN TRANSACTION; +CREATE TABLE parent ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL UNIQUE, + mood TEXT CHECK (mood IN ('sad', 'ok', 'happy')) +); +INSERT INTO parent VALUES(1,'a','ok'); +INSERT INTO parent VALUES(2,'b','sad'); +INSERT INTO parent VALUES(3,'c','happy'); +CREATE TABLE child ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + parent_id INTEGER NOT NULL REFERENCES parent(id) ON DELETE CASCADE, + manager_id INTEGER REFERENCES child(id), + note TEXT, + payload BLOB +); +INSERT INTO child VALUES(1,1,3,'it''s; a "quote" \ back;',X'00ff27'); +INSERT INTO child VALUES(2,1,NULL,replace('line1\nline2;','\n',char(10)),NULL); +INSERT INTO child VALUES(3,2,1,'ünïcødé ;; END;;',NULL); +INSERT INTO child VALUES(4,3,NULL,';',NULL); +CREATE TABLE plain_rowid (k TEXT, v REAL); +INSERT INTO plain_rowid VALUES('x',1.5); +INSERT INTO plain_rowid VALUES('y',NULL); +DELETE FROM sqlite_sequence; +INSERT INTO sqlite_sequence VALUES('parent',5); +INSERT INTO sqlite_sequence VALUES('child',5); +CREATE VIEW v1 AS SELECT p.id, p.name, count(c.id) AS n FROM parent p LEFT JOIN child c ON c.parent_id = p.id GROUP BY p.id, p.name; +CREATE VIEW v2 AS SELECT * FROM v1 WHERE n > 0; +CREATE TRIGGER child_touch BEFORE INSERT ON child BEGIN SELECT 1; END; +CREATE TRIGGER child_touch_after AFTER INSERT ON child BEGIN UPDATE child SET note = coalesce(note, '') || ';' WHERE id = NEW.id; END; +CREATE INDEX child_parent_idx ON child (parent_id); +COMMIT; diff --git a/core/crates/drivers/sqlite/tests/integration/main.rs b/core/crates/drivers/sqlite/tests/integration/main.rs index 3c26ff96ff..b4eac51372 100644 --- a/core/crates/drivers/sqlite/tests/integration/main.rs +++ b/core/crates/drivers/sqlite/tests/integration/main.rs @@ -15,6 +15,8 @@ mod decode; #[cfg(test)] mod dialect; #[cfg(test)] +mod dump; +#[cfg(test)] mod file_locked; #[cfg(test)] mod lock_wait; diff --git a/core/crates/test-support/src/conformance/cases/dump.rs b/core/crates/test-support/src/conformance/cases/dump.rs new file mode 100644 index 0000000000..bd4402c967 --- /dev/null +++ b/core/crates/test-support/src/conformance/cases/dump.rs @@ -0,0 +1,191 @@ +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use async_trait::async_trait; +use tablepro_core::dump::{DumpOutput, DumpOutputClosed, DumpRequest, DumpSkipReason, write_dump}; +use tablepro_core::import::{SqlImportEnd, SqlImportErrorPolicy, SqlImportOptions, run_sql_import}; +use tablepro_core::{Connection, EditorSessionOptions}; +use tokio_util::sync::CancellationToken; + +use crate::conformance::support::{dial, expect, run_completed, run_options, unexpected, within}; +use crate::conformance::{ConformanceFailure, DumpEngine, test_liveness}; +use crate::lock; + +const CASE: &str = "dump_loads_back_into_an_empty_database"; + +/// How long a dump of the sample, or its load, may take. +const STEP_LIMIT: Duration = Duration::from_secs(120); + +/// The sample goes out as a SQL file, loads through the SQL file import +/// into an empty database as one transaction where the engine has one, +/// and a dump of that database is the same file: the same objects in the +/// same order, the same rows and the same counters. +pub async fn dump_loads_back_into_an_empty_database(engine: E) -> Result<(), ConformanceFailure> { + let source = open(&engine, "tp_dump_source").await?; + let editor = source + .open_editor_session(EditorSessionOptions::default(), CancellationToken::new()) + .await?; + for statement in engine.dump_fixture() { + run_completed(CASE, editor.as_ref(), &statement, run_options()).await?; + } + editor.close().await; + let request = DumpRequest { + scope: engine.dump_scope(), + data: true, + }; + let first = dump(source.as_ref(), &request, &engine.expected_skips()).await?; + + let target = open(&engine, "tp_dump_target").await?; + let loader = target + .open_editor_session(EditorSessionOptions::default(), CancellationToken::new()) + .await?; + let dialect = target.dialect(); + let report = tokio::time::timeout( + STEP_LIMIT, + run_sql_import( + loader.as_ref(), + &*dialect, + Box::new(std::io::Cursor::new(first.clone().into_bytes())), + SqlImportOptions { + on_error: SqlImportErrorPolicy::Stop, + one_transaction: true, + }, + &mut |_| {}, + &CancellationToken::new(), + ), + ) + .await + .map_err(|_| unexpected(CASE, format!("the load did not finish within {STEP_LIMIT:?}")))?; + loader.close().await; + expect( + CASE, + report.end == SqlImportEnd::Completed && report.failed == 0, + || format!("the dump did not load back: {:?}\n{first}", report), + )?; + + let second = dump(target.as_ref(), &request, &[]).await?; + expect(CASE, first == second, || first_difference(&first, &second))?; + Ok(()) +} + +/// The file the engine's own tool wrote loads through the import, run +/// both as one transaction and as written, and holds what it should. +pub async fn native_dump_loads_through_the_import(engine: E) -> Result<(), ConformanceFailure> { + const NATIVE: &str = "native_dump_loads_through_the_import"; + let Some(native) = engine.native_dump() else { + return Ok(()); + }; + for (round, one_transaction) in [true, false].into_iter().enumerate() { + let target = open(&engine, &format!("tp_native_dump_{round}")).await?; + let loader = target + .open_editor_session(EditorSessionOptions::default(), CancellationToken::new()) + .await?; + let dialect = target.dialect(); + let report = tokio::time::timeout( + STEP_LIMIT, + run_sql_import( + loader.as_ref(), + &*dialect, + Box::new(std::io::Cursor::new(native.file.as_bytes().to_vec())), + SqlImportOptions { + on_error: SqlImportErrorPolicy::Stop, + one_transaction, + }, + &mut |_| {}, + &CancellationToken::new(), + ), + ) + .await + .map_err(|_| unexpected(NATIVE, "the load did not finish".to_owned()))?; + expect( + NATIVE, + report.end == SqlImportEnd::Completed && report.failed == 0, + || format!("the native dump did not load (one transaction: {one_transaction}): {report:?}"), + )?; + for (sql, expected) in &native.checks { + let collected = run_completed(NATIVE, loader.as_ref(), sql, run_options()).await?; + let found = collected + .rows() + .first() + .and_then(|row| row.first()) + .and_then(tablepro_core::export::value_to_text); + expect(NATIVE, found.as_deref() == Some(expected.as_str()), || { + format!("`{sql}` gave {found:?}, not {expected}") + })?; + } + loader.close().await; + } + Ok(()) +} + +async fn open(engine: &E, name: &str) -> Result, ConformanceFailure> { + let mut options = engine.empty_database(name).await?; + options.liveness = test_liveness(); + Ok(dial(CASE, E::driver(), options).await??) +} + +async fn dump( + connection: &dyn Connection, + request: &DumpRequest, + expected_skips: &[(String, DumpSkipReason)], +) -> Result { + let mut output = Collected::default(); + let report = tokio::time::timeout( + STEP_LIMIT, + write_dump(connection, request, &mut output, within(STEP_LIMIT)), + ) + .await + .map_err(|_| unexpected(CASE, format!("the dump did not finish within {STEP_LIMIT:?}")))? + .map_err(|error| unexpected(CASE, format!("the dump failed: {error}")))?; + let text = String::from_utf8(lock(&output.bytes).clone()) + .map_err(|_| unexpected(CASE, "the dump is not UTF-8".to_owned()))?; + let skipped: Vec<(String, DumpSkipReason)> = report + .skipped + .iter() + .map(|skip| (skip.name.clone(), skip.reason)) + .collect(); + expect(CASE, report.omitted.is_empty() && skipped == expected_skips, || { + format!( + "the dump left values or objects out: {:?} {:?}\n{text}", + report.omitted, report.skipped + ) + })?; + expect(CASE, report.rows > 0 && report.tables > 0, || { + format!("the dump wrote no rows: {report:?}\n{text}") + })?; + Ok(text) +} + +fn first_difference(first: &str, second: &str) -> String { + let (left, right): (Vec<&str>, Vec<&str>) = (first.lines().collect(), second.lines().collect()); + let at = left + .iter() + .zip(&right) + .position(|(a, b)| a != b) + .unwrap_or(left.len().min(right.len())); + let around = |lines: &[&str]| { + lines + .get(at.saturating_sub(3)..(at + 4).min(lines.len())) + .unwrap_or_default() + .join("\n") + }; + format!( + "the reloaded database dumps differently from line {}:\n--- source\n{}\n--- reloaded\n{}\n=== whole source dump\n{first}", + at + 1, + around(&left), + around(&right) + ) +} + +#[derive(Default)] +struct Collected { + bytes: Arc>>, +} + +#[async_trait] +impl DumpOutput for Collected { + async fn write(&mut self, bytes: Vec) -> Result<(), DumpOutputClosed> { + lock(&self.bytes).extend_from_slice(&bytes); + Ok(()) + } +} diff --git a/core/crates/test-support/src/conformance/cases/mod.rs b/core/crates/test-support/src/conformance/cases/mod.rs index 17a8e3d51d..23ba24b6b4 100644 --- a/core/crates/test-support/src/conformance/cases/mod.rs +++ b/core/crates/test-support/src/conformance/cases/mod.rs @@ -7,6 +7,7 @@ pub mod commit_resolution; pub mod ddl; pub mod deadlines; pub mod document; +pub mod dump; pub mod keyspace; pub mod large_table; pub mod liveness; diff --git a/core/crates/test-support/src/conformance/dump_engine.rs b/core/crates/test-support/src/conformance/dump_engine.rs new file mode 100644 index 0000000000..48a21453fc --- /dev/null +++ b/core/crates/test-support/src/conformance/dump_engine.rs @@ -0,0 +1,49 @@ +use async_trait::async_trait; +use tablepro_core::ConnectOptions; +use tablepro_core::dump::{DumpScope, DumpSkipReason}; + +use crate::conformance::{ConformanceEngine, ConformanceFailure}; + +/// An engine the dump round trip writes a database of and loads it back +/// into an empty one. +#[async_trait] +pub trait DumpEngine: ConformanceEngine { + /// Writable options that connect to a database of this name on the + /// engine's server, dropped and made again empty, or to a new file + /// where the engine opens one. + async fn empty_database(&self, name: &str) -> Result; + + /// The statements that fill an empty database with the sample, each + /// sent on its own to an editor session, in order. They make what a + /// restore has to get right on this engine: tables joined by a foreign + /// key, a table whose key points at itself, a counter past the highest + /// key, a sequence where the engine has them, a view over a view, a + /// routine, a trigger that changes the row it fires on, and a value of + /// every kind the value corpus covers that has a literal. + fn dump_fixture(&self) -> Vec; + + /// What the dump covers. + fn dump_scope(&self) -> DumpScope { + DumpScope::Database + } + + /// What the sample's dump reports it writes differently or leaves + /// out, by name: what this engine's file cannot carry. A database the + /// file loaded into has none of it. + fn expected_skips(&self) -> Vec<(String, DumpSkipReason)> { + Vec::new() + } + + /// A file the engine's own dump tool wrote, and reads that each give + /// one value once it loaded, with that value as text. `None` where the + /// engine has no such tool. + fn native_dump(&self) -> Option { + None + } +} + +/// A dump the engine's own tool wrote, as the import must read it. +pub struct NativeDump { + pub file: &'static str, + pub checks: Vec<(String, String)>, +} diff --git a/core/crates/test-support/src/conformance/dump_suite_macro.rs b/core/crates/test-support/src/conformance/dump_suite_macro.rs new file mode 100644 index 0000000000..3ac7ec7fc8 --- /dev/null +++ b/core/crates/test-support/src/conformance/dump_suite_macro.rs @@ -0,0 +1,25 @@ +/// Emit the dump round trip for one engine adapter, which implements +/// `DumpEngine`. `docker: true` marks it `#[ignore = "requires docker"]`. +/// +/// The case fills a database with the adapter's sample, dumps it, loads +/// the file into an empty database through the SQL file import, and +/// dumps that one: the two files must be the same, rows, counters and +/// all. A second case loads the file the engine's own dump tool wrote, +/// where the adapter has one. +#[macro_export] +macro_rules! dump_suite { + (engine: $engine:ty, docker: $docker:tt $(,)?) => { + $crate::__conformance_case! { + $docker, + $engine as ConformanceEngine, + dump_loads_back_into_an_empty_database, + $crate::conformance::cases::dump::dump_loads_back_into_an_empty_database::<$engine> + } + $crate::__conformance_case! { + $docker, + $engine as ConformanceEngine, + native_dump_loads_through_the_import, + $crate::conformance::cases::dump::native_dump_loads_through_the_import::<$engine> + } + }; +} diff --git a/core/crates/test-support/src/conformance/mod.rs b/core/crates/test-support/src/conformance/mod.rs index 872cface8e..85e77bc618 100644 --- a/core/crates/test-support/src/conformance/mod.rs +++ b/core/crates/test-support/src/conformance/mod.rs @@ -26,6 +26,8 @@ mod ddl_conformance_engine; mod ddl_suite_macro; mod document_engine; mod document_suite_macro; +mod dump_engine; +mod dump_suite_macro; mod editor_transaction_engine; #[cfg(test)] mod fake_engine; @@ -56,6 +58,7 @@ pub use connection_suite_flags::ConnectionSuiteFlags; pub use corpus_column::{CorpusColumn, CorpusValue}; pub use ddl_conformance_engine::ConformanceDdlEngine; pub use document_engine::DocumentEngine; +pub use dump_engine::{DumpEngine, NativeDump}; pub use editor_transaction_engine::EditorTransactionEngine; pub use keyspace_engine::KeyspaceEngine; pub use large_table_engine::LargeTableEngine; diff --git a/core/docs/decisions/0011-sql-dump-and-import-in-process.md b/core/docs/decisions/0011-sql-dump-and-import-in-process.md new file mode 100644 index 0000000000..02380bb415 --- /dev/null +++ b/core/docs/decisions/0011-sql-dump-and-import-in-process.md @@ -0,0 +1,43 @@ +# 0011: SQL dumps and SQL file imports run in process + +- **Status**: Accepted +- **Date**: 2026-11-05 + +## Context + +The Linux app's first release adds Export Database, which writes a whole database or one schema, structure and rows, to a `.sql` or `.sql.gz` file, and Import SQL File, which runs such a file against a connection. The Windows app will offer the same. The macOS app has both an in-process SQL export and import, and a separate Backup Dump that runs `pg_dump`, `mysqldump`, `mongodump`, `sqlite3` and `sqlpackage`. + +So either the core writes and reads the file itself, from the drivers it already has, or the apps run each engine's own tool. Measured on 2026-11-05 in containers of the target systems: + +| System | `pg_dump` | `mysqldump` by default | `sqlite3` | DuckDB, ClickHouse, SQL Server tools | +|---|---|---|---|---| +| Ubuntu 26.04 | 18.6, the archive's only major | MySQL 8.4.11 | 3.46.1 | not packaged | +| Ubuntu 26.10 | 18.6 | MySQL 9.7.2 | 3.53.4 | DuckDB 1.5.5 only | +| Fedora 44 | 18.6 | MariaDB 11.8.8 | 3.51.2 | not packaged | +| Fedora 45 | 18.3 | MariaDB 12.3.2 | 3.53.4 | not packaged | + +## Decision + +The core writes the dump and runs the file in process, through the connection's own driver: `tablepro_core::dump` (`write_dump`, each driver's `read_dump_catalog` and `DumpDialect`) and `tablepro_core::import::run_sql_import` with the incremental `ScriptReader`. + +## Rationale + +- **The tools are not there to call.** None is installed with the app. Ubuntu lets one MySQL family in at a time (`mysql-client-core` conflicts with `mariadb-client`), Fedora installs MariaDB's under `mysqldump`, and neither archive packages a tool for DuckDB, ClickHouse or SQL Server. Windows ships none of them. +- **They refuse the servers users have.** `pg_dump` refuses a server newer than itself ("aborting because of server version mismatch", measured with 18.3 against 19beta1), and the distributions carry one major. MySQL 8.4's `mysqldump` fails against MariaDB 11.8 on `COLUMN_STATISTICS`; `mariadb-dump` against MySQL 8.4 writes a file MySQL will not load (ERROR 3105 on a generated column); a default `mysqldump` of a GTID server will not load into another database of that server (ERROR 3546). +- **Three engines have no plain SQL dump tool.** `sqlpackage` writes `.bacpac` and `mssql-scripter` was archived in 2024; ClickHouse's `BACKUP` writes to the server's storage; DuckDB's `EXPORT DATABASE` writes a directory, refuses a self-referencing foreign key and leaves triggers out. +- **The connection would not carry over.** The Linux SSH forward is a Unix socket, over which libpq ignores `sslmode`; neither MySQL client can override the server name a certificate is checked against; on Windows the in-process forward listens only for the app itself ([0009](0009-in-process-ssh-on-windows.md)). The password would go through `PGPASSWORD`, which PostgreSQL does not recommend, or `MYSQL_PWD`, deprecated in MySQL 8.4. +- **The Flatpak cannot run them.** A host binary needs `--talk-name=org.freedesktop.Flatpak`, and Flathub does not accept apps that rely on host components for a core feature. Bundling means building PostgreSQL and a MySQL-family client into the Flatpak, one version for every server. +- **The file loads with the engine's own client anyway.** The dump follows `pg_dump` plain format and `mysqldump`'s header, `DELIMITER` blocks and `GO` batches, so `psql`, `mysql`, `sqlcmd`, `sqlite3` and `clickhouse-client` read it as well as TablePro does. + +## Consequences + +- A dump carries what each driver reads, not everything the engine has. PostgreSQL leaves out ownership, grants, policies, rules, publications and event triggers, as `pg_dump --no-owner --no-acl` would leave the first two; MySQL leaves out events; ClickHouse leaves out user-defined functions, which belong to no database. The Linux README lists them. +- Tables are read one after another, each in a read-only transaction of its own, not at one snapshot, so a database written to during the export can give a file whose tables disagree. +- The import reads files the native tools write: pg_dump's `COPY ... FROM stdin` rows run as INSERTs, its `\restrict` lines are passed over, and another psql command is refused with its name. A MySQL dump that holds raw binary, as `mysqldump` writes without `--hex-blob`, is not UTF-8 and is refused at its line. +- The macOS Backup Dump, which runs the tools, stays a separate feature. A tool-based backup for the `.deb` and `.rpm` builds can come later beside this one; it would not replace it. + +## Alternatives considered + +- **Run the engine's tool where it is installed, the core's writer elsewhere.** Two outputs for one menu item, chosen by what happens to be on `PATH`, and the version and family checks the macOS Backup Dump needed about 3,500 lines of Swift for. +- **Bundle `pg_dump` and `mysqldump` in the Flatpak.** One version for every server, PostgreSQL's and MariaDB's sources built into every release, and nothing for the other engines or for Windows. +- **`COPY` in the PostgreSQL dump.** Faster to load, but only psql reads `COPY ... FROM stdin` from a file; multi-row INSERTs with a column list load through every client and driver. diff --git a/core/docs/decisions/README.md b/core/docs/decisions/README.md index 2c3f0fd4c2..09ebd66a86 100644 --- a/core/docs/decisions/README.md +++ b/core/docs/decisions/README.md @@ -18,3 +18,4 @@ The format, the statuses and how a later ADR changes an earlier one follow [the | [0008](0008-importers-from-other-apps.md) | Importers from other apps live in the core | Accepted | One crate reads each app's files on every platform into one record shape, takes keychains and directories from the app, says what it cannot carry, and reviews the result against the library with the macOS duplicate handling. | | [0009](0009-in-process-ssh-on-windows.md) | SSH on Windows runs in process on russh | Accepted | russh behind `tablepro-ssh`'s `in-process` feature, on every target; Windows uses it, Linux keeps OpenSSH. Forwards are channels with no address, so MySQL and MongoDB relay them through a private listener that serves only this process. | | [0010](0010-macos-on-the-core.md) | The macOS app moves to the core in slices, starting with the connection library | Accepted | UniFFI Swift bindings in an XCFramework of the static library, without the engines, at the app's deployment target; UI, Keychain and the CloudKit transport stay Swift; the core maps the library's CloudKit records both ways with the existing fields and merges a pull field by field. | +| [0011](0011-sql-dump-and-import-in-process.md) | SQL dumps and SQL file imports run in process | Accepted | Export Database writes one `.sql` file from each driver's catalog read in `pg_dump` and `mysqldump` order, and Import SQL File splits a file of any size as the engine's client does and runs it on an editor session; no app calls `pg_dump` or `mysqldump`. | diff --git a/linux/README.md b/linux/README.md index 0f38337814..67f8c00f5f 100644 --- a/linux/README.md +++ b/linux/README.md @@ -339,6 +339,59 @@ The SQL editor can type, delete, paste and move at several places at once. Typing, Return, Tab, BackSpace, Delete and the moves by character, word or line work at every cursor, and one Ctrl+Z takes an edit back at all of them and returns to one cursor. Copy joins the selections with new lines, and pasting as many lines as there are cursors puts one line at each. Ctrl+/ comments every cursor's lines. A click, a find, Format, a query from history and Page Up or Down return to one cursor, as do moving lines (Alt+Up, Alt+Down) and words (Alt+Left, Alt+Right), which act on one line. The completion list, snippets and suggestions while typing work with one cursor, so another cursor ends the snippet being filled in and drops a suggestion, and overwrite (Insert) and Vim mode keep one cursor. +## Export a database and import a SQL file + +**Main menu → Export Database…** writes the database, or one of its +schemas, to a `.sql` file, gzipped as `.sql.gz` if you ask. It holds the +tables, views, routines and triggers, and every table's rows as +multi-row INSERTs, in the order a restore needs: structure, then rows, +then keys, indexes, foreign keys and sequence positions, then views and +routines, and triggers last so loading the rows fires none. The file is +meant for an empty database, and loads with the engine's own client as +well as with TablePro: + +| Engine | Written like | Load it with | Left out | +|---|---|---|---| +| PostgreSQL | `pg_dump` plain format, INSERTs instead of COPY | `psql -v ON_ERROR_STOP=1 -f file.sql` | Owners, grants, policies, rules, publications, event triggers, foreign tables; the rows of extension configuration tables | +| MySQL, MariaDB | `mysqldump`, with its `SET` header and `DELIMITER ;;` blocks | `mysql db < file.sql` | Events; FEDERATED and CONNECT tables, whose rows are another server's; `DEFINER` clauses, so the account that loads the file owns the objects | +| SQL Server | SSMS's Generate Scripts, a `GO` after each batch | `sqlcmd -b -i file.sql` | CLR and encrypted modules, which have no SQL to write; the indexes of indexed views; system versioning, so a temporal table and its history load as ordinary tables with their period times | +| ClickHouse | `SHOW CREATE` for each table and view | `clickhouse-client < file.sql` | User-defined functions, which belong to no database; the rows a materialized view without `TO` keeps itself, so it fills again only from new rows | +| SQLite | `sqlite3 .dump` | `sqlite3 new.db < file.sql` | | +| DuckDB | `EXPORT DATABASE`, in one file | `duckdb new.duckdb < file.sql` | | + +Tables are read one after another, each in a read-only transaction of +its own, so stop writes to the database for a dump whose tables agree. +A value the engine has no SQL for is written as NULL, and the export +says which columns that happened to. Redis, MongoDB, Cassandra and +ScyllaDB have no SQL dump; use `redis-cli --rdb`, `mongodump` or +`nodetool snapshot`. + +**Main menu → Import SQL File…** runs a `.sql` or `.sql.gz` file on the +connection, on a session of its own so what the file sets stays there. +The file is read as it runs, so its size does not matter. It is split +as the engine's client splits it: at `;`, at a MySQL `DELIMITER`, and at +SQL Server `GO` lines, a `GO 3` batch running three times. Files from +`pg_dump` load too: their `COPY ... FROM stdin` rows run as INSERTs, and +the `\restrict` and `\unrestrict` lines that only psql reads are passed +over. Any other psql command, such as `\connect`, is refused with its +line. + +- **Stop at First Error** is on by default, as with `mysql` and with + `psql -v ON_ERROR_STOP=1`. Off, the rest of the file runs and the + summary lists each failure with its line; Copy Details copies them with + their statements. +- **Run as One Transaction** is offered on PostgreSQL, SQL Server, SQLite + and DuckDB, whose CREATE and DROP roll back: a failure or Stop then + undoes everything the file did. The file's own `BEGIN` and `COMMIT` + are passed over inside it, and a `ROLLBACK`, or a SQL Server batch that + holds a `COMMIT` or `ROLLBACK`, stops the import before it runs. MySQL, + MariaDB and ClickHouse commit each statement as it runs, so what ran + before a failure stays. +- On Safe Mode Read-Only, Import SQL File is off. On Confirm + Writes, the import dialog is the one question for the whole file. +- A file must be UTF-8. A MySQL dump of binary columns made without + `--hex-blob` holds raw bytes and is refused at its first such line. + ## SQL Server with Windows integrated auth Pick **Method → Windows (Kerberos)** in the connect dialog. There is no diff --git a/linux/ROADMAP.md b/linux/ROADMAP.md index f92a184d6b..7f32c88699 100644 --- a/linux/ROADMAP.md +++ b/linux/ROADMAP.md @@ -17,7 +17,7 @@ The app runs on GNOME 50 ([ADR 0015](docs/decisions/0015-gnome-50-platform-floor | Import and export | A JSON file with no secrets in it, tags and icons included. On import, an entry with the same id and the same settings updates its name, colour, group, tags and icon; any other is added as a new connection. Tags are matched by id, then by name, so a second import adds none | | Test before saving | Test connects once and reports, then disconnects | | Passwords | Kept in the Secret Service; a prompt asks when one is missing or refused, with Remember Password | -| Safe Mode | A level per connection, set in the connect dialog or the main menu while connected and saved back to the connection. Off asks before DROP, TRUNCATE, and DELETE or UPDATE without WHERE. Confirm Writes also asks before every other write from the editor, a grid save, a structure save, a CSV import or Drop Table, showing the SQL and the connection, with Cancel the default. Read-Only refuses every editor statement that does not read, and opens the connection read-only: SQLite, DuckDB and ClickHouse refuse writes at the engine, PostgreSQL and MySQL run in read-only transactions, SQL Server relies on the login, with the app's own reads in a rolled-back transaction, and Cassandra and ScyllaDB rely on the login. A statement is a read only when its engine's grammar shows it reads, each statement of a script on its own. Moving to or from Read-Only reconnects the window | +| Safe Mode | A level per connection, set in the connect dialog or the main menu while connected and saved back to the connection. Off asks before DROP, TRUNCATE, and DELETE or UPDATE without WHERE. Confirm Writes also asks before every other write from the editor, a grid save, a structure save, a CSV import, a SQL file import or Drop Table, showing the SQL, or the file, and the connection, with Cancel the default. Read-Only refuses every editor statement that does not read, and opens the connection read-only: SQLite, DuckDB and ClickHouse refuse writes at the engine, PostgreSQL and MySQL run in read-only transactions, SQL Server relies on the login, with the app's own reads in a rolled-back transaction, and Cassandra and ScyllaDB rely on the login. A statement is a read only when its engine's grammar shows it reads, each statement of a script on its own. Moving to or from Read-Only reconnects the window | | Query timeout per connection | Custom Query Timeout in the connect dialog sets the editor's timeout for that connection in place of the one in Preferences, from the next connect; 0 turns it off. The app's own calls keep their fixed limits: 15 seconds for metadata, 60 for a page or a count, 120 for a row save or a CSV import, none for a schema change or an export, and on the server 60 for a table-tab statement and 10 for a lock wait | | Database files | SQLite and DuckDB: choose a file, create a new one, open one from Files or a terminal, where the file's header picks the engine, find a moved one with Locate, and choose read-only or read-write for one on a network or FUSE filesystem | | Kerberos for SQL Server | From the ticket cache, in the Flatpak through the KCM socket | @@ -118,6 +118,14 @@ The app runs on GNOME 50 ([ADR 0015](docs/decisions/0015-gnome-50-platform-floor | Formats | CSV with its options, JSON, Markdown, HTML, XML, Excel, and SQL INSERT from a table tab | | Scope | From a table tab, the current page with its pending edits, or every saved row under the filter and sort. From the editor, the rows shown, or every row by running the statement again on a separate read-only connection without the tab's USE or SET; a statement that writes is refused | | Running | Off the GTK thread with progress and Stop; closing or quitting asks about a running export | +| Database | Export Database in the main menu writes a whole database or one schema to a `.sql` or `.sql.gz` file: tables, views, routines and triggers, and every table's rows as multi-row INSERTs, in the order `pg_dump` and `mysqldump` use, so `psql`, `mysql`, `sqlcmd`, `sqlite3` and `clickhouse-client` load it as well as Import SQL File does. Read in process from the connection, not by calling those tools ([core ADR 0011](../core/docs/decisions/0011-sql-dump-and-import-in-process.md)); not offered on Redis, MongoDB, Cassandra and ScyllaDB ([README](README.md#export-a-database-and-import-a-sql-file)) | + +### Import + +| Capability | Detail | +|---|---| +| SQL files | Import SQL File in the main menu runs a `.sql` or `.sql.gz` file of any size on an editor session of its own, split as the engine's client splits it (`DELIMITER`, `GO n`, routine bodies, dollar quotes), with progress, Stop and a summary of what failed and where. pg_dump's `COPY ... FROM stdin` rows run as INSERTs and its `\restrict` lines are passed over | +| Errors and transactions | Stop at First Error, or run the rest and list each failure with its line. On PostgreSQL, SQL Server, SQLite and DuckDB the file can run as one transaction, so a failure or Stop undoes it all; MySQL, MariaDB and ClickHouse commit each statement as it runs. Safe Mode Read-Only offers neither import, and the import dialog is the one question Confirm Writes asks for the whole file | ### Windows and sessions @@ -158,7 +166,7 @@ The app runs on GNOME 50 ([ADR 0015](docs/decisions/0015-gnome-50-platform-floor | Desktop integration | Desktop file with Open With for SQLite files, D-Bus activation, a man page, metainfo with screenshots, and SVG icons | | Translations | Vietnamese ships. The app's own strings go through gettext, and CI fails when `POTFILES.in` is stale or a translation loses a placeholder. CI builds the template with gettext 1.0 and publishes it from main and with each release; translators send pull requests against `po/*.po` ([po/README.md](po/README.md#where-translators-work)) | | Logs | stderr, which the session journals, with JSON lines on request. A panic also writes a crash report under `$XDG_STATE_HOME`, offered with Show Report on the next launch | -| Tests | Unit, gtk (with an accessibility audit of the picker, the connect pages, the workspace, the connect dialog and every preferences page), data and packaging suites; connection, DDL and TLS conformance, and a CSV and SQL INSERT export round trip and a copy and paste round trip, on every SQL engine, keyspace and TLS conformance on Redis, and document and TLS conformance on MongoDB; Docker jobs for PostgreSQL with PgBouncer, MySQL and MariaDB, SQL Server, ClickHouse, Cassandra and ScyllaDB, Redis, MongoDB, SSH, the session and the grid save ([testing.md](docs/testing.md)); every critical audit finding has named guards ([regression-tests.md](docs/regression-tests.md)) | +| Tests | Unit, gtk (with an accessibility audit of the picker, the connect pages, the workspace, the connect dialog and every preferences page), data and packaging suites; connection, DDL and TLS conformance, a CSV and SQL INSERT export round trip, a copy and paste round trip, and a database dump loaded back through Import SQL File and dumped again to the same file, on every SQL engine, keyspace and TLS conformance on Redis, and document and TLS conformance on MongoDB; Docker jobs for PostgreSQL with PgBouncer, MySQL and MariaDB, SQL Server, ClickHouse, Cassandra and ScyllaDB, Redis, MongoDB, SSH, the session and the grid save ([testing.md](docs/testing.md)); every critical audit finding has named guards ([regression-tests.md](docs/regression-tests.md)) | ## Backlog @@ -197,7 +205,7 @@ Microsoft's mssql-tds is checked again each release as a SQL Server client. Toda ### Ideas -Not committed: server monitoring (sessions, locks, slow queries), maintenance commands, backup and restore, replication status, comparing result sets, schema diff and data sync between connections, drawn query plans, scheduled queries, and opt-in usage telemetry. +Not committed: server monitoring (sessions, locks, slow queries), maintenance commands, backup and restore through the engines' own tools (`pg_dump`, `mysqldump`), replication status, comparing result sets, schema diff and data sync between connections, drawn query plans, scheduled queries, and opt-in usage telemetry. ## Release criteria diff --git a/linux/changelog.d/4052.added.md b/linux/changelog.d/4052.added.md new file mode 100644 index 0000000000..557cfda8df --- /dev/null +++ b/linux/changelog.d/4052.added.md @@ -0,0 +1 @@ +Export a whole database or schema to a .sql or .sql.gz file, and import a .sql or .sql.gz file, from the main menu. diff --git a/linux/crates/app/src/services/import_file.rs b/linux/crates/app/src/services/import_file.rs new file mode 100644 index 0000000000..bdfd96a47f --- /dev/null +++ b/linux/crates/app/src/services/import_file.rs @@ -0,0 +1,8 @@ +//! A SQL file the user chose to import, opened for the core's reader. + +use std::fs::File; +use std::path::Path; + +pub fn open_import_file_blocking(path: &Path) -> std::io::Result { + File::open(path) +} diff --git a/linux/crates/app/src/services/mod.rs b/linux/crates/app/src/services/mod.rs index 8a0c3a05e0..2dc37d3e0c 100644 --- a/linux/crates/app/src/services/mod.rs +++ b/linux/crates/app/src/services/mod.rs @@ -18,6 +18,7 @@ pub mod grid; pub mod history_availability; pub mod history_preferences; pub mod history_service; +pub mod import_file; pub(crate) mod library_outline; pub mod mcp_service; pub mod mcp_status; diff --git a/linux/crates/app/src/ui/dump_dialog.rs b/linux/crates/app/src/ui/dump_dialog.rs new file mode 100644 index 0000000000..b375fbe039 --- /dev/null +++ b/linux/crates/app/src/ui/dump_dialog.rs @@ -0,0 +1,243 @@ +//! Export Database: a whole database or one schema, structure and rows, +//! as one SQL file that loads into an empty database. + +use std::rc::Rc; +use std::sync::Arc; + +use gtk4 as gtk; +use gtk4::{gio, glib}; +use libadwaita as adw; +use libadwaita::prelude::*; +use tablepro_core::dump::{DumpRequest, DumpScope}; +use tablepro_session::Session; + +use crate::ui::export::{DumpJob, ExportLedger}; + +pub(crate) struct DumpDialogInit { + pub parent: adw::ApplicationWindow, + pub exports: Rc, + pub session: Arc, + /// The database's name, which the file is named after. + pub database: String, + /// The schemas the user can export one of; empty on an engine whose + /// database is the only choice. + pub schemas: Vec, +} + +/// The rows the dialog reads when Export is clicked. +struct DumpForm { + contents: Option, + schemas: Vec, + data: adw::SwitchRow, + gzip: adw::SwitchRow, +} + +impl DumpForm { + fn scope(&self) -> DumpScope { + let chosen = self + .contents + .as_ref() + .and_then(|row| (row.selected() as usize).checked_sub(1)) + .and_then(|index| self.schemas.get(index)); + match chosen { + Some(schema) => DumpScope::Schema(schema.clone()), + None => DumpScope::Database, + } + } +} + +pub(crate) fn present(init: DumpDialogInit) { + let DumpDialogInit { + parent, + exports, + session, + database, + schemas, + } = init; + let page = adw::PreferencesPage::new(); + let group = adw::PreferencesGroup::builder() + .description(crate::i18n::gettext( + "Tables, views, routines and triggers, with every table’s rows, as one SQL file that loads into an empty database.", + )) + .build(); + let contents = (!schemas.is_empty()).then(|| { + let mut titles = vec![crate::i18n::gettext("Whole Database")]; + titles.extend( + schemas + .iter() + .map(|schema| crate::i18n::gettext_f("Schema “{schema}”", &[("schema", schema)])), + ); + let titles: Vec<&str> = titles.iter().map(String::as_str).collect(); + let row = adw::ComboRow::builder() + .title(crate::i18n::gettext("Contents")) + .model(>k::StringList::new(&titles)) + .build(); + crate::ui::combo_value::show_whole_value(&row); + group.add(&row); + row + }); + let data = adw::SwitchRow::builder() + .title(crate::i18n::gettext("Include Rows")) + .subtitle(crate::i18n::gettext("Off writes the structure alone")) + .active(true) + .build(); + let gzip = adw::SwitchRow::builder() + .title(crate::i18n::gettext("Compress with gzip")) + .subtitle(crate::i18n::gettext("Writes a .sql.gz file")) + .build(); + group.add(&data); + group.add(&gzip); + page.add(&group); + + let export_button = gtk::Button::builder() + .label(crate::i18n::gettext("Export…")) + .halign(gtk::Align::End) + .margin_top(6) + .margin_bottom(12) + .margin_start(12) + .margin_end(12) + .build(); + export_button.add_css_class("suggested-action"); + let toolbar = adw::ToolbarView::new(); + toolbar.add_top_bar(&adw::HeaderBar::new()); + toolbar.set_content(Some(&page)); + toolbar.add_bottom_bar(&export_button); + let dialog = adw::Dialog::builder() + .title(crate::i18n::gettext("Export Database")) + .content_width(480) + .child(&toolbar) + .build(); + dialog.set_default_widget(Some(&export_button)); + + let form = DumpForm { + contents, + schemas, + data, + gzip, + }; + let closing = dialog.downgrade(); + let window = parent.clone(); + export_button.connect_clicked(move |_| { + let request = DumpRequest { + scope: form.scope(), + data: form.data.is_active(), + }; + let gzip = form.gzip.is_active(); + if let Some(dialog) = closing.upgrade() { + dialog.close(); + } + let name = file_name(&database, &request.scope, gzip); + glib::spawn_future_local(choose_file( + window.clone(), + exports.clone(), + Arc::clone(&session), + request, + gzip, + name, + )); + }); + dialog.present(Some(&parent)); +} + +/// The file's suggested name: the database, the schema where one was +/// picked, and the date. +fn file_name(database: &str, scope: &DumpScope, gzip: bool) -> String { + let date = glib::DateTime::now_local() + .ok() + .and_then(|now| now.format("%Y-%m-%d").ok()) + .map(|date| date.to_string()) + .unwrap_or_default(); + let mut stem = sanitized(database); + if let DumpScope::Schema(schema) = scope { + stem.push('-'); + stem.push_str(&sanitized(schema)); + } + if !date.is_empty() { + stem.push('-'); + stem.push_str(&date); + } + match gzip { + true => format!("{stem}.sql.gz"), + false => format!("{stem}.sql"), + } +} + +/// A name a file can carry: no path separator, and none of the +/// characters other systems refuse. +fn sanitized(name: &str) -> String { + let cleaned: String = name + .chars() + .map(|character| match character { + '/' | '\\' | ':' | '*' | '?' | '"' | '<' | '>' | '|' => '_', + other if other.is_control() => '_', + other => other, + }) + .collect(); + match cleaned.trim().trim_matches('.') { + "" => "database".to_owned(), + trimmed => trimmed.to_owned(), + } +} + +async fn choose_file( + parent: adw::ApplicationWindow, + exports: Rc, + session: Arc, + request: DumpRequest, + gzip: bool, + name: String, +) { + let filter = gtk::FileFilter::new(); + match gzip { + true => { + filter.set_name(Some(&crate::i18n::gettext("Gzipped SQL files"))); + filter.add_suffix("gz"); + filter.add_mime_type("application/gzip"); + } + false => { + filter.set_name(Some(&crate::i18n::gettext("SQL files"))); + filter.add_suffix("sql"); + filter.add_mime_type("application/sql"); + } + } + let filters = gio::ListStore::new::(); + filters.append(&filter); + let file_dialog = gtk::FileDialog::builder() + .title(crate::i18n::gettext("Export Database")) + .modal(true) + .initial_name(name) + .default_filter(&filter) + .filters(&filters) + .build(); + // Cancelling the chooser is the only way it fails without a file. + let Ok(destination) = file_dialog.save_future(Some(&parent)).await else { + return; + }; + // The ledger reports how the job ends, or logs why it never began. + let _ = exports.begin_dump(DumpJob { + destination, + session, + request, + gzip, + }); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_file_is_named_after_the_database_and_schema() { + let name = file_name("shop", &DumpScope::Schema("sales/eu".to_owned()), true); + + assert!(name.starts_with("shop-sales_eu-"), "{name}"); + assert!(name.ends_with(".sql.gz"), "{name}"); + assert!(file_name("shop", &DumpScope::Database, false).ends_with(".sql")); + } + + #[test] + fn a_name_with_nothing_a_file_can_carry_becomes_database() { + assert_eq!(sanitized(" .. "), "database"); + assert_eq!(sanitized("a\nb"), "a_b"); + } +} diff --git a/linux/crates/app/src/ui/error_text/mod.rs b/linux/crates/app/src/ui/error_text/mod.rs index 9a91db9fca..64ee708ebf 100644 --- a/linux/crates/app/src/ui/error_text/mod.rs +++ b/linux/crates/app/src/ui/error_text/mod.rs @@ -21,6 +21,7 @@ mod syntax; mod tests; mod tls; mod tls_remedy; +mod transfer; mod unsupported; pub use ddl::{build_error_text, refusal_text, save_blocker_text}; @@ -34,3 +35,4 @@ pub use script::script_diagnostic_message; pub use ssh::ssh_destination_problem_text; pub use syntax::{expression_syntax_message, type_syntax_message}; pub use tls_remedy::{TlsRemedy, connect_remedy}; +pub use transfer::{dump_error_message, sql_file_message, sql_import_fault_message}; diff --git a/linux/crates/app/src/ui/error_text/transfer.rs b/linux/crates/app/src/ui/error_text/transfer.rs new file mode 100644 index 0000000000..daaad391fe --- /dev/null +++ b/linux/crates/app/src/ui/error_text/transfer.rs @@ -0,0 +1,52 @@ +use tablepro_core::dump::DumpError; +use tablepro_core::import::{SqlFileError, SqlImportFault}; +use tablepro_core::sql_syntax::script::{RefusedUnit, ScriptReadError}; + +use super::driver::driver_message; +use super::statement::browse_reason; +use crate::i18n::{gettext, gettext_f}; + +/// Why a database export stopped reading the database. +pub fn dump_error_message(error: &DumpError) -> String { + match error { + DumpError::Driver(error) => driver_message(error), + DumpError::Read(error) => gettext_f( + "A table cannot be read: {reason}.", + &[("reason", &browse_reason(error))], + ), + DumpError::OutputClosed => gettext("The file stopped taking data."), + } +} + +/// Why a SQL file could not be read on. +pub fn sql_file_message(error: &SqlFileError) -> String { + match error { + SqlFileError::Read(detail) => gettext_f("The file cannot be read: {detail}", &[("detail", detail)]), + SqlFileError::NotUtf8 { line } => gettext_f( + "Line {line} is not UTF-8 text. A dump with binary values has to write them as hex, as mysqldump does with --hex-blob.", + &[("line", &line.to_string())], + ), + SqlFileError::Script(ScriptReadError::TooLong { line }) => gettext_f( + "The statement starting on line {line} is longer than 128 MB.", + &[("line", &line.to_string())], + ), + } +} + +/// Why one statement of a SQL file did not run. +pub fn sql_import_fault_message(fault: &SqlImportFault) -> String { + match fault { + SqlImportFault::Statement(error) => driver_message(error), + SqlImportFault::Refused(RefusedUnit::ClientCommand { command }) => gettext_f( + "{command} is a psql command, which only psql runs.", + &[("command", command)], + ), + SqlImportFault::Refused(RefusedUnit::CopyFormat) => { + gettext("Only COPY data in the text format pg_dump writes can be imported, with no options.") + } + SqlImportFault::Refused(RefusedUnit::CopyNotText) => gettext("This COPY row is not UTF-8 text."), + SqlImportFault::FileEndsTransaction => gettext( + "The file ends its own transaction, which would end the one the import runs it in. Turn off Run as One Transaction to run it as written.", + ), + } +} diff --git a/linux/crates/app/src/ui/error_text/unsupported.rs b/linux/crates/app/src/ui/error_text/unsupported.rs index 5ef9cfafc3..27c88417ad 100644 --- a/linux/crates/app/src/ui/error_text/unsupported.rs +++ b/linux/crates/app/src/ui/error_text/unsupported.rs @@ -57,6 +57,7 @@ pub(super) fn unsupported_message(feature: UnsupportedFeature) -> String { "{command} cannot run in the editor: it turns the connection into a stream the editor cannot show.", &[("command", command)], ), + UnsupportedFeature::SqlDump => gettext("This database cannot be exported as a SQL file."), } } @@ -95,6 +96,7 @@ mod tests { UnsupportedFeature::ClrRoutine, UnsupportedFeature::EncryptedDefinition, UnsupportedFeature::EditorCommand { command: "MONITOR" }, + UnsupportedFeature::SqlDump, ]; let messages: Vec = features.iter().map(|feature| unsupported_message(*feature)).collect(); diff --git a/linux/crates/app/src/ui/export/dump_job.rs b/linux/crates/app/src/ui/export/dump_job.rs new file mode 100644 index 0000000000..c59eefecbf --- /dev/null +++ b/linux/crates/app/src/ui/export/dump_job.rs @@ -0,0 +1,116 @@ +use std::sync::Arc; + +use async_trait::async_trait; +use gtk4::gio; +use tablepro_core::dump::{ + DumpError, DumpOutput, DumpOutputClosed, DumpProgress, DumpReport, DumpRequest, GzipChunks, write_dump, +}; +use tablepro_session::runtime::{TaskResult, Tasks}; +use tablepro_session::{OperationError, OperationKind, Session}; +use tokio::sync::mpsc; +use tokio_util::sync::CancellationToken; + +use super::ExportMessage; + +/// A whole database or schema written to one SQL file. +#[derive(Debug)] +pub(crate) struct DumpJob { + pub destination: gio::File, + pub session: Arc, + pub request: DumpRequest, + /// Write the file gzipped, as a `.sql.gz`. + pub gzip: bool, +} + +pub(super) type DumpTask = TaskResult, OperationError>>; + +/// Read the database through `session` and hand the file's bytes to +/// `messages`, gzipped when asked. The bounded channel holds the read back +/// while the disk catches up. +pub(super) fn start_dump_producer( + tasks: &Tasks, + job: &DumpJob, + messages: mpsc::Sender, + cancel: CancellationToken, +) -> DumpTask { + let session = Arc::clone(&job.session); + let request = Arc::new(job.request.clone()); + let gzip = job.gzip; + tasks.spawn_task(async move { + session + .run_cancellable(OperationKind::Export, &cancel, move |lease, options| { + let request = Arc::clone(&request); + let messages = messages.clone(); + async move { + let mut output = ChannelOutput::new(messages, gzip); + let report = write_dump(lease.connection().as_ref(), &request, &mut output, options).await; + let ended = output.finish().await; + Ok(match (report, ended) { + (Ok(report), Ok(())) => Ok(report), + (Ok(_), Err(DumpOutputClosed)) => Err(DumpError::OutputClosed), + (Err(error), _) => Err(error), + }) + } + }) + .await + }) +} + +struct ChannelOutput { + messages: mpsc::Sender, + gzip: Option, + rows: u64, +} + +impl ChannelOutput { + fn new(messages: mpsc::Sender, gzip: bool) -> Self { + Self { + messages, + gzip: gzip.then(GzipChunks::new), + rows: 0, + } + } + + async fn send(&mut self, bytes: Vec) -> Result<(), DumpOutputClosed> { + if bytes.is_empty() { + return Ok(()); + } + self.messages + .send(ExportMessage::Chunk { bytes, rows: self.rows }) + .await + .map_err(|_| DumpOutputClosed) + } + + /// The gzip trailer, once every byte is in. + async fn finish(mut self) -> Result<(), DumpOutputClosed> { + match self.gzip.take() { + Some(gzip) => { + let tail = gzip.finish().map_err(|_| DumpOutputClosed)?; + self.send(tail).await + } + None => Ok(()), + } + } +} + +#[async_trait] +impl DumpOutput for ChannelOutput { + async fn write(&mut self, bytes: Vec) -> Result<(), DumpOutputClosed> { + let bytes = match &mut self.gzip { + Some(gzip) => gzip.compress(&bytes).map_err(|_| DumpOutputClosed)?, + None => bytes, + }; + self.send(bytes).await + } + + fn progress(&mut self, progress: DumpProgress) { + self.rows = progress.rows; + // Progress is only shown, so a full channel drops it rather than + // holding the read. + let _ = self.messages.try_send(ExportMessage::Tables { + done: progress.tables_done, + total: progress.tables_total, + rows: progress.rows, + }); + } +} diff --git a/linux/crates/app/src/ui/export/encoding_sink.rs b/linux/crates/app/src/ui/export/encoding_sink.rs index 24d1ca804b..947fcb8fa8 100644 --- a/linux/crates/app/src/ui/export/encoding_sink.rs +++ b/linux/crates/app/src/ui/export/encoding_sink.rs @@ -168,7 +168,7 @@ mod tests { match message { ExportMessage::Chunk { bytes, rows } => chunks.push((bytes, rows)), ExportMessage::Finished { rows } => finished = Some(rows), - ExportMessage::Progress { .. } => {} + ExportMessage::Progress { .. } | ExportMessage::Tables { .. } => {} ExportMessage::Failed(failure) => panic!("the export failed: {failure}"), } } diff --git a/linux/crates/app/src/ui/export/export_failure_text.rs b/linux/crates/app/src/ui/export/export_failure_text.rs index 6ac1eef582..7115d13382 100644 --- a/linux/crates/app/src/ui/export/export_failure_text.rs +++ b/linux/crates/app/src/ui/export/export_failure_text.rs @@ -5,6 +5,7 @@ use gtk4::{self as gtk, gio, glib}; use libadwaita as adw; use libadwaita::prelude::*; use tablepro_core::ErrorCategory; +use tablepro_core::dump::{DumpObjectKind, DumpReport, DumpSkipReason}; use tablepro_core::export::{EncodeError, RenderError}; use tablepro_session::OperationError; use tablepro_session::runtime::TaskFailure; @@ -31,6 +32,111 @@ pub(crate) fn stopped_text(outcome: AbortOutcome, name: &str) -> String { } } +/// How far a database export has got. +pub(crate) fn tables_progress_text(done: usize, total: usize, rows: u64) -> String { + let rows_text = ngettext_f( + "{rows} row", + "{rows} rows", + plural_count(rows), + &[("rows", &format_thousands(rows))], + ); + ngettext_f( + "{done} of {total} table, {rows_text}", + "{done} of {total} tables, {rows_text}", + plural_count(total as u64), + &[ + ("done", &format_thousands(done as u64)), + ("total", &format_thousands(total as u64)), + ("rows_text", &rows_text), + ], + ) +} + +/// What a database export wrote, for its toast. +pub(crate) fn dumped_text(report: &DumpReport, name: &str) -> String { + ngettext_f( + "Exported {tables} table to {name}", + "Exported {tables} tables to {name}", + plural_count(report.tables as u64), + &[("tables", &format_thousands(report.tables as u64)), ("name", name)], + ) +} + +/// What a database export could not carry, one line each, or `None` +/// when it carried everything. +pub(crate) fn dump_warnings(report: &DumpReport) -> Option { + let mut lines = Vec::new(); + for omitted in &report.omitted { + lines.push(ngettext_f( + "{count} value in {table}.{column} has no SQL form and was written as NULL.", + "{count} values in {table}.{column} have no SQL form and were written as NULL.", + plural_count(omitted.count), + &[ + ("count", &format_thousands(omitted.count)), + ("table", &omitted.table.name), + ("column", &omitted.column), + ], + )); + } + for skipped in &report.skipped { + let special = match skipped.reason { + DumpSkipReason::RowsNotCarried if skipped.kind == DumpObjectKind::Table => Some(gettext_f( + "{name} is in the file without its rows: it has no columns to read them from.", + &[("name", &skipped.name)], + )), + DumpSkipReason::RowsNotCarried => Some(gettext_f( + "{name} is in the file without the rows it stores: a materialized view fills only from rows inserted after it.", + &[("name", &skipped.name)], + )), + DumpSkipReason::RowsFiltered => Some(gettext_f( + "{name} is in the file without its rows: row-level security shows this login only some of them.", + &[("name", &skipped.name)], + )), + DumpSkipReason::ExpressionNotCarried => Some(gettext_f( + "{name} loads with its parent's expression for a generated column whose own expression differs.", + &[("name", &skipped.name)], + )), + DumpSkipReason::VersioningNotCarried => Some(gettext_f( + "{name} loads as an ordinary table with its rows' period times: the file has no system versioning.", + &[("name", &skipped.name)], + )), + DumpSkipReason::NotReadable if skipped.kind == DumpObjectKind::SequenceValue => Some(gettext_f( + "The position of sequence {name} could not be read, so it starts over when the file loads.", + &[("name", &skipped.name)], + )), + _ => None, + }; + if let Some(line) = special { + lines.push(line); + continue; + } + let reason = match skipped.reason { + DumpSkipReason::NoDefinition => gettext("the server gave no SQL for it"), + DumpSkipReason::Encrypted => gettext("its code is encrypted"), + DumpSkipReason::ClrRoutine => gettext("its code is a .NET assembly"), + DumpSkipReason::HiddenSecret => gettext("the server hides a secret in its SQL"), + DumpSkipReason::SplitsApart => gettext("its code holds a line that is only GO"), + DumpSkipReason::NotReadable => gettext("this login may not read it"), + DumpSkipReason::RemoteRows => { + gettext("its rows live on another server, which loading the file would write to") + } + _ => gettext("the export writes no object of its kind"), + }; + lines.push(gettext_f( + "{name} was left out: {reason}.", + &[("name", &skipped.name), ("reason", &reason)], + )); + } + if !report.unordered.is_empty() { + let names: Vec<&str> = report.unordered.iter().map(|table| table.name.as_str()).collect(); + lines.push(gettext_f( + "The foreign keys of {tables} point at each other, so their rows may need foreign key checks off to load.", + &[("tables", &names.join(", "))], + )); + } + (!lines.is_empty()).then(|| lines.join("\n")) +} + /// How far a running export has got, for its row in the operations /// popover. pub(crate) fn progress_text(rows: u64, expected: Option) -> String { @@ -79,6 +185,8 @@ pub(crate) fn failure_text(failure: &ExportFailure, file: Option, &[("name", name)], ), ExportFailure::Image(_) => gettext("The image couldn't be drawn."), + ExportFailure::Dump(error) => crate::ui::error_text::dump_error_message(error), + ExportFailure::DumpSession(error) => operation_error_message(error), ExportFailure::Task(TaskFailure::Panicked(_) | TaskFailure::Aborted) | ExportFailure::Stream(ExportStreamFailure::Task(_)) => gettext("The export stopped unexpectedly."), }; @@ -181,6 +289,23 @@ impl ExportReporter { pub(crate) fn report(&self, file: &gio::File, name: &str, outcome: &ExportOutcome) { match outcome { + ExportOutcome::Dumped(report) => { + let toast = crate::ui::plain_toast(&dumped_text(report, name)); + toast.set_button_label(Some(&gettext("Show in Files"))); + let (shown, reporter) = (file.clone(), self.clone()); + toast.connect_button_clicked(move |_| reporter.show_in_files(&shown)); + self.toast(toast); + if let Some(warnings) = dump_warnings(report) + && let Some(parent) = self.parent.upgrade() + { + let alert = adw::AlertDialog::new(Some(&gettext("Exported with Warnings")), Some(&warnings)); + alert.set_body_use_markup(false); + alert.add_response("close", &gettext("_Close")); + alert.set_default_response(Some("close")); + alert.set_close_response("close"); + alert.present(Some(&parent)); + } + } ExportOutcome::Exported { .. } | ExportOutcome::Saved => { let text = match outcome { ExportOutcome::Exported { rows } => exported_text(*rows, name), diff --git a/linux/crates/app/src/ui/export/export_job_item.rs b/linux/crates/app/src/ui/export/export_job_item.rs index d14ac683b4..6f281d9157 100644 --- a/linux/crates/app/src/ui/export/export_job_item.rs +++ b/linux/crates/app/src/ui/export/export_job_item.rs @@ -24,6 +24,8 @@ mod imp { /// The file is made whole before it is written, so it has no rows /// to count. pub whole: Cell, + /// A database export's tables written, and how many it has. + pub tables: Cell>, } #[glib::object_subclass] @@ -60,12 +62,25 @@ impl ExportJobItem { self.imp().whole.get() } + /// A database export, which counts its tables. The rows that follow + /// tell the popover to show them. + pub(crate) fn set_tables(&self, done: usize, total: usize) { + self.imp().tables.set(Some((done, total))); + } + + pub(crate) fn tables(&self) -> Option<(usize, usize)> { + self.imp().tables.get() + } + pub(crate) fn expected(&self) -> Option { self.imp().expected.get() } /// The share of the rows written, while the total is known. pub(crate) fn fraction(&self) -> Option { + if let Some((done, total)) = self.tables() { + return (total > 0).then(|| (done as f64 / total as f64).min(1.0)); + } let total = self.expected()?.total(); (total > 0).then(|| (self.rows() as f64 / total as f64).min(1.0)) } diff --git a/linux/crates/app/src/ui/export/export_ledger/mod.rs b/linux/crates/app/src/ui/export/export_ledger/mod.rs index c9bec9168e..40c5196f29 100644 --- a/linux/crates/app/src/ui/export/export_ledger/mod.rs +++ b/linux/crates/app/src/ui/export/export_ledger/mod.rs @@ -15,6 +15,7 @@ use tokio::sync::mpsc; use tokio_util::sync::CancellationToken; use uuid::Uuid; +use super::dump_job::{DumpJob, start_dump_producer}; use super::export_producer::{start_query_producer, start_rows_producer}; use super::{ ExportFailure, ExportJob, ExportJobId, ExportJobItem, ExportMessage, ExportOutcome, ExportQuery, ExportReporter, @@ -133,6 +134,19 @@ impl ExportLedger { }) } + /// Start writing a whole database or schema to one SQL file, listed + /// with its tables' progress and Stop. + pub(crate) fn begin_dump(self: &Rc, job: DumpJob) -> Option { + let cancel = CancellationToken::new(); + let item = ExportJobItem::new(&file_name(&job.destination), None, cancel.clone()); + let destination = job.destination.clone(); + let tasks = self.tasks.clone(); + let running = item.clone(); + self.start(item, cancel.clone(), destination, None, async move { + run_dump(&tasks, job, &cancel, &running).await + }) + } + /// Start writing `image`, as `begin` starts an export of rows: listed /// with Stop, holding the process and the session, and asked about /// when the window closes. @@ -355,6 +369,7 @@ async fn run(tasks: &Tasks, job: ExportJob, cancel: &CancellationToken, item: &E item.set_rows(rows); } Some(ExportMessage::Progress { rows }) => item.set_rows(rows), + Some(ExportMessage::Tables { .. }) => {} Some(ExportMessage::Failed(failure)) => { cancel.cancel(); return failed(writer, failure.into()).await; @@ -400,6 +415,65 @@ async fn run(tasks: &Tasks, job: ExportJob, cancel: &CancellationToken, item: &E } } +/// Write a database dump: the producer reads the catalog and every +/// table, and the file takes its bytes as they come. +async fn run_dump(tasks: &Tasks, job: DumpJob, cancel: &CancellationToken, item: &ExportJobItem) -> ExportOutcome { + let writer = match ReplaceFileWriter::open(&job.destination).await { + Ok(writer) => writer, + Err(error) => { + return ExportOutcome::Failed { + failure: error.into(), + file: None, + }; + } + }; + show_display_name(&job.destination, item).await; + if cancel.is_cancelled() { + return ExportOutcome::Stopped(writer.abort().await); + } + let (messages, mut received) = mpsc::channel(CHANNEL_CHUNKS); + let producer = start_dump_producer(tasks, &job, messages, cancel.clone()); + loop { + let Some(next) = until_cancelled(cancel, received.recv()).await else { + return ExportOutcome::Stopped(writer.abort().await); + }; + match next { + Some(ExportMessage::Chunk { bytes, rows }) => { + if let Err(error) = writer.write(bytes).await { + cancel.cancel(); + return failed(writer, error.into()).await; + } + item.set_rows(rows); + } + Some(ExportMessage::Tables { done, total, rows }) => { + item.set_tables(done, total); + item.set_rows(rows); + } + Some(ExportMessage::Progress { .. } | ExportMessage::Finished { .. } | ExportMessage::Failed(_)) => {} + None => break, + } + } + let Some(ended) = until_cancelled(cancel, producer).await else { + return ExportOutcome::Stopped(writer.abort().await); + }; + if cancel.is_cancelled() { + return ExportOutcome::Stopped(writer.abort().await); + } + let report = match ended { + Ok(Ok(Ok(report))) => report, + Ok(Ok(Err(error))) => return failed(writer, ExportFailure::Dump(error)).await, + Ok(Err(error)) => return failed(writer, ExportFailure::DumpSession(error)).await, + Err(failure) => return failed(writer, failure.into()).await, + }; + match writer.finish().await { + Ok(()) => ExportOutcome::Dumped(Box::new(report)), + Err(FinishError { error, file }) => ExportOutcome::Failed { + failure: error.into(), + file: Some(file), + }, + } +} + /// Write an image's bytes once its task has made them. async fn run_image(image: ImageExport, cancel: &CancellationToken, item: &ExportJobItem) -> ExportOutcome { let writer = match ReplaceFileWriter::open(&image.destination).await { diff --git a/linux/crates/app/src/ui/export/export_message.rs b/linux/crates/app/src/ui/export/export_message.rs index 298b0714d1..3d942cd0dd 100644 --- a/linux/crates/app/src/ui/export/export_message.rs +++ b/linux/crates/app/src/ui/export/export_message.rs @@ -20,6 +20,12 @@ pub(crate) enum ExportMessage { Finished { rows: u64, }, + /// How far a database export has got. + Tables { + done: usize, + total: usize, + rows: u64, + }, } /// Why the rows could not be turned into the file's bytes. diff --git a/linux/crates/app/src/ui/export/export_operations_button.rs b/linux/crates/app/src/ui/export/export_operations_button.rs index 40b24525f2..cd84728dd8 100644 --- a/linux/crates/app/src/ui/export/export_operations_button.rs +++ b/linux/crates/app/src/ui/export/export_operations_button.rs @@ -6,7 +6,7 @@ use libadwaita as adw; use libadwaita::subclass::prelude::*; use super::ExportJobItem; -use super::export_failure_text::progress_text; +use super::export_failure_text::{progress_text, tables_progress_text}; use crate::i18n::gettext; mod imp { @@ -154,5 +154,9 @@ fn show_progress(item: &ExportJobItem, progress: >k::ProgressBar, subtitle: &g None if item.rows() > 0 => progress.pulse(), None => {} } - subtitle.set_label(&progress_text(item.rows(), item.expected())); + let text = match item.tables() { + Some((done, total)) => tables_progress_text(done, total, item.rows()), + None => progress_text(item.rows(), item.expected()), + }; + subtitle.set_label(&text); } diff --git a/linux/crates/app/src/ui/export/export_outcome.rs b/linux/crates/app/src/ui/export/export_outcome.rs index 06affba881..ffb9090510 100644 --- a/linux/crates/app/src/ui/export/export_outcome.rs +++ b/linux/crates/app/src/ui/export/export_outcome.rs @@ -15,6 +15,8 @@ pub(crate) enum ExportOutcome { /// A file made whole before it was written, such as an image, which /// has no rows to count. Saved, + /// A whole database or schema, with what it could not carry. + Dumped(Box), Stopped(AbortOutcome), /// `file` says what became of the file, when the export had opened /// it. @@ -27,7 +29,7 @@ pub(crate) enum ExportOutcome { impl From<&ExportOutcome> for WorkOutcome { fn from(outcome: &ExportOutcome) -> Self { match outcome { - ExportOutcome::Exported { .. } | ExportOutcome::Saved => Self::Succeeded, + ExportOutcome::Exported { .. } | ExportOutcome::Saved | ExportOutcome::Dumped(_) => Self::Succeeded, ExportOutcome::Stopped(_) => Self::Cancelled, ExportOutcome::Failed { .. } => Self::Failed, } @@ -54,6 +56,12 @@ pub(crate) enum ExportFailure { /// The image could not be drawn or encoded. #[error("the image could not be made: {0}")] Image(String), + /// Reading the database for a dump failed. + #[error("the database could not be read: {0}")] + Dump(tablepro_core::dump::DumpError), + /// The session the dump read through failed. + #[error("the database could not be read: {0}")] + DumpSession(OperationError), } impl From for ExportFailure { diff --git a/linux/crates/app/src/ui/export/mod.rs b/linux/crates/app/src/ui/export/mod.rs index 36fce8319b..a6a0dbb66f 100644 --- a/linux/crates/app/src/ui/export/mod.rs +++ b/linux/crates/app/src/ui/export/mod.rs @@ -7,6 +7,7 @@ pub(super) mod dialog; mod document_sink; +mod dump_job; mod encoding_sink; mod export_failure_text; mod export_format; @@ -24,6 +25,7 @@ mod export_source; mod replace_file_writer; pub(crate) use document_sink::DocumentSink; +pub(crate) use dump_job::DumpJob; pub(crate) use encoding_sink::EncodingSink; pub(crate) use export_failure_text::{ExportReporter, failure_text}; pub(crate) use export_format::{DocumentFormat, ExportFormat, InsertTarget, RowFormat}; diff --git a/linux/crates/app/src/ui/import_dialog.rs b/linux/crates/app/src/ui/import_dialog.rs index 08173e85ef..5f184131d1 100644 --- a/linux/crates/app/src/ui/import_dialog.rs +++ b/linux/crates/app/src/ui/import_dialog.rs @@ -272,13 +272,14 @@ impl ImportCsvDialog { let suggested = suggest_mapping(&self.sheet.headers, &self.columns); for (column, field) in self.columns.iter().zip(&suggested) { - // The subtitle shows values from the file, which may hold & or <. + // The title and subtitle show names and values from the file, + // which may hold & or <, so markup goes off before they go in. let row = adw::ComboRow::builder() - .use_markup(false) - .title(column.info.name.as_str()) .model(&choices) .sensitive(!column.server_filled) .build(); + row.set_use_markup(false); + row.set_title(column.info.name.as_str()); row.set_selected(field.map_or(0, |index| index as u32 + 1)); let sender = sender.clone(); row.connect_selected_notify(move |_| sender.deliver(ImportCsvDialogInput::MappingChanged)); diff --git a/linux/crates/app/src/ui/mod.rs b/linux/crates/app/src/ui/mod.rs index 8140ea2bb8..0c012f82f9 100644 --- a/linux/crates/app/src/ui/mod.rs +++ b/linux/crates/app/src/ui/mod.rs @@ -13,6 +13,7 @@ mod definition_dialog; mod delete_connection_dialog; mod diagram; mod discard_alert; +mod dump_dialog; mod editor; mod editor_font_provider; pub(crate) mod error_text; @@ -45,6 +46,7 @@ mod session_target; mod session_window; mod shell; mod source_dialog; +mod sql_import_dialog; mod ssh_inputs; mod ssh_section; pub(crate) mod store_sync; diff --git a/linux/crates/app/src/ui/session_window/connections.rs b/linux/crates/app/src/ui/session_window/connections.rs index a75493bc83..be7ab3d488 100644 --- a/linux/crates/app/src/ui/session_window/connections.rs +++ b/linux/crates/app/src/ui/session_window/connections.rs @@ -699,8 +699,9 @@ impl SessionWindow { /// Tick the level of the session in front in the main menu. pub(super) fn show_safe_mode(&self) { - let level = self.front_session().map(|(_, session)| session.safe_mode()); - super::window_actions::show_safe_mode(self.upcast_ref(), level); + let front = self.front_session().map(|(_, session)| session); + super::window_actions::show_safe_mode(self.upcast_ref(), front.as_ref().map(|session| session.safe_mode())); + crate::ui::workspace::show_transfer_actions(self.upcast_ref(), front.as_deref()); } /// Run a change to a saved connection, holding the process until it diff --git a/linux/crates/app/src/ui/sql_import_dialog.rs b/linux/crates/app/src/ui/sql_import_dialog.rs new file mode 100644 index 0000000000..8a1e087561 --- /dev/null +++ b/linux/crates/app/src/ui/sql_import_dialog.rs @@ -0,0 +1,681 @@ +//! Import SQL File: the file and the options, then its progress with +//! Stop, then what it did. One dialog for the three, so the import is +//! never out of sight while it writes. + +use std::cell::RefCell; +use std::rc::Rc; +use std::time::Duration; + +use gtk4 as gtk; +use gtk4::glib; +use libadwaita as adw; +use libadwaita::prelude::*; +use tablepro_core::import::{ + SqlImportEnd, SqlImportErrorPolicy, SqlImportOptions, SqlImportProgress, SqlImportReport, TransactionOutcome, +}; + +use crate::i18n::{gettext, gettext_f, ngettext_f}; +use crate::ui::format::{format_thousands, plural_count}; + +/// Failures listed in the summary; Copy Details carries every one kept. +const LISTED_FAILURES: usize = 100; + +pub(crate) struct SqlImportDialogInit { + pub file_name: String, + pub file_size: u64, + pub connection: String, + /// Whether the engine can run the file as one transaction. + pub transactional: bool, + /// The connection's Safe Mode asks before every write, and this + /// dialog is that question for the whole file. + pub confirms_writes: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Stage { + Choosing, + Running, + Done, +} + +pub(crate) struct SqlImportDialog { + dialog: adw::Dialog, + stack: gtk::Stack, + stop_on_error: adw::SwitchRow, + one_transaction: Option, + progress: gtk::ProgressBar, + counts: gtk::Label, + stop: gtk::Button, + result: adw::StatusPage, + failures: gtk::ListBox, + failures_box: gtk::Box, + file_size: u64, + stage: RefCell, + details: RefCell, +} + +impl SqlImportDialog { + /// The dialog on its first page. `on_import` runs once, with the + /// options, when Import is clicked; `on_stop` when Stop is. + pub(crate) fn new( + init: &SqlImportDialogInit, + on_import: impl Fn(SqlImportOptions) + 'static, + on_stop: impl Fn() + 'static, + ) -> Rc { + let stack = gtk::Stack::builder() + .transition_type(gtk::StackTransitionType::Crossfade) + .vhomogeneous(false) + .build(); + let choosing = ChoosingPage::new(init); + stack.add_named(&choosing.root, Some("choosing")); + let running = RunningPage::new(&init.file_name); + stack.add_named(&running.root, Some("running")); + let done = DonePage::new(); + stack.add_named(&done.root, Some("done")); + + let toolbar = adw::ToolbarView::new(); + toolbar.add_top_bar(&adw::HeaderBar::new()); + toolbar.set_content(Some(&stack)); + let dialog = adw::Dialog::builder() + .title(gettext("Import SQL File")) + .content_width(480) + .child(&toolbar) + .build(); + dialog.set_default_widget(Some(&choosing.import)); + + let this = Rc::new(Self { + dialog, + stack, + stop_on_error: choosing.stop_on_error, + one_transaction: choosing.one_transaction, + progress: running.progress, + counts: running.counts, + stop: running.stop, + result: done.result, + failures: done.failures, + failures_box: done.failures_box, + file_size: init.file_size, + stage: RefCell::new(Stage::Choosing), + details: RefCell::default(), + }); + this.connect(&choosing.import, &done.copy, &done.close, on_import, on_stop); + this + } + + fn connect( + self: &Rc, + import: >k::Button, + copy: >k::Button, + close: >k::Button, + on_import: impl Fn(SqlImportOptions) + 'static, + on_stop: impl Fn() + 'static, + ) { + let started = Rc::downgrade(self); + import.connect_clicked(move |_| { + let Some(this) = started.upgrade() else { + return; + }; + if *this.stage.borrow() != Stage::Choosing { + return; + } + let options = this.options(); + this.show_running(); + on_import(options); + }); + let stopping = Rc::downgrade(self); + self.stop.connect_clicked(move |button| { + button.set_sensitive(false); + if let Some(this) = stopping.upgrade() { + this.counts.set_label(&gettext("Stopping…")); + } + on_stop(); + }); + let closing = self.dialog.downgrade(); + close.connect_clicked(move |_| { + if let Some(dialog) = closing.upgrade() { + dialog.close(); + } + }); + let copying = Rc::downgrade(self); + copy.connect_clicked(move |button| { + if let Some(this) = copying.upgrade() { + let details = this.details.borrow().clone(); + let told = button.clone(); + crate::ui::clipboard::copy_text(button, details, move |message| { + told.announce(message, gtk::AccessibleAnnouncementPriority::Medium); + }); + } + }); + } + + pub(crate) fn present(&self, parent: &impl IsA) { + self.dialog.present(Some(parent)); + } + + pub(crate) fn dialog(&self) -> &adw::Dialog { + &self.dialog + } + + fn options(&self) -> SqlImportOptions { + let stop = self.stop_on_error.is_active(); + SqlImportOptions { + on_error: match stop { + true => SqlImportErrorPolicy::Stop, + false => SqlImportErrorPolicy::Continue, + }, + one_transaction: stop && self.one_transaction.as_ref().is_some_and(adw::SwitchRow::is_active), + } + } + + fn show_running(&self) { + self.stage.replace(Stage::Running); + // Stop is the way out while the file writes; the window's own + // close asks about a running import. + self.dialog.set_can_close(false); + self.stack.set_visible_child_name("running"); + self.dialog.set_default_widget(Some(&self.stop)); + self.show_progress(SqlImportProgress::default()); + } + + pub(crate) fn show_progress(&self, progress: SqlImportProgress) { + if *self.stage.borrow() != Stage::Running { + return; + } + match self.file_size { + 0 => self.progress.pulse(), + size => self + .progress + .set_fraction((progress.bytes_read as f64 / size as f64).clamp(0.0, 1.0)), + } + if self.stop.is_sensitive() { + self.counts + .set_label(&counts_text(progress.statements, progress.failed)); + } + } + + /// The import could not start, or its task failed. + pub(crate) fn show_failure(&self, message: &str) { + self.finish_with("dialog-error-symbolic", &gettext("Import Failed"), message, &[]); + } + + pub(crate) fn show_report(&self, report: &SqlImportReport) { + let (icon, title) = match (&report.end, report.failed) { + (SqlImportEnd::Completed, 0) => ("emblem-ok-symbolic", gettext("Import Finished")), + (SqlImportEnd::Completed, _) => ("dialog-warning-symbolic", gettext("Import Finished with Errors")), + (SqlImportEnd::Cancelled, _) => ("process-stop-symbolic", gettext("Import Stopped")), + _ => ("dialog-error-symbolic", gettext("Import Failed")), + }; + let lines: Vec = report_lines(report); + let failures: Vec<(String, String)> = report + .failures + .iter() + .map(|failure| { + ( + gettext_f("Line {line}", &[("line", &failure.line.to_string())]), + crate::ui::error_text::sql_import_fault_message(&failure.fault), + ) + }) + .collect(); + self.details.replace(details_text(report)); + self.finish_with(icon, &title, &lines.join("\n"), &failures); + } + + fn finish_with(&self, icon: &str, title: &str, description: &str, failures: &[(String, String)]) { + self.stage.replace(Stage::Done); + self.dialog.set_can_close(true); + self.result.set_icon_name(Some(icon)); + self.result.set_title(title); + self.result + .set_description(Some(&glib::markup_escape_text(description))); + while let Some(row) = self.failures.first_child() { + self.failures.remove(&row); + } + for (line, message) in failures.iter().take(LISTED_FAILURES) { + let row = plain_row(line, message); + row.set_subtitle_selectable(true); + self.failures.append(&row); + } + self.failures_box.set_visible(!failures.is_empty()); + self.stack.set_visible_child_name("done"); + } +} + +/// A button at the end of the dialog's bottom bar. +/// The file, the target and the options, with Import. +struct ChoosingPage { + root: gtk::Box, + stop_on_error: adw::SwitchRow, + one_transaction: Option, + import: gtk::Button, +} + +impl ChoosingPage { + fn new(init: &SqlImportDialogInit) -> Self { + let page = adw::PreferencesPage::new(); + let target = adw::PreferencesGroup::new(); + let file = plain_row(&init.file_name, &glib::format_size(init.file_size)); + target.add(&file); + let mut description = gettext_f( + "Every statement in the file runs on “{connection}”.", + &[("connection", &init.connection)], + ); + if init.confirms_writes { + description.push(' '); + description.push_str(&crate::ui::write_confirmation::asks_before_writes(&init.connection)); + } + target.set_description(Some(&glib::markup_escape_text(&description))); + page.add(&target); + + let options = adw::PreferencesGroup::builder().title(gettext("Options")).build(); + let stop_on_error = adw::SwitchRow::builder() + .title(gettext("Stop at First Error")) + .subtitle(gettext("Off runs the rest and lists what failed")) + .active(true) + .build(); + options.add(&stop_on_error); + let one_transaction = init.transactional.then(|| { + let row = adw::SwitchRow::builder() + .title(gettext("Run as One Transaction")) + .subtitle(gettext("A failure or Stop undoes everything the file did")) + .active(true) + .build(); + stop_on_error + .bind_property("active", &row, "sensitive") + .sync_create() + .build(); + options.add(&row); + row + }); + if !init.transactional { + options.set_description(Some(&gettext( + "This database commits each statement as it runs, so a failure keeps what ran before it.", + ))); + } + page.add(&options); + + let import = gtk::Button::builder() + .label(gettext("_Import")) + .use_underline(true) + .halign(gtk::Align::End) + .build(); + import.add_css_class("suggested-action"); + let root = gtk::Box::builder().orientation(gtk::Orientation::Vertical).build(); + root.append(&page); + root.append(&bar(&import)); + Self { + root, + stop_on_error, + one_transaction, + import, + } + } +} + +/// The progress and the counts, with Stop. +struct RunningPage { + root: gtk::Box, + progress: gtk::ProgressBar, + counts: gtk::Label, + stop: gtk::Button, +} + +impl RunningPage { + fn new(file_name: &str) -> Self { + let progress = gtk::ProgressBar::builder().show_text(false).build(); + let counts = gtk::Label::builder().xalign(0.0).wrap(true).build(); + counts.add_css_class("dim-label"); + let heading = gtk::Label::builder() + .label(gettext_f("Importing {name}", &[("name", file_name)])) + .xalign(0.0) + .ellipsize(gtk::pango::EllipsizeMode::Middle) + .build(); + heading.add_css_class("heading"); + progress.update_relation(&[gtk::accessible::Relation::LabelledBy(&[heading.upcast_ref()])]); + let stop = gtk::Button::builder() + .label(gettext("_Stop")) + .use_underline(true) + .build(); + let running = gtk::Box::builder() + .orientation(gtk::Orientation::Vertical) + .spacing(12) + .margin_top(24) + .margin_bottom(12) + .margin_start(24) + .margin_end(24) + .build(); + running.append(&heading); + running.append(&progress); + running.append(&counts); + let root = gtk::Box::builder().orientation(gtk::Orientation::Vertical).build(); + root.append(&running); + root.append(&bar(&stop)); + Self { + root, + progress, + counts, + stop, + } + } +} + +/// What the import did, the statements that failed, Copy Details and +/// Close. +struct DonePage { + root: gtk::Box, + result: adw::StatusPage, + failures: gtk::ListBox, + failures_box: gtk::Box, + copy: gtk::Button, + close: gtk::Button, +} + +impl DonePage { + fn new() -> Self { + let result = adw::StatusPage::builder().build(); + result.add_css_class("compact"); + let failures = gtk::ListBox::builder().selection_mode(gtk::SelectionMode::None).build(); + failures.add_css_class("boxed-list"); + let failures_scroll = gtk::ScrolledWindow::builder() + .hscrollbar_policy(gtk::PolicyType::Never) + .propagate_natural_height(true) + .max_content_height(280) + .child(&failures) + .build(); + let copy = gtk::Button::builder() + .label(gettext("_Copy Details")) + .use_underline(true) + .halign(gtk::Align::Start) + .build(); + let failures_box = gtk::Box::builder() + .orientation(gtk::Orientation::Vertical) + .spacing(12) + .margin_start(12) + .margin_end(12) + .visible(false) + .build(); + failures_box.append(&failures_scroll); + failures_box.append(©); + let close = gtk::Button::builder() + .label(gettext("_Close")) + .use_underline(true) + .halign(gtk::Align::End) + .build(); + let root = gtk::Box::builder().orientation(gtk::Orientation::Vertical).build(); + root.append(&result); + root.append(&failures_box); + root.append(&bar(&close)); + Self { + root, + result, + failures, + failures_box, + copy, + close, + } + } +} + +/// A row whose title and subtitle show as written: a file name, a +/// statement or an error can hold `&` or `<`. Markup goes off before the +/// text goes in, or GTK parses the text as markup first. +fn plain_row(title: &str, subtitle: &str) -> adw::ActionRow { + let row = adw::ActionRow::new(); + row.set_use_markup(false); + row.set_title(title); + row.set_subtitle(subtitle); + row +} + +fn bar(button: >k::Button) -> gtk::Box { + let bar = gtk::Box::builder() + .margin_top(6) + .margin_bottom(12) + .margin_start(12) + .margin_end(12) + .halign(gtk::Align::End) + .build(); + bar.append(button); + bar +} + +fn counts_text(statements: u64, failed: u64) -> String { + let ran = ngettext_f( + "{n} statement run", + "{n} statements run", + plural_count(statements), + &[("n", &format_thousands(statements))], + ); + match failed { + 0 => ran, + failed => ngettext_f( + "{ran}, {n} failed", + "{ran}, {n} failed", + plural_count(failed), + &[("ran", &ran), ("n", &format_thousands(failed))], + ), + } +} + +/// What the summary says, one sentence a line. +fn report_lines(report: &SqlImportReport) -> Vec { + let mut lines = vec![gettext_f( + "{counts} in {time}.", + &[ + ("counts", &counts_text(report.statements, report.failed)), + ("time", &elapsed_text(report.elapsed)), + ], + )]; + match &report.end { + SqlImportEnd::File(error) => lines.push(crate::ui::error_text::sql_file_message(error)), + SqlImportEnd::Session(error) => lines.push(crate::ui::error_text::driver_message(error)), + SqlImportEnd::Stopped => { + if let Some(failure) = report.failures.last() { + lines.push(gettext_f( + "It stopped at line {line}: {reason}", + &[ + ("line", &failure.line.to_string()), + ( + "reason", + &crate::ui::error_text::sql_import_fault_message(&failure.fault), + ), + ], + )); + } + } + SqlImportEnd::Completed | SqlImportEnd::Cancelled => {} + } + match &report.transaction { + TransactionOutcome::Committed => lines.push(gettext("Everything was committed as one transaction.")), + TransactionOutcome::RolledBack => lines.push(gettext("Nothing was kept: the transaction was rolled back.")), + TransactionOutcome::OpenRolledBack => { + lines.push(gettext("The file left a transaction open, and it was rolled back.")) + } + TransactionOutcome::Unknown(error) => lines.push(gettext_f( + "Whether the transaction was kept is not known: {reason}", + &[("reason", &crate::ui::error_text::driver_message(error))], + )), + TransactionOutcome::AsWritten => { + if !matches!(report.end, SqlImportEnd::Completed) && report.statements > 0 { + lines.push(gettext("The statements that ran before it are kept.")); + } + } + } + if report.skipped > 0 { + lines.push(ngettext_f( + "{n} line only a client reads was passed over.", + "{n} lines only a client reads were passed over.", + plural_count(report.skipped), + &[("n", &format_thousands(report.skipped))], + )); + } + lines +} + +/// Every failure kept, with the statement, for the clipboard. +fn details_text(report: &SqlImportReport) -> String { + report + .failures + .iter() + .map(|failure| { + format!( + "{}\n{}\n{}\n", + gettext_f("Line {line}", &[("line", &failure.line.to_string())]), + crate::ui::error_text::sql_import_fault_message(&failure.fault), + failure.statement + ) + }) + .collect::>() + .join("\n") +} + +fn elapsed_text(elapsed: Duration) -> String { + let seconds = elapsed.as_secs(); + match seconds { + 0 => gettext("under a second"), + seconds if seconds < 60 => ngettext_f( + "{n} second", + "{n} seconds", + plural_count(seconds), + &[("n", &seconds.to_string())], + ), + seconds => { + let minutes = seconds / 60; + ngettext_f( + "{n} minute", + "{n} minutes", + plural_count(minutes), + &[("n", &format_thousands(minutes))], + ) + } + } +} + +#[cfg(test)] +mod tests { + use tablepro_core::DriverError; + use tablepro_core::import::{SqlImportFailure, SqlImportFault}; + + use super::*; + + fn report(end: SqlImportEnd, transaction: TransactionOutcome) -> SqlImportReport { + SqlImportReport { + end, + statements: 12, + skipped: 2, + failures: vec![SqlImportFailure { + line: 40, + statement: "INSERT INTO t VALUES (1)".to_owned(), + fault: SqlImportFault::Statement(DriverError::Cancelled), + }], + failed: 1, + transaction, + elapsed: Duration::from_secs(3), + } + } + + #[test] + fn a_stop_names_its_line_and_what_became_of_the_transaction() { + let lines = report_lines(&report(SqlImportEnd::Stopped, TransactionOutcome::RolledBack)); + + assert!(lines.iter().any(|line| line.contains("line 40")), "{lines:?}"); + assert!(lines.iter().any(|line| line.contains("rolled back")), "{lines:?}"); + assert!(lines.iter().any(|line| line.contains("2 lines")), "{lines:?}"); + } + + #[test] + fn a_stop_with_no_transaction_says_what_ran_stays() { + let lines = report_lines(&report(SqlImportEnd::Stopped, TransactionOutcome::AsWritten)); + + assert!(lines.iter().any(|line| line.contains("are kept")), "{lines:?}"); + } + + #[test] + fn the_details_carry_each_failed_statement() { + let details = details_text(&report(SqlImportEnd::Stopped, TransactionOutcome::RolledBack)); + + assert!(details.contains("Line 40")); + assert!(details.contains("INSERT INTO t VALUES (1)")); + } + + #[gtk4::test] + fn a_name_with_markup_characters_shows_as_written() { + adw::init().expect("libadwaita initialises under the test backend"); + let dialog = SqlImportDialog::new( + &SqlImportDialogInit { + file_name: "R&D .sql".to_owned(), + file_size: 100, + connection: "AT&T ".to_owned(), + transactional: true, + confirms_writes: false, + }, + |_| {}, + || {}, + ); + let row = plain_row("R&D .sql", "1 kB"); + + assert!(!row.uses_markup()); + assert_eq!(row.title(), "R&D .sql"); + dialog.finish_with( + "dialog-error-symbolic", + "Import Stopped", + "AT&T ", + &[("Line 1".to_owned(), "near \"<\": syntax error & more".to_owned())], + ); + let failed = dialog + .failures + .first_child() + .and_downcast::() + .expect("a failure row"); + assert_eq!(failed.subtitle().as_deref(), Some("near \"<\": syntax error & more")); + assert!(!failed.uses_markup()); + } + + #[gtk4::test] + fn import_hands_over_the_options_and_turns_to_progress_with_close_off() { + adw::init().expect("libadwaita initialises under the test backend"); + let chosen = Rc::new(RefCell::new(None)); + let kept = chosen.clone(); + let dialog = SqlImportDialog::new( + &SqlImportDialogInit { + file_name: "dump.sql".to_owned(), + file_size: 100, + connection: "Shop".to_owned(), + transactional: true, + confirms_writes: true, + }, + move |options| { + kept.replace(Some(options)); + }, + || {}, + ); + dialog.stop_on_error.set_active(false); + + // An unrealized button ignores activate, so the test clicks. + let import = dialog + .dialog + .default_widget() + .and_downcast::() + .expect("Import is the default button"); + import.emit_clicked(); + + assert_eq!( + *chosen.borrow(), + Some(SqlImportOptions { + on_error: SqlImportErrorPolicy::Continue, + one_transaction: false, + }) + ); + assert!(!dialog.dialog.can_close(), "the dialog closed under a running import"); + dialog.show_progress(SqlImportProgress { + bytes_read: 50, + statements: 3, + failed: 1, + }); + assert!((dialog.progress.fraction() - 0.5).abs() < f64::EPSILON); + assert_eq!(dialog.counts.label(), "3 statements run, 1 failed"); + + dialog.show_report(&report(SqlImportEnd::Completed, TransactionOutcome::AsWritten)); + assert!(dialog.dialog.can_close()); + assert!(dialog.failures_box.is_visible()); + } +} diff --git a/linux/crates/app/src/ui/workspace/database_transfer.rs b/linux/crates/app/src/ui/workspace/database_transfer.rs new file mode 100644 index 0000000000..33b494b337 --- /dev/null +++ b/linux/crates/app/src/ui/workspace/database_transfer.rs @@ -0,0 +1,359 @@ +use std::cell::RefCell; +use std::rc::Rc; +use std::sync::Arc; +use std::time::Duration; + +use gtk4 as gtk; +use gtk4::{gio, glib}; +use libadwaita::prelude::*; +use tablepro_core::dialect::DatabaseLayout; +use tablepro_core::import::{SqlImportOptions, SqlImportProgress, SqlImportReport, run_sql_import}; +use tablepro_core::{EditorSessionOptions, SafeModeLevel}; +use tablepro_session::Session; +use tokio::sync::watch; +use tokio_util::sync::CancellationToken; + +use super::{WorkspaceInput, WorkspaceModel}; +use crate::runtime::DeliverInput; +use crate::services::import_file; +use crate::ui::dump_dialog::DumpDialogInit; +use crate::ui::sql_import_dialog::{SqlImportDialog, SqlImportDialogInit}; + +/// How often the import dialog's progress moves at most. +const PROGRESS_EVERY: Duration = Duration::from_millis(100); + +/// Whether the session's engine writes a SQL dump and runs one back. +pub(crate) fn dumps(session: &Session) -> bool { + session + .current_dialect() + .is_some_and(|dialect| dialect.dump().is_some()) +} + +/// Whether a SQL file may run on the session: its engine has a dump, and +/// neither the connection nor its Safe Mode is read-only. +pub(crate) fn imports_sql(session: &Session) -> bool { + dumps(session) && !session.read_only() && session.safe_mode() != SafeModeLevel::ReadOnly +} + +/// The main menu's database transfers, and the steps they come back +/// with. +#[derive(Debug)] +pub(crate) enum TransferInput { + ExportDatabase, + ImportSqlFile, + /// The file Import SQL File chose, with its size and whether it is + /// a pipe or a device, as GIO read them. + ImportSqlFileChosen { + file: gio::File, + size: u64, + special: bool, + }, +} + +impl WorkspaceModel { + pub(super) fn on_transfer(&self, input: TransferInput) { + match input { + TransferInput::ExportDatabase => self.on_export_database(), + TransferInput::ImportSqlFile => self.on_import_sql_file(), + TransferInput::ImportSqlFileChosen { file, size, special } => { + self.on_import_sql_file_chosen(file, size, special); + } + } + } + + /// Export Database: the scope and options, then the file to write. + fn on_export_database(&self) { + let Some(session) = self.context.active.get() else { + self.show_toast(&crate::i18n::gettext("Connect to a database first.")); + return; + }; + if !dumps(&session) { + self.show_error_alert( + &crate::i18n::gettext("Couldn't Export"), + &crate::i18n::gettext("This database cannot be exported as a SQL file."), + ); + return; + } + let Some(window) = self.window() else { + return; + }; + let exports = self.exports.clone(); + let database = session + .database() + .map_or_else(|| session.metadata().name.clone(), str::to_owned); + let layout = session + .current_dialect() + .map_or(DatabaseLayout::Single, |dialect| dialect.database_layout()); + let reading = session.clone(); + glib::spawn_future_local(async move { + // The catalog's schemas are cached; a read that fails leaves + // the whole database as the only choice. + let schemas = match layout { + DatabaseLayout::WithSchemas => reading + .schemas() + .await + .map(|schemas| { + schemas + .iter() + .filter(|schema| !schema.is_system) + .map(|schema| schema.name.clone()) + .collect() + }) + .unwrap_or_default(), + DatabaseLayout::Single | DatabaseLayout::AsSchemas => Vec::new(), + }; + crate::ui::dump_dialog::present(DumpDialogInit { + parent: window, + exports, + session: reading, + database, + schemas, + }); + }); + } + + /// Import SQL File: the file, then its options, its progress and + /// what it did, in one dialog. + fn on_import_sql_file(&self) { + let Some(session) = self.context.active.get() else { + self.show_toast(&crate::i18n::gettext("Connect to a database first.")); + return; + }; + if !imports_sql(&session) { + let message = match dumps(&session) { + true => crate::i18n::gettext("This connection is read-only."), + false => crate::i18n::gettext("This database cannot run a SQL file."), + }; + self.show_error_alert(&crate::i18n::gettext("Couldn't Import"), &message); + return; + } + let filter = gtk::FileFilter::new(); + filter.set_name(Some(&crate::i18n::gettext("SQL files"))); + filter.add_suffix("sql"); + filter.add_suffix("gz"); + filter.add_mime_type("application/sql"); + filter.add_mime_type("application/gzip"); + let filters = gio::ListStore::new::(); + filters.append(&filter); + let file_dialog = gtk::FileDialog::builder() + .title(crate::i18n::gettext("Import SQL File")) + .modal(true) + .default_filter(&filter) + .filters(&filters) + .build(); + let input = self.input.clone(); + file_dialog.open(self.window().as_ref(), gio::Cancellable::NONE, move |chosen| { + let Ok(file) = chosen else { + return; + }; + glib::spawn_future_local(async move { + let info = file + .query_info_future( + "standard::type,standard::size", + gio::FileQueryInfoFlags::NONE, + glib::Priority::DEFAULT, + ) + .await + .ok(); + let special = info.as_ref().is_some_and(|info| { + info.has_attribute(gio::FILE_ATTRIBUTE_STANDARD_TYPE) && info.file_type() == gio::FileType::Special + }); + let size = info + .filter(|info| info.has_attribute(gio::FILE_ATTRIBUTE_STANDARD_SIZE)) + .and_then(|info| u64::try_from(info.size()).ok()) + .unwrap_or(0); + input.deliver(WorkspaceInput::Transfer(TransferInput::ImportSqlFileChosen { + file, + size, + special, + })); + }); + }); + } + + fn on_import_sql_file_chosen(&self, file: gio::File, size: u64, special: bool) { + let Some(session) = self.context.active.get() else { + return; + }; + let path = match file.path() { + // Opening a pipe or a device waits for a writer that may + // never come. + Some(_) if special => { + self.show_error_alert( + &crate::i18n::gettext("Couldn't Import"), + &crate::i18n::gettext("Choose a file, not a pipe or a device."), + ); + return; + } + Some(path) => path, + None => { + self.show_error_alert( + &crate::i18n::gettext("Couldn't Import"), + &crate::i18n::gettext("Choose a file on this computer."), + ); + return; + } + }; + let file_name = file.basename().map_or_else( + || file.parse_name().to_string(), + |name| name.to_string_lossy().into_owned(), + ); + let transactional = session + .current_dialect() + .and_then(|dialect| dialect.dump().map(|dump| dump.transactional_ddl())) + .unwrap_or(false); + let init = SqlImportDialogInit { + file_name, + file_size: size, + connection: session.metadata().name.clone(), + transactional, + confirms_writes: session.safe_mode() == SafeModeLevel::ConfirmWrites, + }; + let running: Rc>> = Rc::default(); + let stopping = running.clone(); + let started = Rc::new(RefCell::new(None::)); + let starting = started.clone(); + let dialog = SqlImportDialog::new( + &init, + move |options| { + if let Some(start) = starting.borrow_mut().take() { + start.run(options); + } + }, + move || { + if let Some(cancel) = stopping.borrow().as_ref() { + cancel.cancel(); + } + }, + ); + started.replace(Some(StartImport { + session, + path, + running, + dialog: Rc::downgrade(&dialog), + imports: self.imports.clone(), + tasks: self.context.tasks.clone(), + input: self.input.clone(), + })); + if let Some(parent) = self.dialog_parent() { + dialog.present(&parent); + } + // The dialog lives while it is shown: it cannot close during an + // import, and once closed it lets itself go. + let keeper = Rc::new(RefCell::new(Some(dialog.clone()))); + dialog.dialog().connect_closed(move |_| { + keeper.borrow_mut().take(); + }); + } +} + +/// What Import needs once its button is clicked. +struct StartImport { + session: Arc, + path: std::path::PathBuf, + /// The running import's stop, which the dialog's Stop fires. + running: Rc>>, + dialog: std::rc::Weak, + imports: Rc, + tasks: tablepro_session::runtime::Tasks, + input: super::WorkspaceSender, +} + +impl StartImport { + fn run(self, options: SqlImportOptions) { + let Some(ticket) = self.imports.begin() else { + if let Some(dialog) = self.dialog.upgrade() { + dialog.show_failure(&crate::i18n::gettext("The window is closing.")); + } + return; + }; + // The dialog's Stop and the window's close stop the same import. + let cancel = ticket.cancel.clone(); + self.running.replace(Some(cancel.clone())); + let (progress, mut seen) = watch::channel(SqlImportProgress::default()); + let dialog = self.dialog.clone(); + glib::spawn_future_local(async move { + while seen.changed().await.is_ok() { + let latest = *seen.borrow_and_update(); + let Some(dialog) = dialog.upgrade() else { + return; + }; + dialog.show_progress(latest); + drop(dialog); + glib::timeout_future(PROGRESS_EVERY).await; + } + }); + let work = import( + self.session.clone(), + self.tasks.clone(), + self.path, + options, + progress, + cancel, + ); + let imports = self.imports.clone(); + let input = self.input.clone(); + let dialog = self.dialog; + let _delivery = crate::runtime::spawn_and_deliver(&self.tasks, work, move |finished| { + imports.settle(ticket.id); + input.deliver(WorkspaceInput::SchemaChanged { + schema: None, + table: None, + }); + let Some(dialog) = dialog.upgrade() else { + return; + }; + match finished { + Ok(Ok(report)) => dialog.show_report(&report), + Ok(Err(message)) => dialog.show_failure(&message), + Err(failure) => dialog.show_failure(&crate::ui::error_text::task_failure_message(&failure)), + } + }); + } +} + +/// Open an editor session of the import's own and run the file on it, so +/// what the file sets stays on that session, then close it. +async fn import( + session: Arc, + tasks: tablepro_session::runtime::Tasks, + path: std::path::PathBuf, + options: SqlImportOptions, + progress: watch::Sender, + cancel: CancellationToken, +) -> Result { + let Some(dialect) = session.current_dialect() else { + return Err(crate::i18n::gettext("Connect to a database first.")); + }; + let file = tasks + .spawn_blocking_task(move || import_file::open_import_file_blocking(&path)) + .await + .map_err(|failure| crate::ui::error_text::task_failure_message(&failure))? + .map_err(|error| { + crate::i18n::gettext_f("The file cannot be read: {detail}", &[("detail", &error.to_string())]) + })?; + let editor = session + .open_editor_session(EditorSessionOptions::default(), &cancel) + .await + .map_err(|error| crate::ui::error_text::operation_error_message(&error))?; + let mut report_progress = move |latest: SqlImportProgress| { + progress.send_replace(latest); + }; + let report = run_sql_import( + editor.as_ref(), + &*dialect, + Box::new(file), + options, + &mut report_progress, + &cancel, + ) + .await; + editor.close().await; + tracing::info!( + statements = report.statements, + failed = report.failed, + end = ?report.end, + "ran a SQL file" + ); + Ok(report) +} diff --git a/linux/crates/app/src/ui/workspace/import_ledger.rs b/linux/crates/app/src/ui/workspace/import_ledger.rs index 0c5fb8d009..390532d780 100644 --- a/linux/crates/app/src/ui/workspace/import_ledger.rs +++ b/linux/crates/app/src/ui/workspace/import_ledger.rs @@ -12,7 +12,7 @@ use crate::i18n::{gettext, ngettext_f}; use crate::ui::format::plural_count; use crate::workspace::{CloseBlockPrompt, CloseBlocker, InhibitReason, WindowInhibitor}; -/// The CSV imports a workspace has running. +/// The CSV and SQL file imports a workspace has running. /// /// Each one holds the process, so quitting while one writes lets it /// commit or roll back and report, and closing the window asks before @@ -151,8 +151,8 @@ impl CloseBlocker for ImportLedger { (running > 0).then(|| CloseBlockPrompt { heading: gettext("Stop Imports?"), body: ngettext_f( - "{n} CSV import is still running. Stopping it rolls back the rows it wrote, unless the table has no transactions.", - "{n} CSV imports are still running. Stopping them rolls back the rows they wrote, unless the table has no transactions.", + "{n} import is still running. Stopping it rolls back what it wrote, where the database can.", + "{n} imports are still running. Stopping them rolls back what they wrote, where the database can.", plural_count(running as u64), &[("n", &running.to_string())], ), @@ -218,11 +218,7 @@ mod tests { let prompt = ledger.blocking().expect("a running import blocks the close"); assert_eq!(prompt.heading, "Stop Imports?"); - assert!( - prompt.body.starts_with("1 CSV import is still running."), - "{}", - prompt.body - ); + assert!(prompt.body.starts_with("1 import is still running."), "{}", prompt.body); assert_eq!(hold.live(), 1); assert_eq!(session.held.get(), 1); diff --git a/linux/crates/app/src/ui/workspace/mod.rs b/linux/crates/app/src/ui/workspace/mod.rs index d2d5419edb..fc5daa00ce 100644 --- a/linux/crates/app/src/ui/workspace/mod.rs +++ b/linux/crates/app/src/ui/workspace/mod.rs @@ -7,6 +7,7 @@ mod assistant; mod close_flow; +mod database_transfer; mod definitions; mod diagram; mod editor_label; @@ -60,8 +61,10 @@ use tablepro_core::{DataModel, EditorSession}; use tablepro_session::{ObjectListSnapshot, Session, SessionHealth, SessionKey}; use uuid::Uuid; +pub(crate) use self::database_transfer::TransferInput; use self::session_watches::SessionWatches; use self::tab_title::ConnectionLabel; +pub(crate) use self::workspace_actions::show_transfer_actions; pub(crate) use self::workspace_shell::WorkspaceShell; use super::browse_tab::BrowseTab; use super::editor::SqlEditor; @@ -555,6 +558,8 @@ pub(crate) enum WorkspaceInput { bytes: Vec, }, ImportCsvRun(Box), + /// The main menu's Export Database and Import SQL File. + Transfer(TransferInput), OpenHistoryQuery(String), ReplaceActiveTabQuery(String), TabCloseDecided { @@ -1320,6 +1325,7 @@ impl WorkspaceModel { WorkspaceInput::ShowHistory => self.on_show_history(), WorkspaceInput::ShowSavedQueries => self.on_show_saved_queries(), WorkspaceInput::ImportCsvRequested { schema, table } => self.on_import_csv_requested(schema, table), + WorkspaceInput::Transfer(input) => self.on_transfer(input), WorkspaceInput::ImportCsvReady { schema, table, diff --git a/linux/crates/app/src/ui/workspace/workspace_actions.rs b/linux/crates/app/src/ui/workspace/workspace_actions.rs index c4abf6075e..caf79077e7 100644 --- a/linux/crates/app/src/ui/workspace/workspace_actions.rs +++ b/linux/crates/app/src/ui/workspace/workspace_actions.rs @@ -10,7 +10,7 @@ use gtk4::prelude::*; use gtk4::subclass::prelude::*; use libadwaita as adw; -use super::{WorkspaceInput, WorkspaceShell}; +use super::{TransferInput, WorkspaceInput, WorkspaceShell}; use crate::runtime::DeliverInput; use crate::services::history_availability::HistoryAvailability; @@ -22,8 +22,14 @@ type ActionInput = fn() -> WorkspaceInput; /// they are not left greyed out with no reason. const HISTORY_ACTIONS: [&str; 2] = ["show-history", "show-saved-queries"]; -const ENTRIES: [(&str, ActionInput); 6] = [ +const ENTRIES: [(&str, ActionInput); 8] = [ ("new-editor-tab", || WorkspaceInput::NewEditorTab), + (EXPORT_DATABASE, || { + WorkspaceInput::Transfer(TransferInput::ExportDatabase) + }), + (IMPORT_SQL_FILE, || { + WorkspaceInput::Transfer(TransferInput::ImportSqlFile) + }), ("refresh-tables", || WorkspaceInput::RefreshCatalog), ("save-query", || WorkspaceInput::SaveActiveQuery), ("show-history", || WorkspaceInput::ShowHistory), @@ -31,6 +37,8 @@ const ENTRIES: [(&str, ActionInput); 6] = [ (super::health_banner::REVIEW_HOST_KEY, || WorkspaceInput::ReviewHostKey), ]; +const EXPORT_DATABASE: &str = "export-database"; +const IMPORT_SQL_FILE: &str = "import-sql-file"; const REOPEN_CLOSED_TAB: &str = "reopen-closed-tab"; const CLOSE_ALL_TABS: &str = "close-all-tabs"; const SWITCH_RECENT_TAB: &str = "switch-recent-tab"; @@ -116,6 +124,21 @@ pub(super) fn remove(window: &adw::ApplicationWindow) { window.remove_action(SHOW_TABS); } +/// Offer Export Database and Import SQL File where the session in front +/// has a SQL dump, and Import only where it may write. +pub(crate) fn show_transfer_actions(window: &adw::ApplicationWindow, front: Option<&tablepro_session::Session>) { + set_enabled( + window, + EXPORT_DATABASE, + front.is_some_and(super::database_transfer::dumps), + ); + set_enabled( + window, + IMPORT_SQL_FILE, + front.is_some_and(super::database_transfer::imports_sql), + ); +} + fn set_enabled(window: &adw::ApplicationWindow, name: &str, enabled: bool) { if let Some(action) = window.lookup_action(name).and_downcast::() { action.set_enabled(enabled); diff --git a/linux/data/resources/gtk/menus.ui b/linux/data/resources/gtk/menus.ui index 846b59b03b..1ee0fd0a97 100644 --- a/linux/data/resources/gtk/menus.ui +++ b/linux/data/resources/gtk/menus.ui @@ -30,6 +30,16 @@ win.show-saved-queries action-missing + + E_xport Database… + win.export-database + action-missing + + + Import SQL _File… + win.import-sql-file + action-missing + diff --git a/linux/po/POTFILES.in b/linux/po/POTFILES.in index 3628fbd60a..1042895a90 100644 --- a/linux/po/POTFILES.in +++ b/linux/po/POTFILES.in @@ -98,6 +98,7 @@ crates/app/src/services/grid/test_columns.rs crates/app/src/services/history_availability.rs crates/app/src/services/history_preferences.rs crates/app/src/services/history_service.rs +crates/app/src/services/import_file.rs crates/app/src/services/library_outline.rs crates/app/src/services/mcp_service.rs crates/app/src/services/mcp_status.rs @@ -319,6 +320,7 @@ crates/app/src/ui/diagram/tab_imp.rs crates/app/src/ui/diagram/tests.rs crates/app/src/ui/diagram/zoom.rs crates/app/src/ui/discard_alert.rs +crates/app/src/ui/dump_dialog.rs crates/app/src/ui/editor/assistant_actions.rs crates/app/src/ui/editor/buffer_edit.rs crates/app/src/ui/editor/completion/catalog_read.rs @@ -382,9 +384,11 @@ crates/app/src/ui/error_text/syntax.rs crates/app/src/ui/error_text/tests.rs crates/app/src/ui/error_text/tls.rs crates/app/src/ui/error_text/tls_remedy.rs +crates/app/src/ui/error_text/transfer.rs crates/app/src/ui/error_text/unsupported.rs crates/app/src/ui/export/dialog.rs crates/app/src/ui/export/document_sink.rs +crates/app/src/ui/export/dump_job.rs crates/app/src/ui/export/encoding_sink.rs crates/app/src/ui/export/export_failure_text.rs crates/app/src/ui/export/export_format.rs @@ -511,6 +515,7 @@ crates/app/src/ui/session_window/work_notifier.rs crates/app/src/ui/shell/mod.rs crates/app/src/ui/shell/primary_menu_button.rs crates/app/src/ui/source_dialog.rs +crates/app/src/ui/sql_import_dialog.rs crates/app/src/ui/ssh_inputs.rs crates/app/src/ui/ssh_section.rs crates/app/src/ui/store_sync.rs @@ -571,6 +576,7 @@ crates/app/src/ui/welcome_view.rs crates/app/src/ui/window_geometry.rs crates/app/src/ui/workspace/assistant.rs crates/app/src/ui/workspace/close_flow.rs +crates/app/src/ui/workspace/database_transfer.rs crates/app/src/ui/workspace/definitions.rs crates/app/src/ui/workspace/diagram.rs crates/app/src/ui/workspace/editor_label.rs