From abe6a1e1943833d2830941ffed3dedfc58881e63 Mon Sep 17 00:00:00 2001 From: Thomas Berdy Date: Sun, 10 Jul 2022 18:51:13 +0200 Subject: [PATCH 1/3] feat(views): Add schema support for views & MVs Add also populate=False kwarg for create_materialized_view(), which allows MVs to be created WITH NO DATA. BREAKING CHANGE: cascade_on_drop is now None by default, was True before. Also, cascade_on_drop is now a keyword-only argument. --- sqlalchemy_utils/view.py | 138 ++++++++++++++++++++++++++++++++------- tests/test_views.py | 51 ++++++++++++++- 2 files changed, 164 insertions(+), 25 deletions(-) diff --git a/sqlalchemy_utils/view.py b/sqlalchemy_utils/view.py index 96103db1..1dfa02f7 100644 --- a/sqlalchemy_utils/view.py +++ b/sqlalchemy_utils/view.py @@ -5,36 +5,96 @@ from sqlalchemy_utils.functions import get_columns +def _prepare_view_identifier(dialect, view_name, schema=None): + quoted_view_name = dialect.identifier_preparer.quote(view_name) + if schema: + return dialect.identifier_preparer.quote_schema(schema) + '.' + quoted_view_name + else: + return quoted_view_name + + class CreateView(DDLElement): - def __init__(self, name, selectable, materialized=False): + def __init__(self, name, selectable, schema=None): self.name = name self.selectable = selectable - self.materialized = materialized + self.schema = schema @compiler.compiles(CreateView) -def compile_create_materialized_view(element, compiler, **kw): - return 'CREATE {}VIEW {} AS {}'.format( - 'MATERIALIZED ' if element.materialized else '', - compiler.dialect.identifier_preparer.quote(element.name), - compiler.sql_compiler.process(element.selectable, literal_binds=True), +def compile_create_view(element, compiler, **kw): + view_identifier = _prepare_view_identifier( + compiler.dialect, element.name, element.schema + ) + compiled_selectable = compiler.sql_compiler.process( + element.selectable, literal_binds=True ) + return f'CREATE VIEW {view_identifier} AS {compiled_selectable}' class DropView(DDLElement): - def __init__(self, name, materialized=False, cascade=True): + def __init__(self, name, schema=None, cascade=None): self.name = name - self.materialized = materialized + self.schema = schema self.cascade = cascade @compiler.compiles(DropView) +def compile_drop_view(element, compiler, **kw): + view_identifier = _prepare_view_identifier( + compiler.dialect, element.name, element.schema + ) + + stmt = f'DROP VIEW IF EXISTS {view_identifier}' + if element.cascade is True: + stmt += ' CASCADE' + elif element.cascade is False: + stmt += ' RESTRICT' + return stmt + + +class CreateMaterializedView(DDLElement): + def __init__(self, name, selectable, schema=None, populate=None): + self.name = name + self.selectable = selectable + self.schema = schema + self.populate = populate + + +@compiler.compiles(CreateMaterializedView) +def compile_create_materialized_view(element, compiler, **kw): + view_identifier = _prepare_view_identifier( + dialect=compiler.dialect, view_name=element.name, schema=element.schema + ) + compiled_selectable = compiler.sql_compiler.process( + element.selectable, literal_binds=True + ) + + stmt = f'CREATE MATERIALIZED VIEW {view_identifier} AS {compiled_selectable}' + if element.populate is True: + stmt += ' WITH DATA' + elif element.populate is False: + stmt += ' WITH NO DATA' + return stmt + + +class DropMaterializedView(DDLElement): + def __init__(self, name, schema=None, cascade=None): + self.name = name + self.schema = schema + self.cascade = cascade + + +@compiler.compiles(DropMaterializedView) def compile_drop_materialized_view(element, compiler, **kw): - return 'DROP {}VIEW IF EXISTS {} {}'.format( - 'MATERIALIZED ' if element.materialized else '', - compiler.dialect.identifier_preparer.quote(element.name), - 'CASCADE' if element.cascade else '' + view_identifier = _prepare_view_identifier( + dialect=compiler.dialect, view_name=element.name, schema=element.schema ) + stmt = f'DROP MATERIALIZED VIEW IF EXISTS {view_identifier}' + if element.cascade is True: + stmt += ' CASCADE' + elif element.cascade is False: + stmt += ' RESTRICT' + return stmt def create_table_from_selectable( @@ -43,6 +103,7 @@ def create_table_from_selectable( indexes=None, metadata=None, aliases=None, + schema=None, **kwargs ): if indexes is None: @@ -60,7 +121,7 @@ def create_table_from_selectable( ) for c in get_columns(selectable) ] + indexes - table = sa.Table(name, metadata, *args, **kwargs) + table = sa.Table(name, metadata, *args, schema=schema, **kwargs) if not any([c.primary_key for c in get_columns(selectable)]): table.append_constraint( @@ -74,7 +135,11 @@ def create_materialized_view( selectable, metadata, indexes=None, - aliases=None + aliases=None, + *, + schema=None, + populate=None, + cascade_on_drop=None, ): """ Create a view on a given metadata @@ -87,6 +152,17 @@ def create_materialized_view( :param aliases: An optional dictionary containing with keys as column names and values as column aliases. + :param schema: The name of the schema where the view will be created (optional). + :param populate: + Set ``populate=True`` to create the view with ``WITH DATA``. + Set ``populate=False`` to create the view with ``WITH NO DATA``. + Default to ``None`` for no flags. + See also: https://www.postgresql.org/docs/current/sql-createview.html + :param cascade_on_drop: + Set ``cascade_on_drop=True`` to drop the view with ``CASCADE``. + Set ``cascade_on_drop=False`` to create the view with ``RESTRICT``. + Default to ``None`` for no flags. + See also: https://www.postgresql.org/docs/current/sql-dropmaterializedview.html Same as for ``create_view`` except that a ``CREATE MATERIALIZED VIEW`` statement is emitted instead of a ``CREATE VIEW``. @@ -97,13 +173,14 @@ def create_materialized_view( selectable=selectable, indexes=indexes, metadata=None, - aliases=aliases + aliases=aliases, + schema=schema, ) sa.event.listen( metadata, 'after_create', - CreateView(name, selectable, materialized=True) + CreateMaterializedView(name, selectable, schema=schema, populate=populate) ) @sa.event.listens_for(metadata, 'after_create') @@ -114,7 +191,7 @@ def create_indexes(target, connection, **kw): sa.event.listen( metadata, 'before_drop', - DropView(name, materialized=True) + DropMaterializedView(name, schema=schema, cascade=cascade_on_drop) ) return table @@ -123,7 +200,9 @@ def create_view( name, selectable, metadata, - cascade_on_drop=True + *, + schema=None, + cascade_on_drop=None, ): """ Create a view on a given metadata @@ -132,6 +211,11 @@ def create_view( :param metadata: An SQLAlchemy Metadata instance that stores the features of the database being described. + :param schema: The name of the schema where the view will be created (optional). + :param cascade_on_drop: + Set ``cascade_on_drop=True`` to drop the view with ``CASCADE``. + Set ``cascade_on_drop=False`` to create the view with ``RESTRICT``. + Default to ``None`` for no flags. The process for creating a view is similar to the standard way that a table is constructed, except that a selectable is provided instead of @@ -160,10 +244,15 @@ def create_view( table = create_table_from_selectable( name=name, selectable=selectable, - metadata=None + metadata=None, + schema=schema, ) - sa.event.listen(metadata, 'after_create', CreateView(name, selectable)) + sa.event.listen( + metadata, + 'after_create', + CreateView(name, selectable, schema=schema), + ) @sa.event.listens_for(metadata, 'after_create') def create_indexes(target, connection, **kw): @@ -173,12 +262,12 @@ def create_indexes(target, connection, **kw): sa.event.listen( metadata, 'before_drop', - DropView(name, cascade=cascade_on_drop) + DropView(name, schema=schema, cascade=cascade_on_drop) ) return table -def refresh_materialized_view(session, name, concurrently=False): +def refresh_materialized_view(session, name, concurrently=False, *, schema=None): """ Refreshes an already existing materialized view :param session: An SQLAlchemy Session instance. @@ -186,6 +275,7 @@ def refresh_materialized_view(session, name, concurrently=False): :param concurrently: Optional flag that causes the ``CONCURRENTLY`` parameter to be specified when the materialized view is refreshed. + :param schema: The schema of the view to be refreshed (optional). """ # Since session.execute() bypasses autoflush, we must manually flush in # order to include newly-created/modified objects in the refresh. @@ -193,6 +283,6 @@ def refresh_materialized_view(session, name, concurrently=False): session.execute( sa.text('REFRESH MATERIALIZED VIEW {}{}'.format( 'CONCURRENTLY ' if concurrently else '', - session.bind.engine.dialect.identifier_preparer.quote(name) + _prepare_view_identifier(session.bind.engine.dialect, name, schema), )) ) diff --git a/tests/test_views.py b/tests/test_views.py index c4be7099..e76bff23 100644 --- a/tests/test_views.py +++ b/tests/test_views.py @@ -73,7 +73,41 @@ class ArticleView(Base): @pytest.fixture -def init_models(ArticleMV, ArticleView): +def view_schema(Base): + sa.event.listen( + Base.metadata, + "before_create", + sa.DDL("CREATE SCHEMA IF NOT EXISTS views"), + ) + + +@pytest.fixture +def UserMV(Base, User): + class UserMV(Base): + __table__ = create_materialized_view( + name='user-mv', + selectable=sa.select(*_select_args(User.id)), + metadata=Base.metadata, + schema='views', + populate=False, + ) + return UserMV + + +@pytest.fixture +def UserView(Base, User): + class UserView(Base): + __table__ = create_view( + name='user-view', + selectable=sa.select(*_select_args(User.id)), + metadata=Base.metadata, + schema='views', + ) + return UserView + + +@pytest.fixture +def init_models(view_schema, ArticleMV, ArticleView, UserMV, UserView): pass @@ -114,6 +148,21 @@ def test_querying_view( assert row.name == 'Some article' assert row.author_name == 'Some user' + def test_querying_view_in_schema(self, session, User, UserView): + user = User(name='Some user') + session.add(user) + session.commit() + assert session.query(User).first().id == session.query(UserView).first().id + assert 'views."user-view"' in str(session.query(UserView)) + + def test_querying_unpopulated_mv_in_schema(self, session, User, UserMV): + with pytest.raises(sa.exc.OperationalError): + session.query(UserMV).first() + session.rollback() + + refresh_materialized_view(session, 'user-mv', schema='views') + session.query(UserMV).all() + class TrivialViewTestCases: def life_cycle( From 1bdfbc323e70e6dd57b83ae9eeb3f66f76a06e61 Mon Sep 17 00:00:00 2001 From: Thomas Berdy Date: Thu, 14 Jul 2022 19:33:13 +0200 Subject: [PATCH 2/3] Update sqlalchemy_utils/view.py Co-authored-by: Kurt McKee --- sqlalchemy_utils/view.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sqlalchemy_utils/view.py b/sqlalchemy_utils/view.py index 1dfa02f7..e37157a2 100644 --- a/sqlalchemy_utils/view.py +++ b/sqlalchemy_utils/view.py @@ -9,8 +9,8 @@ def _prepare_view_identifier(dialect, view_name, schema=None): quoted_view_name = dialect.identifier_preparer.quote(view_name) if schema: return dialect.identifier_preparer.quote_schema(schema) + '.' + quoted_view_name - else: - return quoted_view_name + + return quoted_view_name class CreateView(DDLElement): From 3a52888dc546a7b7383795a0133bfab9440e2291 Mon Sep 17 00:00:00 2001 From: Thomas Berdy Date: Tue, 19 Jul 2022 20:28:49 +0200 Subject: [PATCH 3/3] chore(views): Add typing annotations --- sqlalchemy_utils/view.py | 117 ++++++++++++++++++++++++++++----------- 1 file changed, 84 insertions(+), 33 deletions(-) diff --git a/sqlalchemy_utils/view.py b/sqlalchemy_utils/view.py index e37157a2..390b6d3e 100644 --- a/sqlalchemy_utils/view.py +++ b/sqlalchemy_utils/view.py @@ -1,11 +1,23 @@ +from typing import Any, Dict, List, Optional, TYPE_CHECKING + import sqlalchemy as sa from sqlalchemy.ext import compiler from sqlalchemy.schema import DDLElement, PrimaryKeyConstraint from sqlalchemy_utils.functions import get_columns +if TYPE_CHECKING: + from sqlalchemy.engine.default import DefaultDialect + from sqlalchemy.orm import Session + from sqlalchemy.sql import Selectable + from sqlalchemy.sql.compiler import SQLCompiler + -def _prepare_view_identifier(dialect, view_name, schema=None): +def _prepare_view_identifier( + dialect: 'DefaultDialect', + view_name: str, + schema: Optional[str] = None, +) -> str: quoted_view_name = dialect.identifier_preparer.quote(view_name) if schema: return dialect.identifier_preparer.quote_schema(schema) + '.' + quoted_view_name @@ -14,14 +26,23 @@ def _prepare_view_identifier(dialect, view_name, schema=None): class CreateView(DDLElement): - def __init__(self, name, selectable, schema=None): + def __init__( + self, + name: str, + selectable: 'Selectable', + schema: Optional[str] = None, + ): self.name = name self.selectable = selectable self.schema = schema @compiler.compiles(CreateView) -def compile_create_view(element, compiler, **kw): +def compile_create_view( + element: 'CreateView', + compiler: 'SQLCompiler', + **kw: Any, +) -> str: view_identifier = _prepare_view_identifier( compiler.dialect, element.name, element.schema ) @@ -32,14 +53,19 @@ def compile_create_view(element, compiler, **kw): class DropView(DDLElement): - def __init__(self, name, schema=None, cascade=None): + def __init__( + self, + name: str, + schema: Optional[str] = None, + cascade: Optional[bool] = None, + ): self.name = name self.schema = schema self.cascade = cascade @compiler.compiles(DropView) -def compile_drop_view(element, compiler, **kw): +def compile_drop_view(element: 'DropView', compiler: 'SQLCompiler', **kw: Any) -> str: view_identifier = _prepare_view_identifier( compiler.dialect, element.name, element.schema ) @@ -53,7 +79,13 @@ def compile_drop_view(element, compiler, **kw): class CreateMaterializedView(DDLElement): - def __init__(self, name, selectable, schema=None, populate=None): + def __init__( + self, + name: str, + selectable: 'Selectable', + schema: Optional[str] = None, + populate: Optional[bool] = None, + ): self.name = name self.selectable = selectable self.schema = schema @@ -61,7 +93,11 @@ def __init__(self, name, selectable, schema=None, populate=None): @compiler.compiles(CreateMaterializedView) -def compile_create_materialized_view(element, compiler, **kw): +def compile_create_materialized_view( + element: 'CreateMaterializedView', + compiler: 'SQLCompiler', + **kw: Any, +) -> str: view_identifier = _prepare_view_identifier( dialect=compiler.dialect, view_name=element.name, schema=element.schema ) @@ -78,14 +114,23 @@ def compile_create_materialized_view(element, compiler, **kw): class DropMaterializedView(DDLElement): - def __init__(self, name, schema=None, cascade=None): + def __init__( + self, + name: str, + schema: Optional[str] = None, + cascade: Optional[bool] = None, + ): self.name = name self.schema = schema self.cascade = cascade @compiler.compiles(DropMaterializedView) -def compile_drop_materialized_view(element, compiler, **kw): +def compile_drop_materialized_view( + element: 'DropMaterializedView', + compiler: 'SQLCompiler', + **kw: Any, +) -> str: view_identifier = _prepare_view_identifier( dialect=compiler.dialect, view_name=element.name, schema=element.schema ) @@ -98,14 +143,14 @@ def compile_drop_materialized_view(element, compiler, **kw): def create_table_from_selectable( - name, - selectable, - indexes=None, - metadata=None, - aliases=None, - schema=None, - **kwargs -): + name: str, + selectable: 'Selectable', + indexes: Optional[List[sa.Index]] = None, + metadata: Optional[sa.MetaData] = None, + aliases: Optional[Dict[str, str]] = None, + schema: Optional[str] = None, + **kwargs: Any, +) -> sa.Table: if indexes is None: indexes = [] if metadata is None: @@ -131,16 +176,16 @@ def create_table_from_selectable( def create_materialized_view( - name, - selectable, - metadata, - indexes=None, - aliases=None, + name: str, + selectable: 'Selectable', + metadata: sa.MetaData, + indexes: Optional[List[sa.Index]] = None, + aliases: Optional[Dict[str, str]] = None, *, - schema=None, - populate=None, - cascade_on_drop=None, -): + schema: Optional[str] = None, + populate: Optional[bool] = None, + cascade_on_drop: Optional[bool] = None, +) -> sa.Table: """ Create a view on a given metadata :param name: The name of the view to create. @@ -197,13 +242,13 @@ def create_indexes(target, connection, **kw): def create_view( - name, - selectable, - metadata, + name: str, + selectable: 'Selectable', + metadata: sa.MetaData, *, - schema=None, - cascade_on_drop=None, -): + schema: Optional[str] = None, + cascade_on_drop: Optional[str] = None, +) -> sa.Table: """ Create a view on a given metadata :param name: The name of the view to create. @@ -267,7 +312,13 @@ def create_indexes(target, connection, **kw): return table -def refresh_materialized_view(session, name, concurrently=False, *, schema=None): +def refresh_materialized_view( + session: 'Session', + name: str, + concurrently: bool = False, + *, + schema: Optional[str] = None, +) -> None: """ Refreshes an already existing materialized view :param session: An SQLAlchemy Session instance.