Merge pull request #53 from bugout-dev/entry-lock

Entry lock

Author: zomglings

alembic/versions/f909b4acb52f_entry_lock.py

@@ -0,0 +1,47 @@
+"""Entry lock
+
+Revision ID: f909b4acb52f
+Revises: 245914e1ddf9
+Create Date: 2022-09-05 10:25:46.047276
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = 'f909b4acb52f'
+down_revision = '245914e1ddf9'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('journal_entry_locks',
+    sa.Column('id', postgresql.UUID(as_uuid=True), nullable=False),
+    sa.Column('journal_entry_id', postgresql.UUID(as_uuid=True), nullable=True),
+    sa.Column('locked_by', sa.String(), nullable=False),
+    sa.Column('locked_at', sa.DateTime(timezone=True), server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), nullable=False),
+    sa.ForeignKeyConstraint(['journal_entry_id'], ['journal_entries.id'], name=op.f('fk_journal_entry_locks_journal_entry_id_journal_entries'), ondelete='CASCADE'),
+    sa.PrimaryKeyConstraint('id', name=op.f('pk_journal_entry_locks')),
+    sa.UniqueConstraint('id', name=op.f('uq_journal_entry_locks_id')),
+    sa.UniqueConstraint('journal_entry_id', name=op.f('uq_journal_entry_locks_journal_entry_id'))
+    )
+    op.create_index(op.f('ix_journal_entry_locks_locked_at'), 'journal_entry_locks', ['locked_at'], unique=False)
+    op.create_index(op.f('ix_journal_entry_locks_locked_by'), 'journal_entry_locks', ['locked_by'], unique=False)
+    op.alter_column('journal_ttls', 'journal_id',
+               existing_type=postgresql.UUID(),
+               nullable=True)
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column('journal_ttls', 'journal_id',
+               existing_type=postgresql.UUID(),
+               nullable=False)
+    op.drop_index(op.f('ix_journal_entry_locks_locked_by'), table_name='journal_entry_locks')
+    op.drop_index(op.f('ix_journal_entry_locks_locked_at'), table_name='journal_entry_locks')
+    op.drop_table('journal_entry_locks')
+    # ### end Alembic commands ###

sample.env

@@ -4,6 +4,7 @@ export BUGOUT_SLACK_CLIENT_SECRET="<slack client secret>"
 export BUGOUT_SLACK_SIGNING_SECRET="<slack signing secret>"
 export BUGOUT_SLACK_VERIFICATION_TOKEN="<slack verification token>"
 export SPIRE_DB_URI="postgresql://<username>:<password>@<db_host>/<db_name>"
+export SPIRE_DB_URI_READ_ONLY="postgresql://<username>:<password>@<db_host>/<db_name>"
 export BUGOUT_OAUTH_COMPLETION_URL="https://bugout.dev"
 export BUGOUT_WEB_URL="https://bugout.dev"
 export BUGOUT_AUTH_URL="http://localhost:7474"
@@ -37,7 +38,7 @@ export BUGOUT_BOT_INSTALLATION_TOKEN_HEADER="<bugout installation token header>"
 export BUGOUT_DRONES_TOKEN="<auth internal drones token>"
 export BUGOUT_DRONES_TOKEN_HEADER="<auth internal drones token header>"
 export SPIRE_OPENAPI_LIST="journals,humbug,preferences,public,go"
-export BUGOUT_REDIS_URL="redis.andrey:6379"
+export BUGOUT_REDIS_URL="http://127.0.0.1:6379"
 export BUGOUT_REDIS_PASSWORD="mypassword"
 export REDIS_REPORTS_QUEUE="<redis key to humbug reports queue>"
 export BUGOUT_HUMBUG_REDIS_TIMEOUT="0.5"

setup.py

@@ -72,4 +72,5 @@
         ],
         "distribute": ["setuptools", "twine", "wheel"],
     },
+    entry_points={"console_scripts": ["journals=spire.journal.cli:main"]},
 )

spire/db.py

@@ -2,15 +2,17 @@
 Spire database connection
 """
 from contextlib import contextmanager
-from datetime import time
-import os
+from typing import Optional
 
 import redis
 from sqlalchemy import create_engine
 from sqlalchemy.orm import sessionmaker, Session
-from sqlalchemy.sql.expression import true
 
 from .utils.settings import (
+    SPIRE_DB_URI,
+    SPIRE_DB_URI_READ_ONLY,
+    SPIRE_DB_POOL_RECYCLE_SECONDS,
+    SPIRE_DB_STATEMENT_TIMEOUT_MILLIS,
     BUGOUT_SPIRE_THREAD_DB_POOL_SIZE,
     BUGOUT_SPIRE_THREAD_DB_MAX_OVERFLOW,
     BUGOUT_REDIS_URL,
@@ -19,14 +21,31 @@
     BUGOUT_HUMBUG_REDIS_CONNECTIONS_PER_PROCESS,
 )
 
-connection_str = os.environ.get("SPIRE_DB_URI")
-if connection_str is None:
-    raise ValueError("SPIRE_DB_URI environment variable not set")
 
-engine = create_engine(
-    connection_str,
+def create_spire_engine(
+    url: Optional[str],
+    pool_size: int,
+    max_overflow: int,
+    statement_timeout: int,
+    pool_recycle: int = SPIRE_DB_POOL_RECYCLE_SECONDS,
+):
+    # Pooling: https://docs.sqlalchemy.org/en/14/core/pooling.html#sqlalchemy.pool.QueuePool
+    # Statement timeout: https://stackoverflow.com/a/44936982
+    return create_engine(
+        url=url,
+        pool_size=pool_size,
+        pool_recycle=pool_recycle,
+        max_overflow=max_overflow,
+        connect_args={"options": f"-c statement_timeout={statement_timeout}"},
+    )
+
+
+engine = create_spire_engine(
+    url=SPIRE_DB_URI,
     pool_size=BUGOUT_SPIRE_THREAD_DB_POOL_SIZE,
     max_overflow=BUGOUT_SPIRE_THREAD_DB_MAX_OVERFLOW,
+    statement_timeout=SPIRE_DB_STATEMENT_TIMEOUT_MILLIS,
+    pool_recycle=SPIRE_DB_POOL_RECYCLE_SECONDS,
 )
 SessionLocal = sessionmaker(bind=engine)
 
@@ -43,6 +62,31 @@ def yield_connection_from_env() -> Session:
         session.close()
 
 
+# Read only database
+RO_engine = create_spire_engine(
+    url=SPIRE_DB_URI_READ_ONLY,
+    pool_size=BUGOUT_SPIRE_THREAD_DB_POOL_SIZE,
+    max_overflow=BUGOUT_SPIRE_THREAD_DB_MAX_OVERFLOW,
+    statement_timeout=SPIRE_DB_STATEMENT_TIMEOUT_MILLIS,
+    pool_recycle=SPIRE_DB_POOL_RECYCLE_SECONDS,
+)
+RO_SessionLocal = sessionmaker(bind=RO_engine)
+
+
+def yield_db_read_only_session() -> Session:
+    """
+    Yields read only database connection (created using environment variables).
+    As per FastAPI docs:
+    https://fastapi.tiangolo.com/tutorial/sql-databases/#create-a-dependency
+    """
+    session = RO_SessionLocal()
+    try:
+        yield session
+    finally:
+        session.close()
+
+
+# Redis
 RedisPool = redis.ConnectionPool.from_url(
     f"redis://:{BUGOUT_REDIS_PASSWORD}@{BUGOUT_REDIS_URL}",
     max_connections=BUGOUT_HUMBUG_REDIS_CONNECTIONS_PER_PROCESS,