Large cleanup and refactoring

Removed use of global state and introduced a ServiceManager to handle health checks, long running background tasks and dependencies between services.

Fixed a ton of bugs highlighted by a stricter type checker.

Renamed the LOADFACTOR_* config parameters.

Removed the 'run' cli for now.

Improved logging when using the cli.

Author: defnull

.github/workflows/docker.yaml

@@ -6,7 +6,8 @@ on:
     tags: [ 'v*' ]
 
 jobs:
-  build-and-push-main:
+
+  build-and-push:
     runs-on: ubuntu-latest
 
     permissions:

bbblb/__init__.py

@@ -1,23 +1,9 @@
 import logging
-from .settings import config
 
 __version__ = "0.0.7"
 VERSION = __version__.split(".", 2)
 VERSION[-1], _, BUILD = VERSION[-1].partition("-")
 
 ROOT_LOGGER = logging.getLogger(__name__)
-ROOT_LOGGER.setLevel(logging.INFO)
-ROOT_LOGGER.propagate = False
-ch = logging.StreamHandler()
-ch.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(name)s %(message)s"))
-ROOT_LOGGER.addHandler(ch)
 
 BRANDING = "BBBLB (AGPL-3, https://github.com/defnull/bbblb)"
-
-
-@config.watch
-def watch_debug_level(name, old, new):
-    if name == "DEBUG":
-        level = logging.DEBUG if new else logging.INFO
-        if level != ROOT_LOGGER.level:
-            ROOT_LOGGER.setLevel(logging.DEBUG if new else logging.INFO)

bbblb/cli/__init__.py

@@ -2,29 +2,20 @@
 import functools
 import importlib
 import pkgutil
-from bbblb.settings import ConfigError, config as cfg
-import bbblb.model
+from bbblb.services import bootstrap
+from bbblb.settings import ConfigError, BBBLBConfig
 import click
 import os
 
 
-def async_command(db=False):
-    """Decorator that wraps coroutine with asyncio.run."""
+def async_command():
+    """Decorator that wraps coroutine with asyncio.run and click.pass_obj."""
 
     def decorator(func):
         @functools.wraps(func)
-        async def wrapper(*a, **ka):
-            try:
-                if db:
-                    await bbblb.model.init_engine(cfg.DB)
-                await func(*a, **ka)
-            finally:
-                if db:
-                    await bbblb.model.dispose_engine()
-
-        @functools.wraps(wrapper)
+        @click.pass_obj
         def sync_wrapper(*args, **kwargs):
-            return asyncio.run(wrapper(*args, **kwargs))
+            return asyncio.run(func(*args, **kwargs))
 
         return sync_wrapper
 
@@ -46,20 +37,47 @@ def sync_wrapper(*args, **kwargs):
     help="Set or unset a BBBLB config parameter",
     multiple=True,
 )
-def main(config_file, config):
+@click.option(
+    "-v", "--verbose", help="Increase verbosity. Can be repeated.", count=True
+)
+@async_command()
+@click.pass_context
+async def main(ctx, obj, config_file, config, verbose):
+    import logging
+
+    logging.basicConfig(
+        format="%(asctime)s %(levelname)s %(name)s %(message)s", level=logging.WARNING
+    )
+
+    if verbose == 0:
+        logging.getLogger("bbblb").setLevel(logging.WARNING)
+    elif verbose == 1:
+        logging.getLogger("bbblb").setLevel(logging.INFO)
+    elif verbose == 2:
+        logging.getLogger("bbblb").setLevel(logging.DEBUG)
+    elif verbose == 3:
+        logging.getLogger("bbblb").setLevel(logging.DEBUG)
+        logging.getLogger("sqlalchemy.engine").setLevel(logging.DEBUG)
+    elif verbose >= 4:
+        logging.root.setLevel(logging.DEBUG)
+
+    config_ = BBBLBConfig()
+
     if config_file:
         os.environ["BBBLB_CONFIG"] = config_file
     for kv in config:
         name, _, value = kv.partition("=")
         name = name.upper()
-        if name not in cfg._options:
+        if name not in config_._options:
             raise ConfigError(f"Unknown config parameter: {name}")
         env_name = f"BBBLB_{name}"
         if value:
             os.environ[env_name] = value
         elif env_name in os.environ:
             del os.environ[env_name]
-    cfg.populate()
+
+    config_.populate()
+    ctx.obj = await bootstrap(config_, autostart=False, logging=False)
 
 
 # Auto-load all modules in the bbblb.cli package to load all commands.

bbblb/cli/db.py

@@ -1,9 +1,9 @@
 import click
-import bbblb
-from bbblb.settings import config
+from bbblb.services import ServiceRegistry
+from bbblb.services.db import check_migration_state, create_database, migrate_db
+from bbblb.settings import BBBLBConfig
 
 from bbblb.cli import async_command, main
-import bbblb.model
 
 
 @main.group()
@@ -15,23 +15,24 @@ def db():
 @click.option(
     "--create", help="Create database if needed (only postgres).", is_flag=True
 )
-@async_command(db=False)
-async def migrate(create: bool):
+@async_command()
+async def migrate(obj: ServiceRegistry, create: bool):
     """
     Migrate database to the current schema version.
 
     WARNING: Make backups!
     """
+    config = await obj.use("config", BBBLBConfig)
 
     try:
         if create:
-            await bbblb.model.create_database(config.DB)
-        current, target = await bbblb.model.check_migration_state(config.DB)
+            await create_database(config.DB)
+        current, target = await check_migration_state(config.DB)
         if current != target:
             click.echo(
                 f"Migrating database schema from {current or 'empty'!r} to {target!r}..."
             )
-            await bbblb.model.migrate_db(config.DB)
+            await migrate_db(config.DB)
             click.echo("Migration complete!")
         else:
             click.echo("Database is up to date. Nothing to do")

bbblb/cli/maketoken.py

@@ -1,11 +1,14 @@
 from bbblb import model
-from bbblb.settings import config as cfg
+from bbblb.services import ServiceRegistry
 import secrets
 import sys
 import time
 import click
 import jwt
 
+from bbblb.services.db import DBContext
+from bbblb.settings import BBBLBConfig
+
 from . import main, async_command
 
 
@@ -24,8 +27,10 @@
 )
 @click.argument("subject")
 @click.argument("scope", nargs=-1)
-@async_command(db=True)
-async def maketoken(subject, expire, server, tenant, scope, verbose):
+@async_command()
+async def maketoken(
+    obj: ServiceRegistry, subject, expire, server, tenant, scope, verbose
+):
     """Generate an Admin Token that can be used to authenticate against the BBBLB API.
 
     The SUBJECT should be a short name or id that identifies the token
@@ -37,38 +42,41 @@ async def maketoken(subject, expire, server, tenant, scope, verbose):
     Tenant or Server tokens do not have scopes, their permissions are hard
     coded because tenants or servers can create their own tokens.
     """
+    config = await obj.use("config", BBBLBConfig)
+    db = await obj.use("db", DBContext)
+
     headers = {}
     payload = {
         "sub": subject,
-        "aud": cfg.DOMAIN,
+        "aud": config.DOMAIN,
         "scope": " ".join(sorted(set(scope))) or "admin",
         "jti": secrets.token_hex(8),
     }
     if expire > 0:
         payload["exp"] = int(time.time() + int(expire))
 
     if server:
-        async with model.session() as session:
+        async with db.session() as session:
             stmt = model.Server.select(domain=server)
             try:
                 server = (await session.execute(stmt)).scalar_one()
             except model.NoResultFound:
-                raise RuntimeError("Server not found in database: {server}")
+                raise RuntimeError(f"Server not found in database: {server}")
         headers["kid"] = f"bbb:{server.domain}"
         del payload["scope"]
         key = server.secret
     elif tenant:
-        async with model.session() as session:
+        async with db.session() as session:
             stmt = model.Tenant.select(name=tenant)
             try:
                 tenant = (await session.execute(stmt)).scalar_one()
             except model.NoResultFound:
-                raise RuntimeError("Tenant not found in database: {tenant}")
+                raise RuntimeError(f"Tenant not found in database: {tenant}")
         headers["kid"] = f"tenant:{tenant.name}"
         del payload["scope"]
         key = tenant.secret
     else:
-        key = cfg.SECRET
+        key = config.SECRET
 
     token = jwt.encode(payload, key, headers=headers)
 

bbblb/cli/override.py

@@ -2,6 +2,9 @@
 from bbblb import model
 import click
 
+from bbblb.services import ServiceRegistry
+from bbblb.services.db import DBContext
+
 from . import main, async_command
 
 
@@ -12,10 +15,12 @@ def override():
 
 @override.command("list")
 @click.argument("tenant", required=False)
-@async_command(db=True)
-async def list_(tenant: str):
+@async_command()
+async def list_(obj: ServiceRegistry, tenant: str):
     """List overrides for all or a specific tenant."""
-    async with model.session() as session:
+    db = await obj.use("db", DBContext)
+
+    async with db.session() as session:
         if tenant:
             stmt = model.Tenant.select(name=tenant)
         else:
@@ -25,9 +30,10 @@ async def list_(tenant: str):
         if tenant and not tenants:
             click.echo(f"Tenant {tenant!r} not found")
             raise SystemExit(1)
-        for tenant in tenants:
-            for key, value in sorted(tenant.overrides.items()):
-                click.echo(f"{tenant.name}: {key}{value}")
+
+        for ten in tenants:
+            for key, value in sorted(ten.overrides.items()):
+                click.echo(f"{ten.name}: {key}{value}")
 
 
 @override.command("set")
@@ -36,8 +42,8 @@ async def list_(tenant: str):
 )
 @click.argument("tenant")
 @click.argument("overrides", nargs=-1, metavar="NAME=VALUE")
-@async_command(db=True)
-async def set_(clear: bool, tenant: str, overrides: list[str]):
+@async_command()
+async def set_(obj: ServiceRegistry, clear: bool, tenant: str, overrides: list[str]):
     """Override create call parameters for a given tenant.
 
     You can define any number of create parameter overrides per tenant as
@@ -51,7 +57,8 @@ async def set_(clear: bool, tenant: str, overrides: list[str]):
     parameters (e.g. duration or maxParticipants), or '+' to add items
     to a comma separated list parameter (e.g. disabledFeatures).
     """
-    async with model.session() as session:
+    db = await obj.use("db", DBContext)
+    async with db.session() as session:
         db_tenant = (
             await session.execute(model.Tenant.select(name=tenant))
         ).scalar_one_or_none()
@@ -66,11 +73,12 @@ async def set_(clear: bool, tenant: str, overrides: list[str]):
             raise SystemExit(1)
 
         for override in overrides:
-            m = re.match("^([a-zA-Z0-9-_]+)([=?<-])(.*)$", override)
+            m = re.match("^([a-zA-Z0-9-_]+)([=?<+])(.*)$", override)
             if not m:
                 click.echo(f"Failed to parse override {override!r}")
                 raise SystemExit(1)
             name, operator, value = m.groups()
+            assert operator in ("=", "?", "<", "+")
             db_tenant.add_override(name, operator, value)
 
         await session.commit()
@@ -80,10 +88,12 @@ async def set_(clear: bool, tenant: str, overrides: list[str]):
 @override.command()
 @click.argument("tenant")
 @click.argument("overrides", nargs=-1, metavar="NAME")
-@async_command(db=True)
-async def unset(tenant: str, overrides: list[str]):
+@async_command()
+async def unset(obj: ServiceRegistry, tenant: str, overrides: list[str]):
     """Remove overrides from a given tenant."""
-    async with model.session() as session:
+    db = await obj.use("db", DBContext)
+
+    async with db.session() as session:
         db_tenant = (
             await session.execute(model.Tenant.select(name=tenant))
         ).scalar_one_or_none()