Convert Docker startup scripts to python

collectoss/api/gunicorn_conf.py

@@ -7,6 +7,7 @@
 
 from collectoss.application.db.lib import get_value
 from collectoss.application.db import dispose_database_engine
+from collectoss.application.environment import SystemEnv
 
 logger = logging.getLogger(__name__)
 
@@ -20,8 +21,8 @@
 workers = multiprocessing.cpu_count() * 2 + 1
 umask = 0o007
 reload = True
-
-is_dev = os.getenv("AUGUR_DEV", 'False').lower() in ('true', '1', 't', 'y', 'yes')
+# this satisfies the type checker
+is_dev = SystemEnv.get_bool("AUGUR_DEV", False)
 
 if is_dev:
 
@@ -40,7 +41,8 @@
 # set the log location for gunicorn    
 logs_directory = get_value('Logging', 'logs_directory')
 
-is_docker = os.getenv("AUGUR_DOCKER_DEPLOY", 'False').lower() in ('true', '1', 't', 'y', 'yes')
+# this syntax satisfies the type checker
+is_docker = SystemEnv.get_bool("AUGUR_DOCKER_DEPLOY", False)
 accesslog = f"{logs_directory}/gunicorn.log"
 errorlog = f"{logs_directory}/gunicorn.log"
 

collectoss/api/routes/auggie.py

@@ -14,6 +14,8 @@
 import requests
 import slack
 
+from collectoss.application.environment import SystemEnv
+
 from ..server import app
 
 
@@ -252,7 +254,7 @@ def get_auggie_user():
     # return Response(response=response, status=200, mimetype="application/json")
     ## From Method
     profile_name = 'collectoss'
-    if os.environ.get('AUGUR_IS_PROD'):
+    if SystemEnv.get('COLLECTOSS_IS_PROD'):
         profile_name = 'default'
     client = boto3.Session(region_name='us-east-1', profile_name=profile_name).client('dynamodb')
     response = client.get_item(
@@ -278,7 +280,7 @@ def update_auggie_user_tracking():
     # return Response(response=response, status=200, mimetype="application/json")
     ## From Method
     profile_name = 'collectoss'
-    if os.environ.get('AUGUR_IS_PROD'):
+    if SystemEnv.get('COLLECTOSS_IS_PROD'):
         profile_name = 'default'
     client = boto3.Session(region_name='us-east-1', profile_name=profile_name).client('dynamodb')
     response = client.update_item(
@@ -326,7 +328,7 @@ def slack_login():
     print("slack_login")
 
     r = requests.get(
-        url=f'https://slack.com/api/oauth.v2.access?code={body["code"]}&client_id={os.environ["AUGGIE_CLIENT_ID"]}&client_secret={os.environ["AUGGIE_CLIENT_SECRET"]}&redirect_uri=http%3A%2F%2Flocalhost%3A8080')
+        url=f'https://slack.com/api/oauth.v2.access?code={body["code"]}&client_id={SystemEnv.get("AUGGIE_CLIENT_ID")}&client_secret={SystemEnv.get("AUGGIE_CLIENT_SECRET")}&redirect_uri=http%3A%2F%2Flocalhost%3A8080')
     data = r.json()
 
     if (data["ok"]):
@@ -340,7 +342,7 @@ def slack_login():
         email = user_response["user"]["email"]
 
         profile_name = 'collectoss'
-        if os.environ.get('AUGUR_IS_PROD'):
+        if SystemEnv.get('COLLECTOSS_IS_PROD'):
             profile_name = 'default'
         print("Making Boto3 Session")
         client = boto3.Session(region_name='us-east-1',

collectoss/api/server.py

@@ -27,6 +27,7 @@
 from collectoss.application.logs import SystemLogger
 from collectoss.application.db.session import DatabaseSession
 from collectoss.application.config import SystemConfig
+from collectoss.application.environment import SystemEnv
 from collectoss.application.db.engine import get_database_string, create_database_engine
 from collectoss.application.db.models import Repo, Issue, PullRequest, Message, PullRequestReview, Commit, IssueAssignee, PullRequestAssignee, PullRequestCommit, PullRequestFile, Contributor, IssueLabel, PullRequestLabel, ContributorsAlias, Release, ClientApplication
 
@@ -300,8 +301,8 @@ def create_cache_manager() -> CacheManager:
     cache_config = {
         'cache.type': 'file',
         # Allow setting cache directories via environment variables
-        'cache.data_dir': Path(env.setdefault("CACHE_DATADIR", 'runtime/cache/')),
-        'cache.lock_dir': Path(env.setdefault("CACHE_LOCKDIR", 'runtime/cache/')),
+        'cache.data_dir': Path(SystemEnv.set_default("CACHE_DATADIR", 'runtime/cache/')),
+        'cache.lock_dir': Path(SystemEnv.set_default("CACHE_LOCKDIR", 'runtime/cache/')),
     }
 
     if not os.path.exists(cache_config['cache.data_dir']):

collectoss/api/view/init.py

@@ -1,13 +1,10 @@
-import os
 from pathlib import Path
-from .server import Environment
 from collectoss.application.logs import SystemLogger
 import secrets, yaml
-
-env = Environment()
+from collectoss.application.environment import SystemEnv
 
 # load configuration files and initialize globals
-configFile = Path(env.setdefault("CONFIG_LOCATION", "config.yml"))
+configFile = Path(SystemEnv.get("CONFIG_LOCATION") or "config.yml")
 
 settings = {}
 

collectoss/api/view/server/__init__.py

@@ -1,2 +1 @@
-from .LoginException import LoginException
-from .Environment import Environment
+from .LoginException import LoginException

collectoss/application/cli/__init__.py

@@ -10,7 +10,9 @@
 
 from collectoss.application.db.engine import DatabaseEngine
 from collectoss.application.db import get_engine, dispose_database_engine
-from sqlalchemy.exc import OperationalError 
+from sqlalchemy.exc import OperationalError
+from collectoss.application.environment import SystemEnv
+
 
 
 def check_connectivity(urls=["http://chaoss.community", "http://github.com", "http://gitlab.com"], timeout=10.0):
@@ -65,11 +67,11 @@ def new_func(ctx, *args, **kwargs):
             return ctx.invoke(function_db_connection, *args, **kwargs)
         except OperationalError as e:
 
-            db_environment_var = os.getenv("AUGUR_DB")
+            db_environment_var = SystemEnv.get("COLLECTOSS_DB")
 
             # determine the location to print in error string
             if db_environment_var:
-                location = f"the AUGUR_DB environment variable\nAUGUR_DB={os.getenv('AUGUR_DB')}"
+                location = f"the COLLECTOSS_DB environment variable\nCOLLECTOSS_DB={SystemEnv.get('COLLECTOSS_DB')}"
             else:
                 with open("db.config.json", 'r') as f:
                     db_config = json.load(f)

collectoss/application/cli/_multicommand.py

@@ -11,7 +11,7 @@
 from pathlib import Path
 # import collectoss.application
 
-CONTEXT_SETTINGS = dict(auto_envvar_prefix='AUGUR')
+CONTEXT_SETTINGS = dict(auto_envvar_prefix='COLLECTOSS')
 
 class CLIMultiCommand(click.MultiCommand):
     def __commands_folder(self):

collectoss/application/cli/api.py

@@ -17,6 +17,8 @@
 from collectoss.application.cli import test_connection, test_db_connection, with_database, DatabaseContext
 from collectoss.application.cli._cli_util import _broadcast_signal_to_processes, raise_open_file_limit, clear_redis_caches, clear_rabbitmq_messages
 from collectoss.application.db.lib import get_value
+from collectoss.application.environment import SystemEnv
+
 
 logger = SystemLogger("collectoss", reset_logfiles=False).get_logger()
 
@@ -36,7 +38,7 @@ def start(ctx, development, port):
     """Start CollectOSS's backend server."""
 
     try:
-        if os.environ.get('AUGUR_DOCKER_DEPLOY') != "1":
+        if SystemEnv.get('COLLECTOSS_DOCKER_DEPLOY') != "1":
             raise_open_file_limit(100000)
     except Exception as e: 
         logger.error(
@@ -46,7 +48,7 @@ def start(ctx, development, port):
         raise e
 
     if development:
-        os.environ["AUGUR_DEV"] = "1"
+        SystemEnv.set("AUGUR_DEV", "1")
         logger.info("Starting in development mode")
 
     try:
@@ -142,7 +144,7 @@ def get_api_processes():
 def is_api_process(process):
 
     command = ''.join(process.info['cmdline'][:]).lower()
-    if os.getenv('VIRTUAL_ENV') in process.info['environ']['VIRTUAL_ENV'] and 'python' in command:
+    if SystemEnv.get('VIRTUAL_ENV') in process.info['environ']['VIRTUAL_ENV'] and 'python' in command:
 
         if process.pid != os.getpid():
 

collectoss/application/cli/backend.py

@@ -15,6 +15,7 @@
 import requests
 from redis.exceptions import ConnectionError as RedisConnectionError
 
+from collectoss.application.environment import SystemEnv
 from collectoss.tasks.start_tasks import collection_monitor, create_collection_status_records
 from collectoss.tasks.git.facade_tasks import clone_repos
 from collectoss.tasks.github.contributors import process_contributors
@@ -29,9 +30,10 @@
 from collectoss.application.cli import test_connection, test_db_connection, with_database, DatabaseContext
 import sqlalchemy as s
 
+from collectoss.util.startup import check_init_schema, check_update_schema, collect_env_variables, print_platform_information, setup_facade_directory, warn_import_repos, merge_config
 from keyman.KeyClient import KeyClient, KeyPublisher
 
-reset_logs = os.getenv("AUGUR_RESET_LOGS", 'True').lower() in ('true', '1', 't', 'y', 'yes')
+reset_logs = SystemEnv.get_bool("AUGUR_RESET_LOGS", True)
 
 logger = SystemLogger("collectoss", reset_logfiles=reset_logs).get_logger()
 
@@ -60,8 +62,23 @@ def start(ctx, disable_collection, development, pidfile, port):
     signal.signal(signal.SIGTERM, manager.shutdown_signal_handler)
     signal.signal(signal.SIGINT, manager.shutdown_signal_handler)
 
+
+    collect_env_variables(logger)
+
+
+    check_init_schema()
+    check_update_schema()
+
+    setup_facade_directory(logger)
+
+    merge_config(ctx.obj.engine, logger)
+
+    warn_import_repos(logger)
+
+    print_platform_information(logger)
+
     try:
-        if os.environ.get('AUGUR_DOCKER_DEPLOY') != "1":
+        if SystemEnv.get('COLLECTOSS_DOCKER_DEPLOY') != "1":
             raise_open_file_limit(100000)
     except Exception as e:
         logger.error(
@@ -71,10 +88,10 @@ def start(ctx, disable_collection, development, pidfile, port):
         raise e
 
     if development:
-        os.environ["AUGUR_DEV"] = "1"
+        SystemEnv.set("AUGUR_DEV", "1")
         logger.info("Starting in development mode")
 
-    os.environ["AUGUR_PIDFILE"] = pidfile
+    SystemEnv.set("AUGUR_PIDFILE", pidfile)
 
     try:
         gunicorn_location = os.getcwd() + "/collectoss/api/gunicorn_conf.py"
@@ -86,10 +103,10 @@ def start(ctx, disable_collection, development, pidfile, port):
     if not port:
         port = get_value("Server", "port")
 
-    os.environ["AUGUR_PORT"] = str(port)
+    SystemEnv.set("AUGUR_PORT", str(port))
 
     if disable_collection:
-        os.environ["AUGUR_DISABLE_COLLECTION"] = "1"
+        SystemEnv.set("AUGUR_DISABLE_COLLECTION", "1")
 
     core_worker_count = get_value("Celery", 'core_worker_count')
     secondary_worker_count = get_value("Celery", 'secondary_worker_count')
@@ -130,7 +147,7 @@ def start(ctx, disable_collection, development, pidfile, port):
     processes = start_celery_worker_processes((core_worker_count, secondary_worker_count, facade_worker_count), disable_collection)
     manager.processes = processes
 
-    celery_beat_schedule_db = os.getenv("CELERYBEAT_SCHEDULE_DB", "celerybeat-schedule.db")
+    celery_beat_schedule_db = SystemEnv.get("CELERYBEAT_SCHEDULE_DB", "celerybeat-schedule.db")
     if os.path.exists(celery_beat_schedule_db):
             logger.info("Deleting old task schedule")
             os.remove(celery_beat_schedule_db)
@@ -144,7 +161,7 @@ def start(ctx, disable_collection, development, pidfile, port):
     manager.keypub = keypub
 
     if not disable_collection:
-        if os.environ.get('AUGUR_DOCKER_DEPLOY') != "1":
+        if SystemEnv.get('COLLECTOSS_DOCKER_DEPLOY') != "1":
             orchestrator = subprocess.Popen("python keyman/Orchestrator.py".split())
 
         # Wait for orchestrator startup
@@ -355,10 +372,10 @@ def export_env(config):
     Exports your GitHub key and database credentials
     """
 
-    export_file = open(os.getenv('AUGUR_EXPORT_FILE', 'collectoss_export_env.sh'), 'w+')
+    export_file = open(SystemEnv.get('COLLECTOSS_EXPORT_FILE') or 'collectoss_export_env.sh', 'w+')
     export_file.write('#!/bin/bash')
     export_file.write('\n')
-    env_file = open(os.getenv('AUGUR_ENV_FILE', 'docker_env.txt'), 'w+')
+    env_file = open(SystemEnv.get('COLLECTOSS_ENV_FILE') or 'docker_env.txt', 'w+')
 
     for env_var in config.get_env_config().items():
         if "LOG" not in env_var[0]:
@@ -403,7 +420,7 @@ def get_backend_processes():
     for process in psutil.process_iter(['cmdline', 'name', 'environ']):
         if process.info['cmdline'] is not None and process.info['environ'] is not None:
             try:
-                if os.getenv('VIRTUAL_ENV') in process.info['environ']['VIRTUAL_ENV'] and 'python' in ''.join(process.info['cmdline'][:]).lower():
+                if SystemEnv.get('VIRTUAL_ENV') in process.info['environ']['VIRTUAL_ENV'] and 'python' in ''.join(process.info['cmdline'][:]).lower():
                     if process.pid != os.getpid():
                         process_list.append(process)
             except (KeyError, FileNotFoundError):

collectoss/application/cli/collection.py

@@ -14,6 +14,7 @@
 import traceback
 import sqlalchemy as s
 
+from collectoss.application.environment import SystemEnv
 from collectoss.tasks.start_tasks import collection_monitor, create_collection_status_records
 from collectoss.tasks.git.facade_tasks import clone_repos
 from collectoss.tasks.github.util.github_api_key_handler import GithubApiKeyHandler
@@ -45,7 +46,7 @@ def start(ctx, development):
     """Start CollectOSS's backend server."""
 
     try:
-        if os.environ.get('AUGUR_DOCKER_DEPLOY') != "1":
+        if SystemEnv.get('COLLECTOSS_DOCKER_DEPLOY') != "1":
             raise_open_file_limit(100000)
     except Exception as e: 
         logger.error(
@@ -75,7 +76,7 @@ def start(ctx, development):
         keypub.publish(key, "gitlab_rest")
 
     if development:
-        os.environ["AUGUR_DEV"] = "1"
+        SystemEnv.set("AUGUR_DEV", "1")
         logger.info("Starting in development mode")
 
     core_worker_count = get_value("Celery", 'core_worker_count')
@@ -237,7 +238,7 @@ def get_collection_processes():
 def is_collection_process(process):
 
     command = ''.join(process.info['cmdline'][:]).lower()
-    if os.getenv('VIRTUAL_ENV') in process.info['environ']['VIRTUAL_ENV'] and 'python' in command:
+    if SystemEnv.get('VIRTUAL_ENV') in process.info['environ']['VIRTUAL_ENV'] and 'python' in command:
         if process.pid != os.getpid():
 
             if "collectossbackendcollection" in command  or "celery_app.celery_appbeat" in command: