env: cleanup and reorder app settings and env varaibles

Author: yomguy

Dockerfile

@@ -52,9 +52,6 @@ RUN /bin/bash /srv/app/bin/setup_plugins.sh
 # Install Vamp plugins
 RUN /bin/bash /srv/app/bin/install_vamp_plugins.sh
 
-# Install bower
-#RUN npm install -g bower
-
 # Install timeside
 WORKDIR /srv/lib/timeside
 RUN pip3 install -U setuptools pip numpy
@@ -64,5 +61,9 @@ RUN pip3 install -r requirements.txt
 COPY . /srv/lib/timeside/
 RUN pip3 install -e .
 
+# Install npm modules
+#RUN npm install -g bower
+RUN npm install --prefix /srv/app
+
 WORKDIR /srv/app
 

app/bin/app.sh

@@ -3,10 +3,6 @@
 SCRIPT_DIR="$(dirname "$0")"
 source "$SCRIPT_DIR/app_base.sh"
 
-# fix log access
-mkdir -p '/var/log/app/'
-chown -R $uid:$gid '/var/log/app/'
-
 # waiting for db
 su $uid -g $gid -s /bin/bash -c "python3 $manage wait-for-db"
 
@@ -22,17 +18,14 @@ su $uid -g $gid -s /bin/bash -c "python3 $manage timeside-analysis-subprocessor-
 #     python $manage update_index --workers $processes &
 # fi
 
-# NPM modules install
-npm install --prefix /srv/app
-
 # app start
 if [ "$1" = "--runserver" ]
 then
     su $uid -g $gid -s /bin/bash -c "python3 $manage runserver 0.0.0.0:8000"
 else
 
     # fix static access
-    chown -R $uid:$gid '/srv/static'
+    chown -R $uid:$gid $static
 
     # collect static files
     su $uid -g $gid -s /bin/bash -c "python3 $manage collectstatic --noinput"
@@ -42,5 +35,5 @@ else
 
     uwsgi --socket :$port --wsgi-file $wsgi --chdir $app --master \
         --processes $processes --threads $threads \
-        --uid $uid --gid $gid --logto $log --touch-reload $wsgi
+        --uid $uid --gid $gid --logto $app_log_file --touch-reload $wsgi
 fi

app/bin/app_base.sh

@@ -1,37 +1,49 @@
 #!/bin/bash
 
-# paths
-app='/srv/app'
-src='/srv/src'
-manage=$app'/manage.py'
-wsgi=$app'/wsgi.py'
-static='/srv/static/'
-media='/srv/media/'
-log='/var/log/uwsgi/app.log'
-worker_logfile='/var/log/celery/worker.log'
-loglevel='DEBUG'
-
 # uwsgi params
 port=8000
 processes=4
 threads=4
 autoreload=3
-# uwsgi and celery params
+
+# uid / gid params for app and worker
 uid='www-data'
 gid='www-data'
 
-# install more apps
-# pip install -U django-cors-headers
-# ...
+# paths
+app='/srv/app'
+src='/srv/src'
+static='/srv/static/'
+media='/srv/media/'
+log='/var/log/app'
+
+# entrypoints
+manage=$app'/manage.py'
+wsgi=$app'/wsgi.py'
+
+# levels
+log_level='DEBUG'
+
+# log paths
+app_log_dir='/var/log/app'
+app_log_file=$app_log_dir'/app.log'
+worker_log_dir='/var/log/celery'
+worker_log_file=$worker_log_dir'/worker.log'
+
+# check and fix dirs
+mkdir -p $app_log_dir
+chown -R $uid:$gid $app_log_dir
+mkdir -p $worker_log_dir
+chown -R $uid:$gid $worker_log_dir
 
-# always take the last youtube-dl version
+# install the last version of those packages
 pip3 install -U youtube-dl
 
 # Install plugins
 bash /srv/app/bin/setup_plugins.sh
 
 # fix media access rights
-find $media -maxdepth 1 -path ${media}import -prune -o -type d -not -user www-data -exec chown www-data:www-data {} \;
+find $media -maxdepth 1 -path ${media}import -prune -o -type d -not -user $uid -exec chown $uid:$gid {} \;
 
 # wait for other services
 # bash $app/bin/wait.sh

app/bin/worker.sh

@@ -3,9 +3,5 @@
 SCRIPT_DIR="$(dirname "$0")"
 source "$SCRIPT_DIR/app_base.sh"
 
-# fix celery log access
-mkdir -p '/var/log/celery/'
-chown -R $uid:$gid '/var/log/celery/'
-
-python3 manage.py timeside-celery-worker --loglevel $loglevel --logfile $worker_logfile --uid $uid --gid $gid
+python3 manage.py timeside-celery-worker --loglevel $log_level --logfile $worker_log_file --uid $uid --gid $gid
 

app/settings.py

@@ -1,16 +1,17 @@
+import os
+import sys
 import environ
 
+from worker import app
+
+
+sys.dont_write_bytecode = True
+
 # set default values and casting
-env = environ.Env(DEBUG=(bool, False),
-                  CELERY_TASK_ALWAYS_EAGER=(bool, False),
-                  )
+env = environ.Env()
+
 # Django settings for server project.
 DEBUG = env('DEBUG')  # False if not in os.environ
-DEBUG=True
-
-import os
-import sys
-sys.dont_write_bytecode = True
 
 ADMINS = (
     ('Guillaume Pellerin', 'guillaume.pellerin@ircam.fr'),
@@ -20,9 +21,9 @@
 
 MANAGERS = ADMINS
 
-# Full filesystem path to the project.
-# project_root = environ.Path(__file__) - 1
-# data_root = project_root - 2
+EMAIL_HOST = env('EMAIL_HOST')
+EMAIL_SUBJECT_PREFIX = env('EMAIL_SUBJECT_PREFIX')
+DEFAULT_FROM_EMAIL = env('DEFAULT_FROM_EMAIL')
 
 DATABASES = {
     'default': {
@@ -66,6 +67,10 @@
 # If you set this to False, Django will not use timezone-aware datetimes.
 USE_TZ = True
 
+
+# App path
+APP_PATH = '/srv/app/'
+
 # Absolute filesystem path to the directory that will hold user-uploaded files.
 # Example: "/home/media/media.lawrence.com/media/"
 MEDIA_ROOT = '/srv/media/'
@@ -85,8 +90,11 @@
 # Example: "http://media.lawrence.com/static/"
 STATIC_URL = '/static/'
 
+ROOT_URLCONF = 'urls'
+
 # django-npm
-NPM_ROOT_PATH = '/srv/app/'
+NPM_ROOT_PATH = APP_PATH
+
 
 # Additional locations of static files
 STATICFILES_DIRS = (
@@ -95,6 +103,9 @@
     # Don't forget to use absolute paths, not relative paths.
 )
 
+# Python dotted path to the WSGI application used by Django's runserver.
+WSGI_APPLICATION = 'wsgi.application'
+
 # List of finder classes that know how to find static files in
 # various locations.
 STATICFILES_FINDERS = (
@@ -127,6 +138,7 @@
         },
     },
 ]
+
 MIDDLEWARE = (
     'corsheaders.middleware.CorsMiddleware',
     'django.middleware.common.CommonMiddleware',
@@ -140,13 +152,6 @@
     'debug_toolbar.middleware.DebugToolbarMiddleware',
 )
 
-ROOT_URLCONF = 'urls'
-
-# Python dotted path to the WSGI application used by Django's runserver.
-WSGI_APPLICATION = 'wsgi.application'
-
-
-
 INSTALLED_APPS = (
     'django.contrib.auth',
     'django.contrib.contenttypes',
@@ -233,19 +238,15 @@
 }
 
 CELERY_IMPORTS = ("timeside.server.tasks",)
-
-CELERY_BACKEND_URL = 'redis://redis:6379/0'
+CELERY_BACKEND_URL = 'redis://broker:6379/0'
 CELERY_BROKER_TRANSPORT = 'redis'
-CELERY_BROKER_URL = 'redis://redis:6379/0'
+CELERY_BROKER_URL = 'redis://broker:6379/0'
 CELERY_RESULT_BACKEND = 'django-db'
 CELERY_CACHE_BACKEND = 'django-cache'
 CELERY_TASK_SERIALIZER = "json"
 CELERY_ACCEPT_CONTENT = ['application/json']
 CELERY_TASK_ALWAYS_EAGER = env('CELERY_TASK_ALWAYS_EAGER')  # If this is True, all tasks will be executed locally by blocking until the task returns.
 
-
-from worker import app
-
 BOWER_COMPONENTS_ROOT = '/srv/static/'
 BOWER_PATH = '/usr/local/bin/bower'
 BOWER_INSTALLED_APPS = (
@@ -263,9 +264,10 @@
 CORS_ALLOW_CREDENTIALS = True
 
 SESSION_COOKIE_SAMESITE = None
-SESSION_COOKIE_SECURE = True
+SESSION_COOKIE_SECURE = False if os.environ.get('DEBUG') == 'True' else True
+
 CSRF_COOKIE_SAMESITE = None
-CSRF_COOKIE_SECURE = True
+CSRF_COOKIE_SECURE = False if os.environ.get('DEBUG') == 'True' else True
 
 X_FRAME_OPTIONS = 'ALLOWALL'
 XS_SHARING_ALLOWED_METHODS = ['POST','GET','OPTIONS', 'PUT', 'DELETE']
@@ -275,11 +277,4 @@
         'SHOW_TOOLBAR_CALLBACK': lambda x : True
     }
 
-
-LOGIN_REDIRECT_URL = '/'
-
-EMAIL_HOST = '134.158.33.163'
-EMAIL_SUBJECT_PREFIX = '[TimeSide-SANDBOX]'
-DEFAULT_FROM_EMAIL = 'root@cchum-kvm-telemeta-dev.in2p3.fr'
-
 TIMESIDE_DEFAULT_DECODER = 'aubio_decoder'

docker-compose.yml

@@ -56,7 +56,6 @@ app:
     - ./app:/srv/app
     - ./timeside:/srv/lib/timeside/timeside
     - ./lib/plugins:/srv/lib/plugins
-    - ./etc/apt/sources.list:/etc/apt/sources.list
   volumes_from:
     - var
   env_file:
@@ -67,7 +66,7 @@ app:
   expose:
     - "8000"
   ports:
-    - "9070:8000"
+    - "9030:8000"
 
 
 worker:

env/debug.env

@@ -1,15 +1,16 @@
-# DATABASE
-
-POSTGRES_PASSWORD=mysecretpassword
-
-# -- DJANGO
-
 DEBUG=True
 
 SECRET_KEY=ghv8us2587n97dq&w$c((o5rj_$-9#d-8j#57y_a9og8wux1h7
 
-# replace broker by localhost if you start your app outside docker-compose
-CELERY_BROKER_URL=redis://broker:6379/0
-
 # If this is True, all tasks will be executed locally by blocking until the task returns.
 CELERY_TASK_ALWAYS_EAGER=False
+
+# CACHE
+bower_storage__packages=/srv/cache/bower
+PIP_DOWNLOAD_CACHE=/srv/cache/pip
+CONDA_ENVS_PATH=/srv/cache/miniconda/envs:/opt/miniconda/envs
+
+# EMAIL
+EMAIL_HOST = 'localhost'
+EMAIL_SUBJECT_PREFIX = '[TimeSide]'
+DEFAULT_FROM_EMAIL = 'no-reply@example.com'

env/debug.yml

@@ -24,20 +24,9 @@
 app:
   env_file:
     - env/debug.env
-  volumes:
-    - ./app:/srv/app
-    - .:/srv/lib/timeside
-    - ./timeside:/srv/lib/timeside/timeside
-    - ./lib/plugins:/srv/lib/plugins
-    - ./etc/apt/sources.list:/etc/apt/sources.list
   command: /bin/bash bin/app.sh --runserver
-  ports:
-    - 9030:8000
 
 worker:
   env_file:
     - env/debug.env
-  volumes:
-    - ./lib:/srv/lib/plugins
-    - ./:/srv/lib/timeside/
 

env/prod.env

@@ -1,16 +1,9 @@
-# DATABASE
-
 POSTGRES_PASSWORD=mysecretpassword
 
-# DJANGO
-
 DEBUG=False
 
 SECRET_KEY=ghv8us2587n97dq&w$c((o5rj_$-9#d-8j#57y_a9og8wux1h7
 
-# replace broker by localhost if you start your app outside docker-compose
-CELERY_BROKER_URL=redis://broker:6379/0
-
 # If this is True, all tasks will be executed locally by blocking until the task returns.
 CELERY_TASK_ALWAYS_EAGER=False
 
@@ -20,3 +13,8 @@ PYTHONWARNINGS=ignore::DeprecationWarning
 bower_storage__packages=/srv/cache/bower
 PIP_DOWNLOAD_CACHE=/srv/cache/pip
 CONDA_ENVS_PATH=/srv/cache/miniconda/envs:/opt/miniconda/envs
+
+# EMAIL
+EMAIL_HOST = 'localhost'
+EMAIL_SUBJECT_PREFIX = '[TimeSide]'
+DEFAULT_FROM_EMAIL = 'no-reply@example.com'