BK-2085 Update celery, remove djcelery

Author: ride90

docs/settings.py

@@ -38,8 +38,8 @@
     'django.contrib.messages',
     'django.contrib.staticfiles',
 
+    'django_celery_results',
     'compressor',
-    'djcelery',
 
     # list of booki apps
     'booki.editor',

lib/booktype/apps/convert/tasks.py

@@ -21,6 +21,7 @@
 
 from uuid import uuid4
 from celery import group
+from celery.result import allow_join_result
 
 from booktype.convert import loader
 from booktype.convert.runner import run_conversion
@@ -93,6 +94,8 @@ def convert(request_data, base_path):
         }
     """
 
+    # TODO we should use a chain of tasks
+
     assets = AssetCollection(base_path)
 
     assets.add_urls(request_data.assets)
@@ -118,9 +121,13 @@ def convert(request_data, base_path):
         )
         subtasks.append(subtask)
 
-    job = group(subtasks)
+    job = group(subtasks, disable_sync_subtasks=False)
     result = job.apply_async()
-    result.join(propagate=False)
+
+    # TODO we should use chain here
+    # http://docs.celeryq.org/en/latest/userguide/tasks.html#task-synchronous-subtasks
+    with allow_join_result():
+        result.join(propagate=False)
 
     subtasks_info = {async.task_id: async for async in result.children}
     celery.current_task.update_state(state="PROGRESS", meta=subtasks_info)

lib/booktype/skeleton/base_settings.py.original

@@ -2,9 +2,6 @@
 import os
 from unipath import Path
 
-import djcelery
-djcelery.setup_loader()
-
 # DJANGO ADMIN
 ADMINS = (
     # ('Your Name', 'your_email@domain.com'),
@@ -264,8 +261,8 @@ INSTALLED_APPS = (
     'django.contrib.messages',
     'django.contrib.staticfiles',
 
+    'django_celery_results',
     'compressor',
-    'djcelery',
     'rest_framework',
     'rest_framework.authtoken',
     'rest_framework_swagger',
@@ -327,8 +324,11 @@ REST_FRAMEWORK = {
 }
 
 # CELERY
-BROKER_URL = os.environ.get('BOOKTYPE_BROKER_URL', 'amqp://guest:guest@localhost:5672/')
+CELERY_BROKER_URL = os.environ.get('BOOKTYPE_BROKER_URL', 'amqp://guest:guest@localhost:5672/')
 CELERY_ACCEPT_CONTENT = ['pickle', 'json', 'msgpack', 'yaml']
+CELERY_TASK_SERIALIZER = 'pickle'
+CELERY_RESULT_SERIALIZER = 'pickle'
+CELERY_RESULT_BACKEND = 'django-db'
 
 # set of default roles and corresponding permissions
 BOOKTYPE_DEFAULT_ROLES = {

lib/booktype/skeleton/prod_settings.py.original

@@ -41,7 +41,7 @@ REDIS_PASSWORD = os.environ.get('BOOKTYPE_REDIS_PASSWORD')
 REDIS_DB = os.environ.get('BOOKTYPE_REDIS_DB', 0)
 
 # CELERY
-BROKER_URL = os.environ.get('BOOKTYPE_BROKER_URL', 'amqp://guest:guest@localhost:5672/')
+CELERY_BROKER_URL = os.environ.get('BOOKTYPE_BROKER_URL', 'amqp://guest:guest@localhost:5672/')
 
 # DATABASE
 DATABASES = {

lib/booktype/skeleton/stage_settings.py.original

@@ -41,7 +41,7 @@ REDIS_PASSWORD = os.environ.get('BOOKTYPE_REDIS_PASSWORD')
 REDIS_DB = os.environ.get('BOOKTYPE_REDIS_DB', 0)
 
 # CELERY
-BROKER_URL = os.environ.get('BOOKTYPE_BROKER_URL', 'amqp://guest:guest@localhost:5672/')
+CELERY_BROKER_URL = os.environ.get('BOOKTYPE_BROKER_URL', 'amqp://guest:guest@localhost:5672/')
 
 # DATABASE
 DATABASES = {

requirements/_base.txt

@@ -1,7 +1,7 @@
 Django==1.11
 django-compressor==2.0
-celery>=3.1.18,<3.2
-django-celery==3.1.17
+celery==4.1.1
+django-celery-results==1.0.1
 Unidecode
 lxml
 Pillow==2.9.0

scripts/createbooktype

@@ -21,7 +21,6 @@ import sys
 import os
 import argparse
 import string
-import unipath
 import random
 
 
@@ -193,17 +192,6 @@ def check_redis_available():
         logln("[OK]")
 
 
-def check_south_available():
-    log("+ Trying to import South module.  ")
-
-    try:
-        import south
-    except ImportError:
-        raise InstallError()
-    else:
-        logln("[OK]")
-
-
 def check_unidecode_available():
     log("+ Trying to import Unidecode module.  ")
 
@@ -588,10 +576,6 @@ sys.path.insert(0, '%(booki_path)s/')
 
 os.environ["DJANGO_SETTINGS_MODULE"] = "%(project_name)s_site.settings.%(profile)s"
 
-# Initialise celery
-import djcelery
-djcelery.setup_loader()
-
 # This application object is used by any WSGI server configured to use this
 # file. This includes Django's development server, if the WSGI_APPLICATION
 # setting points here.
@@ -609,6 +593,71 @@ application = get_wsgi_application()
     else:
         logln("[OK]")
 
+def include_celery(destination):
+    log("+ Import celery in {}_site __init__ file".format(get_project_name(destination)))
+    try:
+        f = open('%s/%s_site/__init__.py' % (destination, get_project_name(destination)), 'wt')
+        f.write('''
+from __future__ import absolute_import, unicode_literals
+
+# TODO we should have ability to change profile mode for celery
+# in general we need ENV-variables driven settings, but it will require bigger changes
+from .celery import app as celery_app
+
+__all__ = ['celery_app']
+
+    ''')
+        f.close()
+    except OSError:
+        raise InstallError()
+    else:
+        logln("[OK]")
+
+
+def create_celery(destination):
+    log("+ Creating celery.py file.  ".format(profile))
+
+    import booki
+    booki_path = os.path.abspath(os.path.dirname(booki.__file__) + '/..')
+
+    try:
+        f = open('%s/%s_site/celery.py' % (destination, get_project_name(destination)), 'wt')
+        f.write('''
+from __future__ import absolute_import, unicode_literals
+
+import os
+import sys
+from unipath import Path
+from celery import Celery
+
+BASE_DIR = Path(os.path.abspath(__file__)).ancestor(2)
+
+sys.path.insert(0, BASE_DIR)
+sys.path.insert(0, BASE_DIR.child("lib"))
+sys.path.insert(0, '{booki_path}')
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', '{project_name}_site.settings.dev')
+
+app = Celery('celery_booktype')
+
+# Using a string here means the worker doesn't have to serialize
+# the configuration object to child processes.
+# - namespace='CELERY' means all celery-related configuration keys
+#   should have a `CELERY_` prefix.
+app.config_from_object('django.conf:settings', namespace='CELERY')
+
+# Load task modules from all registered Django app configs.
+app.autodiscover_tasks()
+    '''.format(booki_path=booki_path, project_name=get_project_name(destination)))
+        f.close()
+    except OSError:
+        raise InstallError()
+    else:
+        logln("[OK]")
+
+
+
+
 
 def create_supervisor(destination, profile, virtual_env):
     log("+ Creating supervisor_{}.conf file.".format(profile))
@@ -902,7 +951,6 @@ if __name__ == '__main__':
             check_lxml_available()
             check_pil_available()
             check_redis_available()
-            # check_south_available()
             check_unidecode_available()
 
         # show info about user id, group id and etc...
@@ -954,10 +1002,14 @@ if __name__ == '__main__':
         # create urls
         create_urls(project_destination)
 
-        # create wsgi file
+        # create wsgi files
         for profile in PROFILES:
             create_wsgi(project_destination, profile, args.virtual_env)
 
+        # create celery files
+        include_celery(project_destination)
+        create_celery(project_destination)
+
         # create supervisor file for wsgi (stage and prod only)
         for profile in PROFILES[1:]:
             create_supervisor(project_destination, profile, args.virtual_env)

tests/func_settings.py

@@ -101,7 +101,7 @@
     'django.contrib.messages',
     'django.contrib.staticfiles',
 
-    'djcelery',
+    'django_celery_results',
     'compressor',
 
     # list of booki apps

tests/selenium_settings.py

@@ -139,7 +139,7 @@
     'django.contrib.staticfiles',
 
     'compressor',
-    'djcelery',
+    'django_celery_results',
 
     # list of booki apps
     'booki.editor',

tests/settings.py

@@ -93,7 +93,7 @@
     'django.contrib.messages',
     'django.contrib.staticfiles',
 
-    'djcelery',
+    'django_celery_results',
 
     # list of booki apps
     'booki.editor',