Initial commit to support Celery 4.3.0

Merges 55e9b2263c from versions/next
with code from GitLab issue #594 and GitLab merge request !55.

Thanks to Jakob Haufe (@sur5r) and Jesaja Everling (@jeverling)
for much of the research and code updates.

Signed-off-by: Roberto Rosario <roberto.rosario@mayan-edms.com>

mayan/apps/common/dependencies.py

@@ -117,31 +117,39 @@ PythonDependency(
         Celery under the GPL license.  The BSD license, unlike the GPL,
         let you distribute a modified version without making your
         changes open source.
-    ''', module=__name__, name='celery', version_string='==3.1.24'
+    ''', module=__name__, name='celery', version_string='==4.3.0'
 )
 PythonDependency(
     copyright_text='''
-        Copyright (c) 2012-2013 GoPivotal, Inc.  All Rights Reserved.
+        Copyright (c) 2015-2016 Ask Solem.  All Rights Reserved.
+        Copyright (c) 2012-2014 GoPivotal, Inc.  All Rights Reserved.
         Copyright (c) 2009-2012 Ask Solem.  All Rights Reserved.
-        All rights reserved.
+
+        django-celery-beat is licensed under The BSD License (3 Clause, also known as
+        the new BSD license).  The license is an OSI approved Open Source
+        license and is GPL-compatible(1).
+
+        The license text can also be found here:
+        http://www.opensource.org/licenses/BSD-3-Clause
+
+        License
+        =======
 
         Redistribution and use in source and binary forms, with or without
         modification, are permitted provided that the following conditions are met:
-
-        * Redistributions of source code must retain the above copyright notice,
-        this list of conditions and the following disclaimer.
-        * Redistributions in binary form must reproduce the above copyright
-        notice, this list of conditions and the following disclaimer in the
-        documentation and/or other materials provided with the distribution.
-
-        Neither the name of Ask Solem nor the names of its contributors may be used
-        to endorse or promote products derived from this software without specific
-        prior written permission.
+            * Redistributions of source code must retain the above copyright
+              notice, this list of conditions and the following disclaimer.
+            * Redistributions in binary form must reproduce the above copyright
+              notice, this list of conditions and the following disclaimer in the
+              documentation and/or other materials provided with the distribution.
+            * Neither the name of Ask Solem nor the
+              names of its contributors may be used to endorse or promote products
+              derived from this software without specific prior written permission.
 
         THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
         AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
         THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-        PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
+        PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL Ask Solem OR CONTRIBUTORS
         BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
         CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
         SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
@@ -149,7 +157,25 @@ PythonDependency(
         CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
         ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
         POSSIBILITY OF SUCH DAMAGE.
-    ''', module=__name__, name='django-celery', version_string='==3.2.1'
+
+        Documentation License
+        =====================
+
+        The documentation portion of django-celery-beat (the rendered contents of the
+        "docs" directory of a software distribution or checkout) is supplied
+        under the "Creative Commons Attribution-ShareAlike 4.0
+        International" (CC BY-SA 4.0) License as described by
+        http://creativecommons.org/licenses/by-sa/4.0/
+
+        Footnotes
+        =========
+        (1) A GPL-compatible license makes it possible to
+            combine django-celery-beat with other software that is released
+            under the GPL, it does not mean that we're distributing
+            django-celery-beat under the GPL license.  The BSD license, unlike the GPL,
+            let you distribute a modified version without making your
+            changes open source.
+    ''', module=__name__, name='django-celery-beat', version_string='==1.5.0'
 )
 PythonDependency(
     module=__name__, name='django-downloadview', version_string='==1.9'

mayan/apps/mayan_statistics/classes.py

@@ -107,7 +107,7 @@ class Statistic(object):
             day_of_month=day_of_month, month_of_year=month_of_year,
         )
 
-        app.conf.CELERYBEAT_SCHEDULE.update(
+        app.conf.beat_schedule.update(
             {
                 self.get_task_name(): {
                     'task': task_execute_statistic.dotted_path,
@@ -117,7 +117,7 @@ class Statistic(object):
             }
         )
 
-        app.conf.CELERY_ROUTES.update(
+        app.conf.task_routes.update(
             {
                 self.get_task_name(): {
                     'queue': queue_statistics.name

mayan/apps/platform/classes.py

@@ -140,9 +140,9 @@ class PlatformTemplateSupervisord(PlatformTemplate):
             environment_name='MAYAN_ALLOWED_HOSTS'
         ),
         YAMLVariable(
-            name='BROKER_URL',
+            name='CELERY_BROKER_URL',
             default='redis://127.0.0.1:6379/0',
-            environment_name='MAYAN_BROKER_URL'
+            environment_name='MAYAN_CELERY_BROKER_URL'
         ),
         YAMLVariable(
             name='CELERY_RESULT_BACKEND',

mayan/apps/platform/templates/platform/supervisord.tmpl

@@ -1,11 +1,11 @@
 [supervisord]
 environment=
-    PYTHONPATH={{ INSTALLATION_PATH }}/lib/python2.7/site-packages:{{ MEDIA_ROOT }}/mayan_settings,
+    PYTHONPATH={{ INSTALLATION_PATH }}/lib/python3.7/site-packages:{{ MEDIA_ROOT }}/mayan_settings,
     DJANGO_SETTINGS_MODULE=mayan.settings.production,
     MAYAN_MEDIA_ROOT="{{ MEDIA_ROOT }}",
     MAYAN_ALLOWED_HOSTS="{{ ALLOWED_HOSTS }}",
     MAYAN_CELERY_RESULT_BACKEND="{{ CELERY_RESULT_BACKEND }}",
-    MAYAN_BROKER_URL="{{ BROKER_URL }}",
+    MAYAN_CELERY_BROKER_URL="{{ CELERY_BROKER_URL }}",
     MAYAN_DATABASES="{{ DATABASES }}"
 
 [program:mayan-gunicorn]
@@ -18,7 +18,7 @@ user = mayan
 [program:mayan-worker-{{ worker.name }}]
 autorestart = true
 autostart = true
-command = nice -n {{ worker.nice_level }} {{ INSTALLATION_PATH }}/bin/mayan-edms.py celery worker -Ofair -l ERROR -Q {% for queue in worker.queues %}{{ queue.name }}{% if not forloop.last %},{% endif %}{% endfor %} -n mayan-worker-{{ worker.name }}.%%h --concurrency=1
+command = nice -n {{ worker.nice_level }} {{ INSTALLATION_PATH }}/bin/celery worker -A mayan -Ofair -l ERROR -Q {% for queue in worker.queues %}{{ queue.name }}{% if not forloop.last %},{% endif %}{% endfor %} -n mayan-worker-{{ worker.name }}.%%h --concurrency=1
 killasgroup = true
 numprocs = 1
 priority = 998
@@ -30,7 +30,7 @@ user = mayan
 [program:mayan-celery-beat]
 autorestart = true
 autostart = true
-command = nice -n 1 {{ INSTALLATION_PATH }}/bin/mayan-edms.py celery beat --pidfile= -l ERROR
+command = nice -n 1 {{ INSTALLATION_PATH }}/bin/celery beat -A mayan --pidfile= -l ERROR
 killasgroup = true
 numprocs = 1
 priority = 998

mayan/apps/platform/templates/platform/supervisord_docker.tmpl

@@ -1,3 +1,7 @@
+[supervisord]
+environment=
+    DJANGO_SETTINGS_MODULE="%(ENV_MAYAN_SETTINGS_MODULE)s"
+
 [program:mayan-gunicorn]
 autorestart = false
 autostart = true
@@ -12,7 +16,7 @@ user = mayan
 [program:redis]
 autorestart = false
 autostart = true
-command = /bin/bash -c "if [ ${MAYAN_BROKER_URL} == ${MAYAN_DEFAULT_BROKER_URL} ] && [ ${MAYAN_CELERY_RESULT_BACKEND} == ${MAYAN_DEFAULT_CELERY_RESULT_BACKEND} ];then /usr/bin/redis-server /etc/redis/;fi"
+command = /bin/bash -c "if [ ${MAYAN_CELERY_BROKER_URL} == ${MAYAN_DEFAULT_CELERY_BROKER_URL} ] && [ ${MAYAN_CELERY_RESULT_BACKEND} == ${MAYAN_DEFAULT_CELERY_RESULT_BACKEND} ];then /usr/bin/redis-server /etc/redis/;fi"
 stderr_logfile = /dev/fd/2
 stderr_logfile_maxbytes = 0
 stdout_logfile = /dev/fd/1
@@ -23,7 +27,7 @@ user = root
 [program:mayan-worker-{{ worker.name }}]
 autorestart = false
 autostart = true
-command = nice -n {{ worker.nice_level }} /bin/bash -c "${MAYAN_BIN} celery --settings=${MAYAN_SETTINGS_MODULE} worker -Ofair -l ERROR -Q {% for queue in worker.queues %}{{ queue.name }}{% if not forloop.last %},{% endif %}{% endfor %} -n mayan-worker-{{ worker.name }}.%%h ${MAYAN_WORKER_{{ worker.name|upper }}_CONCURRENCY}"
+command = nice -n {{ worker.nice_level }} /bin/bash -c "${MAYAN_PYTHON_BIN_DIR}celery worker -A mayan -Ofair -l ERROR -Q {% for queue in worker.queues %}{{ queue.name }}{% if not forloop.last %},{% endif %}{% endfor %} -n mayan-worker-{{ worker.name }}.%%h ${MAYAN_WORKER_{{ worker.name|upper }}_CONCURRENCY}"
 killasgroup = true
 numprocs = 1
 priority = 998
@@ -39,7 +43,7 @@ user = mayan
 [program:mayan-celery-beat]
 autorestart = false
 autostart = true
-command = nice -n 1 /bin/bash -c "${MAYAN_BIN} celery --settings=${MAYAN_SETTINGS_MODULE} beat --pidfile= -l ERROR"
+command = nice -n 1 /bin/bash -c "${MAYAN_PYTHON_BIN_DIR}celery -A mayan beat --pidfile= -l ERROR"
 killasgroup = true
 numprocs = 1
 priority = 998

mayan/apps/smart_settings/utils.py

@@ -34,7 +34,11 @@ def get_environment_variables():
 
 
 def get_environment_setting(name, fallback_default=None):
-    value = os.environ.get('MAYAN_{}'.format(name), get_default(name=name, fallback_default=fallback_default))
+    value = os.environ.get(
+        'MAYAN_{}'.format(name), get_default(
+            name=name, fallback_default=fallback_default
+        )
+    )
 
     if value:
         return yaml.load(stream=value, Loader=SafeLoader)

mayan/apps/sources/models/base.py

@@ -7,7 +7,7 @@ from django.db import models, transaction
 from django.utils.encoding import force_text, python_2_unicode_compatible
 from django.utils.translation import ugettext_lazy as _
 
-from djcelery.models import PeriodicTask, IntervalSchedule
+from django_celery_beat.models import PeriodicTask, IntervalSchedule
 from model_utils.managers import InheritanceManager
 
 from mayan.apps.common.compressed_files import Archive

mayan/apps/task_manager/classes.py

@@ -156,13 +156,13 @@ class CeleryQueue(object):
         if self.transient:
             kwargs['delivery_mode'] = 1
 
-        celery_app.conf.CELERY_QUEUES.append(Queue(**kwargs))
+        celery_app.conf.task_queues.append(Queue(**kwargs))
 
         if self.default_queue:
-            celery_app.conf.CELERY_DEFAULT_QUEUE = self.name
+            celery_app.conf.task_default_queue = self.name
 
         for task_type in self.task_types:
-            celery_app.conf.CELERY_ROUTES.update(
+            celery_app.conf.task_routes.update(
                 {
                     task_type.dotted_path: {
                         'queue': self.name
@@ -171,7 +171,7 @@ class CeleryQueue(object):
             )
 
             if task_type.schedule:
-                celery_app.conf.CELERYBEAT_SCHEDULE.update(
+                celery_app.conf.beat_schedule.update(
                     {
                         task_type.name: {
                             'task': task_type.dotted_path,

mayan/apps/task_manager/settings.py

@@ -10,7 +10,7 @@ __all__ = ()
 namespace = Namespace(label=_('Celery'), name='celery')
 
 setting_celery_broker_url = namespace.add_setting(
-    global_name='BROKER_URL', default=None,
+    global_name='CELERY_BROKER_URL', default=None,
     help_text=_(
         'Default: "amqp://". Default broker URL. This must be a URL in '
         'the form of: transport://userid:password@hostname:port/virtual_host '