Initial commit to support Celery 4.3.0

Merges 55e9b2263c from versions/next
with code from GitLab issue #594 and GitLab merge request !55.

Thanks to Jakob Haufe (@sur5r) and Jesaja Everling (@jeverling)
for much of the research and code updates.

Signed-off-by: Roberto Rosario <roberto.rosario@mayan-edms.com>

mayan/apps/platform/classes.py

@@ -140,9 +140,9 @@ class PlatformTemplateSupervisord(PlatformTemplate):
             environment_name='MAYAN_ALLOWED_HOSTS'
         ),
         YAMLVariable(
-            name='BROKER_URL',
+            name='CELERY_BROKER_URL',
             default='redis://127.0.0.1:6379/0',
-            environment_name='MAYAN_BROKER_URL'
+            environment_name='MAYAN_CELERY_BROKER_URL'
         ),
         YAMLVariable(
             name='CELERY_RESULT_BACKEND',

mayan/apps/platform/templates/platform/supervisord.tmpl

@@ -1,11 +1,11 @@
 [supervisord]
 environment=
-    PYTHONPATH={{ INSTALLATION_PATH }}/lib/python2.7/site-packages:{{ MEDIA_ROOT }}/mayan_settings,
+    PYTHONPATH={{ INSTALLATION_PATH }}/lib/python3.7/site-packages:{{ MEDIA_ROOT }}/mayan_settings,
     DJANGO_SETTINGS_MODULE=mayan.settings.production,
     MAYAN_MEDIA_ROOT="{{ MEDIA_ROOT }}",
     MAYAN_ALLOWED_HOSTS="{{ ALLOWED_HOSTS }}",
     MAYAN_CELERY_RESULT_BACKEND="{{ CELERY_RESULT_BACKEND }}",
-    MAYAN_BROKER_URL="{{ BROKER_URL }}",
+    MAYAN_CELERY_BROKER_URL="{{ CELERY_BROKER_URL }}",
     MAYAN_DATABASES="{{ DATABASES }}"
 
 [program:mayan-gunicorn]
@@ -18,7 +18,7 @@ user = mayan
 [program:mayan-worker-{{ worker.name }}]
 autorestart = true
 autostart = true
-command = nice -n {{ worker.nice_level }} {{ INSTALLATION_PATH }}/bin/mayan-edms.py celery worker -Ofair -l ERROR -Q {% for queue in worker.queues %}{{ queue.name }}{% if not forloop.last %},{% endif %}{% endfor %} -n mayan-worker-{{ worker.name }}.%%h --concurrency=1
+command = nice -n {{ worker.nice_level }} {{ INSTALLATION_PATH }}/bin/celery worker -A mayan -Ofair -l ERROR -Q {% for queue in worker.queues %}{{ queue.name }}{% if not forloop.last %},{% endif %}{% endfor %} -n mayan-worker-{{ worker.name }}.%%h --concurrency=1
 killasgroup = true
 numprocs = 1
 priority = 998
@@ -30,7 +30,7 @@ user = mayan
 [program:mayan-celery-beat]
 autorestart = true
 autostart = true
-command = nice -n 1 {{ INSTALLATION_PATH }}/bin/mayan-edms.py celery beat --pidfile= -l ERROR
+command = nice -n 1 {{ INSTALLATION_PATH }}/bin/celery beat -A mayan --pidfile= -l ERROR
 killasgroup = true
 numprocs = 1
 priority = 998

mayan/apps/platform/templates/platform/supervisord_docker.tmpl

@@ -1,3 +1,7 @@
+[supervisord]
+environment=
+    DJANGO_SETTINGS_MODULE="%(ENV_MAYAN_SETTINGS_MODULE)s"
+
 [program:mayan-gunicorn]
 autorestart = false
 autostart = true
@@ -12,7 +16,7 @@ user = mayan
 [program:redis]
 autorestart = false
 autostart = true
-command = /bin/bash -c "if [ ${MAYAN_BROKER_URL} == ${MAYAN_DEFAULT_BROKER_URL} ] && [ ${MAYAN_CELERY_RESULT_BACKEND} == ${MAYAN_DEFAULT_CELERY_RESULT_BACKEND} ];then /usr/bin/redis-server /etc/redis/;fi"
+command = /bin/bash -c "if [ ${MAYAN_CELERY_BROKER_URL} == ${MAYAN_DEFAULT_CELERY_BROKER_URL} ] && [ ${MAYAN_CELERY_RESULT_BACKEND} == ${MAYAN_DEFAULT_CELERY_RESULT_BACKEND} ];then /usr/bin/redis-server /etc/redis/;fi"
 stderr_logfile = /dev/fd/2
 stderr_logfile_maxbytes = 0
 stdout_logfile = /dev/fd/1
@@ -23,7 +27,7 @@ user = root
 [program:mayan-worker-{{ worker.name }}]
 autorestart = false
 autostart = true
-command = nice -n {{ worker.nice_level }} /bin/bash -c "${MAYAN_BIN} celery --settings=${MAYAN_SETTINGS_MODULE} worker -Ofair -l ERROR -Q {% for queue in worker.queues %}{{ queue.name }}{% if not forloop.last %},{% endif %}{% endfor %} -n mayan-worker-{{ worker.name }}.%%h ${MAYAN_WORKER_{{ worker.name|upper }}_CONCURRENCY}"
+command = nice -n {{ worker.nice_level }} /bin/bash -c "${MAYAN_PYTHON_BIN_DIR}celery worker -A mayan -Ofair -l ERROR -Q {% for queue in worker.queues %}{{ queue.name }}{% if not forloop.last %},{% endif %}{% endfor %} -n mayan-worker-{{ worker.name }}.%%h ${MAYAN_WORKER_{{ worker.name|upper }}_CONCURRENCY}"
 killasgroup = true
 numprocs = 1
 priority = 998
@@ -39,7 +43,7 @@ user = mayan
 [program:mayan-celery-beat]
 autorestart = false
 autostart = true
-command = nice -n 1 /bin/bash -c "${MAYAN_BIN} celery --settings=${MAYAN_SETTINGS_MODULE} beat --pidfile= -l ERROR"
+command = nice -n 1 /bin/bash -c "${MAYAN_PYTHON_BIN_DIR}celery -A mayan beat --pidfile= -l ERROR"
 killasgroup = true
 numprocs = 1
 priority = 998