Celery: Update Celery to version 4.1.1

Upgrade Celery version used from 3.1.26 to 4.1.1. The following settings have been renamed: CELERY_ALWAYS_EAGER to CELERY_TASK_ALWAYS_EAGER, BROKER_URL to CELERY_BROKER_URL. Signed-off-by: Roberto Rosario <Roberto.Rosario@mayan-edms.com>
2018-12-08 22:47:15 -04:00
parent 034e0668f4
commit 55e9b2263c
33 changed files with 88 additions and 89 deletions
--- a/HISTORY.rst
+++ b/HISTORY.rst
@@ -172,6 +172,10 @@
  - Documents: 0042 to 0043
  - Tags: 0001 to 0008

+- Upgraded from Celery 3.1.26 to 4.1.1. The following settings have been 
+  renamed: CELERY_ALWAYS_EAGER to CELERY_TASK_ALWAYS_EAGER, 
+  BROKER_URL to CELERY_BROKER_URL.
+
 3.1.9 (2018-11-01)
 ==================
 - Convert the furl instance to text to allow serializing it into
--- a/2
+++ b/2
@@ -283,7 +283,7 @@ test-with-docker-frontend:
 	./manage.py runserver --settings=mayan.settings.staging.docker

 test-with-docker-worker:
-	./manage.py celery worker --settings=mayan.settings.staging.docker -B -l INFO -O fair
+	./manage.py celery worker --settings=mayan.settings.staging.docker -B -l INFO

 docker-mysql-on:
 	docker run -d --name mysql -p 3306:3306 -e MYSQL_ALLOW_EMPTY_PASSWORD=True -e MYSQL_DATABASE=mayan_edms mysql
--- a/contrib/scripts/install/dialog.sh
+++ b/contrib/scripts/install/dialog.sh
@@ -1452,8 +1452,8 @@ cat > /etc/supervisor/conf.d/mayan.conf <<EOF
 [supervisord]
 environment=
        MAYAN_ALLOWED_HOSTS="*", # Allow access to other network hosts other than localhost
+        MAYAN_CELERY_BROKER_URL="redis://127.0.0.1:6379/0",
        MAYAN_CELERY_RESULT_BACKEND="redis://127.0.0.1:6379/0",
-        MAYAN_BROKER_URL="redis://127.0.0.1:6379/0",
        PYTHONPATH=${MAYAN_INSTALLATION_FOLDER}/lib/python2.7/site-packages:$MAYAN_MEDIA_ROOT,
        MAYAN_DATABASE_ENGINE=django.db.backends.postgresql,
        MAYAN_DATABASE_HOST=127.0.0.1,
@@ -1473,7 +1473,7 @@ user = mayan
 [program:mayan-worker-fast]
 autorestart = true
 autostart = true
-command = nice -n 1 ${MAYAN_BIN} celery worker -Ofair -l ERROR -Q converter -n mayan-worker-fast.%%h --concurrency=1
+command = nice -n 1 ${MAYAN_BIN} celery worker -l ERROR -Q converter -n mayan-worker-fast.%%h --concurrency=1
 killasgroup = true
 numprocs = 1
 priority = 998
@@ -1484,7 +1484,7 @@ user = mayan
 [program:mayan-worker-medium]
 autorestart = true
 autostart = true
-command = nice -n 18 ${MAYAN_BIN} celery worker -Ofair -l ERROR -Q checkouts_periodic,documents_periodic,indexing,metadata,sources,sources_periodic,uploads,documents -n mayan-worker-medium.%%h --concurrency=1
+command = nice -n 18 ${MAYAN_BIN} celery worker -l ERROR -Q checkouts_periodic,documents_periodic,indexing,metadata,sources,sources_periodic,uploads,documents -n mayan-worker-medium.%%h --concurrency=1
 killasgroup = true
 numprocs = 1
 priority = 998
@@ -1495,7 +1495,7 @@ user = mayan
 [program:mayan-worker-slow]
 autorestart = true
 autostart = true
-command = nice -n 19 ${MAYAN_BIN} celery worker -Ofair -l ERROR -Q mailing,tools,statistics,parsing,ocr -n mayan-worker-slow.%%h --concurrency=1
+command = nice -n 19 ${MAYAN_BIN} celery worker -l ERROR -Q mailing,tools,statistics,parsing,ocr -n mayan-worker-slow.%%h --concurrency=1
 killasgroup = true
 numprocs = 1
 priority = 998
--- a/contrib/scripts/install/production.sh
+++ b/contrib/scripts/install/production.sh
@@ -133,7 +133,7 @@ EOF
 echo -e "\n -> Creating the supervisor file for the Celery worker, /etc/supervisor/conf.d/mayan-celery.conf \n"
 cat > /etc/supervisor/conf.d/mayan-celery.conf << EOF
 [program:mayan-worker]
-command = ${INSTALLATION_DIRECTORY}bin/python ${INSTALLATION_DIRECTORY}bin/mayan-edms.py celery --settings=mayan.settings.production worker -Ofair -l ERROR
+command = ${INSTALLATION_DIRECTORY}bin/python ${INSTALLATION_DIRECTORY}bin/mayan-edms.py celery --settings=mayan.settings.production worker -l ERROR
 directory = ${INSTALLATION_DIRECTORY}
 user = www-data
 stdout_logfile = /var/log/mayan/worker-stdout.log
--- a/docker/README.md
+++ b/docker/README.md
@@ -126,13 +126,13 @@ Defaults to `None`. This optional environment variable is used to set the hostna

 Defaults to `None`. This optional environment variable is used to set the port number to use when connecting to the database. An empty string means the default port. Not used with SQLite. For more information read the pertinent Django documentation page: [Settings, PORT](https://docs.djangoproject.com/en/1.11/ref/settings/#port)

-### `MAYAN_BROKER_URL`
+### `MAYAN_CELERY_BROKER_URL`

 Defaults to 'redis://127.0.0.1:6379/0'. This optional environment variable is determines the broker that Celery will use to relay task messages between the frontend code and the background workers. For more information read the pertinent Celery Kombu documentation page: [Broker URL](http://kombu.readthedocs.io/en/latest/userguide/connections.html#connection-urls)

 This Docker image supports using Redis and RabbitMQ as brokers.

-Caveat: If the `MAYAN_BROKER_URL` and `MAYAN_CELERY_RESULT_BACKEND` environment variables are specified, the built-in Redis server inside the container will be disabled.
+Caveat: If the `MAYAN_CELERY_BROKER_URL` and `MAYAN_CELERY_RESULT_BACKEND` environment variables are specified, the built-in Redis server inside the container will be disabled.

 ### `MAYAN_CELERY_RESULT_BACKEND`

@@ -140,7 +140,7 @@ Defaults to 'redis://127.0.0.1:6379/0'. This optional environment variable is de

 This Docker image supports using Redis and RabbitMQ as result backends.

-Caveat: If the `MAYAN_BROKER_URL` and `MAYAN_CELERY_RESULT_BACKEND` environment variables are specified, the built-in Redis server inside the container will be disabled.
+Caveat: If the `MAYAN_CELERY_BROKER_URL` and `MAYAN_CELERY_RESULT_BACKEND` environment variables are specified, the built-in Redis server inside the container will be disabled.

 ### `MAYAN_NGINX_CLIENT_MAX_BODY_SIZE`

--- a/docker/README.md.tmpl
+++ b/docker/README.md.tmpl
@@ -126,13 +126,13 @@ Defaults to `None`. This optional environment variable is used to set the hostna

 Defaults to `None`. This optional environment variable is used to set the port number to use when connecting to the database. An empty string means the default port. Not used with SQLite. For more information read the pertinent Django documentation page: [Settings, PORT](https://docs.djangoproject.com/en/1.11/ref/settings/#port)

-### `MAYAN_BROKER_URL`
+### `MAYAN_CELERY_BROKER_URL`

 Defaults to 'redis://127.0.0.1:6379/0'. This optional environment variable is determines the broker that Celery will use to relay task messages between the frontend code and the background workers. For more information read the pertinent Celery Kombu documentation page: [Broker URL](http://kombu.readthedocs.io/en/latest/userguide/connections.html#connection-urls)

 This Docker image supports using Redis and RabbitMQ as brokers.

-Caveat: If the `MAYAN_BROKER_URL` and `MAYAN_CELERY_RESULT_BACKEND` environment variables are specified, the built-in Redis server inside the container will be disabled.
+Caveat: If the `MAYAN_CELERY_BROKER_URL` and `MAYAN_CELERY_RESULT_BACKEND` environment variables are specified, the built-in Redis server inside the container will be disabled.

 ### `MAYAN_CELERY_RESULT_BACKEND`

@@ -140,7 +140,7 @@ Defaults to 'redis://127.0.0.1:6379/0'. This optional environment variable is de

 This Docker image supports using Redis and RabbitMQ as result backends.

-Caveat: If the `MAYAN_BROKER_URL` and `MAYAN_CELERY_RESULT_BACKEND` environment variables are specified, the built-in Redis server inside the container will be disabled.
+Caveat: If the `MAYAN_CELERY_BROKER_URL` and `MAYAN_CELERY_RESULT_BACKEND` environment variables are specified, the built-in Redis server inside the container will be disabled.

 ### `MAYAN_NGINX_CLIENT_MAX_BODY_SIZE`

--- a/docker/docker-compose-development.yml
+++ b/docker/docker-compose-development.yml
@@ -59,7 +59,7 @@ services:
      results:
        condition: service_healthy
    environment:
-      MAYAN_BROKER_URL: amqp://mayan:mayan@broker:5672/mayan
+      MAYAN_CELERY_BROKER_URL: amqp://mayan:mayan@broker:5672/mayan
      MAYAN_CELERY_RESULT_BACKEND: redis://results:6379/0
      MAYAN_DATABASE_ENGINE: django.db.backends.postgresql
      MAYAN_DATABASE_HOST: db
--- a/docker/docker-compose.yml
+++ b/docker/docker-compose.yml
@@ -45,7 +45,7 @@ services:
      results:
        condition: service_healthy
    environment:
-      MAYAN_BROKER_URL: amqp://mayan:mayan@broker:5672/mayan
+      MAYAN_CELERY_BROKER_URL: amqp://mayan:mayan@broker:5672/mayan
      MAYAN_CELERY_RESULT_BACKEND: redis://results:6379/0
      MAYAN_DATABASE_ENGINE: django.db.backends.postgresql
      MAYAN_DATABASE_HOST: db
--- a/docker/entrypoint.sh
+++ b/docker/entrypoint.sh
@@ -6,12 +6,12 @@ INSTALL_FLAG=/var/lib/mayan/system/SECRET_KEY
 CONCURRENCY_ARGUMENT=--concurrency=
 export DOCKER_ROOT=/opt/mayan-edms

-export MAYAN_DEFAULT_BROKER_URL=redis://127.0.0.1:6379/0
+export MAYAN_DEFAULT_CELERY_BROKER_URL=redis://127.0.0.1:6379/0
 export MAYAN_DEFAULT_CELERY_RESULT_BACKEND=redis://127.0.0.1:6379/0

 export MAYAN_ALLOWED_HOSTS='["*"]'
 export MAYAN_BIN=/opt/mayan-edms/bin/mayan-edms.py
-export MAYAN_BROKER_URL=${MAYAN_BROKER_URL:-${MAYAN_DEFAULT_BROKER_URL}}
+export MAYAN_CELERY_BROKER_URL=${MAYAN_CELERY_BROKER_URL:-${MAYAN_DEFAULT_CELERY_BROKER_URL}}
 export MAYAN_CELERY_RESULT_BACKEND=${MAYAN_CELERY_RESULT_BACKEND:-${MAYAN_DEFAULT_CELERY_RESULT_BACKEND}}
 export MAYAN_INSTALL_DIR=/opt/mayan-edms
 export MAYAN_PYTHON_BIN_DIR=/opt/mayan-edms/bin/
--- a/docker/etc/supervisor/mayan.conf
+++ b/docker/etc/supervisor/mayan.conf
@@ -12,7 +12,7 @@ user = mayan
 [program:redis]
 autorestart = false
 autostart = true
-command = /bin/bash -c "if [ ${MAYAN_BROKER_URL} == ${MAYAN_DEFAULT_BROKER_URL} ] && [ ${MAYAN_CELERY_RESULT_BACKEND} == ${MAYAN_DEFAULT_CELERY_RESULT_BACKEND} ];then /usr/bin/redis-server /etc/redis/;fi"
+command = /bin/bash -c "if [ ${MAYAN_CELERY_BROKER_URL} == ${MAYAN_DEFAULT_BROKER_URL} ] && [ ${MAYAN_CELERY_RESULT_BACKEND} == ${MAYAN_DEFAULT_CELERY_RESULT_BACKEND} ];then /usr/bin/redis-server /etc/redis/;fi"
 stderr_logfile = /dev/fd/2
 stderr_logfile_maxbytes = 0
 stdout_logfile = /dev/fd/1
@@ -22,7 +22,7 @@ user = root
 [program:mayan-worker-fast]
 autorestart = false
 autostart = true
-command = nice -n 1 /bin/bash -c "${MAYAN_BIN} celery --settings=${MAYAN_SETTINGS_MODULE} worker -Ofair -l ERROR -Q converter,document_states_fast,sources_fast -n mayan-worker-fast.%%h ${MAYAN_WORKER_FAST_CONCURRENCY}"
+command = nice -n 1 /bin/bash -c "${MAYAN_BIN} celery --settings=${MAYAN_SETTINGS_MODULE} worker -l ERROR -Q converter,document_states_fast,sources_fast -n mayan-worker-fast.%%h ${MAYAN_WORKER_FAST_CONCURRENCY}"
 killasgroup = true
 numprocs = 1
 priority = 998
@@ -37,7 +37,7 @@ user = mayan
 [program:mayan-worker-medium]
 autorestart = false
 autostart = true
-command = nice -n 18 /bin/bash -c "${MAYAN_BIN} celery --settings=${MAYAN_SETTINGS_MODULE} worker -Ofair -l ERROR -Q checkouts_periodic,documents_periodic,indexing,metadata,sources,sources_periodic,uploads,documents -n mayan-worker-medium.%%h ${MAYAN_WORKER_MEDIUM_CONCURRENCY}"
+command = nice -n 18 /bin/bash -c "${MAYAN_BIN} celery --settings=${MAYAN_SETTINGS_MODULE} worker -l ERROR -Q checkouts_periodic,documents_periodic,indexing,metadata,sources,sources_periodic,uploads,documents -n mayan-worker-medium.%%h ${MAYAN_WORKER_MEDIUM_CONCURRENCY}"
 killasgroup = true
 numprocs = 1
 priority = 998
@@ -52,7 +52,7 @@ user = mayan
 [program:mayan-worker-slow]
 autorestart = false
 autostart = true
-command = nice -n 19 /bin/bash -c "${MAYAN_BIN} celery --settings=${MAYAN_SETTINGS_MODULE} worker -Ofair -l ERROR -Q mailing,tools,statistics,parsing,ocr -n mayan-worker-slow.%%h ${MAYAN_WORKER_SLOW_CONCURRENCY}"
+command = nice -n 19 /bin/bash -c "${MAYAN_BIN} celery --settings=${MAYAN_SETTINGS_MODULE} worker -l ERROR -Q mailing,tools,statistics,parsing,ocr -n mayan-worker-slow.%%h ${MAYAN_WORKER_SLOW_CONCURRENCY}"
 killasgroup = true
 numprocs = 1
 priority = 998
--- a/docs/chapters/deploying.rst
+++ b/docs/chapters/deploying.rst
@@ -91,8 +91,8 @@ Create the supervisor file at ``/etc/supervisor/conf.d/mayan.conf``:
    [supervisord]
    environment=
        MAYAN_ALLOWED_HOSTS='["*"]',  # Allow access to other network hosts other than localhost
+        MAYAN_CELERY_BROKER_URL="redis://127.0.0.1:6379/0",
        MAYAN_CELERY_RESULT_BACKEND="redis://127.0.0.1:6379/0",
-        MAYAN_BROKER_URL="redis://127.0.0.1:6379/0",
        PYTHONPATH=/opt/mayan-edms/lib/python2.7/site-packages:/opt/mayan-edms/data,
        MAYAN_MEDIA_ROOT=/opt/mayan-edms/media,
        MAYAN_DATABASES='{default: {ENGINE: django.db.backends.postgresql, HOST: 127.0.0.1, NAME: mayan, PASSWORD: mayanuserpass, USER: mayan, CONN_MAX_AGE: 60}}',
@@ -107,7 +107,7 @@ Create the supervisor file at ``/etc/supervisor/conf.d/mayan.conf``:
    [program:mayan-worker-fast]
    autorestart = true
    autostart = true
-    command = nice -n 1 /opt/mayan-edms/bin/mayan-edms.py celery worker -Ofair -l ERROR -Q converter,document_states_fast,sources_fast -n mayan-worker-fast.%%h --concurrency=1
+    command = nice -n 1 /opt/mayan-edms/bin/mayan-edms.py celery worker -l ERROR -Q converter,document_states_fast,sources_fast -n mayan-worker-fast.%%h --concurrency=1
    killasgroup = true
    numprocs = 1
    priority = 998
@@ -118,7 +118,7 @@ Create the supervisor file at ``/etc/supervisor/conf.d/mayan.conf``:
    [program:mayan-worker-medium]
    autorestart = true
    autostart = true
-    command = nice -n 18 /opt/mayan-edms/bin/mayan-edms.py celery worker -Ofair -l ERROR -Q checkouts_periodic,documents_periodic,indexing,metadata,sources,sources_periodic,uploads,documents -n mayan-worker-medium.%%h --concurrency=1
+    command = nice -n 18 /opt/mayan-edms/bin/mayan-edms.py celery worker -l ERROR -Q checkouts_periodic,documents_periodic,indexing,metadata,sources,sources_periodic,uploads,documents -n mayan-worker-medium.%%h --concurrency=1
    killasgroup = true
    numprocs = 1
    priority = 998
@@ -129,7 +129,7 @@ Create the supervisor file at ``/etc/supervisor/conf.d/mayan.conf``:
    [program:mayan-worker-slow]
    autorestart = true
    autostart = true
-    command = nice -n 19 /opt/mayan-edms/bin/mayan-edms.py celery worker -Ofair -l ERROR -Q mailing,tools,statistics,parsing,ocr -n mayan-worker-slow.%%h --concurrency=1
+    command = nice -n 19 /opt/mayan-edms/bin/mayan-edms.py celery worker -l ERROR -Q mailing,tools,statistics,parsing,ocr -n mayan-worker-slow.%%h --concurrency=1
    killasgroup = true
    numprocs = 1
    priority = 998
@@ -260,8 +260,8 @@ Create the supervisor file at ``/etc/supervisor/conf.d/mayan.conf``:
    [supervisord]
    environment=
        MAYAN_ALLOWED_HOSTS='["*"]',  # Allow access to other network hosts other than localhost
+        MAYAN_CELERY_BROKER_URL="amqp://mayan:mayanrabbitmqpassword@localhost:5672/mayan",
        MAYAN_CELERY_RESULT_BACKEND="redis://127.0.0.1:6379/0",
-        MAYAN_BROKER_URL="amqp://mayan:mayanrabbitmqpassword@localhost:5672/mayan",
        PYTHONPATH=/opt/mayan-edms/lib/python2.7/site-packages:/opt/mayan-edms/data,
        MAYAN_MEDIA_ROOT=/opt/mayan-edms/media,
        MAYAN_DATABASES='{default: {ENGINE: django.db.backends.postgresql, HOST: 127.0.0.1, NAME: mayan, PASSWORD: mayanuserpass, USER: mayan, CONN_MAX_AGE: 60}}',
@@ -276,7 +276,7 @@ Create the supervisor file at ``/etc/supervisor/conf.d/mayan.conf``:
    [program:mayan-worker-fast]
    autorestart = true
    autostart = true
-    command = nice -n 1 /opt/mayan-edms/bin/mayan-edms.py celery worker -Ofair -l ERROR -Q converter,document_states_fast,sources_fast -n mayan-worker-fast.%%h
+    command = nice -n 1 /opt/mayan-edms/bin/mayan-edms.py celery worker -l ERROR -Q converter,document_states_fast,sources_fast -n mayan-worker-fast.%%h
    killasgroup = true
    numprocs = 1
    priority = 998
@@ -287,7 +287,7 @@ Create the supervisor file at ``/etc/supervisor/conf.d/mayan.conf``:
    [program:mayan-worker-medium]
    autorestart = true
    autostart = true
-    command = nice -n 18 /opt/mayan-edms/bin/mayan-edms.py celery worker -Ofair -l ERROR -Q checkouts_periodic,documents_periodic,indexing,metadata,sources,sources_periodic,uploads,documents -n mayan-worker-medium.%%h --concurrency=1
+    command = nice -n 18 /opt/mayan-edms/bin/mayan-edms.py celery worker -l ERROR -Q checkouts_periodic,documents_periodic,indexing,metadata,sources,sources_periodic,uploads,documents -n mayan-worker-medium.%%h --concurrency=1
    killasgroup = true
    numprocs = 1
    priority = 998
@@ -298,7 +298,7 @@ Create the supervisor file at ``/etc/supervisor/conf.d/mayan.conf``:
    [program:mayan-worker-slow]
    autorestart = true
    autostart = true
-    command = nice -n 19 /opt/mayan-edms/bin/mayan-edms.py celery worker -Ofair -l ERROR -Q mailing,tools,statistics,parsing,ocr -n mayan-worker-slow.%%h --concurrency=1
+    command = nice -n 19 /opt/mayan-edms/bin/mayan-edms.py celery worker -l ERROR -Q mailing,tools,statistics,parsing,ocr -n mayan-worker-slow.%%h --concurrency=1
    killasgroup = true
    numprocs = 1
    priority = 998
--- a/docs/chapters/docker.rst
+++ b/docs/chapters/docker.rst
@@ -55,7 +55,7 @@ the default port. Not used with SQLite. For more information read the
 pertinent Django documentation page:
 :django-docs:`Settings, PORT <ref/settings/#port>`

-``MAYAN_BROKER_URL``
+``MAYAN_CELERY_BROKER_URL``

 This optional environment variable determines the broker that Celery will use
 to relay task messages between the frontend code and the background workers.
@@ -65,7 +65,7 @@ For more information read the pertinent Celery Kombu documentation page: `Broker

 This Docker image supports using Redis and RabbitMQ as brokers.

-Caveat: If the `MAYAN_BROKER_URL` and `MAYAN_CELERY_RESULT_BACKEND` environment
+Caveat: If the `MAYAN_CELERY_BROKER_URL` and `MAYAN_CELERY_RESULT_BACKEND` environment
 variables are specified, the built-in Redis server inside the container will
 be disabled.

@@ -80,7 +80,7 @@ code. For more information read the pertinent Celery Kombu documentation page:

 This Docker image supports using Redis and RabbitMQ as result backends.

-Caveat: If the `MAYAN_BROKER_URL` and `MAYAN_CELERY_RESULT_BACKEND` environment
+Caveat: If the `MAYAN_CELERY_BROKER_URL` and `MAYAN_CELERY_RESULT_BACKEND` environment
 variables are specified, the built-in Redis server inside the container will
 be disabled.

--- a/docs/chapters/scaling_up.rst
+++ b/docs/chapters/scaling_up.rst
@@ -116,11 +116,11 @@ For the Docker image, launch a separate RabbitMQ container

    docker run -d --name mayan-edms-rabbitmq -e RABBITMQ_DEFAULT_USER=mayan -e RABBITMQ_DEFAULT_PASS=mayanrabbitmqpassword -e RABBITMQ_DEFAULT_VHOST=mayan rabbitmq:3

-Pass the MAYAN_BROKER_URL environment variable (https://kombu.readthedocs.io/en/latest/userguide/connections.html#connection-urls)
+Pass the MAYAN_CELERY_BROKER_URL environment variable (https://kombu.readthedocs.io/en/latest/userguide/connections.html#connection-urls)
 to the Mayan EDMS container so that it uses the RabbitMQ container the
 message broker::

-    -e MAYAN_BROKER_URL="amqp://mayan:mayanrabbitmqpassword@localhost:5672/mayan",
+    -e MAYAN_CELERY_BROKER_URL="amqp://mayan:mayanrabbitmqpassword@localhost:5672/mayan",

 When tasks finish, they leave behind a return status or the result of a
 calculation, these are stored for a while so that whoever requested the
--- a/mayan/apps/checkouts/apps.py
+++ b/mayan/apps/checkouts/apps.py
@@ -95,7 +95,7 @@ class CheckoutsApp(MayanAppConfig):
            )
        )

-        app.conf.CELERYBEAT_SCHEDULE.update(
+        app.conf.beat_schedule.update(
            {
                'task_check_expired_check_outs': {
                    'task': 'mayan.apps.checkouts.tasks.task_check_expired_check_outs',
@@ -106,14 +106,14 @@ class CheckoutsApp(MayanAppConfig):
            }
        )

-        app.conf.CELERY_QUEUES.append(
+        app.conf.task_queues.append(
            Queue(
                'checkouts_periodic', Exchange('checkouts_periodic'),
                routing_key='checkouts_periodic', delivery_mode=1
            ),
        )

-        app.conf.CELERY_ROUTES.update(
+        app.conf.task_routes.update(
            {
                'mayan.apps.checkouts.tasks.task_check_expired_check_outs': {
                    'queue': 'checkouts_periodic'
--- a/mayan/apps/common/apps.py
+++ b/mayan/apps/common/apps.py
@@ -92,7 +92,7 @@ class CommonApp(MayanAppConfig):
            name='main_menu', template_name='appearance/main_menu.html'
        )

-        app.conf.CELERYBEAT_SCHEDULE.update(
+        app.conf.beat_schedule.update(
            {
                'task_delete_stale_uploads': {
                    'task': 'mayan.apps.common.tasks.task_delete_stale_uploads',
@@ -103,7 +103,7 @@ class CommonApp(MayanAppConfig):
            }
        )

-        app.conf.CELERY_QUEUES.extend(
+        app.conf.task_queues.extend(
            (
                Queue('default', Exchange('default'), routing_key='default'),
                Queue('tools', Exchange('tools'), routing_key='tools'),
@@ -114,9 +114,9 @@ class CommonApp(MayanAppConfig):
            )
        )

-        app.conf.CELERY_DEFAULT_QUEUE = 'default'
+        app.conf.task_default_queue = 'default'

-        app.conf.CELERY_ROUTES.update(
+        app.conf.task_routes.update(
            {
                'mayan.apps.common.tasks.task_delete_stale_uploads': {
                    'queue': 'common_periodic'
--- a/mayan/apps/common/settings.py
+++ b/mayan/apps/common/settings.py
@@ -405,8 +405,8 @@ setting_django_wsgi_application = namespace.add_setting(
 namespace = Namespace(name='celery', label=_('Celery'))

 setting_celery_always_eager = namespace.add_setting(
-    global_name='CELERY_ALWAYS_EAGER',
-    default=settings.CELERY_ALWAYS_EAGER,
+    global_name='CELERY_TASK_ALWAYS_EAGER',
+    default=settings.CELERY_TASK_ALWAYS_EAGER,
    help_text=_(
        'If this is True, all tasks will be executed locally by blocking '
        'until the task returns. apply_async() and Task.delay() will return '
@@ -417,7 +417,7 @@ setting_celery_always_eager = namespace.add_setting(
    )
 )
 setting_celery_broker_url = namespace.add_setting(
-    global_name='BROKER_URL', default=settings.BROKER_URL,
+    global_name='CELERY_BROKER_URL', default=settings.CELERY_BROKER_URL,
    help_text=_(
        'Default: "amqp://". Default broker URL. This must be a URL in '
        'the form of: transport://userid:password@hostname:port/virtual_host '
--- a/mayan/apps/document_indexing/apps.py
+++ b/mayan/apps/document_indexing/apps.py
@@ -161,11 +161,11 @@ class DocumentIndexingApp(MayanAppConfig):
            )
        )

-        app.conf.CELERY_QUEUES.append(
+        app.conf.task_queues.append(
            Queue('indexing', Exchange('indexing'), routing_key='indexing'),
        )

-        app.conf.CELERY_ROUTES.update(
+        app.conf.task_routes.update(
            {
                'mayan.apps.document_indexing.tasks.task_delete_empty': {
                    'queue': 'indexing'
--- a/mayan/apps/document_parsing/apps.py
+++ b/mayan/apps/document_parsing/apps.py
@@ -143,11 +143,11 @@ class DocumentParsingApp(MayanAppConfig):
            attribute='result'
        )

-        app.conf.CELERY_QUEUES.append(
+        app.conf.task_queues.append(
            Queue('parsing', Exchange('parsing'), routing_key='parsing'),
        )

-        app.conf.CELERY_ROUTES.update(
+        app.conf.task_routes.update(
            {
                'mayan.apps.document_parsing.tasks.task_parse_document_version': {
                    'queue': 'parsing'
--- a/mayan/apps/document_signatures/apps.py
+++ b/mayan/apps/document_signatures/apps.py
@@ -101,13 +101,13 @@ class DocumentSignaturesApp(MayanAppConfig):
            ).get_signature_type_display()
        )

-        app.conf.CELERY_QUEUES.append(
+        app.conf.task_queues.append(
            Queue(
                'signatures', Exchange('signatures'), routing_key='signatures'
            ),
        )

-        app.conf.CELERY_ROUTES.update(
+        app.conf.task_routes.update(
            {
                'mayan.apps.document_signatures.tasks.task_verify_key_signatures': {
                    'queue': 'signatures'
--- a/mayan/apps/document_states/apps.py
+++ b/mayan/apps/document_states/apps.py
@@ -242,7 +242,7 @@ class DocumentStatesApp(MayanAppConfig):
            )
        )

-        app.conf.CELERY_QUEUES.extend(
+        app.conf.task_queues.extend(
            (
                Queue(
                    'document_states', Exchange('document_states'),
@@ -250,7 +250,8 @@ class DocumentStatesApp(MayanAppConfig):
                ),
            )
        )
-        app.conf.CELERY_QUEUES.extend(
+
+        app.conf.task_queues.extend(
            (
                Queue(
                    'document_states_fast', Exchange('document_states_fast'),
@@ -259,7 +260,7 @@ class DocumentStatesApp(MayanAppConfig):
            )
        )

-        app.conf.CELERY_ROUTES.update(
+        app.conf.task_routes.update(
            {
                'mayan.apps.document_states.tasks.task_generate_document_state_image': {
                    'queue': 'document_states'
--- a/mayan/apps/documents/apps.py
+++ b/mayan/apps/documents/apps.py
@@ -340,7 +340,7 @@ class DocumentsApp(MayanAppConfig):
            func=lambda context: context['object'].documents.count()
        )

-        app.conf.CELERYBEAT_SCHEDULE.update(
+        app.conf.beat_schedule.update(
            {
                'task_check_delete_periods': {
                    'task': 'mayan.apps.documents.tasks.task_check_delete_periods',
@@ -359,7 +359,7 @@ class DocumentsApp(MayanAppConfig):
            }
        )

-        app.conf.CELERY_QUEUES.extend(
+        app.conf.task_queues.extend(
            (
                Queue(
                    'converter', Exchange('converter'),
@@ -376,7 +376,7 @@ class DocumentsApp(MayanAppConfig):
            )
        )

-        app.conf.CELERY_ROUTES.update(
+        app.conf.task_routes.update(
            {
                'mayan.apps.documents.tasks.task_check_delete_periods': {
                    'queue': 'documents_periodic'
--- a/mayan/apps/mailer/apps.py
+++ b/mayan/apps/mailer/apps.py
@@ -92,11 +92,11 @@ class MailerApp(MayanAppConfig):
            )
        )

-        app.conf.CELERY_QUEUES.append(
+        app.conf.task_queues.append(
            Queue('mailing', Exchange('mailing'), routing_key='mailing'),
        )

-        app.conf.CELERY_ROUTES.update(
+        app.conf.task_routes.update(
            {
                'mayan.apps.mailer.tasks.task_send_document': {
                    'queue': 'mailing'
--- a/mayan/apps/mayan_statistics/apps.py
+++ b/mayan/apps/mayan_statistics/apps.py
@@ -36,7 +36,7 @@ class StatisticsApp(MayanAppConfig):
            attribute='schedule',
        )

-        app.conf.CELERY_QUEUES.extend(
+        app.conf.task_queues.extend(
            (
                Queue(
                    'statistics', Exchange('statistics'),
@@ -45,7 +45,7 @@ class StatisticsApp(MayanAppConfig):
            )
        )

-        app.conf.CELERY_ROUTES.update(
+        app.conf.task_routes.update(
            {
                'mayan.apps.mayan_statistics.tasks.task_execute_statistic': {
                    'queue': 'statistics'
--- a/mayan/apps/mayan_statistics/classes.py
+++ b/mayan/apps/mayan_statistics/classes.py
@@ -93,7 +93,7 @@ class Statistic(object):
            day_of_month=day_of_month, month_of_year=month_of_year,
        )

-        app.conf.CELERYBEAT_SCHEDULE.update(
+        app.conf.beat_schedule.update(
            {
                self.get_task_name(): {
                    'task': 'mayan_statistics.tasks.task_execute_statistic',
@@ -103,7 +103,7 @@ class Statistic(object):
            }
        )

-        app.conf.CELERY_ROUTES.update(
+        app.conf.task_routes.update(
            {
                self.get_task_name(): {
                    'queue': 'statistics'
--- a/mayan/apps/metadata/apps.py
+++ b/mayan/apps/metadata/apps.py
@@ -171,11 +171,11 @@ class MetadataApp(MayanAppConfig):
            ).render()
        )

-        app.conf.CELERY_QUEUES.append(
+        app.conf.task_queues.append(
            Queue('metadata', Exchange('metadata'), routing_key='metadata'),
        )

-        app.conf.CELERY_ROUTES.update(
+        app.conf.task_routes.update(
            {
                'mayan.apps.metadata.tasks.task_remove_metadata_type': {
                    'queue': 'metadata'
--- a/mayan/apps/ocr/apps.py
+++ b/mayan/apps/ocr/apps.py
@@ -140,11 +140,11 @@ class OCRApp(MayanAppConfig):
            attribute='result'
        )

-        app.conf.CELERY_QUEUES.append(
+        app.conf.task_queues.append(
            Queue('ocr', Exchange('ocr'), routing_key='ocr'),
        )

-        app.conf.CELERY_ROUTES.update(
+        app.conf.task_routes.update(
            {
                'mayan.apps.ocr.tasks.task_do_ocr': {
                    'queue': 'ocr'
--- a/mayan/apps/sources/apps.py
+++ b/mayan/apps/sources/apps.py
@@ -90,7 +90,7 @@ class SourcesApp(MayanAppConfig):
            func=lambda context: context['object'].message
        )

-        app.conf.CELERY_QUEUES.extend(
+        app.conf.task_queues.extend(
            (
                Queue(
                    'sources', Exchange('sources'), routing_key='sources'
@@ -106,7 +106,7 @@ class SourcesApp(MayanAppConfig):
            )
        )

-        app.conf.CELERY_ROUTES.update(
+        app.conf.task_routes.update(
            {
                'mayan.apps.sources.tasks.task_check_interval_source': {
                    'queue': 'sources_periodic'
--- a/mayan/celery.py
+++ b/mayan/celery.py
@@ -2,13 +2,11 @@ from __future__ import absolute_import, unicode_literals

 import os

-from django.conf import settings
-
 from .runtime import celery_class

 os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mayan.settings.production')

 app = celery_class('mayan')

-app.config_from_object('django.conf:settings')
-app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
+app.config_from_object('django.conf:settings', namespace='CELERY')
+app.autodiscover_tasks()
--- a/mayan/settings/base.py
+++ b/mayan/settings/base.py
@@ -273,17 +273,16 @@ PAGINATION_SETTINGS = {
 # ----------- Celery ----------

 CELERY_ACCEPT_CONTENT = ('json',)
-CELERY_ALWAYS_EAGER = False
-CELERY_CREATE_MISSING_QUEUES = False
-CELERY_DISABLE_RATE_LIMITS = True
-CELERY_EAGER_PROPAGATES_EXCEPTIONS = True
+CELERY_BEAT_SCHEDULER = 'djcelery.schedulers.DatabaseScheduler'
 CELERY_ENABLE_UTC = True
-CELERY_QUEUES = []
 CELERY_RESULT_SERIALIZER = 'json'
-CELERY_ROUTES = {}
+CELERY_TASK_CREATE_MISSING_QUEUES = False
+CELERY_TASK_EAGER_PROPAGATES = False
+CELERY_TASK_QUEUES = []
+CELERY_TASK_ROUTES = {}
 CELERY_TASK_SERIALIZER = 'json'
 CELERY_TIMEZONE = 'UTC'
-CELERYBEAT_SCHEDULER = 'djcelery.schedulers.DatabaseScheduler'
+CELERY_WORKER_DISABLE_RATE_LIMITS = True

 # ------------ CORS ------------

@@ -312,9 +311,9 @@ AJAX_REDIRECT_CODE = 278

 # ----- Celery -----

-BROKER_URL = os.environ.get('MAYAN_BROKER_URL')
-CELERY_ALWAYS_EAGER = yaml_loads(
-    os.environ.get('MAYAN_CELERY_ALWAYS_EAGER', 'true')
+CELERY_BROKER_URL = os.environ.get('MAYAN_CELERY_BROKER_URL')
+CELERY_TASK_ALWAYS_EAGER = yaml_loads(
+    os.environ.get('MAYAN_CELERY_TASK_ALWAYS_EAGER', 'false')
 )
 CELERY_RESULT_BACKEND = os.environ.get('MAYAN_CELERY_RESULT_BACKEND')

--- a/mayan/settings/development.py
+++ b/mayan/settings/development.py
@@ -6,8 +6,8 @@ ALLOWED_HOSTS = ['*']

 DEBUG = True

-CELERY_ALWAYS_EAGER = True
-CELERY_EAGER_PROPAGATES_EXCEPTIONS = CELERY_ALWAYS_EAGER
+CELERY_TASK_ALWAYS_EAGER = True
+CELERY_TASK_EAGER_PROPAGATES = True

 if 'rosetta' not in INSTALLED_APPS:
    try:
--- a/mayan/settings/production.py
+++ b/mayan/settings/production.py
@@ -2,8 +2,6 @@ from __future__ import absolute_import, unicode_literals

 from . import *  # NOQA

-CELERY_ALWAYS_EAGER = False
-
 TEMPLATES[0]['OPTIONS']['loaders'] = (
    (
        'django.template.loaders.cached.Loader', (
--- a/mayan/settings/testing/base.py
+++ b/mayan/settings/testing/base.py
@@ -33,9 +33,8 @@ TEMPLATES[0]['OPTIONS']['loaders'] = (
    ),
 )

-CELERY_ALWAYS_EAGER = True
-CELERY_EAGER_PROPAGATES_EXCEPTIONS = True
-BROKER_BACKEND = 'memory'
+CELERY_TASK_ALWAYS_EAGER = True
+CELERY_TASK_EAGER_PROPAGATES = True

 # Remove middlewares not used for tests
 MIDDLEWARE = [
--- a/requirements/base.txt
+++ b/requirements/base.txt
@@ -1,7 +1,7 @@
 Pillow==5.3.0
 PyYAML==3.13

-celery==3.1.26post2
+celery==4.1.1

 django-activity-stream==0.6.5
 django-autoadmin==1.1.1