diff --git a/HISTORY.rst b/HISTORY.rst index ba28e4be73..26d2f4bd18 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -172,6 +172,10 @@ - Documents: 0042 to 0043 - Tags: 0001 to 0008 +- Upgraded from Celery 3.1.26 to 4.1.1. The following settings have been + renamed: CELERY_ALWAYS_EAGER to CELERY_TASK_ALWAYS_EAGER, + BROKER_URL to CELERY_BROKER_URL. + 3.1.9 (2018-11-01) ================== - Convert the furl instance to text to allow serializing it into diff --git a/Makefile b/Makefile index def162954d..6735854b4e 100644 --- a/Makefile +++ b/Makefile @@ -283,7 +283,7 @@ test-with-docker-frontend: ./manage.py runserver --settings=mayan.settings.staging.docker test-with-docker-worker: - ./manage.py celery worker --settings=mayan.settings.staging.docker -B -l INFO -O fair + ./manage.py celery worker --settings=mayan.settings.staging.docker -B -l INFO docker-mysql-on: docker run -d --name mysql -p 3306:3306 -e MYSQL_ALLOW_EMPTY_PASSWORD=True -e MYSQL_DATABASE=mayan_edms mysql diff --git a/contrib/scripts/install/dialog.sh b/contrib/scripts/install/dialog.sh index f3fa23b38d..cacf60d54b 100644 --- a/contrib/scripts/install/dialog.sh +++ b/contrib/scripts/install/dialog.sh @@ -1452,8 +1452,8 @@ cat > /etc/supervisor/conf.d/mayan.conf < Creating the supervisor file for the Celery worker, /etc/supervisor/conf.d/mayan-celery.conf \n" cat > /etc/supervisor/conf.d/mayan-celery.conf << EOF [program:mayan-worker] -command = ${INSTALLATION_DIRECTORY}bin/python ${INSTALLATION_DIRECTORY}bin/mayan-edms.py celery --settings=mayan.settings.production worker -Ofair -l ERROR +command = ${INSTALLATION_DIRECTORY}bin/python ${INSTALLATION_DIRECTORY}bin/mayan-edms.py celery --settings=mayan.settings.production worker -l ERROR directory = ${INSTALLATION_DIRECTORY} user = www-data stdout_logfile = /var/log/mayan/worker-stdout.log diff --git a/docker/README.md b/docker/README.md index ddfb10413d..a35eb17ed8 100755 --- a/docker/README.md +++ b/docker/README.md @@ -126,13 +126,13 @@ Defaults to `None`. This optional environment variable is used to set the hostna Defaults to `None`. This optional environment variable is used to set the port number to use when connecting to the database. An empty string means the default port. Not used with SQLite. For more information read the pertinent Django documentation page: [Settings, PORT](https://docs.djangoproject.com/en/1.11/ref/settings/#port) -### `MAYAN_BROKER_URL` +### `MAYAN_CELERY_BROKER_URL` Defaults to 'redis://127.0.0.1:6379/0'. This optional environment variable is determines the broker that Celery will use to relay task messages between the frontend code and the background workers. For more information read the pertinent Celery Kombu documentation page: [Broker URL](http://kombu.readthedocs.io/en/latest/userguide/connections.html#connection-urls) This Docker image supports using Redis and RabbitMQ as brokers. -Caveat: If the `MAYAN_BROKER_URL` and `MAYAN_CELERY_RESULT_BACKEND` environment variables are specified, the built-in Redis server inside the container will be disabled. +Caveat: If the `MAYAN_CELERY_BROKER_URL` and `MAYAN_CELERY_RESULT_BACKEND` environment variables are specified, the built-in Redis server inside the container will be disabled. ### `MAYAN_CELERY_RESULT_BACKEND` @@ -140,7 +140,7 @@ Defaults to 'redis://127.0.0.1:6379/0'. This optional environment variable is de This Docker image supports using Redis and RabbitMQ as result backends. -Caveat: If the `MAYAN_BROKER_URL` and `MAYAN_CELERY_RESULT_BACKEND` environment variables are specified, the built-in Redis server inside the container will be disabled. +Caveat: If the `MAYAN_CELERY_BROKER_URL` and `MAYAN_CELERY_RESULT_BACKEND` environment variables are specified, the built-in Redis server inside the container will be disabled. ### `MAYAN_NGINX_CLIENT_MAX_BODY_SIZE` diff --git a/docker/README.md.tmpl b/docker/README.md.tmpl index fea2c45309..df6d39c022 100755 --- a/docker/README.md.tmpl +++ b/docker/README.md.tmpl @@ -126,13 +126,13 @@ Defaults to `None`. This optional environment variable is used to set the hostna Defaults to `None`. This optional environment variable is used to set the port number to use when connecting to the database. An empty string means the default port. Not used with SQLite. For more information read the pertinent Django documentation page: [Settings, PORT](https://docs.djangoproject.com/en/1.11/ref/settings/#port) -### `MAYAN_BROKER_URL` +### `MAYAN_CELERY_BROKER_URL` Defaults to 'redis://127.0.0.1:6379/0'. This optional environment variable is determines the broker that Celery will use to relay task messages between the frontend code and the background workers. For more information read the pertinent Celery Kombu documentation page: [Broker URL](http://kombu.readthedocs.io/en/latest/userguide/connections.html#connection-urls) This Docker image supports using Redis and RabbitMQ as brokers. -Caveat: If the `MAYAN_BROKER_URL` and `MAYAN_CELERY_RESULT_BACKEND` environment variables are specified, the built-in Redis server inside the container will be disabled. +Caveat: If the `MAYAN_CELERY_BROKER_URL` and `MAYAN_CELERY_RESULT_BACKEND` environment variables are specified, the built-in Redis server inside the container will be disabled. ### `MAYAN_CELERY_RESULT_BACKEND` @@ -140,7 +140,7 @@ Defaults to 'redis://127.0.0.1:6379/0'. This optional environment variable is de This Docker image supports using Redis and RabbitMQ as result backends. -Caveat: If the `MAYAN_BROKER_URL` and `MAYAN_CELERY_RESULT_BACKEND` environment variables are specified, the built-in Redis server inside the container will be disabled. +Caveat: If the `MAYAN_CELERY_BROKER_URL` and `MAYAN_CELERY_RESULT_BACKEND` environment variables are specified, the built-in Redis server inside the container will be disabled. ### `MAYAN_NGINX_CLIENT_MAX_BODY_SIZE` diff --git a/docker/docker-compose-development.yml b/docker/docker-compose-development.yml index 48c71afe26..561af37d3f 100755 --- a/docker/docker-compose-development.yml +++ b/docker/docker-compose-development.yml @@ -59,7 +59,7 @@ services: results: condition: service_healthy environment: - MAYAN_BROKER_URL: amqp://mayan:mayan@broker:5672/mayan + MAYAN_CELERY_BROKER_URL: amqp://mayan:mayan@broker:5672/mayan MAYAN_CELERY_RESULT_BACKEND: redis://results:6379/0 MAYAN_DATABASE_ENGINE: django.db.backends.postgresql MAYAN_DATABASE_HOST: db diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 978b417148..06886a95c3 100755 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -45,7 +45,7 @@ services: results: condition: service_healthy environment: - MAYAN_BROKER_URL: amqp://mayan:mayan@broker:5672/mayan + MAYAN_CELERY_BROKER_URL: amqp://mayan:mayan@broker:5672/mayan MAYAN_CELERY_RESULT_BACKEND: redis://results:6379/0 MAYAN_DATABASE_ENGINE: django.db.backends.postgresql MAYAN_DATABASE_HOST: db diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 2c76ba45d5..3ce98e5eac 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -6,12 +6,12 @@ INSTALL_FLAG=/var/lib/mayan/system/SECRET_KEY CONCURRENCY_ARGUMENT=--concurrency= export DOCKER_ROOT=/opt/mayan-edms -export MAYAN_DEFAULT_BROKER_URL=redis://127.0.0.1:6379/0 +export MAYAN_DEFAULT_CELERY_BROKER_URL=redis://127.0.0.1:6379/0 export MAYAN_DEFAULT_CELERY_RESULT_BACKEND=redis://127.0.0.1:6379/0 export MAYAN_ALLOWED_HOSTS='["*"]' export MAYAN_BIN=/opt/mayan-edms/bin/mayan-edms.py -export MAYAN_BROKER_URL=${MAYAN_BROKER_URL:-${MAYAN_DEFAULT_BROKER_URL}} +export MAYAN_CELERY_BROKER_URL=${MAYAN_CELERY_BROKER_URL:-${MAYAN_DEFAULT_CELERY_BROKER_URL}} export MAYAN_CELERY_RESULT_BACKEND=${MAYAN_CELERY_RESULT_BACKEND:-${MAYAN_DEFAULT_CELERY_RESULT_BACKEND}} export MAYAN_INSTALL_DIR=/opt/mayan-edms export MAYAN_PYTHON_BIN_DIR=/opt/mayan-edms/bin/ diff --git a/docker/etc/supervisor/mayan.conf b/docker/etc/supervisor/mayan.conf index fb7e54e5e2..cae69cfe3e 100644 --- a/docker/etc/supervisor/mayan.conf +++ b/docker/etc/supervisor/mayan.conf @@ -12,7 +12,7 @@ user = mayan [program:redis] autorestart = false autostart = true -command = /bin/bash -c "if [ ${MAYAN_BROKER_URL} == ${MAYAN_DEFAULT_BROKER_URL} ] && [ ${MAYAN_CELERY_RESULT_BACKEND} == ${MAYAN_DEFAULT_CELERY_RESULT_BACKEND} ];then /usr/bin/redis-server /etc/redis/;fi" +command = /bin/bash -c "if [ ${MAYAN_CELERY_BROKER_URL} == ${MAYAN_DEFAULT_BROKER_URL} ] && [ ${MAYAN_CELERY_RESULT_BACKEND} == ${MAYAN_DEFAULT_CELERY_RESULT_BACKEND} ];then /usr/bin/redis-server /etc/redis/;fi" stderr_logfile = /dev/fd/2 stderr_logfile_maxbytes = 0 stdout_logfile = /dev/fd/1 @@ -22,7 +22,7 @@ user = root [program:mayan-worker-fast] autorestart = false autostart = true -command = nice -n 1 /bin/bash -c "${MAYAN_BIN} celery --settings=${MAYAN_SETTINGS_MODULE} worker -Ofair -l ERROR -Q converter,document_states_fast,sources_fast -n mayan-worker-fast.%%h ${MAYAN_WORKER_FAST_CONCURRENCY}" +command = nice -n 1 /bin/bash -c "${MAYAN_BIN} celery --settings=${MAYAN_SETTINGS_MODULE} worker -l ERROR -Q converter,document_states_fast,sources_fast -n mayan-worker-fast.%%h ${MAYAN_WORKER_FAST_CONCURRENCY}" killasgroup = true numprocs = 1 priority = 998 @@ -37,7 +37,7 @@ user = mayan [program:mayan-worker-medium] autorestart = false autostart = true -command = nice -n 18 /bin/bash -c "${MAYAN_BIN} celery --settings=${MAYAN_SETTINGS_MODULE} worker -Ofair -l ERROR -Q checkouts_periodic,documents_periodic,indexing,metadata,sources,sources_periodic,uploads,documents -n mayan-worker-medium.%%h ${MAYAN_WORKER_MEDIUM_CONCURRENCY}" +command = nice -n 18 /bin/bash -c "${MAYAN_BIN} celery --settings=${MAYAN_SETTINGS_MODULE} worker -l ERROR -Q checkouts_periodic,documents_periodic,indexing,metadata,sources,sources_periodic,uploads,documents -n mayan-worker-medium.%%h ${MAYAN_WORKER_MEDIUM_CONCURRENCY}" killasgroup = true numprocs = 1 priority = 998 @@ -52,7 +52,7 @@ user = mayan [program:mayan-worker-slow] autorestart = false autostart = true -command = nice -n 19 /bin/bash -c "${MAYAN_BIN} celery --settings=${MAYAN_SETTINGS_MODULE} worker -Ofair -l ERROR -Q mailing,tools,statistics,parsing,ocr -n mayan-worker-slow.%%h ${MAYAN_WORKER_SLOW_CONCURRENCY}" +command = nice -n 19 /bin/bash -c "${MAYAN_BIN} celery --settings=${MAYAN_SETTINGS_MODULE} worker -l ERROR -Q mailing,tools,statistics,parsing,ocr -n mayan-worker-slow.%%h ${MAYAN_WORKER_SLOW_CONCURRENCY}" killasgroup = true numprocs = 1 priority = 998 diff --git a/docs/chapters/deploying.rst b/docs/chapters/deploying.rst index 5c3dedb81c..512cc08a6d 100644 --- a/docs/chapters/deploying.rst +++ b/docs/chapters/deploying.rst @@ -91,8 +91,8 @@ Create the supervisor file at ``/etc/supervisor/conf.d/mayan.conf``: [supervisord] environment= MAYAN_ALLOWED_HOSTS='["*"]', # Allow access to other network hosts other than localhost + MAYAN_CELERY_BROKER_URL="redis://127.0.0.1:6379/0", MAYAN_CELERY_RESULT_BACKEND="redis://127.0.0.1:6379/0", - MAYAN_BROKER_URL="redis://127.0.0.1:6379/0", PYTHONPATH=/opt/mayan-edms/lib/python2.7/site-packages:/opt/mayan-edms/data, MAYAN_MEDIA_ROOT=/opt/mayan-edms/media, MAYAN_DATABASES='{default: {ENGINE: django.db.backends.postgresql, HOST: 127.0.0.1, NAME: mayan, PASSWORD: mayanuserpass, USER: mayan, CONN_MAX_AGE: 60}}', @@ -107,7 +107,7 @@ Create the supervisor file at ``/etc/supervisor/conf.d/mayan.conf``: [program:mayan-worker-fast] autorestart = true autostart = true - command = nice -n 1 /opt/mayan-edms/bin/mayan-edms.py celery worker -Ofair -l ERROR -Q converter,document_states_fast,sources_fast -n mayan-worker-fast.%%h --concurrency=1 + command = nice -n 1 /opt/mayan-edms/bin/mayan-edms.py celery worker -l ERROR -Q converter,document_states_fast,sources_fast -n mayan-worker-fast.%%h --concurrency=1 killasgroup = true numprocs = 1 priority = 998 @@ -118,7 +118,7 @@ Create the supervisor file at ``/etc/supervisor/conf.d/mayan.conf``: [program:mayan-worker-medium] autorestart = true autostart = true - command = nice -n 18 /opt/mayan-edms/bin/mayan-edms.py celery worker -Ofair -l ERROR -Q checkouts_periodic,documents_periodic,indexing,metadata,sources,sources_periodic,uploads,documents -n mayan-worker-medium.%%h --concurrency=1 + command = nice -n 18 /opt/mayan-edms/bin/mayan-edms.py celery worker -l ERROR -Q checkouts_periodic,documents_periodic,indexing,metadata,sources,sources_periodic,uploads,documents -n mayan-worker-medium.%%h --concurrency=1 killasgroup = true numprocs = 1 priority = 998 @@ -129,7 +129,7 @@ Create the supervisor file at ``/etc/supervisor/conf.d/mayan.conf``: [program:mayan-worker-slow] autorestart = true autostart = true - command = nice -n 19 /opt/mayan-edms/bin/mayan-edms.py celery worker -Ofair -l ERROR -Q mailing,tools,statistics,parsing,ocr -n mayan-worker-slow.%%h --concurrency=1 + command = nice -n 19 /opt/mayan-edms/bin/mayan-edms.py celery worker -l ERROR -Q mailing,tools,statistics,parsing,ocr -n mayan-worker-slow.%%h --concurrency=1 killasgroup = true numprocs = 1 priority = 998 @@ -260,8 +260,8 @@ Create the supervisor file at ``/etc/supervisor/conf.d/mayan.conf``: [supervisord] environment= MAYAN_ALLOWED_HOSTS='["*"]', # Allow access to other network hosts other than localhost + MAYAN_CELERY_BROKER_URL="amqp://mayan:mayanrabbitmqpassword@localhost:5672/mayan", MAYAN_CELERY_RESULT_BACKEND="redis://127.0.0.1:6379/0", - MAYAN_BROKER_URL="amqp://mayan:mayanrabbitmqpassword@localhost:5672/mayan", PYTHONPATH=/opt/mayan-edms/lib/python2.7/site-packages:/opt/mayan-edms/data, MAYAN_MEDIA_ROOT=/opt/mayan-edms/media, MAYAN_DATABASES='{default: {ENGINE: django.db.backends.postgresql, HOST: 127.0.0.1, NAME: mayan, PASSWORD: mayanuserpass, USER: mayan, CONN_MAX_AGE: 60}}', @@ -276,7 +276,7 @@ Create the supervisor file at ``/etc/supervisor/conf.d/mayan.conf``: [program:mayan-worker-fast] autorestart = true autostart = true - command = nice -n 1 /opt/mayan-edms/bin/mayan-edms.py celery worker -Ofair -l ERROR -Q converter,document_states_fast,sources_fast -n mayan-worker-fast.%%h + command = nice -n 1 /opt/mayan-edms/bin/mayan-edms.py celery worker -l ERROR -Q converter,document_states_fast,sources_fast -n mayan-worker-fast.%%h killasgroup = true numprocs = 1 priority = 998 @@ -287,7 +287,7 @@ Create the supervisor file at ``/etc/supervisor/conf.d/mayan.conf``: [program:mayan-worker-medium] autorestart = true autostart = true - command = nice -n 18 /opt/mayan-edms/bin/mayan-edms.py celery worker -Ofair -l ERROR -Q checkouts_periodic,documents_periodic,indexing,metadata,sources,sources_periodic,uploads,documents -n mayan-worker-medium.%%h --concurrency=1 + command = nice -n 18 /opt/mayan-edms/bin/mayan-edms.py celery worker -l ERROR -Q checkouts_periodic,documents_periodic,indexing,metadata,sources,sources_periodic,uploads,documents -n mayan-worker-medium.%%h --concurrency=1 killasgroup = true numprocs = 1 priority = 998 @@ -298,7 +298,7 @@ Create the supervisor file at ``/etc/supervisor/conf.d/mayan.conf``: [program:mayan-worker-slow] autorestart = true autostart = true - command = nice -n 19 /opt/mayan-edms/bin/mayan-edms.py celery worker -Ofair -l ERROR -Q mailing,tools,statistics,parsing,ocr -n mayan-worker-slow.%%h --concurrency=1 + command = nice -n 19 /opt/mayan-edms/bin/mayan-edms.py celery worker -l ERROR -Q mailing,tools,statistics,parsing,ocr -n mayan-worker-slow.%%h --concurrency=1 killasgroup = true numprocs = 1 priority = 998 diff --git a/docs/chapters/docker.rst b/docs/chapters/docker.rst index 2f5c7ef45f..743bf50bb2 100644 --- a/docs/chapters/docker.rst +++ b/docs/chapters/docker.rst @@ -55,7 +55,7 @@ the default port. Not used with SQLite. For more information read the pertinent Django documentation page: :django-docs:`Settings, PORT ` -``MAYAN_BROKER_URL`` +``MAYAN_CELERY_BROKER_URL`` This optional environment variable determines the broker that Celery will use to relay task messages between the frontend code and the background workers. @@ -65,7 +65,7 @@ For more information read the pertinent Celery Kombu documentation page: `Broker This Docker image supports using Redis and RabbitMQ as brokers. -Caveat: If the `MAYAN_BROKER_URL` and `MAYAN_CELERY_RESULT_BACKEND` environment +Caveat: If the `MAYAN_CELERY_BROKER_URL` and `MAYAN_CELERY_RESULT_BACKEND` environment variables are specified, the built-in Redis server inside the container will be disabled. @@ -80,7 +80,7 @@ code. For more information read the pertinent Celery Kombu documentation page: This Docker image supports using Redis and RabbitMQ as result backends. -Caveat: If the `MAYAN_BROKER_URL` and `MAYAN_CELERY_RESULT_BACKEND` environment +Caveat: If the `MAYAN_CELERY_BROKER_URL` and `MAYAN_CELERY_RESULT_BACKEND` environment variables are specified, the built-in Redis server inside the container will be disabled. diff --git a/docs/chapters/scaling_up.rst b/docs/chapters/scaling_up.rst index c15c548c6b..923a78096e 100644 --- a/docs/chapters/scaling_up.rst +++ b/docs/chapters/scaling_up.rst @@ -116,11 +116,11 @@ For the Docker image, launch a separate RabbitMQ container docker run -d --name mayan-edms-rabbitmq -e RABBITMQ_DEFAULT_USER=mayan -e RABBITMQ_DEFAULT_PASS=mayanrabbitmqpassword -e RABBITMQ_DEFAULT_VHOST=mayan rabbitmq:3 -Pass the MAYAN_BROKER_URL environment variable (https://kombu.readthedocs.io/en/latest/userguide/connections.html#connection-urls) +Pass the MAYAN_CELERY_BROKER_URL environment variable (https://kombu.readthedocs.io/en/latest/userguide/connections.html#connection-urls) to the Mayan EDMS container so that it uses the RabbitMQ container the message broker:: - -e MAYAN_BROKER_URL="amqp://mayan:mayanrabbitmqpassword@localhost:5672/mayan", + -e MAYAN_CELERY_BROKER_URL="amqp://mayan:mayanrabbitmqpassword@localhost:5672/mayan", When tasks finish, they leave behind a return status or the result of a calculation, these are stored for a while so that whoever requested the diff --git a/mayan/apps/checkouts/apps.py b/mayan/apps/checkouts/apps.py index fdda1de151..eb5be40c57 100644 --- a/mayan/apps/checkouts/apps.py +++ b/mayan/apps/checkouts/apps.py @@ -95,7 +95,7 @@ class CheckoutsApp(MayanAppConfig): ) ) - app.conf.CELERYBEAT_SCHEDULE.update( + app.conf.beat_schedule.update( { 'task_check_expired_check_outs': { 'task': 'mayan.apps.checkouts.tasks.task_check_expired_check_outs', @@ -106,14 +106,14 @@ class CheckoutsApp(MayanAppConfig): } ) - app.conf.CELERY_QUEUES.append( + app.conf.task_queues.append( Queue( 'checkouts_periodic', Exchange('checkouts_periodic'), routing_key='checkouts_periodic', delivery_mode=1 ), ) - app.conf.CELERY_ROUTES.update( + app.conf.task_routes.update( { 'mayan.apps.checkouts.tasks.task_check_expired_check_outs': { 'queue': 'checkouts_periodic' diff --git a/mayan/apps/common/apps.py b/mayan/apps/common/apps.py index 827d16642d..e80bc02070 100644 --- a/mayan/apps/common/apps.py +++ b/mayan/apps/common/apps.py @@ -92,7 +92,7 @@ class CommonApp(MayanAppConfig): name='main_menu', template_name='appearance/main_menu.html' ) - app.conf.CELERYBEAT_SCHEDULE.update( + app.conf.beat_schedule.update( { 'task_delete_stale_uploads': { 'task': 'mayan.apps.common.tasks.task_delete_stale_uploads', @@ -103,7 +103,7 @@ class CommonApp(MayanAppConfig): } ) - app.conf.CELERY_QUEUES.extend( + app.conf.task_queues.extend( ( Queue('default', Exchange('default'), routing_key='default'), Queue('tools', Exchange('tools'), routing_key='tools'), @@ -114,9 +114,9 @@ class CommonApp(MayanAppConfig): ) ) - app.conf.CELERY_DEFAULT_QUEUE = 'default' + app.conf.task_default_queue = 'default' - app.conf.CELERY_ROUTES.update( + app.conf.task_routes.update( { 'mayan.apps.common.tasks.task_delete_stale_uploads': { 'queue': 'common_periodic' diff --git a/mayan/apps/common/settings.py b/mayan/apps/common/settings.py index 87243bc927..2206192210 100644 --- a/mayan/apps/common/settings.py +++ b/mayan/apps/common/settings.py @@ -405,8 +405,8 @@ setting_django_wsgi_application = namespace.add_setting( namespace = Namespace(name='celery', label=_('Celery')) setting_celery_always_eager = namespace.add_setting( - global_name='CELERY_ALWAYS_EAGER', - default=settings.CELERY_ALWAYS_EAGER, + global_name='CELERY_TASK_ALWAYS_EAGER', + default=settings.CELERY_TASK_ALWAYS_EAGER, help_text=_( 'If this is True, all tasks will be executed locally by blocking ' 'until the task returns. apply_async() and Task.delay() will return ' @@ -417,7 +417,7 @@ setting_celery_always_eager = namespace.add_setting( ) ) setting_celery_broker_url = namespace.add_setting( - global_name='BROKER_URL', default=settings.BROKER_URL, + global_name='CELERY_BROKER_URL', default=settings.CELERY_BROKER_URL, help_text=_( 'Default: "amqp://". Default broker URL. This must be a URL in ' 'the form of: transport://userid:password@hostname:port/virtual_host ' diff --git a/mayan/apps/document_indexing/apps.py b/mayan/apps/document_indexing/apps.py index 168c60f1db..fa5642964f 100644 --- a/mayan/apps/document_indexing/apps.py +++ b/mayan/apps/document_indexing/apps.py @@ -161,11 +161,11 @@ class DocumentIndexingApp(MayanAppConfig): ) ) - app.conf.CELERY_QUEUES.append( + app.conf.task_queues.append( Queue('indexing', Exchange('indexing'), routing_key='indexing'), ) - app.conf.CELERY_ROUTES.update( + app.conf.task_routes.update( { 'mayan.apps.document_indexing.tasks.task_delete_empty': { 'queue': 'indexing' diff --git a/mayan/apps/document_parsing/apps.py b/mayan/apps/document_parsing/apps.py index 8c78d8340e..10829c5e7f 100644 --- a/mayan/apps/document_parsing/apps.py +++ b/mayan/apps/document_parsing/apps.py @@ -143,11 +143,11 @@ class DocumentParsingApp(MayanAppConfig): attribute='result' ) - app.conf.CELERY_QUEUES.append( + app.conf.task_queues.append( Queue('parsing', Exchange('parsing'), routing_key='parsing'), ) - app.conf.CELERY_ROUTES.update( + app.conf.task_routes.update( { 'mayan.apps.document_parsing.tasks.task_parse_document_version': { 'queue': 'parsing' diff --git a/mayan/apps/document_signatures/apps.py b/mayan/apps/document_signatures/apps.py index a71cf62545..7180136382 100644 --- a/mayan/apps/document_signatures/apps.py +++ b/mayan/apps/document_signatures/apps.py @@ -101,13 +101,13 @@ class DocumentSignaturesApp(MayanAppConfig): ).get_signature_type_display() ) - app.conf.CELERY_QUEUES.append( + app.conf.task_queues.append( Queue( 'signatures', Exchange('signatures'), routing_key='signatures' ), ) - app.conf.CELERY_ROUTES.update( + app.conf.task_routes.update( { 'mayan.apps.document_signatures.tasks.task_verify_key_signatures': { 'queue': 'signatures' diff --git a/mayan/apps/document_states/apps.py b/mayan/apps/document_states/apps.py index e4a0a15899..1895341fb6 100644 --- a/mayan/apps/document_states/apps.py +++ b/mayan/apps/document_states/apps.py @@ -242,7 +242,7 @@ class DocumentStatesApp(MayanAppConfig): ) ) - app.conf.CELERY_QUEUES.extend( + app.conf.task_queues.extend( ( Queue( 'document_states', Exchange('document_states'), @@ -250,7 +250,8 @@ class DocumentStatesApp(MayanAppConfig): ), ) ) - app.conf.CELERY_QUEUES.extend( + + app.conf.task_queues.extend( ( Queue( 'document_states_fast', Exchange('document_states_fast'), @@ -259,7 +260,7 @@ class DocumentStatesApp(MayanAppConfig): ) ) - app.conf.CELERY_ROUTES.update( + app.conf.task_routes.update( { 'mayan.apps.document_states.tasks.task_generate_document_state_image': { 'queue': 'document_states' diff --git a/mayan/apps/documents/apps.py b/mayan/apps/documents/apps.py index 724858c4f2..d316a28990 100644 --- a/mayan/apps/documents/apps.py +++ b/mayan/apps/documents/apps.py @@ -340,7 +340,7 @@ class DocumentsApp(MayanAppConfig): func=lambda context: context['object'].documents.count() ) - app.conf.CELERYBEAT_SCHEDULE.update( + app.conf.beat_schedule.update( { 'task_check_delete_periods': { 'task': 'mayan.apps.documents.tasks.task_check_delete_periods', @@ -359,7 +359,7 @@ class DocumentsApp(MayanAppConfig): } ) - app.conf.CELERY_QUEUES.extend( + app.conf.task_queues.extend( ( Queue( 'converter', Exchange('converter'), @@ -376,7 +376,7 @@ class DocumentsApp(MayanAppConfig): ) ) - app.conf.CELERY_ROUTES.update( + app.conf.task_routes.update( { 'mayan.apps.documents.tasks.task_check_delete_periods': { 'queue': 'documents_periodic' diff --git a/mayan/apps/mailer/apps.py b/mayan/apps/mailer/apps.py index 47374e6412..aaad974d2a 100644 --- a/mayan/apps/mailer/apps.py +++ b/mayan/apps/mailer/apps.py @@ -92,11 +92,11 @@ class MailerApp(MayanAppConfig): ) ) - app.conf.CELERY_QUEUES.append( + app.conf.task_queues.append( Queue('mailing', Exchange('mailing'), routing_key='mailing'), ) - app.conf.CELERY_ROUTES.update( + app.conf.task_routes.update( { 'mayan.apps.mailer.tasks.task_send_document': { 'queue': 'mailing' diff --git a/mayan/apps/mayan_statistics/apps.py b/mayan/apps/mayan_statistics/apps.py index 564bc1e593..8de4d63780 100644 --- a/mayan/apps/mayan_statistics/apps.py +++ b/mayan/apps/mayan_statistics/apps.py @@ -36,7 +36,7 @@ class StatisticsApp(MayanAppConfig): attribute='schedule', ) - app.conf.CELERY_QUEUES.extend( + app.conf.task_queues.extend( ( Queue( 'statistics', Exchange('statistics'), @@ -45,7 +45,7 @@ class StatisticsApp(MayanAppConfig): ) ) - app.conf.CELERY_ROUTES.update( + app.conf.task_routes.update( { 'mayan.apps.mayan_statistics.tasks.task_execute_statistic': { 'queue': 'statistics' diff --git a/mayan/apps/mayan_statistics/classes.py b/mayan/apps/mayan_statistics/classes.py index 031ffde7a6..06fe1b32bd 100644 --- a/mayan/apps/mayan_statistics/classes.py +++ b/mayan/apps/mayan_statistics/classes.py @@ -93,7 +93,7 @@ class Statistic(object): day_of_month=day_of_month, month_of_year=month_of_year, ) - app.conf.CELERYBEAT_SCHEDULE.update( + app.conf.beat_schedule.update( { self.get_task_name(): { 'task': 'mayan_statistics.tasks.task_execute_statistic', @@ -103,7 +103,7 @@ class Statistic(object): } ) - app.conf.CELERY_ROUTES.update( + app.conf.task_routes.update( { self.get_task_name(): { 'queue': 'statistics' diff --git a/mayan/apps/metadata/apps.py b/mayan/apps/metadata/apps.py index c8fe69daa6..097403a8ce 100644 --- a/mayan/apps/metadata/apps.py +++ b/mayan/apps/metadata/apps.py @@ -171,11 +171,11 @@ class MetadataApp(MayanAppConfig): ).render() ) - app.conf.CELERY_QUEUES.append( + app.conf.task_queues.append( Queue('metadata', Exchange('metadata'), routing_key='metadata'), ) - app.conf.CELERY_ROUTES.update( + app.conf.task_routes.update( { 'mayan.apps.metadata.tasks.task_remove_metadata_type': { 'queue': 'metadata' diff --git a/mayan/apps/ocr/apps.py b/mayan/apps/ocr/apps.py index 764124d8da..3bed51bcda 100644 --- a/mayan/apps/ocr/apps.py +++ b/mayan/apps/ocr/apps.py @@ -140,11 +140,11 @@ class OCRApp(MayanAppConfig): attribute='result' ) - app.conf.CELERY_QUEUES.append( + app.conf.task_queues.append( Queue('ocr', Exchange('ocr'), routing_key='ocr'), ) - app.conf.CELERY_ROUTES.update( + app.conf.task_routes.update( { 'mayan.apps.ocr.tasks.task_do_ocr': { 'queue': 'ocr' diff --git a/mayan/apps/sources/apps.py b/mayan/apps/sources/apps.py index 2e621e3b88..c392845240 100644 --- a/mayan/apps/sources/apps.py +++ b/mayan/apps/sources/apps.py @@ -90,7 +90,7 @@ class SourcesApp(MayanAppConfig): func=lambda context: context['object'].message ) - app.conf.CELERY_QUEUES.extend( + app.conf.task_queues.extend( ( Queue( 'sources', Exchange('sources'), routing_key='sources' @@ -106,7 +106,7 @@ class SourcesApp(MayanAppConfig): ) ) - app.conf.CELERY_ROUTES.update( + app.conf.task_routes.update( { 'mayan.apps.sources.tasks.task_check_interval_source': { 'queue': 'sources_periodic' diff --git a/mayan/celery.py b/mayan/celery.py index ca8fc9cdad..c94e633e6e 100644 --- a/mayan/celery.py +++ b/mayan/celery.py @@ -2,13 +2,11 @@ from __future__ import absolute_import, unicode_literals import os -from django.conf import settings - from .runtime import celery_class os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mayan.settings.production') app = celery_class('mayan') -app.config_from_object('django.conf:settings') -app.autodiscover_tasks(lambda: settings.INSTALLED_APPS) +app.config_from_object('django.conf:settings', namespace='CELERY') +app.autodiscover_tasks() diff --git a/mayan/settings/base.py b/mayan/settings/base.py index 2070e1e61e..88d5d6d98a 100644 --- a/mayan/settings/base.py +++ b/mayan/settings/base.py @@ -273,17 +273,16 @@ PAGINATION_SETTINGS = { # ----------- Celery ---------- CELERY_ACCEPT_CONTENT = ('json',) -CELERY_ALWAYS_EAGER = False -CELERY_CREATE_MISSING_QUEUES = False -CELERY_DISABLE_RATE_LIMITS = True -CELERY_EAGER_PROPAGATES_EXCEPTIONS = True +CELERY_BEAT_SCHEDULER = 'djcelery.schedulers.DatabaseScheduler' CELERY_ENABLE_UTC = True -CELERY_QUEUES = [] CELERY_RESULT_SERIALIZER = 'json' -CELERY_ROUTES = {} +CELERY_TASK_CREATE_MISSING_QUEUES = False +CELERY_TASK_EAGER_PROPAGATES = False +CELERY_TASK_QUEUES = [] +CELERY_TASK_ROUTES = {} CELERY_TASK_SERIALIZER = 'json' CELERY_TIMEZONE = 'UTC' -CELERYBEAT_SCHEDULER = 'djcelery.schedulers.DatabaseScheduler' +CELERY_WORKER_DISABLE_RATE_LIMITS = True # ------------ CORS ------------ @@ -312,9 +311,9 @@ AJAX_REDIRECT_CODE = 278 # ----- Celery ----- -BROKER_URL = os.environ.get('MAYAN_BROKER_URL') -CELERY_ALWAYS_EAGER = yaml_loads( - os.environ.get('MAYAN_CELERY_ALWAYS_EAGER', 'true') +CELERY_BROKER_URL = os.environ.get('MAYAN_CELERY_BROKER_URL') +CELERY_TASK_ALWAYS_EAGER = yaml_loads( + os.environ.get('MAYAN_CELERY_TASK_ALWAYS_EAGER', 'false') ) CELERY_RESULT_BACKEND = os.environ.get('MAYAN_CELERY_RESULT_BACKEND') diff --git a/mayan/settings/development.py b/mayan/settings/development.py index 193270208d..9091ecc30d 100644 --- a/mayan/settings/development.py +++ b/mayan/settings/development.py @@ -6,8 +6,8 @@ ALLOWED_HOSTS = ['*'] DEBUG = True -CELERY_ALWAYS_EAGER = True -CELERY_EAGER_PROPAGATES_EXCEPTIONS = CELERY_ALWAYS_EAGER +CELERY_TASK_ALWAYS_EAGER = True +CELERY_TASK_EAGER_PROPAGATES = True if 'rosetta' not in INSTALLED_APPS: try: diff --git a/mayan/settings/production.py b/mayan/settings/production.py index f5cc3f6af5..20be234b5f 100644 --- a/mayan/settings/production.py +++ b/mayan/settings/production.py @@ -2,8 +2,6 @@ from __future__ import absolute_import, unicode_literals from . import * # NOQA -CELERY_ALWAYS_EAGER = False - TEMPLATES[0]['OPTIONS']['loaders'] = ( ( 'django.template.loaders.cached.Loader', ( diff --git a/mayan/settings/testing/base.py b/mayan/settings/testing/base.py index 0b89bf6e0a..a32a38ba5c 100644 --- a/mayan/settings/testing/base.py +++ b/mayan/settings/testing/base.py @@ -33,9 +33,8 @@ TEMPLATES[0]['OPTIONS']['loaders'] = ( ), ) -CELERY_ALWAYS_EAGER = True -CELERY_EAGER_PROPAGATES_EXCEPTIONS = True -BROKER_BACKEND = 'memory' +CELERY_TASK_ALWAYS_EAGER = True +CELERY_TASK_EAGER_PROPAGATES = True # Remove middlewares not used for tests MIDDLEWARE = [ diff --git a/requirements/base.txt b/requirements/base.txt index 3f05d8b4f9..7fc9f1733b 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,7 +1,7 @@ Pillow==5.3.0 PyYAML==3.13 -celery==3.1.26post2 +celery==4.1.1 django-activity-stream==0.6.5 django-autoadmin==1.1.1