diff --git a/HISTORY.rst b/HISTORY.rst index 94752844c8..0e9da0cbd6 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -5,6 +5,9 @@ * Add no results help text to the document type -> metadata type association view. * Expose the Django INSTALLED_APPS setting. +* Add support for changing the concurrency of the Celery workers in the + Docker image. Add environment variables MAYAN_WORKER_FAST_CONCURRENCY, + MAYAN_WORKER_MEDIUM_CONCURRENCY and MAYAN_WORKER_SLOW_CONCURRENCY. 3.1.2 (2018-09-21) ================== diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index cb2c34ede2..2932bddac6 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -3,6 +3,7 @@ set -e echo "mayan: starting entrypoint.sh" INSTALL_FLAG=/var/lib/mayan/media/system/SECRET_KEY +CONCURRENCY_ARGUMENT=--concurrency= export DOCKER_ROOT=/opt/mayan-edms export MAYAN_DEFAULT_BROKER_URL=redis://127.0.0.1:6379/0 @@ -21,6 +22,31 @@ export MAYAN_GUNICORN_BIN=${MAYAN_PYTHON_BIN_DIR}gunicorn export MAYAN_GUNICORN_WORKERS=${MAYAN_GUNICORN_WORKERS:-2} export MAYAN_PIP_BIN=${MAYAN_PYTHON_BIN_DIR}pip +MAYAN_WORKER_FAST_CONCURRENCY=${MAYAN_WORKER_FAST_CONCURRENCY:-1} +MAYAN_WORKER_MEDIUM_CONCURRENCY=${MAYAN_WORKER_MEDIUM_CONCURRENCY:-1} +MAYAN_WORKER_SLOW_CONCURRENCY=${MAYAN_WORKER_SLOW_CONCURRENCY:-1} + +if [ "$MAYAN_WORKER_FAST_CONCURRENCY" -eq 0 ]; then + MAYAN_WORKER_FAST_CONCURRENCY= +else + MAYAN_WORKER_FAST_CONCURRENCY="${CONCURRENCY_ARGUMENT}${MAYAN_WORKER_FAST_CONCURRENCY}" +fi +export MAYAN_WORKER_FAST_CONCURRENCY + +if [ "$MAYAN_WORKER_MEDIUM_CONCURRENCY" -eq 0 ]; then + MAYAN_WORKER_MEDIUM_CONCURRENCY= +else + MAYAN_WORKER_MEDIUM_CONCURRENCY="${CONCURRENCY_ARGUMENT}${MAYAN_WORKER_MEDIUM_CONCURRENCY}" +fi +export MAYAN_WORKER_MEDIUM_CONCURRENCY + +if [ "$MAYAN_WORKER_SLOW_CONCURRENCY" -eq 0 ]; then + MAYAN_WORKER_SLOW_CONCURRENCY= +else + MAYAN_WORKER_SLOW_CONCURRENCY="${CONCURRENCY_ARGUMENT}${MAYAN_WORKER_SLOW_CONCURRENCY}" +fi +export MAYAN_WORKER_SLOW_CONCURRENCY + export CELERY_ALWAYS_EAGER=False export PYTHONPATH=$PYTHONPATH:$MAYAN_MEDIA_ROOT diff --git a/docker/etc/supervisor/mayan.conf b/docker/etc/supervisor/mayan.conf index e09c5c3d14..2384831653 100644 --- a/docker/etc/supervisor/mayan.conf +++ b/docker/etc/supervisor/mayan.conf @@ -22,7 +22,7 @@ user = root [program:mayan-worker-fast] autorestart = false autostart = true -command = nice -n 1 /bin/bash -c "${MAYAN_BIN} celery --settings=${MAYAN_SETTINGS_MODULE} worker -Ofair -l ERROR -Q converter -n mayan-worker-fast.%%h --concurrency=1" +command = nice -n 1 /bin/bash -c "${MAYAN_BIN} celery --settings=${MAYAN_SETTINGS_MODULE} worker -Ofair -l ERROR -Q converter,sources_fast -n mayan-worker-fast.%%h ${MAYAN_WORKERS_FAST_CONCURRENCY}" killasgroup = true numprocs = 1 priority = 998 @@ -37,7 +37,7 @@ user = mayan [program:mayan-worker-medium] autorestart = false autostart = true -command = nice -n 18 /bin/bash -c "${MAYAN_BIN} celery --settings=${MAYAN_SETTINGS_MODULE} worker -Ofair -l ERROR -Q checkouts_periodic,documents_periodic,indexing,metadata,sources,sources_periodic,uploads,documents -n mayan-worker-medium.%%h --concurrency=1" +command = nice -n 18 /bin/bash -c "${MAYAN_BIN} celery --settings=${MAYAN_SETTINGS_MODULE} worker -Ofair -l ERROR -Q checkouts_periodic,documents_periodic,indexing,metadata,sources,sources_periodic,uploads,documents -n mayan-worker-medium.%%h ${MAYAN_WORKERS_MEDIUM_CONCURRENCY}" killasgroup = true numprocs = 1 priority = 998 @@ -52,7 +52,7 @@ user = mayan [program:mayan-worker-slow] autorestart = false autostart = true -command = nice -n 19 /bin/bash -c "${MAYAN_BIN} celery --settings=${MAYAN_SETTINGS_MODULE} worker -Ofair -l ERROR -Q mailing,tools,statistics,parsing,ocr -n mayan-worker-slow.%%h --concurrency=1" +command = nice -n 19 /bin/bash -c "${MAYAN_BIN} celery --settings=${MAYAN_SETTINGS_MODULE} worker -Ofair -l ERROR -Q mailing,tools,statistics,parsing,ocr -n mayan-worker-slow.%%h ${MAYAN_WORKERS_SLOW_CONCURRENCY}" killasgroup = true numprocs = 1 priority = 998