Unify supervisor configuration files. Lower nice levels of most workers. Merge OCR and slow workers. Lower gunicorn workers to 2. Call all supervisor processes using a shell to expand the install and mayan binary locations and avoid hardcoding.

Signed-off-by: Roberto Rosario <roberto.rosario.gonzalez@gmail.com>
2018-05-04 00:48:03 -04:00
parent 9fad6c37bf
commit 55359c72cf
7 changed files with 92 additions and 110 deletions
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -39,6 +39,7 @@ apt-get install -y --no-install-recommends \
        python-setuptools \
        python-wheel \
        redis-server \
+        sane-utils \
        supervisor \
        tesseract-ocr \
        zlib1g-dev \
@@ -76,7 +77,7 @@ WORKDIR /code

 COPY . /code

-RUN apt-get update && apt-get install make python-dev python-pip -y 
+RUN apt-get update && apt-get install make python-dev python-pip -y

 RUN pip install -r requirements/build.txt

@@ -99,14 +100,7 @@ RUN pip install *.whl && \
 rm *.whl

 # Setup supervisor
-#RUN mkdir /etc/supervisor.d/
-COPY docker/etc/supervisor/beat.conf /etc/supervisor/conf.d
-COPY docker/etc/supervisor/gunicorn.conf /etc/supervisor/conf.d
-COPY docker/etc/supervisor/redis.conf /etc/supervisor/conf.d
-COPY docker/etc/supervisor/workers.conf /etc/supervisor/conf.d
-
-# Create the directory for the logs
-RUN mkdir /var/log/mayan
+COPY docker/etc/supervisor/mayan.conf /etc/supervisor/conf.d

 # Fix ownership
 RUN chown -R mayan:mayan $PROJECT_INSTALL_DIR
--- a/docker/entrypoint.sh
+++ b/docker/entrypoint.sh
@@ -1,44 +1,52 @@
 #!/bin/sh

 set -e
-echo "* start"
+echo "mayan: starting entrypoint.sh"
 INSTALL_FLAG=/var/lib/mayan/media/system/SECRET_KEY

-export MAYAN_MEDIA_ROOT=/var/lib/mayan
-export MAYAN_GUNICORN_WORKERS=${MAYAN_GUNICORN_WORKERS:-3}
+export MAYAN_DEFAULT_BROKER_URL=redis://127.0.0.1:6379/0
+export MAYAN_DEFAULT_CELERY_RESULT_BACKEND=redis://127.0.0.1:6379/0
 export MAYAN_ALLOWED_HOSTS=*
-export CELERY_ALWAYS_EAGER=True
-export MAYAN_CELERY_RESULT_BACKEND=${MAYAN_CELERY_RESULT_BACKEND:-redis://127.0.0.1:6379/0}
+export MAYAN_BIN=/usr/local/lib/python2.7/dist-packages/mayan/bin/mayan-edms.py
 export MAYAN_BROKER_URL=${MAYAN_BROKER_URL:-redis://127.0.0.1:6379/0}
+export MAYAN_CELERY_RESULT_BACKEND=${MAYAN_CELERY_RESULT_BACKEND:-redis://127.0.0.1:6379/0}
+export MAYAN_GUNICORN_WORKERS=${MAYAN_GUNICORN_WORKERS:-2}
+export MAYAN_INSTALL_DIR=/usr/local/lib/python2.7/dist-packages/mayan
+export MAYAN_MEDIA_ROOT=/var/lib/mayan
+export MAYAN_SETTINGS_FILE=${MAYAN_SETTINGS_FILE:-mayan.settings.production}
+
+export CELERY_ALWAYS_EAGER=False
+export PYTHONPATH=$PYTHONPATH:$MAYAN_MEDIA_ROOT/settings

 chown mayan:mayan /var/lib/mayan -R

 initialize() {
-    echo "* initialize"
+    echo "mayan: initialize()"
    su mayan -c "mayan-edms.py initialsetup --force"
    su mayan -c "mayan-edms.py collectstatic --noinput --clear"
 }

 upgrade() {
-    echo "* upgrade"
+    echo "mayan: upgrade()"
    su mayan -c "mayan-edms.py performupgrade"
    su mayan -c "mayan-edms.py collectstatic --noinput --clear"
 }

 start() {
+    echo "mayan: start()"
    rm -rf /var/run/supervisor.sock
    exec /usr/bin/supervisord -nc /etc/supervisor/supervisord.conf
 }

 os_package_installs() {
-    echo "* os_package_installs"
+    echo "mayan: os_package_installs()"
    if [ "${MAYAN_APT_INSTALLS}" ]; then
        apt-get-install $MAYAN_APT_INSTALLS
    fi
 }

 pip_installs() {
-    echo "* pip_installs"
+    echo "mayan: pip_installs()"
    if [ "${MAYAN_PIP_INSTALLS}" ]; then
        pip install $MAYAN_PIP_INSTALLS
    fi
--- a/docker/etc/supervisor/beat.conf
+++ b/docker/etc/supervisor/beat.conf
@@ -1,15 +0,0 @@
-[program:celery-beat]
-autorestart = false
-autostart = true
-command = python /usr/local/lib/python2.7/dist-packages/mayan/bin/mayan-edms.py celery --settings=mayan.settings.production beat -l ERROR
-directory = /usr/local/lib/python2.7/dist-packages/mayan/
-killasgroup = true
-numprocs = 1
-priority = 998
-stderr_logfile = /dev/fd/2
-stderr_logfile_maxbytes = 0
-stdout_logfile = /dev/fd/1
-stdout_logfile_maxbytes = 0
-startsecs = 10
-stopwaitsecs = 1
-user = mayan
--- a/docker/etc/supervisor/gunicorn.conf
+++ b/docker/etc/supervisor/gunicorn.conf
@@ -1,9 +0,0 @@
-[program:gunicorn]
-autorestart = false
-autostart = true
-command = /bin/bash -c "gunicorn -w ${MAYAN_GUNICORN_WORKERS:-3} mayan.wsgi --max-requests 1000 --max-requests-jitter 50 --worker-class gevent --bind 0.0.0.0:8000"
-#command = gunicorn -w ${MAYAN_GUNICORN_WORKERS:-3} mayan.wsgi --max-requests 1000 --max-requests-jitter 50 --worker-class gevent --bind 0.0.0.0:8000
-#command = /bin/bash -c "if [ -z ${MAYAN_BROKER_URL} ] || [ -z ${MAYAN_CELERY_RESULT_BACKEND} ];then /usr/bin/redis-server /etc/redis/;fi"
-directory = /tmp
-redirect_stderr = true
-user = mayan
--- a/docker/etc/supervisor/mayan.conf
+++ b/docker/etc/supervisor/mayan.conf
@@ -0,0 +1,72 @@
+[program:gunicorn]
+autorestart = false
+autostart = true
+command = /bin/bash -c "cd ${MAYAN_INSTALL_DIR}; gunicorn -w ${MAYAN_GUNICORN_WORKERS} mayan.wsgi --max-requests 500 --max-requests-jitter 50 --worker-class gevent --bind 0.0.0.0:8000  --env DJANGO_SETTINGS_MODULE=${MAYAN_SETTINGS_FILE}"
+redirect_stderr = true
+user = mayan
+
+[program:redis]
+autorestart = false
+autostart = true
+command = /bin/bash -c "if [ ${MAYAN_BROKER_URL} != ${MAYAN_DEFAULT_BROKER_URL} ] || [ ${MAYAN_CELERY_RESULT_BACKEND} != ${MAYAN_DEFAULT_CELERY_RESULT_BACKEND} ];then /usr/bin/redis-server /etc/redis/;fi"
+user = root
+
+[program:mayan-worker-fast]
+autorestart = false
+autostart = true
+command = nice -n 1 /bin/bash -c "cd ${MAYAN_INSTALL_DIR}; python ${MAYAN_BIN} celery --settings=${MAYAN_SETTINGS_FILE} worker -Ofair -l ERROR -Q converter -n mayan-worker-fast.%%h --concurrency=1"
+killasgroup = true
+numprocs = 1
+priority = 998
+startsecs = 10
+stderr_logfile = /dev/fd/2
+stderr_logfile_maxbytes = 0
+stdout_logfile = /dev/fd/1
+stdout_logfile_maxbytes = 0
+stopwaitsecs = 1
+user = mayan
+
+[program:mayan-worker-medium]
+autorestart = false
+autostart = true
+command = nice -n 18 /bin/bash -c "cd  ${MAYAN_INSTALL_DIR}; python ${MAYAN_BIN} celery --settings=${MAYAN_SETTINGS_FILE} worker -Ofair -l ERROR -Q checkouts_periodic,documents_periodic,indexing,metadata,sources,sources_periodic,uploads,documents -n mayan-worker-medium.%%h --concurrency=1"
+killasgroup = true
+numprocs = 1
+priority = 998
+startsecs = 10
+stderr_logfile = /dev/fd/2
+stderr_logfile_maxbytes = 0
+stdout_logfile = /dev/fd/1
+stdout_logfile_maxbytes = 0
+stopwaitsecs = 1
+user = mayan
+
+[program:mayan-worker-slow]
+autorestart = false
+autostart = true
+command = nice -n 19 /bin/bash -c "cd ${MAYAN_INSTALL_DIR}; python ${MAYAN_BIN} celery --settings=${MAYAN_SETTINGS_FILE} worker -Ofair -l ERROR -Q mailing,tools,statistics,parsing,ocr -n mayan-worker-slow.%%h --concurrency=1"
+killasgroup = true
+numprocs = 1
+priority = 998
+startsecs = 10
+stderr_logfile = /dev/fd/2
+stderr_logfile_maxbytes = 0
+stdout_logfile = /dev/fd/1
+stdout_logfile_maxbytes = 0
+stopwaitsecs = 1
+user = mayan
+
+[program:celery-beat]
+autorestart = false
+autostart = true
+command = nice -n 1 /bin/bash -c "cd ${MAYAN_INSTALL_DIR}; python ${MAYAN_BIN} celery --settings=${MAYAN_SETTINGS_FILE} beat -l ERROR"
+killasgroup = true
+numprocs = 1
+priority = 998
+stderr_logfile = /dev/fd/2
+stderr_logfile_maxbytes = 0
+stdout_logfile = /dev/fd/1
+stdout_logfile_maxbytes = 0
+startsecs = 10
+stopwaitsecs = 1
+user = mayan
--- a/docker/etc/supervisor/redis.conf
+++ b/docker/etc/supervisor/redis.conf
@@ -1,5 +0,0 @@
-[program:redis]
-autorestart = false
-autostart = true
-command = /bin/bash -c "if [ -z ${MAYAN_BROKER_URL} ] || [ -z ${MAYAN_CELERY_RESULT_BACKEND} ];then /usr/bin/redis-server /etc/redis/;fi"
-user = root
--- a/docker/etc/supervisor/workers.conf
+++ b/docker/etc/supervisor/workers.conf
@@ -1,63 +0,0 @@
-[program:mayan-worker-fast]
-autorestart = false
-autostart = true
-command = python /usr/local/lib/python2.7/dist-packages/mayan/bin/mayan-edms.py celery --settings=mayan.settings.production worker -Ofair -l ERROR -Q converter -n mayan-worker-fast.%%h --concurrency=1
-directory = /usr/local/lib/python2.7/dist-packages/mayan/
-killasgroup = true
-numprocs = 1
-priority = 998
-startsecs = 10
-stderr_logfile = /dev/fd/2
-stderr_logfile_maxbytes = 0
-stdout_logfile = /dev/fd/1
-stdout_logfile_maxbytes = 0
-stopwaitsecs = 1
-user = mayan
-
-[program:mayan-worker-medium]
-autorestart = false
-autostart = true
-command = python /usr/local/lib/python2.7/dist-packages/mayan/bin/mayan-edms.py celery --settings=mayan.settings.production worker -Ofair -l ERROR -Q checkouts_periodic,documents_periodic,indexing,metadata,sources,sources_periodic,uploads,documents -n mayan-worker-medium.%%h --concurrency=1
-directory = /usr/local/lib/python2.7/dist-packages/mayan/
-killasgroup = true
-numprocs = 1
-priority = 998
-startsecs = 10
-stderr_logfile = /dev/fd/2
-stderr_logfile_maxbytes = 0
-stdout_logfile = /dev/fd/1
-stdout_logfile_maxbytes = 0
-stopwaitsecs = 1
-user = mayan
-
-[program:mayan-worker-slow]
-autorestart = false
-autostart = true
-command = python /usr/local/lib/python2.7/dist-packages/mayan/bin/mayan-edms.py celery --settings=mayan.settings.production worker -Ofair -l ERROR -Q mailing,tools,statistics -n mayan-worker-slow.%%h --concurrency=1
-directory = /usr/local/lib/python2.7/dist-packages/mayan/
-killasgroup = true
-numprocs = 1
-priority = 998
-startsecs = 10
-stderr_logfile = /dev/fd/2
-stderr_logfile_maxbytes = 0
-stdout_logfile = /dev/fd/1
-stdout_logfile_maxbytes = 0
-stopwaitsecs = 1
-user = mayan
-
-[program:mayan-worker-ocr]
-autorestart = false
-autostart = true
-command = nice -n 19 python /usr/local/lib/python2.7/dist-packages/mayan/bin/mayan-edms.py celery --settings=mayan.settings.production worker -Ofair -l ERROR -Q parsing,ocr -n mayan-worker-ocr.%%h --concurrency=1
-directory = /usr/local/lib/python2.7/dist-packages/mayan/
-killasgroup = true
-numprocs = 1
-priority = 998
-startsecs = 10
-stderr_logfile = /dev/fd/2
-stderr_logfile_maxbytes = 0
-stdout_logfile = /dev/fd/1
-stdout_logfile_maxbytes = 0
-stopwaitsecs = 1
-user = mayan