Unify supervisor configuration files. Lower nice levels of most workers. Merge OCR and slow workers. Lower gunicorn workers to 2. Call all supervisor processes using a shell to expand the install and mayan binary locations and avoid hardcoding.
Signed-off-by: Roberto Rosario <roberto.rosario.gonzalez@gmail.com>
This commit is contained in:
+3
-9
@@ -39,6 +39,7 @@ apt-get install -y --no-install-recommends \
|
|||||||
python-setuptools \
|
python-setuptools \
|
||||||
python-wheel \
|
python-wheel \
|
||||||
redis-server \
|
redis-server \
|
||||||
|
sane-utils \
|
||||||
supervisor \
|
supervisor \
|
||||||
tesseract-ocr \
|
tesseract-ocr \
|
||||||
zlib1g-dev \
|
zlib1g-dev \
|
||||||
@@ -76,7 +77,7 @@ WORKDIR /code
|
|||||||
|
|
||||||
COPY . /code
|
COPY . /code
|
||||||
|
|
||||||
RUN apt-get update && apt-get install make python-dev python-pip -y
|
RUN apt-get update && apt-get install make python-dev python-pip -y
|
||||||
|
|
||||||
RUN pip install -r requirements/build.txt
|
RUN pip install -r requirements/build.txt
|
||||||
|
|
||||||
@@ -99,14 +100,7 @@ RUN pip install *.whl && \
|
|||||||
rm *.whl
|
rm *.whl
|
||||||
|
|
||||||
# Setup supervisor
|
# Setup supervisor
|
||||||
#RUN mkdir /etc/supervisor.d/
|
COPY docker/etc/supervisor/mayan.conf /etc/supervisor/conf.d
|
||||||
COPY docker/etc/supervisor/beat.conf /etc/supervisor/conf.d
|
|
||||||
COPY docker/etc/supervisor/gunicorn.conf /etc/supervisor/conf.d
|
|
||||||
COPY docker/etc/supervisor/redis.conf /etc/supervisor/conf.d
|
|
||||||
COPY docker/etc/supervisor/workers.conf /etc/supervisor/conf.d
|
|
||||||
|
|
||||||
# Create the directory for the logs
|
|
||||||
RUN mkdir /var/log/mayan
|
|
||||||
|
|
||||||
# Fix ownership
|
# Fix ownership
|
||||||
RUN chown -R mayan:mayan $PROJECT_INSTALL_DIR
|
RUN chown -R mayan:mayan $PROJECT_INSTALL_DIR
|
||||||
|
|||||||
+17
-9
@@ -1,44 +1,52 @@
|
|||||||
#!/bin/sh
|
#!/bin/sh
|
||||||
|
|
||||||
set -e
|
set -e
|
||||||
echo "* start"
|
echo "mayan: starting entrypoint.sh"
|
||||||
INSTALL_FLAG=/var/lib/mayan/media/system/SECRET_KEY
|
INSTALL_FLAG=/var/lib/mayan/media/system/SECRET_KEY
|
||||||
|
|
||||||
export MAYAN_MEDIA_ROOT=/var/lib/mayan
|
export MAYAN_DEFAULT_BROKER_URL=redis://127.0.0.1:6379/0
|
||||||
export MAYAN_GUNICORN_WORKERS=${MAYAN_GUNICORN_WORKERS:-3}
|
export MAYAN_DEFAULT_CELERY_RESULT_BACKEND=redis://127.0.0.1:6379/0
|
||||||
export MAYAN_ALLOWED_HOSTS=*
|
export MAYAN_ALLOWED_HOSTS=*
|
||||||
export CELERY_ALWAYS_EAGER=True
|
export MAYAN_BIN=/usr/local/lib/python2.7/dist-packages/mayan/bin/mayan-edms.py
|
||||||
export MAYAN_CELERY_RESULT_BACKEND=${MAYAN_CELERY_RESULT_BACKEND:-redis://127.0.0.1:6379/0}
|
|
||||||
export MAYAN_BROKER_URL=${MAYAN_BROKER_URL:-redis://127.0.0.1:6379/0}
|
export MAYAN_BROKER_URL=${MAYAN_BROKER_URL:-redis://127.0.0.1:6379/0}
|
||||||
|
export MAYAN_CELERY_RESULT_BACKEND=${MAYAN_CELERY_RESULT_BACKEND:-redis://127.0.0.1:6379/0}
|
||||||
|
export MAYAN_GUNICORN_WORKERS=${MAYAN_GUNICORN_WORKERS:-2}
|
||||||
|
export MAYAN_INSTALL_DIR=/usr/local/lib/python2.7/dist-packages/mayan
|
||||||
|
export MAYAN_MEDIA_ROOT=/var/lib/mayan
|
||||||
|
export MAYAN_SETTINGS_FILE=${MAYAN_SETTINGS_FILE:-mayan.settings.production}
|
||||||
|
|
||||||
|
export CELERY_ALWAYS_EAGER=False
|
||||||
|
export PYTHONPATH=$PYTHONPATH:$MAYAN_MEDIA_ROOT/settings
|
||||||
|
|
||||||
chown mayan:mayan /var/lib/mayan -R
|
chown mayan:mayan /var/lib/mayan -R
|
||||||
|
|
||||||
initialize() {
|
initialize() {
|
||||||
echo "* initialize"
|
echo "mayan: initialize()"
|
||||||
su mayan -c "mayan-edms.py initialsetup --force"
|
su mayan -c "mayan-edms.py initialsetup --force"
|
||||||
su mayan -c "mayan-edms.py collectstatic --noinput --clear"
|
su mayan -c "mayan-edms.py collectstatic --noinput --clear"
|
||||||
}
|
}
|
||||||
|
|
||||||
upgrade() {
|
upgrade() {
|
||||||
echo "* upgrade"
|
echo "mayan: upgrade()"
|
||||||
su mayan -c "mayan-edms.py performupgrade"
|
su mayan -c "mayan-edms.py performupgrade"
|
||||||
su mayan -c "mayan-edms.py collectstatic --noinput --clear"
|
su mayan -c "mayan-edms.py collectstatic --noinput --clear"
|
||||||
}
|
}
|
||||||
|
|
||||||
start() {
|
start() {
|
||||||
|
echo "mayan: start()"
|
||||||
rm -rf /var/run/supervisor.sock
|
rm -rf /var/run/supervisor.sock
|
||||||
exec /usr/bin/supervisord -nc /etc/supervisor/supervisord.conf
|
exec /usr/bin/supervisord -nc /etc/supervisor/supervisord.conf
|
||||||
}
|
}
|
||||||
|
|
||||||
os_package_installs() {
|
os_package_installs() {
|
||||||
echo "* os_package_installs"
|
echo "mayan: os_package_installs()"
|
||||||
if [ "${MAYAN_APT_INSTALLS}" ]; then
|
if [ "${MAYAN_APT_INSTALLS}" ]; then
|
||||||
apt-get-install $MAYAN_APT_INSTALLS
|
apt-get-install $MAYAN_APT_INSTALLS
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
pip_installs() {
|
pip_installs() {
|
||||||
echo "* pip_installs"
|
echo "mayan: pip_installs()"
|
||||||
if [ "${MAYAN_PIP_INSTALLS}" ]; then
|
if [ "${MAYAN_PIP_INSTALLS}" ]; then
|
||||||
pip install $MAYAN_PIP_INSTALLS
|
pip install $MAYAN_PIP_INSTALLS
|
||||||
fi
|
fi
|
||||||
|
|||||||
@@ -1,15 +0,0 @@
|
|||||||
[program:celery-beat]
|
|
||||||
autorestart = false
|
|
||||||
autostart = true
|
|
||||||
command = python /usr/local/lib/python2.7/dist-packages/mayan/bin/mayan-edms.py celery --settings=mayan.settings.production beat -l ERROR
|
|
||||||
directory = /usr/local/lib/python2.7/dist-packages/mayan/
|
|
||||||
killasgroup = true
|
|
||||||
numprocs = 1
|
|
||||||
priority = 998
|
|
||||||
stderr_logfile = /dev/fd/2
|
|
||||||
stderr_logfile_maxbytes = 0
|
|
||||||
stdout_logfile = /dev/fd/1
|
|
||||||
stdout_logfile_maxbytes = 0
|
|
||||||
startsecs = 10
|
|
||||||
stopwaitsecs = 1
|
|
||||||
user = mayan
|
|
||||||
@@ -1,9 +0,0 @@
|
|||||||
[program:gunicorn]
|
|
||||||
autorestart = false
|
|
||||||
autostart = true
|
|
||||||
command = /bin/bash -c "gunicorn -w ${MAYAN_GUNICORN_WORKERS:-3} mayan.wsgi --max-requests 1000 --max-requests-jitter 50 --worker-class gevent --bind 0.0.0.0:8000"
|
|
||||||
#command = gunicorn -w ${MAYAN_GUNICORN_WORKERS:-3} mayan.wsgi --max-requests 1000 --max-requests-jitter 50 --worker-class gevent --bind 0.0.0.0:8000
|
|
||||||
#command = /bin/bash -c "if [ -z ${MAYAN_BROKER_URL} ] || [ -z ${MAYAN_CELERY_RESULT_BACKEND} ];then /usr/bin/redis-server /etc/redis/;fi"
|
|
||||||
directory = /tmp
|
|
||||||
redirect_stderr = true
|
|
||||||
user = mayan
|
|
||||||
@@ -0,0 +1,72 @@
|
|||||||
|
[program:gunicorn]
|
||||||
|
autorestart = false
|
||||||
|
autostart = true
|
||||||
|
command = /bin/bash -c "cd ${MAYAN_INSTALL_DIR}; gunicorn -w ${MAYAN_GUNICORN_WORKERS} mayan.wsgi --max-requests 500 --max-requests-jitter 50 --worker-class gevent --bind 0.0.0.0:8000 --env DJANGO_SETTINGS_MODULE=${MAYAN_SETTINGS_FILE}"
|
||||||
|
redirect_stderr = true
|
||||||
|
user = mayan
|
||||||
|
|
||||||
|
[program:redis]
|
||||||
|
autorestart = false
|
||||||
|
autostart = true
|
||||||
|
command = /bin/bash -c "if [ ${MAYAN_BROKER_URL} != ${MAYAN_DEFAULT_BROKER_URL} ] || [ ${MAYAN_CELERY_RESULT_BACKEND} != ${MAYAN_DEFAULT_CELERY_RESULT_BACKEND} ];then /usr/bin/redis-server /etc/redis/;fi"
|
||||||
|
user = root
|
||||||
|
|
||||||
|
[program:mayan-worker-fast]
|
||||||
|
autorestart = false
|
||||||
|
autostart = true
|
||||||
|
command = nice -n 1 /bin/bash -c "cd ${MAYAN_INSTALL_DIR}; python ${MAYAN_BIN} celery --settings=${MAYAN_SETTINGS_FILE} worker -Ofair -l ERROR -Q converter -n mayan-worker-fast.%%h --concurrency=1"
|
||||||
|
killasgroup = true
|
||||||
|
numprocs = 1
|
||||||
|
priority = 998
|
||||||
|
startsecs = 10
|
||||||
|
stderr_logfile = /dev/fd/2
|
||||||
|
stderr_logfile_maxbytes = 0
|
||||||
|
stdout_logfile = /dev/fd/1
|
||||||
|
stdout_logfile_maxbytes = 0
|
||||||
|
stopwaitsecs = 1
|
||||||
|
user = mayan
|
||||||
|
|
||||||
|
[program:mayan-worker-medium]
|
||||||
|
autorestart = false
|
||||||
|
autostart = true
|
||||||
|
command = nice -n 18 /bin/bash -c "cd ${MAYAN_INSTALL_DIR}; python ${MAYAN_BIN} celery --settings=${MAYAN_SETTINGS_FILE} worker -Ofair -l ERROR -Q checkouts_periodic,documents_periodic,indexing,metadata,sources,sources_periodic,uploads,documents -n mayan-worker-medium.%%h --concurrency=1"
|
||||||
|
killasgroup = true
|
||||||
|
numprocs = 1
|
||||||
|
priority = 998
|
||||||
|
startsecs = 10
|
||||||
|
stderr_logfile = /dev/fd/2
|
||||||
|
stderr_logfile_maxbytes = 0
|
||||||
|
stdout_logfile = /dev/fd/1
|
||||||
|
stdout_logfile_maxbytes = 0
|
||||||
|
stopwaitsecs = 1
|
||||||
|
user = mayan
|
||||||
|
|
||||||
|
[program:mayan-worker-slow]
|
||||||
|
autorestart = false
|
||||||
|
autostart = true
|
||||||
|
command = nice -n 19 /bin/bash -c "cd ${MAYAN_INSTALL_DIR}; python ${MAYAN_BIN} celery --settings=${MAYAN_SETTINGS_FILE} worker -Ofair -l ERROR -Q mailing,tools,statistics,parsing,ocr -n mayan-worker-slow.%%h --concurrency=1"
|
||||||
|
killasgroup = true
|
||||||
|
numprocs = 1
|
||||||
|
priority = 998
|
||||||
|
startsecs = 10
|
||||||
|
stderr_logfile = /dev/fd/2
|
||||||
|
stderr_logfile_maxbytes = 0
|
||||||
|
stdout_logfile = /dev/fd/1
|
||||||
|
stdout_logfile_maxbytes = 0
|
||||||
|
stopwaitsecs = 1
|
||||||
|
user = mayan
|
||||||
|
|
||||||
|
[program:celery-beat]
|
||||||
|
autorestart = false
|
||||||
|
autostart = true
|
||||||
|
command = nice -n 1 /bin/bash -c "cd ${MAYAN_INSTALL_DIR}; python ${MAYAN_BIN} celery --settings=${MAYAN_SETTINGS_FILE} beat -l ERROR"
|
||||||
|
killasgroup = true
|
||||||
|
numprocs = 1
|
||||||
|
priority = 998
|
||||||
|
stderr_logfile = /dev/fd/2
|
||||||
|
stderr_logfile_maxbytes = 0
|
||||||
|
stdout_logfile = /dev/fd/1
|
||||||
|
stdout_logfile_maxbytes = 0
|
||||||
|
startsecs = 10
|
||||||
|
stopwaitsecs = 1
|
||||||
|
user = mayan
|
||||||
@@ -1,5 +0,0 @@
|
|||||||
[program:redis]
|
|
||||||
autorestart = false
|
|
||||||
autostart = true
|
|
||||||
command = /bin/bash -c "if [ -z ${MAYAN_BROKER_URL} ] || [ -z ${MAYAN_CELERY_RESULT_BACKEND} ];then /usr/bin/redis-server /etc/redis/;fi"
|
|
||||||
user = root
|
|
||||||
@@ -1,63 +0,0 @@
|
|||||||
[program:mayan-worker-fast]
|
|
||||||
autorestart = false
|
|
||||||
autostart = true
|
|
||||||
command = python /usr/local/lib/python2.7/dist-packages/mayan/bin/mayan-edms.py celery --settings=mayan.settings.production worker -Ofair -l ERROR -Q converter -n mayan-worker-fast.%%h --concurrency=1
|
|
||||||
directory = /usr/local/lib/python2.7/dist-packages/mayan/
|
|
||||||
killasgroup = true
|
|
||||||
numprocs = 1
|
|
||||||
priority = 998
|
|
||||||
startsecs = 10
|
|
||||||
stderr_logfile = /dev/fd/2
|
|
||||||
stderr_logfile_maxbytes = 0
|
|
||||||
stdout_logfile = /dev/fd/1
|
|
||||||
stdout_logfile_maxbytes = 0
|
|
||||||
stopwaitsecs = 1
|
|
||||||
user = mayan
|
|
||||||
|
|
||||||
[program:mayan-worker-medium]
|
|
||||||
autorestart = false
|
|
||||||
autostart = true
|
|
||||||
command = python /usr/local/lib/python2.7/dist-packages/mayan/bin/mayan-edms.py celery --settings=mayan.settings.production worker -Ofair -l ERROR -Q checkouts_periodic,documents_periodic,indexing,metadata,sources,sources_periodic,uploads,documents -n mayan-worker-medium.%%h --concurrency=1
|
|
||||||
directory = /usr/local/lib/python2.7/dist-packages/mayan/
|
|
||||||
killasgroup = true
|
|
||||||
numprocs = 1
|
|
||||||
priority = 998
|
|
||||||
startsecs = 10
|
|
||||||
stderr_logfile = /dev/fd/2
|
|
||||||
stderr_logfile_maxbytes = 0
|
|
||||||
stdout_logfile = /dev/fd/1
|
|
||||||
stdout_logfile_maxbytes = 0
|
|
||||||
stopwaitsecs = 1
|
|
||||||
user = mayan
|
|
||||||
|
|
||||||
[program:mayan-worker-slow]
|
|
||||||
autorestart = false
|
|
||||||
autostart = true
|
|
||||||
command = python /usr/local/lib/python2.7/dist-packages/mayan/bin/mayan-edms.py celery --settings=mayan.settings.production worker -Ofair -l ERROR -Q mailing,tools,statistics -n mayan-worker-slow.%%h --concurrency=1
|
|
||||||
directory = /usr/local/lib/python2.7/dist-packages/mayan/
|
|
||||||
killasgroup = true
|
|
||||||
numprocs = 1
|
|
||||||
priority = 998
|
|
||||||
startsecs = 10
|
|
||||||
stderr_logfile = /dev/fd/2
|
|
||||||
stderr_logfile_maxbytes = 0
|
|
||||||
stdout_logfile = /dev/fd/1
|
|
||||||
stdout_logfile_maxbytes = 0
|
|
||||||
stopwaitsecs = 1
|
|
||||||
user = mayan
|
|
||||||
|
|
||||||
[program:mayan-worker-ocr]
|
|
||||||
autorestart = false
|
|
||||||
autostart = true
|
|
||||||
command = nice -n 19 python /usr/local/lib/python2.7/dist-packages/mayan/bin/mayan-edms.py celery --settings=mayan.settings.production worker -Ofair -l ERROR -Q parsing,ocr -n mayan-worker-ocr.%%h --concurrency=1
|
|
||||||
directory = /usr/local/lib/python2.7/dist-packages/mayan/
|
|
||||||
killasgroup = true
|
|
||||||
numprocs = 1
|
|
||||||
priority = 998
|
|
||||||
startsecs = 10
|
|
||||||
stderr_logfile = /dev/fd/2
|
|
||||||
stderr_logfile_maxbytes = 0
|
|
||||||
stdout_logfile = /dev/fd/1
|
|
||||||
stdout_logfile_maxbytes = 0
|
|
||||||
stopwaitsecs = 1
|
|
||||||
user = mayan
|
|
||||||
Reference in New Issue
Block a user