From 1e1b4dedf4c5cc43eb045c6e903924e87dcd2140 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 23 Jul 2019 21:22:30 -0400 Subject: [PATCH] Update Docker make file - Include PIP proxies. - Add docker compose targets. Signed-off-by: Roberto Rosario --- docker/Dockerfile | 2 +- docker/Makefile | 19 +++++++++++++-- docker/rootfs/usr/local/bin/run_celery.sh | 5 ++++ docker/rootfs/usr/local/bin/run_frontend.sh | 7 ++++++ docker/rootfs/usr/local/bin/run_worker.sh | 8 +++++++ mayan/apps/platform/classes.py | 23 +++++++++++++++++++ .../platform/supervisord_docker.tmpl | 16 +------------ .../templates/platform/worker_queues.tmpl | 1 + mayan/apps/task_manager/classes.py | 4 ++++ 9 files changed, 67 insertions(+), 18 deletions(-) create mode 100755 docker/rootfs/usr/local/bin/run_celery.sh create mode 100755 docker/rootfs/usr/local/bin/run_frontend.sh create mode 100755 docker/rootfs/usr/local/bin/run_worker.sh create mode 100644 mayan/apps/platform/templates/platform/worker_queues.tmpl diff --git a/docker/Dockerfile b/docker/Dockerfile index c9fa36c609..58372d50a3 100755 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -144,7 +144,7 @@ VOLUME ["/var/lib/mayan"] ENTRYPOINT ["entrypoint.sh"] EXPOSE 8000 -CMD ["mayan"] +CMD ["run_all"] RUN ${PROJECT_INSTALL_DIR}/bin/mayan-edms.py platformtemplate supervisord_docker > /etc/supervisor/conf.d/mayan.conf \ && apt-get clean autoclean \ diff --git a/docker/Makefile b/docker/Makefile index bd2743759b..cbb25827b9 100755 --- a/docker/Makefile +++ b/docker/Makefile @@ -1,4 +1,9 @@ -APT_PROXY ?= `/sbin/ip route|awk '/docker0/ { print $$9 }'`:3142 +HOST_IP = `/sbin/ip route|awk '/docker0/ { print $$9 }'` + +APT_PROXY ?= $(HOST_IP):3142 +PIP_INDEX_URL ?= http://$(HOST_IP):3141/root/pypi/+simple/ +PIP_TRUSTED_HOST ?= $(HOST_IP) + IMAGE_VERSION ?= `cat docker/rootfs/version` CONSOLE_COLUMNS ?= `echo $$(tput cols)` CONSOLE_LINES ?= `echo $$(tput lines)` @@ -7,7 +12,7 @@ docker-build: ## Build a new image locally. docker build -t mayanedms/mayanedms:$(IMAGE_VERSION) -f docker/Dockerfile . docker-build-with-proxy: ## Build a new image locally using an APT proxy as APT_PROXY. - docker build -t mayanedms/mayanedms:$(IMAGE_VERSION) -f docker/Dockerfile --build-arg APT_PROXY=$(APT_PROXY) . + docker build -t mayanedms/mayanedms:$(IMAGE_VERSION) -f docker/Dockerfile --build-arg APT_PROXY=$(APT_PROXY) --build-arg PIP_INDEX_URL=$(PIP_INDEX_URL) --build-arg PIP_TRUSTED_HOST=$(PIP_TRUSTED_HOST) --build-arg HTTP_PROXY=$(HTTP_PROXY) --build-arg HTTPS_PROXY=$(HTTPS_PROXY) . docker-shell: ## Launch a bash instance inside a running container. Pass the container name via DOCKER_CONTAINER. docker exec -e TERM=$(TERM) -e "COLUMNS=$(CONSOLE_COLUMNS)" -e "LINES=$(CONSOLE_LINES)" -it $(DOCKER_CONTAINER) /bin/bash @@ -23,3 +28,13 @@ docker-test-cleanup: ## Delete the test container and the test volume. docker-test-all: ## Build and executed the test suite in a test container. docker-test-all: docker-build-with-proxy docker run --rm run-tests + +docker-compose-build: + docker-compose -f docker/docker-compose.yml -p mayan-edms build + +docker-compose-build-with-proxy: + docker-compose -f docker/docker-compose.yml -p mayan-edms build --build-arg APT_PROXY=$(APT_PROXY) --build-arg PIP_INDEX_URL=$(PIP_INDEX_URL) --build-arg PIP_TRUSTED_HOST=$(PIP_TRUSTED_HOST) --build-arg HTTP_PROXY=$(HTTP_PROXY) --build-arg HTTPS_PROXY=$(HTTPS_PROXY) + +docker-compose-up: + docker-compose -f docker/docker-compose.yml -p mayan-edms up + diff --git a/docker/rootfs/usr/local/bin/run_celery.sh b/docker/rootfs/usr/local/bin/run_celery.sh new file mode 100755 index 0000000000..a4aba0c8a2 --- /dev/null +++ b/docker/rootfs/usr/local/bin/run_celery.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +# Use -A and not --app. Both are the same but behave differently +# -A can be located before the command while --app cannot. +su mayan -c "${MAYAN_PYTHON_BIN_DIR}celery -A mayan $@" diff --git a/docker/rootfs/usr/local/bin/run_frontend.sh b/docker/rootfs/usr/local/bin/run_frontend.sh new file mode 100755 index 0000000000..f3c3967ffb --- /dev/null +++ b/docker/rootfs/usr/local/bin/run_frontend.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +MAYAN_GUNICORN_MAX_REQUESTS=${MAYAN_GUNICORN_MAX_REQUESTS:-500} +MAYAN_GUNICORN_MAX_REQUESTS_JITTERS=${MAYAN_GUNICORN_MAX_REQUESTS_JITTERS:-50} +MAYAN_GUNICORN_WORKER_CLASS=${MAYAN_GUNICORN_WORKER_CLASS:-sync} + +su mayan -c "${MAYAN_PYTHON_BIN_DIR}gunicorn -w ${MAYAN_GUNICORN_WORKERS} mayan.wsgi --max-requests ${MAYAN_GUNICORN_MAX_REQUESTS} --max-requests-jitter ${MAYAN_GUNICORN_MAX_REQUESTS_JITTERS} --worker-class ${MAYAN_GUNICORN_WORKER_CLASS} --bind 0.0.0.0:8000 --timeout ${MAYAN_GUNICORN_TIMEOUT}" diff --git a/docker/rootfs/usr/local/bin/run_worker.sh b/docker/rootfs/usr/local/bin/run_worker.sh new file mode 100755 index 0000000000..dd5db6068d --- /dev/null +++ b/docker/rootfs/usr/local/bin/run_worker.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +QUEUE_LIST=`MAYAN_WORKER_NAME=$1 su mayan -c "${MAYAN_PYTHON_BIN_DIR}mayan-edms.py platformtemplate worker_queues"` + +# Use -A and not --app. Both are the same but behave differently +# -A can be located before the command while --app cannot. +# Pass ${@:2} to allow overriding the defaults arguments +su mayan -c "${MAYAN_PYTHON_BIN_DIR}celery -A mayan worker -Ofair -l ERROR -Q $QUEUE_LIST ${@:2}" diff --git a/mayan/apps/platform/classes.py b/mayan/apps/platform/classes.py index 8db3ebdb37..8fc3a92cb6 100644 --- a/mayan/apps/platform/classes.py +++ b/mayan/apps/platform/classes.py @@ -181,5 +181,28 @@ class PlatformTemplateSupervisordDocker(PlatformTemplate): return {'workers': Worker.all()} +class PlatformTemplateWorkerQueues(PlatformTemplate): + label = _('Template showing the queues of a worker.') + name = 'worker_queues' + + variables = ( + Variable( + name='WORKER_NAME', default=None, + environment_name='MAYAN_WORKER_NAME' + ), + ) + + def get_context(self): + worker_name = self.get_variables_context().get('WORKER_NAME') + queues = Worker.get(name=worker_name).queues + + return { + 'queues': queues, 'queue_names': sorted( + map(lambda x: x.name, queues) + ) + } + + PlatformTemplate.register(klass=PlatformTemplateSupervisord) PlatformTemplate.register(klass=PlatformTemplateSupervisordDocker) +PlatformTemplate.register(klass=PlatformTemplateWorkerQueues) diff --git a/mayan/apps/platform/templates/platform/supervisord_docker.tmpl b/mayan/apps/platform/templates/platform/supervisord_docker.tmpl index 3448c3b618..7ca6278191 100644 --- a/mayan/apps/platform/templates/platform/supervisord_docker.tmpl +++ b/mayan/apps/platform/templates/platform/supervisord_docker.tmpl @@ -1,11 +1,7 @@ -[supervisord] -environment= - DJANGO_SETTINGS_MODULE="%(ENV_MAYAN_SETTINGS_MODULE)s" - [program:mayan-gunicorn] autorestart = false autostart = true -command = /bin/bash -c "${MAYAN_GUNICORN_BIN} -w ${MAYAN_GUNICORN_WORKERS} mayan.wsgi --max-requests 500 --max-requests-jitter 50 --worker-class gevent --bind 0.0.0.0:8000 --env DJANGO_SETTINGS_MODULE=${MAYAN_SETTINGS_MODULE}" --timeout ${MAYAN_GUNICORN_TIMEOUT} +command = /bin/bash -c "${MAYAN_GUNICORN_BIN} -w ${MAYAN_GUNICORN_WORKERS} mayan.wsgi --max-requests 500 --max-requests-jitter 50 --worker-class sync --bind 0.0.0.0:8000 --env DJANGO_SETTINGS_MODULE=${MAYAN_SETTINGS_MODULE}" --timeout ${MAYAN_GUNICORN_TIMEOUT} redirect_stderr = true stderr_logfile = /dev/fd/2 stderr_logfile_maxbytes = 0 @@ -13,16 +9,6 @@ stdout_logfile = /dev/fd/1 stdout_logfile_maxbytes = 0 user = mayan -[program:redis] -autorestart = false -autostart = true -command = /bin/bash -c "if [ ${MAYAN_CELERY_BROKER_URL} == ${MAYAN_DEFAULT_CELERY_BROKER_URL} ] && [ ${MAYAN_CELERY_RESULT_BACKEND} == ${MAYAN_DEFAULT_CELERY_RESULT_BACKEND} ];then /usr/bin/redis-server /etc/redis/;fi" -stderr_logfile = /dev/fd/2 -stderr_logfile_maxbytes = 0 -stdout_logfile = /dev/fd/1 -stdout_logfile_maxbytes = 0 -user = root - {% for worker in workers %} [program:mayan-worker-{{ worker.name }}] autorestart = false diff --git a/mayan/apps/platform/templates/platform/worker_queues.tmpl b/mayan/apps/platform/templates/platform/worker_queues.tmpl new file mode 100644 index 0000000000..871cafccb8 --- /dev/null +++ b/mayan/apps/platform/templates/platform/worker_queues.tmpl @@ -0,0 +1 @@ +{{ queue_names|join:"," }} diff --git a/mayan/apps/task_manager/classes.py b/mayan/apps/task_manager/classes.py index b075acfb91..2fd398b125 100644 --- a/mayan/apps/task_manager/classes.py +++ b/mayan/apps/task_manager/classes.py @@ -188,6 +188,10 @@ class Worker(object): def all(cls): return cls._registry.values() + @classmethod + def get(cls, name): + return cls._registry[name] + def __init__(self, name, label=None, nice_level=0): self.name = name self.label = label