Update Docker make file

- Include PIP proxies.
- Add docker compose targets.

Signed-off-by: Roberto Rosario <roberto.rosario@mayan-edms.com>

docker/Dockerfile

@@ -144,7 +144,7 @@ VOLUME ["/var/lib/mayan"]
 ENTRYPOINT ["entrypoint.sh"]
 
 EXPOSE 8000
-CMD ["mayan"]
+CMD ["run_all"]
 
 RUN ${PROJECT_INSTALL_DIR}/bin/mayan-edms.py platformtemplate supervisord_docker > /etc/supervisor/conf.d/mayan.conf \
 && apt-get clean autoclean \

docker/Makefile

@@ -1,4 +1,9 @@
-APT_PROXY ?= `/sbin/ip route|awk '/docker0/ { print $$9 }'`:3142
+HOST_IP = `/sbin/ip route|awk '/docker0/ { print $$9 }'`
+
+APT_PROXY ?= $(HOST_IP):3142
+PIP_INDEX_URL ?= http://$(HOST_IP):3141/root/pypi/+simple/
+PIP_TRUSTED_HOST ?= $(HOST_IP)
+
 IMAGE_VERSION ?= `cat docker/rootfs/version`
 CONSOLE_COLUMNS ?= `echo $$(tput cols)`
 CONSOLE_LINES ?= `echo $$(tput lines)`
@@ -7,7 +12,7 @@ docker-build: ## Build a new image locally.
 	docker build -t mayanedms/mayanedms:$(IMAGE_VERSION) -f docker/Dockerfile .
 
 docker-build-with-proxy: ## Build a new image locally using an APT proxy as APT_PROXY.
-	docker build -t mayanedms/mayanedms:$(IMAGE_VERSION) -f docker/Dockerfile --build-arg APT_PROXY=$(APT_PROXY) .
+	docker build -t mayanedms/mayanedms:$(IMAGE_VERSION) -f docker/Dockerfile --build-arg APT_PROXY=$(APT_PROXY) --build-arg PIP_INDEX_URL=$(PIP_INDEX_URL) --build-arg PIP_TRUSTED_HOST=$(PIP_TRUSTED_HOST) --build-arg HTTP_PROXY=$(HTTP_PROXY) --build-arg HTTPS_PROXY=$(HTTPS_PROXY) .
 
 docker-shell: ## Launch a bash instance inside a running container. Pass the container name via DOCKER_CONTAINER.
 	docker exec -e TERM=$(TERM) -e "COLUMNS=$(CONSOLE_COLUMNS)" -e "LINES=$(CONSOLE_LINES)" -it $(DOCKER_CONTAINER) /bin/bash
@@ -23,3 +28,13 @@ docker-test-cleanup: ## Delete the test container and the test volume.
 docker-test-all: ## Build and executed the test suite in a test container.
 docker-test-all: docker-build-with-proxy
 	docker run --rm run-tests
+
+docker-compose-build:
+	docker-compose -f docker/docker-compose.yml -p mayan-edms build
+
+docker-compose-build-with-proxy:
+	docker-compose -f docker/docker-compose.yml -p mayan-edms build --build-arg APT_PROXY=$(APT_PROXY) --build-arg PIP_INDEX_URL=$(PIP_INDEX_URL) --build-arg PIP_TRUSTED_HOST=$(PIP_TRUSTED_HOST) --build-arg HTTP_PROXY=$(HTTP_PROXY) --build-arg HTTPS_PROXY=$(HTTPS_PROXY)
+
+docker-compose-up:
+	docker-compose -f docker/docker-compose.yml -p mayan-edms up
+

docker/rootfs/usr/local/bin/run_celery.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+# Use -A and not --app. Both are the same but behave differently
+# -A can be located before the command while --app cannot.
+su mayan -c "${MAYAN_PYTHON_BIN_DIR}celery -A mayan $@"

docker/rootfs/usr/local/bin/run_frontend.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+MAYAN_GUNICORN_MAX_REQUESTS=${MAYAN_GUNICORN_MAX_REQUESTS:-500}
+MAYAN_GUNICORN_MAX_REQUESTS_JITTERS=${MAYAN_GUNICORN_MAX_REQUESTS_JITTERS:-50}
+MAYAN_GUNICORN_WORKER_CLASS=${MAYAN_GUNICORN_WORKER_CLASS:-sync}
+
+su mayan -c "${MAYAN_PYTHON_BIN_DIR}gunicorn -w ${MAYAN_GUNICORN_WORKERS} mayan.wsgi --max-requests ${MAYAN_GUNICORN_MAX_REQUESTS} --max-requests-jitter ${MAYAN_GUNICORN_MAX_REQUESTS_JITTERS} --worker-class ${MAYAN_GUNICORN_WORKER_CLASS} --bind 0.0.0.0:8000 --timeout ${MAYAN_GUNICORN_TIMEOUT}"

docker/rootfs/usr/local/bin/run_worker.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+QUEUE_LIST=`MAYAN_WORKER_NAME=$1 su mayan -c "${MAYAN_PYTHON_BIN_DIR}mayan-edms.py platformtemplate worker_queues"`
+
+# Use -A and not --app. Both are the same but behave differently
+# -A can be located before the command while --app cannot.
+# Pass ${@:2} to allow overriding the defaults arguments
+su mayan -c "${MAYAN_PYTHON_BIN_DIR}celery -A mayan worker -Ofair -l ERROR -Q $QUEUE_LIST ${@:2}"

mayan/apps/platform/classes.py

@@ -181,5 +181,28 @@ class PlatformTemplateSupervisordDocker(PlatformTemplate):
         return {'workers': Worker.all()}
 
 
+class PlatformTemplateWorkerQueues(PlatformTemplate):
+    label = _('Template showing the queues of a worker.')
+    name = 'worker_queues'
+
+    variables = (
+        Variable(
+            name='WORKER_NAME', default=None,
+            environment_name='MAYAN_WORKER_NAME'
+        ),
+    )
+
+    def get_context(self):
+        worker_name = self.get_variables_context().get('WORKER_NAME')
+        queues = Worker.get(name=worker_name).queues
+
+        return {
+            'queues': queues, 'queue_names': sorted(
+                map(lambda x: x.name, queues)
+            )
+        }
+
+
 PlatformTemplate.register(klass=PlatformTemplateSupervisord)
 PlatformTemplate.register(klass=PlatformTemplateSupervisordDocker)
+PlatformTemplate.register(klass=PlatformTemplateWorkerQueues)

mayan/apps/platform/templates/platform/supervisord_docker.tmpl

@@ -1,11 +1,7 @@
-[supervisord]
-environment=
-    DJANGO_SETTINGS_MODULE="%(ENV_MAYAN_SETTINGS_MODULE)s"
-
 [program:mayan-gunicorn]
 autorestart = false
 autostart = true
-command = /bin/bash -c "${MAYAN_GUNICORN_BIN} -w ${MAYAN_GUNICORN_WORKERS} mayan.wsgi --max-requests 500 --max-requests-jitter 50 --worker-class gevent --bind 0.0.0.0:8000 --env DJANGO_SETTINGS_MODULE=${MAYAN_SETTINGS_MODULE}" --timeout ${MAYAN_GUNICORN_TIMEOUT}
+command = /bin/bash -c "${MAYAN_GUNICORN_BIN} -w ${MAYAN_GUNICORN_WORKERS} mayan.wsgi --max-requests 500 --max-requests-jitter 50 --worker-class sync --bind 0.0.0.0:8000 --env DJANGO_SETTINGS_MODULE=${MAYAN_SETTINGS_MODULE}" --timeout ${MAYAN_GUNICORN_TIMEOUT}
 redirect_stderr = true
 stderr_logfile = /dev/fd/2
 stderr_logfile_maxbytes = 0
@@ -13,16 +9,6 @@ stdout_logfile = /dev/fd/1
 stdout_logfile_maxbytes = 0
 user = mayan
 
-[program:redis]
-autorestart = false
-autostart = true
-command = /bin/bash -c "if [ ${MAYAN_CELERY_BROKER_URL} == ${MAYAN_DEFAULT_CELERY_BROKER_URL} ] && [ ${MAYAN_CELERY_RESULT_BACKEND} == ${MAYAN_DEFAULT_CELERY_RESULT_BACKEND} ];then /usr/bin/redis-server /etc/redis/;fi"
-stderr_logfile = /dev/fd/2
-stderr_logfile_maxbytes = 0
-stdout_logfile = /dev/fd/1
-stdout_logfile_maxbytes = 0
-user = root
-
 {% for worker in workers %}
 [program:mayan-worker-{{ worker.name }}]
 autorestart = false

mayan/apps/platform/templates/platform/worker_queues.tmpl

@@ -0,0 +1 @@
+{{ queue_names|join:"," }}

mayan/apps/task_manager/classes.py

@@ -188,6 +188,10 @@ class Worker(object):
     def all(cls):
         return cls._registry.values()
 
+    @classmethod
+    def get(cls, name):
+        return cls._registry[name]
+
     def __init__(self, name, label=None, nice_level=0):
         self.name = name
         self.label = label