Add top level config.env file

Update the following elements to use the top level config
environment file:
- CI
- Makefile
- Docker makefile
- Documenation

Signed-off-by: Roberto Rosario <roberto.rosario@mayan-edms.com>

.gitlab-ci.yml

@@ -170,7 +170,8 @@ test-mysql:
     - mysql:8.0.3
   script:
     - apt-get install -qq libmysqlclient-dev mysql-client
-    - pip install mysqlclient
+    - set -a && . ./config.env && set +a
+    - pip install mysqlclient==$PYTHON_MYSQL_VERSION
     - mysql -h"$MYSQL_PORT_3306_TCP_ADDR" -P"$MYSQL_PORT_3306_TCP_PORT" -uroot -p"$MYSQL_ENV_MYSQL_ROOT_PASSWORD" -e "set global character_set_server=utf8mb4;"
     - python manage.py test --mayan-apps --settings=mayan.settings.testing.gitlab-ci.db_mysql --nomigrations
   tags:
@@ -185,7 +186,8 @@ test-postgres:
     - postgres
   script:
     - apt-get install -qq libpq-dev
-    - pip install psycopg2
+    - set -a && . ./config.env && set +a
+    - pip install psycopg2==$PYTHON_PSYCOPG2_VERSION
     - python manage.py test --mayan-apps --settings=mayan.settings.testing.gitlab-ci.db_postgres --nomigrations
   tags:
     - postgres

Makefile

@@ -1,3 +1,5 @@
+#!make
+include config.env
 .PHONY: clean-pyc clean-build
 
 help:
@@ -35,7 +37,7 @@ test-launch-postgres:
 	@docker volume rm test-postgres || true
 	docker run -d --name test-postgres -p 5432:5432 -v test-postgres:/var/lib/postgresql/data healthcheck/postgres
 	sudo apt-get install -q libpq-dev
-	pip install psycopg2
+	pip install psycopg2==$(PYTHON_PSYCOPG2_VERSION)
 	while ! docker inspect --format='{{json .State.Health}}' test-postgres|grep 'Status":"healthy"'; do sleep 1; done
 
 test-with-postgres: ## MODULE=<python module name> - Run tests for a single app, module or test class against a Postgres database container.
@@ -55,7 +57,7 @@ test-launch-mysql:
 	@docker volume rm test-mysql || true
 	docker run -d --name test-mysql -p 3306:3306 -e MYSQL_ALLOW_EMPTY_PASSWORD=True -e MYSQL_DATABASE=mayan -v test-mysql:/var/lib/mysql healthcheck/mysql
 	sudo apt-get install -q libmysqlclient-dev mysql-client
-	pip install mysqlclient
+	pip install mysqlclient==$(PYTHON_MYSQL_VERSION)
 	while ! docker inspect --format='{{json .State.Health}}' test-mysql|grep 'Status":"healthy"'; do sleep 1; done
 	mysql -h 127.0.0.1 -P 3306 -uroot  -e "set global character_set_server=utf8mb4;"
 
@@ -77,7 +79,7 @@ test-launch-oracle:
 	@docker volume rm test-oracle || true
 	docker run -d --name test-oracle -p 49160:22 -p 49161:1521 -e ORACLE_ALLOW_REMOTE=true -v test-oracle:/u01/app/oracle wnameless/oracle-xe-11g
 	# https://gist.github.com/kimus/10012910
-	pip install cx_Oracle
+	pip install cx_Oracle==$(PYTHON_ORACLE_VERSION)
 	while ! nc -z 127.0.0.1 49161; do sleep 1; done
 	sleep 10
 

config.env

@@ -0,0 +1,9 @@
+DOCKER_POSTGRES_IMAGE_VERSION=postgres:9.6
+DOCKER_RABBITMQ_IMAGE_VERSION=rabbitmq:3
+DOCKER_REDIS_IMAGE_VERSION=redis:5.0
+PYTHON_LIBRABBITMQ_VERSION=1.6.1
+PYTHON_MYSQL_VERSION=1.2.5
+PYTHON_ORACLE_VERSION=7.2.3
+PYTHON_PSYCOPG2_VERSION=2.7.3.2
+PYTHON_PSUTIL_VERSION=5.6.2
+PYTHON_REDIS_VERSION=2.10.6

docker/Dockerfile

@@ -8,6 +8,8 @@ FROM debian:9.8-slim as BASE_IMAGE
 
 LABEL maintainer="Roberto Rosario roberto.rosario@mayan-edms.com"
 
+COPY config.env /config.env
+
 ENV PYTHONUNBUFFERED=1 \
     LC_ALL=C.UTF-8 \
     PROJECT_INSTALL_DIR=/opt/mayan-edms
@@ -63,7 +65,7 @@ echo "save \"\"" >> /etc/redis/redis.conf \
 && echo "databases 1" >> /etc/redis/redis.conf
 
 ####
-# BUILDER_IMAGE - This image buildS the Python package and is discarded afterwards
+# BUILDER_IMAGE - This image builds the Python package and is discarded afterwards
 ####
 
 # Reuse image
@@ -104,17 +106,21 @@ apt-get install -y --no-install-recommends \
 && chown -R mayan:mayan /src
 
 USER mayan
-RUN python -m virtualenv "${PROJECT_INSTALL_DIR}" \
+
+RUN set -a \
+&& . /config.env \
+&& set +a \
+&& python -m virtualenv "${PROJECT_INSTALL_DIR}" \
 && . "${PROJECT_INSTALL_DIR}/bin/activate" \
 && pip install --no-cache-dir --no-use-pep517 \
-    librabbitmq==1.6.1 \
-    mysql-python==1.2.5 \
-    psycopg2==2.7.3.2 \
-    redis==2.10.6 \
+    librabbitmq==$PYTHON_LIBRABBITMQ_VERSION \
+    mysql-python==$PYTHON_MYSQL_VERSION \
+    psycopg2==$PYTHON_PSYCOPG2_VERSION \
+    redis==$PYTHON_REDIS_VERSION \
 # psutil is needed by ARM builds otherwise gevent and gunicorn fail to start
 && UNAME=`uname -m` && if [ "${UNAME#*arm}" != $UNAME ]; then \
     pip install --no-cache-dir --no-use-pep517 \
-    psutil==5.6.2 \
+    psutil==$PYTHON_PSUTIL_VERSION \
 ; fi \
 # Install the Python packages needed to build Mayan EDMS
 && pip install --no-cache-dir --no-use-pep517 -r /src/requirements/build.txt \

docker/Makefile

@@ -1,3 +1,6 @@
+#!make
+include config.env
+
 APT_PROXY ?= `/sbin/ip route|awk '/docker0/ { print $$9 }'`:3142
 CONSOLE_COLUMNS ?= `echo $$(tput cols)`
 CONSOLE_LINES ?= `echo $$(tput lines)`
@@ -48,13 +51,13 @@ docker-staging-container-postgresql-start:
     -e POSTGRES_DB=mayan \
     -e POSTGRES_PASSWORD=mayanuserpass \
     -v mayan-staging-postgres:/var/lib/postgresql/data \
-    postgres:9.6-alpine
+    $(DOCKER_POSTGRES_IMAGE_VERSION)
 
 docker-staging-container-redis-start:
 	docker run -d \
     --name mayan-staging-redis \
     --network=mayan-staging \
-    redis:5.0-alpine \
+    $(DOCKER_REDIS_IMAGE_VERSION) \
     redis-server \
     --databases \
     "2" \

docs/chapters/backups.rst

@@ -59,7 +59,7 @@ Example::
     -e POSTGRES_DB=mayan \
     -e POSTGRES_PASSWORD=mayanuserpass \
     -v /docker-volumes/mayan-edms/postgres-new:/var/lib/postgresql/data \
-    -d postgres:9.6
+    |DOCKER_POSTGRES_IMAGE_VERSION|
 
     docker exec -i mayan-edms-pg-new pg_restore -U mayan -d mayan < 2018-06-07_17-09-34.dump
 

docs/chapters/deploying.rst

@@ -84,14 +84,14 @@ For another setup that offers more performance and scalability refer to the
 ------------------------------------------------------
    ::
 
-       sudo -u mayan /opt/mayan-edms/bin/pip install --no-cache-dir --no-use-pep517 psycopg2==2.7.3.2 redis==2.10.6
+       sudo -u mayan /opt/mayan-edms/bin/pip install --no-cache-dir --no-use-pep517 psycopg2==|PYTHON_PSYCOPG2_VERSION| redis==|PYTHON_REDIS_VERSION|
 
    .. note::
 
        Platforms with the ARM CPU might also need additional requirements.
        ::
 
-           sudo -u mayan /opt/mayan-edms/bin/pip install --no-cache-dir --no-use-pep517 psutil==5.6.2
+           sudo -u mayan /opt/mayan-edms/bin/pip install --no-cache-dir --no-use-pep517 psutil==|PYTHON_PSUTIL_VERSION|
 
 
 8. Create the database for the installation:
@@ -206,7 +206,7 @@ of a restart or power failure. The Gunicorn workers are increased to 3.
 ------------------------------------------
    ::
 
-       sudo -u mayan /opt/mayan-edms/bin/pip install --no-cache-dir --no-use-pep517 librabbitmq==2.0.0
+       sudo -u mayan /opt/mayan-edms/bin/pip install --no-cache-dir --no-use-pep517 librabbitmq==|PYTHON_LIBRABBITMQ_VERSION|
 
 
 3. Create the RabbitMQ user and vhost:

docs/chapters/docker.rst

@@ -23,7 +23,7 @@ tag here, remember to do so in the next steps also.::
 
 Then download version 9.6 of the Docker PostgreSQL image::
 
-    docker pull postgres:9.6
+    docker pull |DOCKER_POSTGRES_IMAGE_VERSION|
 
 Create and run a PostgreSQL container::
 
@@ -35,7 +35,7 @@ Create and run a PostgreSQL container::
     -e POSTGRES_DB=mayan \
     -e POSTGRES_PASSWORD=mayanuserpass \
     -v /docker-volumes/mayan-edms/postgres:/var/lib/postgresql/data \
-    postgres:9.6
+    |DOCKER_POSTGRES_IMAGE_VERSION|
 
 The PostgreSQL container will have one database named ``mayan``, with an user
 named ``mayan`` too, with a password of ``mayanuserpass``. The container will
@@ -97,7 +97,7 @@ binding (``-p 5432:5432``)::
     -e POSTGRES_DB=mayan \
     -e POSTGRES_PASSWORD=mayanuserpass \
     -v /docker-volumes/mayan-edms/postgres:/var/lib/postgresql/data \
-    postgres:9.6
+    |DOCKER_POSTGRES_IMAGE_VERSION|
 
 Launch the Mayan EDMS container with the network option and change the
 database hostname to the PostgreSQL container name (``mayan-edms-postgres``)

docs/chapters/scaling_up.rst

@@ -92,7 +92,7 @@ section for the required changes.
 For the Docker image, launch a separate RabbitMQ container
 (https://hub.docker.com/_/rabbitmq/)::
 
-    docker run -d --name mayan-edms-rabbitmq -e RABBITMQ_DEFAULT_USER=mayan -e RABBITMQ_DEFAULT_PASS=mayanrabbitmqpassword -e RABBITMQ_DEFAULT_VHOST=mayan rabbitmq:3
+    docker run -d --name mayan-edms-rabbitmq -e RABBITMQ_DEFAULT_USER=mayan -e RABBITMQ_DEFAULT_PASS=mayanrabbitmqpassword -e RABBITMQ_DEFAULT_VHOST=mayan |DOCKER_RABBITMQ_IMAGE_VERSION|
 
 Pass the MAYAN_BROKER_URL environment variable (https://kombu.readthedocs.io/en/latest/userguide/connections.html#connection-urls)
 to the Mayan EDMS container so that it uses the RabbitMQ container the

docs/conf.py

@@ -199,8 +199,10 @@ html_show_sphinx = False
 # Grouping the document tree into LaTeX files. List of tuples
 # (source start file, target name, title, author, documentclass [howto/manual]).
 latex_documents = [
-  ('index', 'MayanEDMS.tex', 'Mayan EDMS Documentation',
-   mayan.__author__, 'manual'),
+    (
+        'index', 'MayanEDMS.tex', 'Mayan EDMS Documentation',
+        mayan.__author__, 'manual'
+    ),
 ]
 
 # The name of an image file (relative to this directory) to place at the top of
@@ -255,5 +257,25 @@ extlinks = {
 }
 
 
+def _load_env_file(filename='../config.env'):
+    result = []
+    with open(filename) as file_object:
+        for line in file_object:
+            if not line.startswith('#'):
+                key, value = line.strip().split('=')
+                result.append(('|{}|'.format(key), value))
+
+    return result
+
+
+def GlobalSubstitution(app, docname, source):
+    for old, new in global_subtitutions:
+        source[0] = source[0].replace(old, new)
+
+
 def setup(app):
     app.add_stylesheet('css/custom.css')
+    app.connect('source-read', GlobalSubstitution)
+
+
+global_subtitutions = _load_env_file()