Add top level config.env file

Update the following elements to use the top level config
environment file:
- CI
- Makefile
- Docker makefile
- Documenation

Signed-off-by: Roberto Rosario <roberto.rosario@mayan-edms.com>

.gitlab-ci.yml

@@ -170,7 +170,8 @@ test-mysql:
     - mysql:8.0.3
   script:
     - apt-get install -qq libmysqlclient-dev mysql-client
-    - pip install mysqlclient
+    - set -a && . ./config.env && set +a
+    - pip install mysqlclient==$PYTHON_MYSQL_VERSION
     - mysql -h"$MYSQL_PORT_3306_TCP_ADDR" -P"$MYSQL_PORT_3306_TCP_PORT" -uroot -p"$MYSQL_ENV_MYSQL_ROOT_PASSWORD" -e "set global character_set_server=utf8mb4;"
     - python manage.py test --mayan-apps --settings=mayan.settings.testing.gitlab-ci.db_mysql --nomigrations
   tags:
@@ -185,7 +186,8 @@ test-postgres:
     - postgres
   script:
     - apt-get install -qq libpq-dev
-    - pip install psycopg2
+    - set -a && . ./config.env && set +a
+    - pip install psycopg2==$PYTHON_PSYCOPG2_VERSION
     - python manage.py test --mayan-apps --settings=mayan.settings.testing.gitlab-ci.db_postgres --nomigrations
   tags:
     - postgres

Makefile

@@ -1,3 +1,5 @@
+#!make
+include config.env
 .PHONY: clean-pyc clean-build
 
 help:
@@ -35,7 +37,7 @@ test-launch-postgres:
 	@docker volume rm test-postgres || true
 	docker run -d --name test-postgres -p 5432:5432 -v test-postgres:/var/lib/postgresql/data healthcheck/postgres
 	sudo apt-get install -q libpq-dev
-	pip install psycopg2
+	pip install psycopg2==$(PYTHON_PSYCOPG2_VERSION)
 	while ! docker inspect --format='{{json .State.Health}}' test-postgres|grep 'Status":"healthy"'; do sleep 1; done
 
 test-with-postgres: ## MODULE=<python module name> - Run tests for a single app, module or test class against a Postgres database container.
@@ -55,7 +57,7 @@ test-launch-mysql:
 	@docker volume rm test-mysql || true
 	docker run -d --name test-mysql -p 3306:3306 -e MYSQL_ALLOW_EMPTY_PASSWORD=True -e MYSQL_DATABASE=mayan -v test-mysql:/var/lib/mysql healthcheck/mysql
 	sudo apt-get install -q libmysqlclient-dev mysql-client
-	pip install mysqlclient
+	pip install mysqlclient==$(PYTHON_MYSQL_VERSION)
 	while ! docker inspect --format='{{json .State.Health}}' test-mysql|grep 'Status":"healthy"'; do sleep 1; done
 	mysql -h 127.0.0.1 -P 3306 -uroot  -e "set global character_set_server=utf8mb4;"
 
@@ -77,7 +79,7 @@ test-launch-oracle:
 	@docker volume rm test-oracle || true
 	docker run -d --name test-oracle -p 49160:22 -p 49161:1521 -e ORACLE_ALLOW_REMOTE=true -v test-oracle:/u01/app/oracle wnameless/oracle-xe-11g
 	# https://gist.github.com/kimus/10012910
-	pip install cx_Oracle
+	pip install cx_Oracle==$(PYTHON_ORACLE_VERSION)
 	while ! nc -z 127.0.0.1 49161; do sleep 1; done
 	sleep 10
 

config.env

@@ -0,0 +1,9 @@
+DOCKER_POSTGRES_IMAGE_VERSION=postgres:9.6
+DOCKER_RABBITMQ_IMAGE_VERSION=rabbitmq:3
+DOCKER_REDIS_IMAGE_VERSION=redis:5.0
+PYTHON_LIBRABBITMQ_VERSION=1.6.1
+PYTHON_MYSQL_VERSION=1.2.5
+PYTHON_ORACLE_VERSION=7.2.3
+PYTHON_PSYCOPG2_VERSION=2.7.3.2
+PYTHON_PSUTIL_VERSION=5.6.2
+PYTHON_REDIS_VERSION=2.10.6

docker/Dockerfile

@@ -8,6 +8,8 @@ FROM debian:9.8-slim as BASE_IMAGE
 
 LABEL maintainer="Roberto Rosario roberto.rosario@mayan-edms.com"
 
+COPY config.env /config.env
+
 ENV PYTHONUNBUFFERED=1 \
     LC_ALL=C.UTF-8 \
     PROJECT_INSTALL_DIR=/opt/mayan-edms
@@ -63,7 +65,7 @@ echo "save \"\"" >> /etc/redis/redis.conf \
 && echo "databases 1" >> /etc/redis/redis.conf
 
 ####
-# BUILDER_IMAGE - This image buildS the Python package and is discarded afterwards
+# BUILDER_IMAGE - This image builds the Python package and is discarded afterwards
 ####
 
 # Reuse image
@@ -104,17 +106,21 @@ apt-get install -y --no-install-recommends \
 && chown -R mayan:mayan /src
 
 USER mayan
-RUN python -m virtualenv "${PROJECT_INSTALL_DIR}" \
+
+RUN set -a \
+&& . /config.env \
+&& set +a \
+&& python -m virtualenv "${PROJECT_INSTALL_DIR}" \
 && . "${PROJECT_INSTALL_DIR}/bin/activate" \
 && pip install --no-cache-dir --no-use-pep517 \
-    librabbitmq==1.6.1 \
+    librabbitmq==$PYTHON_LIBRABBITMQ_VERSION \
-    mysql-python==1.2.5 \
+    mysql-python==$PYTHON_MYSQL_VERSION \
-    psycopg2==2.7.3.2 \
+    psycopg2==$PYTHON_PSYCOPG2_VERSION \
-    redis==2.10.6 \
+    redis==$PYTHON_REDIS_VERSION \
 # psutil is needed by ARM builds otherwise gevent and gunicorn fail to start
 && UNAME=`uname -m` && if [ "${UNAME#*arm}" != $UNAME ]; then \
     pip install --no-cache-dir --no-use-pep517 \
-    psutil==5.6.2 \
+    psutil==$PYTHON_PSUTIL_VERSION \
 ; fi \
 # Install the Python packages needed to build Mayan EDMS
 && pip install --no-cache-dir --no-use-pep517 -r /src/requirements/build.txt \

docker/Makefile

@@ -1,3 +1,6 @@
+#!make
+include config.env
+
 APT_PROXY ?= `/sbin/ip route|awk '/docker0/ { print $$9 }'`:3142
 CONSOLE_COLUMNS ?= `echo $$(tput cols)`
 CONSOLE_LINES ?= `echo $$(tput lines)`
@@ -48,13 +51,13 @@ docker-staging-container-postgresql-start:
     -e POSTGRES_DB=mayan \
     -e POSTGRES_PASSWORD=mayanuserpass \
     -v mayan-staging-postgres:/var/lib/postgresql/data \
-    postgres:9.6-alpine
+    $(DOCKER_POSTGRES_IMAGE_VERSION)
 
 docker-staging-container-redis-start:
 	docker run -d \
     --name mayan-staging-redis \
     --network=mayan-staging \
-    redis:5.0-alpine \
+    $(DOCKER_REDIS_IMAGE_VERSION) \
     redis-server \
     --databases \
     "2" \

docs/chapters/backups.rst

@@ -59,7 +59,7 @@ Example::
     -e POSTGRES_DB=mayan \
     -e POSTGRES_PASSWORD=mayanuserpass \
     -v /docker-volumes/mayan-edms/postgres-new:/var/lib/postgresql/data \
-    -d postgres:9.6
+    |DOCKER_POSTGRES_IMAGE_VERSION|
 
     docker exec -i mayan-edms-pg-new pg_restore -U mayan -d mayan < 2018-06-07_17-09-34.dump
 

docs/chapters/deploying.rst

@@ -84,14 +84,14 @@ For another setup that offers more performance and scalability refer to the
 ------------------------------------------------------
    ::
 
-       sudo -u mayan /opt/mayan-edms/bin/pip install --no-cache-dir --no-use-pep517 psycopg2==2.7.3.2 redis==2.10.6
+       sudo -u mayan /opt/mayan-edms/bin/pip install --no-cache-dir --no-use-pep517 psycopg2==|PYTHON_PSYCOPG2_VERSION| redis==|PYTHON_REDIS_VERSION|
 
    .. note::
 
        Platforms with the ARM CPU might also need additional requirements.
        ::
 
-           sudo -u mayan /opt/mayan-edms/bin/pip install --no-cache-dir --no-use-pep517 psutil==5.6.2
+           sudo -u mayan /opt/mayan-edms/bin/pip install --no-cache-dir --no-use-pep517 psutil==|PYTHON_PSUTIL_VERSION|
 
 
 8. Create the database for the installation:
@@ -206,7 +206,7 @@ of a restart or power failure. The Gunicorn workers are increased to 3.
 ------------------------------------------
    ::
 
-       sudo -u mayan /opt/mayan-edms/bin/pip install --no-cache-dir --no-use-pep517 librabbitmq==2.0.0
+       sudo -u mayan /opt/mayan-edms/bin/pip install --no-cache-dir --no-use-pep517 librabbitmq==|PYTHON_LIBRABBITMQ_VERSION|
 
 
 3. Create the RabbitMQ user and vhost:

docs/chapters/docker.rst

@@ -23,7 +23,7 @@ tag here, remember to do so in the next steps also.::
 
 Then download version 9.6 of the Docker PostgreSQL image::
 
-    docker pull postgres:9.6
+    docker pull |DOCKER_POSTGRES_IMAGE_VERSION|
 
 Create and run a PostgreSQL container::
 
@@ -35,7 +35,7 @@ Create and run a PostgreSQL container::
     -e POSTGRES_DB=mayan \
     -e POSTGRES_PASSWORD=mayanuserpass \
     -v /docker-volumes/mayan-edms/postgres:/var/lib/postgresql/data \
-    postgres:9.6
+    |DOCKER_POSTGRES_IMAGE_VERSION|
 
 The PostgreSQL container will have one database named ``mayan``, with an user
 named ``mayan`` too, with a password of ``mayanuserpass``. The container will
@@ -97,7 +97,7 @@ binding (``-p 5432:5432``)::
     -e POSTGRES_DB=mayan \
     -e POSTGRES_PASSWORD=mayanuserpass \
     -v /docker-volumes/mayan-edms/postgres:/var/lib/postgresql/data \
-    postgres:9.6
+    |DOCKER_POSTGRES_IMAGE_VERSION|
 
 Launch the Mayan EDMS container with the network option and change the
 database hostname to the PostgreSQL container name (``mayan-edms-postgres``)

docs/chapters/scaling_up.rst

@@ -92,7 +92,7 @@ section for the required changes.
 For the Docker image, launch a separate RabbitMQ container
 (https://hub.docker.com/_/rabbitmq/)::
 
-    docker run -d --name mayan-edms-rabbitmq -e RABBITMQ_DEFAULT_USER=mayan -e RABBITMQ_DEFAULT_PASS=mayanrabbitmqpassword -e RABBITMQ_DEFAULT_VHOST=mayan rabbitmq:3
+    docker run -d --name mayan-edms-rabbitmq -e RABBITMQ_DEFAULT_USER=mayan -e RABBITMQ_DEFAULT_PASS=mayanrabbitmqpassword -e RABBITMQ_DEFAULT_VHOST=mayan |DOCKER_RABBITMQ_IMAGE_VERSION|
 
 Pass the MAYAN_BROKER_URL environment variable (https://kombu.readthedocs.io/en/latest/userguide/connections.html#connection-urls)
 to the Mayan EDMS container so that it uses the RabbitMQ container the

docs/conf.py

@@ -29,12 +29,12 @@ sys.path.append(
 # -- General configuration -----------------------------------------------------
 
 # If your documentation needs a minimal Sphinx version, state it here.
-#needs_sphinx = '1.0'
+# needs_sphinx = '1.0'
 
 # Add any Sphinx extension module names here, as strings. They can be extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
-#extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode']
+# extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode']
-#extensions = ["djangodocs", "sphinx.ext.intersphinx"]
+# extensions = ["djangodocs", "sphinx.ext.intersphinx"]
 extensions = [
     'sphinx.ext.extlinks', 'sphinxcontrib.blockdiag',
     'sphinxcontrib.spelling'
@@ -72,20 +72,20 @@ release = version
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
-#language = None
+# language = None
 
 # There are two options for replacing |today|: either, you set today to some
 # non-false value, then it is used:
-#today = ''
+# today = ''
 # Else, today_fmt is used as the format for a strftime call.
-#today_fmt = '%B %d, %Y'
+# today_fmt = '%B %d, %Y'
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
 exclude_patterns = ['_build']
 
 # The reST default role (used for this markup: `text`) to use for all documents.
-#default_role = None
+# default_role = None
 
 # If true, '()' will be appended to :func: etc. cross-reference text.
 add_function_parentheses = True
@@ -102,7 +102,7 @@ show_authors = False
 pygments_style = 'sphinx'
 
 # A list of ignored prefixes for module index sorting.
-#modindex_common_prefix = []
+# modindex_common_prefix = []
 
 
 # -- Options for HTML output ---------------------------------------------------
@@ -119,23 +119,23 @@ html_theme_options = {
 }
 
 # Add any paths that contain custom themes here, relative to this directory.
-#html_theme_path = []
+# html_theme_path = []
 
 # The name for this set of Sphinx documents.  If None, it defaults to
 # "<project> v<release> documentation".
-#html_title = None
+# html_title = None
 
 # A shorter title for the navigation bar.  Default is the same as html_title.
-#html_short_title = None
+# html_short_title = None
 
 # The name of an image file (relative to this directory) to place at the top
 # of the sidebar.
-#html_logo = None
+# html_logo = None
 
 # The name of an image file (within the static path) to use as favicon of the
 # docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
 # pixels large.
-#html_favicon = None
+# html_favicon = None
 
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
@@ -148,40 +148,40 @@ html_last_updated_fmt = '%b %d, %Y'
 
 # If true, SmartyPants will be used to convert quotes and dashes to
 # typographically correct entities.
-#html_use_smartypants = True
+# html_use_smartypants = True
 
 # Custom sidebar templates, maps document names to template names.
-#html_sidebars = {}
+# html_sidebars = {}
 
 # Additional templates that should be rendered to pages, maps page names to
 # template names.
-#html_additional_pages = {}
+# html_additional_pages = {}
 
 # If false, no module index is generated.
-#html_domain_indices = True
+# html_domain_indices = True
 
 # If false, no index is generated.
-#html_use_index = True
+# html_use_index = True
 
 # If true, the index is split into individual pages for each letter.
-#html_split_index = False
+# html_split_index = False
 
 # If true, links to the reST sources are added to the pages.
 html_show_sourcelink = False
 
 # If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
-#html_show_sphinx = True
+# html_show_sphinx = True
 
 # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
-#html_show_copyright = True
+# html_show_copyright = True
 
 # If true, an OpenSearch description file will be output, and all pages will
 # contain a <link> tag referring to it.  The value of this option must be the
 # base URL from which the finished HTML is served.
-#html_use_opensearch = ''
+# html_use_opensearch = ''
 
 # This is the file name suffix for HTML files (e.g. ".xhtml").
-#html_file_suffix = None
+# html_file_suffix = None
 
 # Output file base name for HTML help builder.
 htmlhelp_basename = 'MayanEDMSdoc'
@@ -191,40 +191,42 @@ html_show_sphinx = False
 # -- Options for LaTeX output --------------------------------------------------
 
 # The paper size ('letter' or 'a4').
-#latex_paper_size = 'letter'
+# latex_paper_size = 'letter'
 
 # The font size ('10pt', '11pt' or '12pt').
-#latex_font_size = '10pt'
+# latex_font_size = '10pt'
 
 # Grouping the document tree into LaTeX files. List of tuples
 # (source start file, target name, title, author, documentclass [howto/manual]).
 latex_documents = [
-  ('index', 'MayanEDMS.tex', 'Mayan EDMS Documentation',
+    (
-   mayan.__author__, 'manual'),
+        'index', 'MayanEDMS.tex', 'Mayan EDMS Documentation',
+        mayan.__author__, 'manual'
+    ),
 ]
 
 # The name of an image file (relative to this directory) to place at the top of
 # the title page.
-#latex_logo = None
+# latex_logo = None
 
 # For "manual" documents, if this is true, then toplevel headings are parts,
 # not chapters.
-#latex_use_parts = False
+# latex_use_parts = False
 
 # If true, show page references after internal links.
-#latex_show_pagerefs = False
+# latex_show_pagerefs = False
 
 # If true, show URL addresses after external links.
-#latex_show_urls = False
+# latex_show_urls = False
 
 # Additional stuff for the LaTeX preamble.
-#latex_preamble = ''
+# latex_preamble = ''
 
 # Documents to append as an appendix to all manuals.
-#latex_appendices = []
+# latex_appendices = []
 
 # If false, no module index is generated.
-#latex_domain_indices = True
+# latex_domain_indices = True
 
 
 # -- Options for manual page output --------------------------------------------
@@ -255,5 +257,25 @@ extlinks = {
 }
 
 
+def _load_env_file(filename='../config.env'):
+    result = []
+    with open(filename) as file_object:
+        for line in file_object:
+            if not line.startswith('#'):
+                key, value = line.strip().split('=')
+                result.append(('|{}|'.format(key), value))
+
+    return result
+
+
+def GlobalSubstitution(app, docname, source):
+    for old, new in global_subtitutions:
+        source[0] = source[0].replace(old, new)
+
+
 def setup(app):
     app.add_stylesheet('css/custom.css')
+    app.connect('source-read', GlobalSubstitution)
+
+
+global_subtitutions = _load_env_file()