Add top level config.env file

Update the following elements to use the top level config environment file: - CI - Makefile - Docker makefile - Documenation Signed-off-by: Roberto Rosario <roberto.rosario@mayan-edms.com>
2019-11-02 02:15:05 -04:00
parent 0ab9c18733
commit 1252263f93
10 changed files with 100 additions and 56 deletions
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -170,7 +170,8 @@ test-mysql:
    - mysql:8.0.3
  script:
    - apt-get install -qq libmysqlclient-dev mysql-client
-    - pip install mysqlclient
+    - set -a && . ./config.env && set +a
+    - pip install mysqlclient==$PYTHON_MYSQL_VERSION
    - mysql -h"$MYSQL_PORT_3306_TCP_ADDR" -P"$MYSQL_PORT_3306_TCP_PORT" -uroot -p"$MYSQL_ENV_MYSQL_ROOT_PASSWORD" -e "set global character_set_server=utf8mb4;"
    - python manage.py test --mayan-apps --settings=mayan.settings.testing.gitlab-ci.db_mysql --nomigrations
  tags:
@@ -185,7 +186,8 @@ test-postgres:
    - postgres
  script:
    - apt-get install -qq libpq-dev
-    - pip install psycopg2
+    - set -a && . ./config.env && set +a
+    - pip install psycopg2==$PYTHON_PSYCOPG2_VERSION
    - python manage.py test --mayan-apps --settings=mayan.settings.testing.gitlab-ci.db_postgres --nomigrations
  tags:
    - postgres
--- a/8
+++ b/8
@@ -1,3 +1,5 @@
+#!make
+include config.env
 .PHONY: clean-pyc clean-build

 help:
@@ -35,7 +37,7 @@ test-launch-postgres:
 	@docker volume rm test-postgres || true
 	docker run -d --name test-postgres -p 5432:5432 -v test-postgres:/var/lib/postgresql/data healthcheck/postgres
 	sudo apt-get install -q libpq-dev
-	pip install psycopg2
+	pip install psycopg2==$(PYTHON_PSYCOPG2_VERSION)
 	while ! docker inspect --format='{{json .State.Health}}' test-postgres|grep 'Status":"healthy"'; do sleep 1; done

 test-with-postgres: ## MODULE=<python module name> - Run tests for a single app, module or test class against a Postgres database container.
@@ -55,7 +57,7 @@ test-launch-mysql:
 	@docker volume rm test-mysql || true
 	docker run -d --name test-mysql -p 3306:3306 -e MYSQL_ALLOW_EMPTY_PASSWORD=True -e MYSQL_DATABASE=mayan -v test-mysql:/var/lib/mysql healthcheck/mysql
 	sudo apt-get install -q libmysqlclient-dev mysql-client
-	pip install mysqlclient
+	pip install mysqlclient==$(PYTHON_MYSQL_VERSION)
 	while ! docker inspect --format='{{json .State.Health}}' test-mysql|grep 'Status":"healthy"'; do sleep 1; done
 	mysql -h 127.0.0.1 -P 3306 -uroot  -e "set global character_set_server=utf8mb4;"

@@ -77,7 +79,7 @@ test-launch-oracle:
 	@docker volume rm test-oracle || true
 	docker run -d --name test-oracle -p 49160:22 -p 49161:1521 -e ORACLE_ALLOW_REMOTE=true -v test-oracle:/u01/app/oracle wnameless/oracle-xe-11g
 	# https://gist.github.com/kimus/10012910
-	pip install cx_Oracle
+	pip install cx_Oracle==$(PYTHON_ORACLE_VERSION)
 	while ! nc -z 127.0.0.1 49161; do sleep 1; done
 	sleep 10

--- a/config.env
+++ b/config.env
@@ -0,0 +1,9 @@
+DOCKER_POSTGRES_IMAGE_VERSION=postgres:9.6
+DOCKER_RABBITMQ_IMAGE_VERSION=rabbitmq:3
+DOCKER_REDIS_IMAGE_VERSION=redis:5.0
+PYTHON_LIBRABBITMQ_VERSION=1.6.1
+PYTHON_MYSQL_VERSION=1.2.5
+PYTHON_ORACLE_VERSION=7.2.3
+PYTHON_PSYCOPG2_VERSION=2.7.3.2
+PYTHON_PSUTIL_VERSION=5.6.2
+PYTHON_REDIS_VERSION=2.10.6
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -8,6 +8,8 @@ FROM debian:9.8-slim as BASE_IMAGE

 LABEL maintainer="Roberto Rosario roberto.rosario@mayan-edms.com"

+COPY config.env /config.env
+
 ENV PYTHONUNBUFFERED=1 \
    LC_ALL=C.UTF-8 \
    PROJECT_INSTALL_DIR=/opt/mayan-edms
@@ -63,7 +65,7 @@ echo "save \"\"" >> /etc/redis/redis.conf \
 && echo "databases 1" >> /etc/redis/redis.conf

 ####
-# BUILDER_IMAGE - This image buildS the Python package and is discarded afterwards
+# BUILDER_IMAGE - This image builds the Python package and is discarded afterwards
 ####

 # Reuse image
@@ -104,17 +106,21 @@ apt-get install -y --no-install-recommends \
 && chown -R mayan:mayan /src

 USER mayan
-RUN python -m virtualenv "${PROJECT_INSTALL_DIR}" \
+
+RUN set -a \
+&& . /config.env \
+&& set +a \
+&& python -m virtualenv "${PROJECT_INSTALL_DIR}" \
 && . "${PROJECT_INSTALL_DIR}/bin/activate" \
 && pip install --no-cache-dir --no-use-pep517 \
-    librabbitmq==1.6.1 \
-    mysql-python==1.2.5 \
-    psycopg2==2.7.3.2 \
-    redis==2.10.6 \
+    librabbitmq==$PYTHON_LIBRABBITMQ_VERSION \
+    mysql-python==$PYTHON_MYSQL_VERSION \
+    psycopg2==$PYTHON_PSYCOPG2_VERSION \
+    redis==$PYTHON_REDIS_VERSION \
 # psutil is needed by ARM builds otherwise gevent and gunicorn fail to start
 && UNAME=`uname -m` && if [ "${UNAME#*arm}" != $UNAME ]; then \
    pip install --no-cache-dir --no-use-pep517 \
-    psutil==5.6.2 \
+    psutil==$PYTHON_PSUTIL_VERSION \
 ; fi \
 # Install the Python packages needed to build Mayan EDMS
 && pip install --no-cache-dir --no-use-pep517 -r /src/requirements/build.txt \
--- a/docker/Makefile
+++ b/docker/Makefile
@@ -1,3 +1,6 @@
+#!make
+include config.env
+
 APT_PROXY ?= `/sbin/ip route|awk '/docker0/ { print $$9 }'`:3142
 CONSOLE_COLUMNS ?= `echo $$(tput cols)`
 CONSOLE_LINES ?= `echo $$(tput lines)`
@@ -48,13 +51,13 @@ docker-staging-container-postgresql-start:
    -e POSTGRES_DB=mayan \
    -e POSTGRES_PASSWORD=mayanuserpass \
    -v mayan-staging-postgres:/var/lib/postgresql/data \
-    postgres:9.6-alpine
+    $(DOCKER_POSTGRES_IMAGE_VERSION)

 docker-staging-container-redis-start:
 	docker run -d \
    --name mayan-staging-redis \
    --network=mayan-staging \
-    redis:5.0-alpine \
+    $(DOCKER_REDIS_IMAGE_VERSION) \
    redis-server \
    --databases \
    "2" \
--- a/docs/chapters/backups.rst
+++ b/docs/chapters/backups.rst
@@ -59,7 +59,7 @@ Example::
    -e POSTGRES_DB=mayan \
    -e POSTGRES_PASSWORD=mayanuserpass \
    -v /docker-volumes/mayan-edms/postgres-new:/var/lib/postgresql/data \
-    -d postgres:9.6
+    |DOCKER_POSTGRES_IMAGE_VERSION|

    docker exec -i mayan-edms-pg-new pg_restore -U mayan -d mayan < 2018-06-07_17-09-34.dump

--- a/docs/chapters/deploying.rst
+++ b/docs/chapters/deploying.rst
@@ -84,14 +84,14 @@ For another setup that offers more performance and scalability refer to the
 ------------------------------------------------------
   ::

-       sudo -u mayan /opt/mayan-edms/bin/pip install --no-cache-dir --no-use-pep517 psycopg2==2.7.3.2 redis==2.10.6
+       sudo -u mayan /opt/mayan-edms/bin/pip install --no-cache-dir --no-use-pep517 psycopg2==|PYTHON_PSYCOPG2_VERSION| redis==|PYTHON_REDIS_VERSION|

   .. note::

       Platforms with the ARM CPU might also need additional requirements.
       ::

-           sudo -u mayan /opt/mayan-edms/bin/pip install --no-cache-dir --no-use-pep517 psutil==5.6.2
+           sudo -u mayan /opt/mayan-edms/bin/pip install --no-cache-dir --no-use-pep517 psutil==|PYTHON_PSUTIL_VERSION|


 8. Create the database for the installation:
@@ -206,7 +206,7 @@ of a restart or power failure. The Gunicorn workers are increased to 3.
 ------------------------------------------
   ::

-       sudo -u mayan /opt/mayan-edms/bin/pip install --no-cache-dir --no-use-pep517 librabbitmq==2.0.0
+       sudo -u mayan /opt/mayan-edms/bin/pip install --no-cache-dir --no-use-pep517 librabbitmq==|PYTHON_LIBRABBITMQ_VERSION|


 3. Create the RabbitMQ user and vhost:
--- a/docs/chapters/docker.rst
+++ b/docs/chapters/docker.rst
@@ -23,7 +23,7 @@ tag here, remember to do so in the next steps also.::

 Then download version 9.6 of the Docker PostgreSQL image::

-    docker pull postgres:9.6
+    docker pull |DOCKER_POSTGRES_IMAGE_VERSION|

 Create and run a PostgreSQL container::

@@ -35,7 +35,7 @@ Create and run a PostgreSQL container::
    -e POSTGRES_DB=mayan \
    -e POSTGRES_PASSWORD=mayanuserpass \
    -v /docker-volumes/mayan-edms/postgres:/var/lib/postgresql/data \
-    postgres:9.6
+    |DOCKER_POSTGRES_IMAGE_VERSION|

 The PostgreSQL container will have one database named ``mayan``, with an user
 named ``mayan`` too, with a password of ``mayanuserpass``. The container will
@@ -97,7 +97,7 @@ binding (``-p 5432:5432``)::
    -e POSTGRES_DB=mayan \
    -e POSTGRES_PASSWORD=mayanuserpass \
    -v /docker-volumes/mayan-edms/postgres:/var/lib/postgresql/data \
-    postgres:9.6
+    |DOCKER_POSTGRES_IMAGE_VERSION|

 Launch the Mayan EDMS container with the network option and change the
 database hostname to the PostgreSQL container name (``mayan-edms-postgres``)
--- a/docs/chapters/scaling_up.rst
+++ b/docs/chapters/scaling_up.rst
@@ -92,7 +92,7 @@ section for the required changes.
 For the Docker image, launch a separate RabbitMQ container
 (https://hub.docker.com/_/rabbitmq/)::

-    docker run -d --name mayan-edms-rabbitmq -e RABBITMQ_DEFAULT_USER=mayan -e RABBITMQ_DEFAULT_PASS=mayanrabbitmqpassword -e RABBITMQ_DEFAULT_VHOST=mayan rabbitmq:3
+    docker run -d --name mayan-edms-rabbitmq -e RABBITMQ_DEFAULT_USER=mayan -e RABBITMQ_DEFAULT_PASS=mayanrabbitmqpassword -e RABBITMQ_DEFAULT_VHOST=mayan |DOCKER_RABBITMQ_IMAGE_VERSION|

 Pass the MAYAN_BROKER_URL environment variable (https://kombu.readthedocs.io/en/latest/userguide/connections.html#connection-urls)
 to the Mayan EDMS container so that it uses the RabbitMQ container the
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -29,12 +29,12 @@ sys.path.append(
 # -- General configuration -----------------------------------------------------

 # If your documentation needs a minimal Sphinx version, state it here.
-#needs_sphinx = '1.0'
+# needs_sphinx = '1.0'

 # Add any Sphinx extension module names here, as strings. They can be extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
-#extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode']
-#extensions = ["djangodocs", "sphinx.ext.intersphinx"]
+# extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode']
+# extensions = ["djangodocs", "sphinx.ext.intersphinx"]
 extensions = [
    'sphinx.ext.extlinks', 'sphinxcontrib.blockdiag',
    'sphinxcontrib.spelling'
@@ -72,20 +72,20 @@ release = version

 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
-#language = None
+# language = None

 # There are two options for replacing |today|: either, you set today to some
 # non-false value, then it is used:
-#today = ''
+# today = ''
 # Else, today_fmt is used as the format for a strftime call.
-#today_fmt = '%B %d, %Y'
+# today_fmt = '%B %d, %Y'

 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
 exclude_patterns = ['_build']

 # The reST default role (used for this markup: `text`) to use for all documents.
-#default_role = None
+# default_role = None

 # If true, '()' will be appended to :func: etc. cross-reference text.
 add_function_parentheses = True
@@ -102,7 +102,7 @@ show_authors = False
 pygments_style = 'sphinx'

 # A list of ignored prefixes for module index sorting.
-#modindex_common_prefix = []
+# modindex_common_prefix = []


 # -- Options for HTML output ---------------------------------------------------
@@ -119,23 +119,23 @@ html_theme_options = {
 }

 # Add any paths that contain custom themes here, relative to this directory.
-#html_theme_path = []
+# html_theme_path = []

 # The name for this set of Sphinx documents.  If None, it defaults to
 # "<project> v<release> documentation".
-#html_title = None
+# html_title = None

 # A shorter title for the navigation bar.  Default is the same as html_title.
-#html_short_title = None
+# html_short_title = None

 # The name of an image file (relative to this directory) to place at the top
 # of the sidebar.
-#html_logo = None
+# html_logo = None

 # The name of an image file (within the static path) to use as favicon of the
 # docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
 # pixels large.
-#html_favicon = None
+# html_favicon = None

 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
@@ -148,40 +148,40 @@ html_last_updated_fmt = '%b %d, %Y'

 # If true, SmartyPants will be used to convert quotes and dashes to
 # typographically correct entities.
-#html_use_smartypants = True
+# html_use_smartypants = True

 # Custom sidebar templates, maps document names to template names.
-#html_sidebars = {}
+# html_sidebars = {}

 # Additional templates that should be rendered to pages, maps page names to
 # template names.
-#html_additional_pages = {}
+# html_additional_pages = {}

 # If false, no module index is generated.
-#html_domain_indices = True
+# html_domain_indices = True

 # If false, no index is generated.
-#html_use_index = True
+# html_use_index = True

 # If true, the index is split into individual pages for each letter.
-#html_split_index = False
+# html_split_index = False

 # If true, links to the reST sources are added to the pages.
 html_show_sourcelink = False

 # If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
-#html_show_sphinx = True
+# html_show_sphinx = True

 # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
-#html_show_copyright = True
+# html_show_copyright = True

 # If true, an OpenSearch description file will be output, and all pages will
 # contain a <link> tag referring to it.  The value of this option must be the
 # base URL from which the finished HTML is served.
-#html_use_opensearch = ''
+# html_use_opensearch = ''

 # This is the file name suffix for HTML files (e.g. ".xhtml").
-#html_file_suffix = None
+# html_file_suffix = None

 # Output file base name for HTML help builder.
 htmlhelp_basename = 'MayanEDMSdoc'
@@ -191,40 +191,42 @@ html_show_sphinx = False
 # -- Options for LaTeX output --------------------------------------------------

 # The paper size ('letter' or 'a4').
-#latex_paper_size = 'letter'
+# latex_paper_size = 'letter'

 # The font size ('10pt', '11pt' or '12pt').
-#latex_font_size = '10pt'
+# latex_font_size = '10pt'

 # Grouping the document tree into LaTeX files. List of tuples
 # (source start file, target name, title, author, documentclass [howto/manual]).
 latex_documents = [
-  ('index', 'MayanEDMS.tex', 'Mayan EDMS Documentation',
-   mayan.__author__, 'manual'),
+    (
+        'index', 'MayanEDMS.tex', 'Mayan EDMS Documentation',
+        mayan.__author__, 'manual'
+    ),
 ]

 # The name of an image file (relative to this directory) to place at the top of
 # the title page.
-#latex_logo = None
+# latex_logo = None

 # For "manual" documents, if this is true, then toplevel headings are parts,
 # not chapters.
-#latex_use_parts = False
+# latex_use_parts = False

 # If true, show page references after internal links.
-#latex_show_pagerefs = False
+# latex_show_pagerefs = False

 # If true, show URL addresses after external links.
-#latex_show_urls = False
+# latex_show_urls = False

 # Additional stuff for the LaTeX preamble.
-#latex_preamble = ''
+# latex_preamble = ''

 # Documents to append as an appendix to all manuals.
-#latex_appendices = []
+# latex_appendices = []

 # If false, no module index is generated.
-#latex_domain_indices = True
+# latex_domain_indices = True


 # -- Options for manual page output --------------------------------------------
@@ -255,5 +257,25 @@ extlinks = {
 }


+def _load_env_file(filename='../config.env'):
+    result = []
+    with open(filename) as file_object:
+        for line in file_object:
+            if not line.startswith('#'):
+                key, value = line.strip().split('=')
+                result.append(('|{}|'.format(key), value))
+
+    return result
+
+
+def GlobalSubstitution(app, docname, source):
+    for old, new in global_subtitutions:
+        source[0] = source[0].replace(old, new)
+
+
 def setup(app):
    app.add_stylesheet('css/custom.css')
+    app.connect('source-read', GlobalSubstitution)
+
+
+global_subtitutions = _load_env_file()