diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a5b56f4358..e9585fa24b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -170,7 +170,8 @@ test-mysql: - mysql:8.0.3 script: - apt-get install -qq libmysqlclient-dev mysql-client - - pip install mysqlclient + - set -a && . ./config.env && set +a + - pip install mysqlclient==$PYTHON_MYSQL_VERSION - mysql -h"$MYSQL_PORT_3306_TCP_ADDR" -P"$MYSQL_PORT_3306_TCP_PORT" -uroot -p"$MYSQL_ENV_MYSQL_ROOT_PASSWORD" -e "set global character_set_server=utf8mb4;" - python manage.py test --mayan-apps --settings=mayan.settings.testing.gitlab-ci.db_mysql --nomigrations tags: @@ -185,7 +186,8 @@ test-postgres: - postgres script: - apt-get install -qq libpq-dev - - pip install psycopg2 + - set -a && . ./config.env && set +a + - pip install psycopg2==$PYTHON_PSYCOPG2_VERSION - python manage.py test --mayan-apps --settings=mayan.settings.testing.gitlab-ci.db_postgres --nomigrations tags: - postgres diff --git a/Makefile b/Makefile index 07d5650c5f..f49736c6dd 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,5 @@ +#!make +include config.env .PHONY: clean-pyc clean-build help: @@ -35,7 +37,7 @@ test-launch-postgres: @docker volume rm test-postgres || true docker run -d --name test-postgres -p 5432:5432 -v test-postgres:/var/lib/postgresql/data healthcheck/postgres sudo apt-get install -q libpq-dev - pip install psycopg2 + pip install psycopg2==$(PYTHON_PSYCOPG2_VERSION) while ! docker inspect --format='{{json .State.Health}}' test-postgres|grep 'Status":"healthy"'; do sleep 1; done test-with-postgres: ## MODULE= - Run tests for a single app, module or test class against a Postgres database container. @@ -55,7 +57,7 @@ test-launch-mysql: @docker volume rm test-mysql || true docker run -d --name test-mysql -p 3306:3306 -e MYSQL_ALLOW_EMPTY_PASSWORD=True -e MYSQL_DATABASE=mayan -v test-mysql:/var/lib/mysql healthcheck/mysql sudo apt-get install -q libmysqlclient-dev mysql-client - pip install mysqlclient + pip install mysqlclient==$(PYTHON_MYSQL_VERSION) while ! docker inspect --format='{{json .State.Health}}' test-mysql|grep 'Status":"healthy"'; do sleep 1; done mysql -h 127.0.0.1 -P 3306 -uroot -e "set global character_set_server=utf8mb4;" @@ -77,7 +79,7 @@ test-launch-oracle: @docker volume rm test-oracle || true docker run -d --name test-oracle -p 49160:22 -p 49161:1521 -e ORACLE_ALLOW_REMOTE=true -v test-oracle:/u01/app/oracle wnameless/oracle-xe-11g # https://gist.github.com/kimus/10012910 - pip install cx_Oracle + pip install cx_Oracle==$(PYTHON_ORACLE_VERSION) while ! nc -z 127.0.0.1 49161; do sleep 1; done sleep 10 diff --git a/config.env b/config.env new file mode 100644 index 0000000000..da8df9da35 --- /dev/null +++ b/config.env @@ -0,0 +1,9 @@ +DOCKER_POSTGRES_IMAGE_VERSION=postgres:9.6 +DOCKER_RABBITMQ_IMAGE_VERSION=rabbitmq:3 +DOCKER_REDIS_IMAGE_VERSION=redis:5.0 +PYTHON_LIBRABBITMQ_VERSION=1.6.1 +PYTHON_MYSQL_VERSION=1.2.5 +PYTHON_ORACLE_VERSION=7.2.3 +PYTHON_PSYCOPG2_VERSION=2.7.3.2 +PYTHON_PSUTIL_VERSION=5.6.2 +PYTHON_REDIS_VERSION=2.10.6 diff --git a/docker/Dockerfile b/docker/Dockerfile index bd503964c5..8384ac883e 100755 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -8,6 +8,8 @@ FROM debian:9.8-slim as BASE_IMAGE LABEL maintainer="Roberto Rosario roberto.rosario@mayan-edms.com" +COPY config.env /config.env + ENV PYTHONUNBUFFERED=1 \ LC_ALL=C.UTF-8 \ PROJECT_INSTALL_DIR=/opt/mayan-edms @@ -63,7 +65,7 @@ echo "save \"\"" >> /etc/redis/redis.conf \ && echo "databases 1" >> /etc/redis/redis.conf #### -# BUILDER_IMAGE - This image buildS the Python package and is discarded afterwards +# BUILDER_IMAGE - This image builds the Python package and is discarded afterwards #### # Reuse image @@ -104,17 +106,21 @@ apt-get install -y --no-install-recommends \ && chown -R mayan:mayan /src USER mayan -RUN python -m virtualenv "${PROJECT_INSTALL_DIR}" \ + +RUN set -a \ +&& . /config.env \ +&& set +a \ +&& python -m virtualenv "${PROJECT_INSTALL_DIR}" \ && . "${PROJECT_INSTALL_DIR}/bin/activate" \ && pip install --no-cache-dir --no-use-pep517 \ - librabbitmq==1.6.1 \ - mysql-python==1.2.5 \ - psycopg2==2.7.3.2 \ - redis==2.10.6 \ + librabbitmq==$PYTHON_LIBRABBITMQ_VERSION \ + mysql-python==$PYTHON_MYSQL_VERSION \ + psycopg2==$PYTHON_PSYCOPG2_VERSION \ + redis==$PYTHON_REDIS_VERSION \ # psutil is needed by ARM builds otherwise gevent and gunicorn fail to start && UNAME=`uname -m` && if [ "${UNAME#*arm}" != $UNAME ]; then \ pip install --no-cache-dir --no-use-pep517 \ - psutil==5.6.2 \ + psutil==$PYTHON_PSUTIL_VERSION \ ; fi \ # Install the Python packages needed to build Mayan EDMS && pip install --no-cache-dir --no-use-pep517 -r /src/requirements/build.txt \ diff --git a/docker/Makefile b/docker/Makefile index d3e49fed4b..9a8d9162a0 100755 --- a/docker/Makefile +++ b/docker/Makefile @@ -1,3 +1,6 @@ +#!make +include config.env + APT_PROXY ?= `/sbin/ip route|awk '/docker0/ { print $$9 }'`:3142 CONSOLE_COLUMNS ?= `echo $$(tput cols)` CONSOLE_LINES ?= `echo $$(tput lines)` @@ -48,13 +51,13 @@ docker-staging-container-postgresql-start: -e POSTGRES_DB=mayan \ -e POSTGRES_PASSWORD=mayanuserpass \ -v mayan-staging-postgres:/var/lib/postgresql/data \ - postgres:9.6-alpine + $(DOCKER_POSTGRES_IMAGE_VERSION) docker-staging-container-redis-start: docker run -d \ --name mayan-staging-redis \ --network=mayan-staging \ - redis:5.0-alpine \ + $(DOCKER_REDIS_IMAGE_VERSION) \ redis-server \ --databases \ "2" \ diff --git a/docs/chapters/backups.rst b/docs/chapters/backups.rst index 5d65284308..496ce9c766 100644 --- a/docs/chapters/backups.rst +++ b/docs/chapters/backups.rst @@ -59,7 +59,7 @@ Example:: -e POSTGRES_DB=mayan \ -e POSTGRES_PASSWORD=mayanuserpass \ -v /docker-volumes/mayan-edms/postgres-new:/var/lib/postgresql/data \ - -d postgres:9.6 + |DOCKER_POSTGRES_IMAGE_VERSION| docker exec -i mayan-edms-pg-new pg_restore -U mayan -d mayan < 2018-06-07_17-09-34.dump diff --git a/docs/chapters/deploying.rst b/docs/chapters/deploying.rst index 425f3274f9..a14f694f16 100644 --- a/docs/chapters/deploying.rst +++ b/docs/chapters/deploying.rst @@ -84,14 +84,14 @@ For another setup that offers more performance and scalability refer to the ------------------------------------------------------ :: - sudo -u mayan /opt/mayan-edms/bin/pip install --no-cache-dir --no-use-pep517 psycopg2==2.7.3.2 redis==2.10.6 + sudo -u mayan /opt/mayan-edms/bin/pip install --no-cache-dir --no-use-pep517 psycopg2==|PYTHON_PSYCOPG2_VERSION| redis==|PYTHON_REDIS_VERSION| .. note:: Platforms with the ARM CPU might also need additional requirements. :: - sudo -u mayan /opt/mayan-edms/bin/pip install --no-cache-dir --no-use-pep517 psutil==5.6.2 + sudo -u mayan /opt/mayan-edms/bin/pip install --no-cache-dir --no-use-pep517 psutil==|PYTHON_PSUTIL_VERSION| 8. Create the database for the installation: @@ -206,7 +206,7 @@ of a restart or power failure. The Gunicorn workers are increased to 3. ------------------------------------------ :: - sudo -u mayan /opt/mayan-edms/bin/pip install --no-cache-dir --no-use-pep517 librabbitmq==2.0.0 + sudo -u mayan /opt/mayan-edms/bin/pip install --no-cache-dir --no-use-pep517 librabbitmq==|PYTHON_LIBRABBITMQ_VERSION| 3. Create the RabbitMQ user and vhost: diff --git a/docs/chapters/docker.rst b/docs/chapters/docker.rst index 9fa58333cf..aca7fbd0f2 100644 --- a/docs/chapters/docker.rst +++ b/docs/chapters/docker.rst @@ -23,7 +23,7 @@ tag here, remember to do so in the next steps also.:: Then download version 9.6 of the Docker PostgreSQL image:: - docker pull postgres:9.6 + docker pull |DOCKER_POSTGRES_IMAGE_VERSION| Create and run a PostgreSQL container:: @@ -35,7 +35,7 @@ Create and run a PostgreSQL container:: -e POSTGRES_DB=mayan \ -e POSTGRES_PASSWORD=mayanuserpass \ -v /docker-volumes/mayan-edms/postgres:/var/lib/postgresql/data \ - postgres:9.6 + |DOCKER_POSTGRES_IMAGE_VERSION| The PostgreSQL container will have one database named ``mayan``, with an user named ``mayan`` too, with a password of ``mayanuserpass``. The container will @@ -97,7 +97,7 @@ binding (``-p 5432:5432``):: -e POSTGRES_DB=mayan \ -e POSTGRES_PASSWORD=mayanuserpass \ -v /docker-volumes/mayan-edms/postgres:/var/lib/postgresql/data \ - postgres:9.6 + |DOCKER_POSTGRES_IMAGE_VERSION| Launch the Mayan EDMS container with the network option and change the database hostname to the PostgreSQL container name (``mayan-edms-postgres``) diff --git a/docs/chapters/scaling_up.rst b/docs/chapters/scaling_up.rst index 01a59cecb3..f7eb580164 100644 --- a/docs/chapters/scaling_up.rst +++ b/docs/chapters/scaling_up.rst @@ -92,7 +92,7 @@ section for the required changes. For the Docker image, launch a separate RabbitMQ container (https://hub.docker.com/_/rabbitmq/):: - docker run -d --name mayan-edms-rabbitmq -e RABBITMQ_DEFAULT_USER=mayan -e RABBITMQ_DEFAULT_PASS=mayanrabbitmqpassword -e RABBITMQ_DEFAULT_VHOST=mayan rabbitmq:3 + docker run -d --name mayan-edms-rabbitmq -e RABBITMQ_DEFAULT_USER=mayan -e RABBITMQ_DEFAULT_PASS=mayanrabbitmqpassword -e RABBITMQ_DEFAULT_VHOST=mayan |DOCKER_RABBITMQ_IMAGE_VERSION| Pass the MAYAN_BROKER_URL environment variable (https://kombu.readthedocs.io/en/latest/userguide/connections.html#connection-urls) to the Mayan EDMS container so that it uses the RabbitMQ container the diff --git a/docs/conf.py b/docs/conf.py index 520ff4791e..5605895f04 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -29,12 +29,12 @@ sys.path.append( # -- General configuration ----------------------------------------------------- # If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' +# needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -#extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode'] -#extensions = ["djangodocs", "sphinx.ext.intersphinx"] +# extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode'] +# extensions = ["djangodocs", "sphinx.ext.intersphinx"] extensions = [ 'sphinx.ext.extlinks', 'sphinxcontrib.blockdiag', 'sphinxcontrib.spelling' @@ -72,20 +72,20 @@ release = version # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. -#language = None +# language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: -#today = '' +# today = '' # Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' +# today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. exclude_patterns = ['_build'] # The reST default role (used for this markup: `text`) to use for all documents. -#default_role = None +# default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. add_function_parentheses = True @@ -102,7 +102,7 @@ show_authors = False pygments_style = 'sphinx' # A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] +# modindex_common_prefix = [] # -- Options for HTML output --------------------------------------------------- @@ -119,23 +119,23 @@ html_theme_options = { } # Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = [] +# html_theme_path = [] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". -#html_title = None +# html_title = None # A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None +# html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. -#html_logo = None +# html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. -#html_favicon = None +# html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, @@ -148,40 +148,40 @@ html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. -#html_use_smartypants = True +# html_use_smartypants = True # Custom sidebar templates, maps document names to template names. -#html_sidebars = {} +# html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. -#html_additional_pages = {} +# html_additional_pages = {} # If false, no module index is generated. -#html_domain_indices = True +# html_domain_indices = True # If false, no index is generated. -#html_use_index = True +# html_use_index = True # If true, the index is split into individual pages for each letter. -#html_split_index = False +# html_split_index = False # If true, links to the reST sources are added to the pages. html_show_sourcelink = False # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -#html_show_sphinx = True +# html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -#html_show_copyright = True +# html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. -#html_use_opensearch = '' +# html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None +# html_file_suffix = None # Output file base name for HTML help builder. htmlhelp_basename = 'MayanEDMSdoc' @@ -191,40 +191,42 @@ html_show_sphinx = False # -- Options for LaTeX output -------------------------------------------------- # The paper size ('letter' or 'a4'). -#latex_paper_size = 'letter' +# latex_paper_size = 'letter' # The font size ('10pt', '11pt' or '12pt'). -#latex_font_size = '10pt' +# latex_font_size = '10pt' # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ - ('index', 'MayanEDMS.tex', 'Mayan EDMS Documentation', - mayan.__author__, 'manual'), + ( + 'index', 'MayanEDMS.tex', 'Mayan EDMS Documentation', + mayan.__author__, 'manual' + ), ] # The name of an image file (relative to this directory) to place at the top of # the title page. -#latex_logo = None +# latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. -#latex_use_parts = False +# latex_use_parts = False # If true, show page references after internal links. -#latex_show_pagerefs = False +# latex_show_pagerefs = False # If true, show URL addresses after external links. -#latex_show_urls = False +# latex_show_urls = False # Additional stuff for the LaTeX preamble. -#latex_preamble = '' +# latex_preamble = '' # Documents to append as an appendix to all manuals. -#latex_appendices = [] +# latex_appendices = [] # If false, no module index is generated. -#latex_domain_indices = True +# latex_domain_indices = True # -- Options for manual page output -------------------------------------------- @@ -255,5 +257,25 @@ extlinks = { } +def _load_env_file(filename='../config.env'): + result = [] + with open(filename) as file_object: + for line in file_object: + if not line.startswith('#'): + key, value = line.strip().split('=') + result.append(('|{}|'.format(key), value)) + + return result + + +def GlobalSubstitution(app, docname, source): + for old, new in global_subtitutions: + source[0] = source[0].replace(old, new) + + def setup(app): app.add_stylesheet('css/custom.css') + app.connect('source-read', GlobalSubstitution) + + +global_subtitutions = _load_env_file()