Initial generic error log refactor

Signed-off-by: Roberto Rosario <Roberto.Rosario.Gonzalez@gmail.com>

.dockerignore

@@ -4,7 +4,8 @@ HISTORY.md
 mayan/media
 db.sqlite*
 docker
-!docker/rootfs
+!docker/etc
+!docker/entrypoint.sh
+!docker/version
 ./.*
 docs
-

.drone.yml

@@ -1,31 +0,0 @@
-kind: pipeline
-name: default
-
-steps:
-  - name: docker  
-    image: plugins/docker
-    settings:
-      registry: registry.d1v3.de
-      repo: registry.d1v3.de/mayan-edms
-      username:
-        from_secret: docker_username
-      password:
-        from_secret: docker_password
-      dockerfile: docker/Dockerfile
-      tags: latest
-      auto_tag: true
-
-
----
-kind: secret
-name: docker_username
-get:
-    path: kv/data/drone/docker
-    name: username
----
-
-kind: secret
-name: docker_password
-get:
-    path: kv/data/drone/docker
-    name: token

.flake8

@@ -1,3 +1,2 @@
 [flake8]
-exclude = node_modules,static,templates
-ignore = E501, N801, N802, N804, N805, N806
+ignore = E501

.github/FUNDING.yml

@@ -1 +0,0 @@
-custom: https://www.mayan-edms.com/donation/

.gitignore

@@ -7,7 +7,6 @@
 .coverage
 .coverage.tox*
 .idea/
-*.swp
 .tox/
 .vagrant
 _build/
@@ -30,8 +29,4 @@ static_collected/
 /venv/
 /venv3/
 /whoosh_index/
-google_fonts/
 node_modules/
-docs/build/
-docs/_templates/layout.html
-docs/_templates/footer.html

.gitlab-ci.yml

@@ -17,9 +17,9 @@ job_docker_build:
     - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
   script:
     - docker build --pull -t "$CI_REGISTRY_IMAGE" -f docker/Dockerfile .
-    - VERSION=`cat docker/rootfs/version`
+    - VERSION=`cat docker/version`
     - docker tag "$CI_REGISTRY_IMAGE" "$CI_REGISTRY_IMAGE:$VERSION"
-    - docker run --rm "$CI_REGISTRY_IMAGE:$VERSION" run_tests
+    - docker run --rm "$CI_REGISTRY_IMAGE:$VERSION" run-tests
     - docker push "$CI_REGISTRY_IMAGE:$VERSION"
     - docker push "$CI_REGISTRY_IMAGE:latest"
     - docker tag "$CI_REGISTRY_IMAGE:$VERSION" registry-1.docker.io/mayanedms/mayanedms:"$VERSION"
@@ -37,7 +37,7 @@ job_docker_push:
   before_script:
     - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
   script:
-    - VERSION=`cat docker/rootfs/version`
+    - VERSION=`cat docker/version`
     - docker pull "$CI_REGISTRY_IMAGE:$VERSION"
     - docker tag "$CI_REGISTRY_IMAGE:$VERSION" registry-1.docker.io/mayanedms/mayanedms:"$VERSION"
     - docker tag "$CI_REGISTRY_IMAGE:$VERSION" registry-1.docker.io/mayanedms/mayanedms:latest
@@ -58,7 +58,7 @@ job_docker_nightly:
     - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
   script:
     - docker build --pull -t "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG" -f docker/Dockerfile .
-    - docker run --rm "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG" run_tests
+    - docker run --rm "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG" run-tests
     - docker push "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG"
   only:
     - nightly
@@ -119,7 +119,7 @@ job_build_python:
     - update-locale LANG=en_US.UTF-8
     - export LC_ALL=en_US.UTF-8
     - pip install -r requirements/build.txt
-    - make python-wheel
+    - make wheel
   only:
     - releases/all
     - releases/python
@@ -140,7 +140,7 @@ job_push_python:
 
 .test_base: &test_base
   stage: test
-  image: ubuntu:18.04
+  image: ubuntu:16.04
   cache:
     paths:
       - ~/.cache/pip/
@@ -152,14 +152,13 @@ job_push_python:
     - locale-gen en_US.UTF-8
     - update-locale LANG=en_US.UTF-8
     - export LC_ALL=en_US.UTF-8
-    - apt-get install -qq curl exiftool firefox-geckodriver gcc ghostscript gnupg1 graphviz libfuse2 libjpeg-dev libmagic1 libpng-dev libtiff-dev poppler-utils libreoffice poppler-utils python-dev python-virtualenv python3-dev tesseract-ocr tesseract-ocr-deu
-    - virtualenv venv -p /usr/bin/python3
-    - . venv/bin/activate
-    - pip install -r requirements.txt -r requirements/testing-base.txt
+    - apt-get install -qq curl gcc ghostscript gpgv gnupg graphviz libfuse2 libjpeg-dev libmagic1 libpng-dev libtiff-dev poppler-utils libreoffice poppler-utils python-dev python-pip tesseract-ocr tesseract-ocr-deu
+    - pip install -r requirements/testing.txt
   only:
     - releases/all
     - releases/docker
     - releases/python
+    - master
     - staging
     - nightly
 
@@ -171,10 +170,8 @@ test-mysql:
   services:
     - mysql:8.0.3
   script:
-    - apt-get install -qq libmysqlclient-dev libssl-dev mysql-client
-    - . venv/bin/activate
-    - set -a && . ./config.env && set +a
-    - pip install mysqlclient==$PYTHON_MYSQL_VERSION
+    - apt-get install -qq libmysqlclient-dev mysql-client
+    - pip install mysql-python
     - mysql -h"$MYSQL_PORT_3306_TCP_ADDR" -P"$MYSQL_PORT_3306_TCP_PORT" -uroot -p"$MYSQL_ENV_MYSQL_ROOT_PASSWORD" -e "set global character_set_server=utf8mb4;"
     - python manage.py test --mayan-apps --settings=mayan.settings.testing.gitlab-ci.db_mysql --nomigrations
   tags:
@@ -189,9 +186,7 @@ test-postgres:
     - postgres
   script:
     - apt-get install -qq libpq-dev
-    - . venv/bin/activate
-    - set -a && . ./config.env && set +a
-    - pip install psycopg2==$PYTHON_PSYCOPG2_VERSION
+    - pip install psycopg2
     - python manage.py test --mayan-apps --settings=mayan.settings.testing.gitlab-ci.db_postgres --nomigrations
   tags:
     - postgres
@@ -199,7 +194,6 @@ test-postgres:
 test-sqlite:
   <<: *test_base
   script:
-    - . venv/bin/activate
     - python manage.py test --mayan-apps --settings=mayan.settings.testing.gitlab-ci --nomigrations
 
 deploy_demo:

.isort.cfg

@@ -0,0 +1,6 @@
+[settings]
+default_section = THIRDPARTY
+known_first_party = mayan
+known_django = django
+multi_line_output = 5
+sections = FUTURE,STDLIB,DJANGO,THIRDPARTY,FIRSTPARTY,LOCALFOLDER

.tx/config

@@ -55,18 +55,18 @@ source_lang = en
 source_file = mayan/apps/dashboards/locale/en/LC_MESSAGES/django.po
 type = PO
 
-[mayan-edms.dependencies-3-0]
-file_filter = mayan/apps/dependencies/locale/<lang>/LC_MESSAGES/django.po
-source_lang = en
-source_file = mayan/apps/dependencies/locale/en/LC_MESSAGES/django.po
-type = PO
-
 [mayan-edms.django_gpg-2-0]
 file_filter = mayan/apps/django_gpg/locale/<lang>/LC_MESSAGES/django.po
 source_lang = en
 source_file = mayan/apps/django_gpg/locale/en/LC_MESSAGES/django.po
 type = PO
 
+[mayan-edms.documents-2-0]
+file_filter = mayan/apps/documents/locale/<lang>/LC_MESSAGES/django.po
+source_lang = en
+source_file = mayan/apps/documents/locale/en/LC_MESSAGES/django.po
+type = PO
+
 [mayan-edms.document_comments-2-0]
 file_filter = mayan/apps/document_comments/locale/<lang>/LC_MESSAGES/django.po
 source_lang = en
@@ -97,12 +97,6 @@ source_lang = en
 source_file = mayan/apps/document_states/locale/en/LC_MESSAGES/django.po
 type = PO
 
-[mayan-edms.documents-2-0]
-file_filter = mayan/apps/documents/locale/<lang>/LC_MESSAGES/django.po
-source_lang = en
-source_file = mayan/apps/documents/locale/en/LC_MESSAGES/django.po
-type = PO
-
 [mayan-edms.dynamic_search-2-0]
 file_filter = mayan/apps/dynamic_search/locale/<lang>/LC_MESSAGES/django.po
 source_lang = en
@@ -115,18 +109,12 @@ source_lang = en
 source_file = mayan/apps/events/locale/en/LC_MESSAGES/django.po
 type = PO
 
-[mayan-edms.file_caching-3-0]
+[mayan-edms.file_caching-2-0]
 file_filter = mayan/apps/file_caching/locale/<lang>/LC_MESSAGES/django.po
 source_lang = en
 source_file = mayan/apps/file_caching/locale/en/LC_MESSAGES/django.po
 type = PO
 
-[mayan-edms.file_metadata-3-0]
-file_filter = mayan/apps/file_metadata/locale/<lang>/LC_MESSAGES/django.po
-source_lang = en
-source_file = mayan/apps/file_metadata/locale/en/LC_MESSAGES/django.po
-type = PO
-
 [mayan-edms.linking-2-0]
 file_filter = mayan/apps/linking/locale/<lang>/LC_MESSAGES/django.po
 source_lang = en
@@ -175,12 +163,6 @@ source_lang = en
 source_file = mayan/apps/permissions/locale/en/LC_MESSAGES/django.po
 type = PO
 
-[mayan-edms.platform-2-0]
-file_filter = mayan/apps/platform/locale/<lang>/LC_MESSAGES/django.po
-source_lang = en
-source_file = mayan/apps/platform/locale/en/LC_MESSAGES/django.po
-type = PO
-
 [mayan-edms.rest_api-2-0]
 file_filter = mayan/apps/rest_api/locale/<lang>/LC_MESSAGES/django.po
 source_lang = en
@@ -223,21 +205,8 @@ source_lang = en
 source_file = mayan/apps/task_manager/locale/en/LC_MESSAGES/django.po
 type = PO
 
-[mayan-edms.templating-3-0]
-file_filter = mayan/apps/templating/locale/<lang>/LC_MESSAGES/django.po
-source_lang = en
-source_file = mayan/apps/templating/locale/en/LC_MESSAGES/django.po
-type = PO
-
 [mayan-edms.user_management-2-0]
 file_filter = mayan/apps/user_management/locale/<lang>/LC_MESSAGES/django.po
 source_lang = en
 source_file = mayan/apps/user_management/locale/en/LC_MESSAGES/django.po
 type = PO
-
-[mayan-edms.web_link-3-0]
-file_filter = mayan/apps/web_links/locale/<lang>/LC_MESSAGES/django.po
-source_lang = en
-source_file = mayan/apps/web_links/locale/en/LC_MESSAGES/django.po
-type = PO
-

CONTRIBUTING.md

@@ -4,30 +4,26 @@ Contributing to Mayan EDMS
 Issues
 ------
 
-- The issue system is for reporting problems with the program code. For
-questions use the forum at: https://forum.mayan-edms.com/
-- Do not open issues asking for **support or consulting**. For paid support
-and consultation visit: https://www.mayan-edms.com/support/
-- The issue must be related to the code only, do not open issues for problems
-with databases, deployments, webservers, cloud providers, etc.
-- Before submitting a new issue, check for **existing issues** first and
-join the discussion.
+- Before submitting a new issue, check for **open issues** first and join the
+discussion.
 - If your issue is not attended in a while, **be patient**, the core team is
 small and the codebase big.
-- Try to reproduce the issue using a **separate, clean installation**,
-sometimes the issue can be caused by an error in a configuration file and
-not with the code itself.
-- **Do not upload** sensitive, private or classified information or files
-with your issue. If the issue is triggered by a user file, create a dummy
-file with the same properties that can trigger the issue and upload that
-file instead.
+- Try to reproduce the issue using a **separate, clean installation**, sometimes
+the issue can be caused by an error in a configuration file and not with the
+code itself.
+- **Do not upload** sensitive, private or classified information or files with
+your issue. If the issue is triggered by a user file, create a dummy file with the
+same properties that can trigger the issue and upload that file instead.
 - Add steps that trigger the issue in a **repeatable manner**.
 - **Screenshots** go a long way in helping understand problems.
+- The issue must be related to the code only, do not open issues for problems
+with deployments, webservers, cloud providers, etc.
+- Do not open issues asking for **support or consulting**.
 
 Code
 ----
 
-1. Complete and mail, or scan and email the corresponding Contributor Assignment Agreement: [Mayan EDMS Individual Contributor Assignment Agreement](https://gitlab.com/mayan-edms/mayan-edms/blob/master/docs/chapters/caa_individual.rst) or [Mayan EDMS Entity Contributor Assignment Agreement](https://gitlab.com/mayan-edms/mayan-edms/blob/master/docs/chapters/caa_entity.rst).
+1. Complete and mail, or scan and email the corresponding Contributor Assignment Agreement: [Mayan EDMS Individual Contributor Assignment Agreement](https://docs.mayan-edms.com/topics/caa_individual.rst) or [Mayan EDMS Entity Contributor Assignment Agreement](https://docs.mayan-edms.com/topics/topics/caa_entity.rst).
 1. Fork [the repository](http://gitlab.com/mayan-edms/mayan-edms).
 1. Choose the version for which you want to develop. The code is divided in the
 following branches:

LICENSE

@@ -1,4 +1,4 @@
-   Copyright 2011 Roberto Rosario
+   Copyright 2011-2018 Roberto Rosario
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

MANIFEST.in

@@ -1,6 +1,3 @@
 include README.md LICENSE HISTORY.rst mayan/LICENSE
-recursive-include mayan/apps *.txt *.html *.css *.ico *.png *.jpg *.js *.mo *.ttf *.woff *.woff2 *.gif *.eot *.svg *.doc *.pdf *.tiff *.sig *.asc *.gpg *.zip *.tar *.gz *.bz2 *.tmpl
-global-exclude mayan/settings/local.py *.po
-prune mayan/apps/*/static/*/node_modules/*
-prune mayan/settings/travis/*
-prune mayan/media/*
+recursive-include mayan/apps *.txt *.html *.css *.ico *.png *.jpg *.js *.po *.mo *.ttf *.woff *.woff2 *.gif *.eot *.svg *.doc *.pdf *.tiff *.sig *.asc *.gpg *.zip *.tar *.gz *.bz2 package.json
+global-exclude mayan/settings/local.py mayan/settings/travis/* mayan/media/*

Makefile

@@ -1,61 +1,90 @@
-#!make
-include config.env
+.PHONY: clean-pyc clean-build
 
-.PHONY: clean clean-pyc clean-build test
+help: docker-help
+	@echo
+	@echo "**** Main makefile ****"
+	@echo "clean-build - Remove build artifacts."
+	@echo "clean-pyc - Remove Python artifacts."
+	@echo "clean - Remove Python and build artifacts."
+	@echo "generate-setup - Create and updated setup.py"
+	@echo "check-readme - Checks validity of the README.rst file for PyPI publication."
+	@echo "check-missing_migrations - Make sure all models have proper migrations."
 
-help:
-	@echo "Usage: make <target>\n"
-	@awk 'BEGIN {FS = ":.*##"} /^[0-9a-zA-Z_-]+:.*?## / { printf "  * %-40s -%s\n", $$1, $$2 }' $(MAKEFILE_LIST)|sort
+	@echo "test-all - Run all tests."
+	@echo "test MODULE=<python module name> - Run tests for a single app, module or test class."
+	@echo "test-with-postgres-all - Run all tests against a Postgres database container."
+	@echo "test-postgres MODULE=<python module name> - Run tests for a single app, module or test class against a Postgres database container."
+	@echo "test-with-mysql-all - Run all tests against a MySQL database container."
+	@echo "test-mysql MODULE=<python module name> - Run tests for a single app, module or test class against a MySQL database container."
+	@echo "test-with-oracle-all - Run all tests against a Oracle database container."
+	@echo "test-oracle MODULE=<python module name> - Run tests for a single app, module or test class against a Oracle database container."
+
+	@echo "docs-serve - Run the livehtml documentation generator."
+
+	@echo "translations-make - Refresh all translation files."
+	@echo "translations-compile - Compile all translation files."
+	@echo "translations-push - Upload all translation files to Transifex."
+	@echo "translations-pull - Download all translation files from Transifex."
+
+	@echo "sdist - Build the source distribution package."
+	@echo "wheel - Build the wheel distribution package."
+	@echo "release - Package (sdist and wheel) and upload a release."
+	@echo "test-release - Package (sdist and wheel) and upload to the PyPI test server."
+	@echo "release-test-via-docker-ubuntu - Package (sdist and wheel) and upload to the PyPI test server using an Ubuntu Docker builder."
+	@echo "release-via-docker-ubuntu - Package (sdist and wheel) and upload to PyPI using an Ubuntu Docker builder."
+	@echo "test-sdist-via-docker-ubuntu - Make an sdist packange and test it using an Ubuntu Docker container."
+	@echo "test-wheel-via-docker-ubuntu - Make a wheel package and test it using an Ubuntu Docker container."
+	@echo "runserver - Run the development server."
+	@echo "runserver_plus - Run the Django extension's development server."
+	@echo "shell_plus - Run the shell_plus command."
+
+	@echo "test-with-docker-services-on - Launch and initialize production-like services using Docker (Postgres and Redis)."
+	@echo "test-with-docker-services-off - Stop and delete the Docker production-like services."
+	@echo "test-with-docker-frontend - Launch a front end instance that uses the production-like services."
+	@echo "test-with-docker-worker - Launch a worker instance that uses the production-like services."
+	@echo "docker-mysql-on - Launch and initialize a MySQL Docker container."
+	@echo "docker-mysql-off - Stop and delete the MySQL Docker container."
+	@echo "docker-postgres-on - Launch and initialize a PostgreSQL Docker container."
+	@echo "docker-postgres-off - Stop and delete the PostgreSQL Docker container."
+
+	@echo "safety-check - Run a package safety check."
 
 # Cleaning
 
-clean: ## Remove Python and build artifacts.
 clean: clean-build clean-pyc
 
-clean-build: ## Remove build artifacts.
+clean-build:
 	rm -fr build/
 	rm -fr dist/
 	rm -fr *.egg-info
 
-clean-pyc: ## Remove Python artifacts.
+clean-pyc:
 	find . -name '*.pyc' -exec rm -f {} +
 	find . -name '*.pyo' -exec rm -f {} +
 	find . -name '*~' -exec rm -f {} +
-	find . -name '__pycache__' -exec rm -R -f {} +
+
 
 # Testing
 
-test: clean-pyc
-test: ## MODULE=<python module name> - Run tests for a single app, module or test class.
-	./manage.py test $(MODULE) --settings=mayan.settings.testing.development --nomigrations $(ARGUMENTS)
+test:
+	./manage.py test $(MODULE) --settings=mayan.settings.testing.development --nomigrations
 
-test-debug: clean-pyc
-test-debug: ## MODULE=<python module name> - Run tests for a single app, module or test class, in debug mode.
-	./manage.py test $(MODULE) --settings=mayan.settings.testing.development --nomigrations --debug-mode $(ARGUMENTS)
-
-test-all: ## Run all tests.
-test-all: clean-pyc
-	./manage.py test --mayan-apps --settings=mayan.settings.testing.development --nomigrations $(ARGUMENTS)
-
-test-all-debug: ## Run all tests in debug mode.
-test-all-debug: clean-pyc
-	./manage.py test --mayan-apps --settings=mayan.settings.testing.development --nomigrations --debug-mode $(ARGUMENTS)
+test-all:
+	./manage.py test --mayan-apps --settings=mayan.settings.testing.development --nomigrations
 
 test-launch-postgres:
 	@docker rm -f test-postgres || true
 	@docker volume rm test-postgres || true
-	docker run -d --name test-postgres -p 5432:5432 -v test-postgres:/var/lib/postgresql/data $(DOCKER_POSTGRES_IMAGE_VERSION)
-	sudo apt-get install -q libpq-dev
-	pip install psycopg2==$(PYTHON_PSYCOPG2_VERSION)
-	while ! nc -z 127.0.0.1 5432; do sleep 1; done
+	docker run -d --name test-postgres -p 5432:5432 -v test-postgres:/var/lib/postgresql/data healthcheck/postgres
+	sudo apt-get install -qq libpq-dev
+	pip install psycopg2
+	while ! docker inspect --format='{{json .State.Health}}' test-postgres|grep 'Status":"healthy"'; do sleep 1; done
 
-test-with-postgres: ## MODULE=<python module name> - Run tests for a single app, module or test class against a Postgres database container.
 test-with-postgres: test-launch-postgres
 	./manage.py test $(MODULE) --settings=mayan.settings.testing.docker.db_postgres --nomigrations
 	@docker rm -f test-postgres || true
 	@docker volume rm test-postgres || true
 
-test-with-postgres-all: ## Run all tests against a Postgres database container.
 test-with-postgres-all: test-launch-postgres
 	./manage.py test --mayan-apps --settings=mayan.settings.testing.docker.db_postgres --nomigrations
 	@docker rm -f test-postgres || true
@@ -64,20 +93,17 @@ test-with-postgres-all: test-launch-postgres
 test-launch-mysql:
 	@docker rm -f test-mysql || true
 	@docker volume rm test-mysql || true
-	docker run -d --name test-mysql -p 3306:3306 -e MYSQL_ALLOW_EMPTY_PASSWORD=True -e MYSQL_DATABASE=mayan -v test-mysql:/var/lib/mysql $(DOCKER_MYSQL_IMAGE_VERSION)
-	sudo apt-get install -q libmysqlclient-dev mysql-client
-	pip install mysqlclient==$(PYTHON_MYSQL_VERSION)
-	while ! nc -z 127.0.0.1 3306; do sleep 1; done
+	docker run -d --name test-mysql -p 3306:3306 -e MYSQL_ALLOW_EMPTY_PASSWORD=True -e MYSQL_DATABASE=mayan -v test-mysql:/var/lib/mysql healthcheck/mysql
+	sudo apt-get install -qq libmysqlclient-dev mysql-client
+	pip install mysql-python
+	while ! docker inspect --format='{{json .State.Health}}' test-mysql|grep 'Status":"healthy"'; do sleep 1; done
 	mysql -h 127.0.0.1 -P 3306 -uroot  -e "set global character_set_server=utf8mb4;"
 
-test-with-mysql: ## MODULE=<python module name> - Run tests for a single app, module or test class against a MySQL database container.
 test-with-mysql: test-launch-mysql
 	./manage.py test $(MODULE) --settings=mayan.settings.testing.docker.db_mysql --nomigrations
 	@docker rm -f test-mysql || true
 	@docker volume rm test-mysql || true
 
-
-test-with-mysql-all: ## Run all tests against a MySQL database container.
 test-with-mysql-all: test-launch-mysql
 	./manage.py test --mayan-apps --settings=mayan.settings.testing.docker.db_mysql --nomigrations
 	@docker rm -f test-mysql || true
@@ -86,19 +112,17 @@ test-with-mysql-all: test-launch-mysql
 test-launch-oracle:
 	@docker rm -f test-oracle || true
 	@docker volume rm test-oracle || true
-	docker run -d --name test-oracle -p 49160:22 -p 49161:1521 -e ORACLE_ALLOW_REMOTE=true -v test-oracle:/u01/app/oracle $(DOCKER_ORACLE_IMAGE_VERSION)
+	docker run -d --name test-oracle -p 49160:22 -p 49161:1521 -e ORACLE_ALLOW_REMOTE=true -v test-oracle:/u01/app/oracle wnameless/oracle-xe-11g
 	# https://gist.github.com/kimus/10012910
-	pip install cx_Oracle==$(PYTHON_ORACLE_VERSION)
+	pip install cx_Oracle
 	while ! nc -z 127.0.0.1 49161; do sleep 1; done
 	sleep 10
 
-test-with-oracle: ## MODULE=<python module name> - Run tests for a single app, module or test class against a Oracle database container.
 test-with-oracle: test-launch-oracle
 	./manage.py test $(MODULE) --settings=mayan.settings.testing.docker.db_oracle --nomigrations
 	@docker rm -f test-oracle || true
 	@docker volume rm test-oracle || true
 
-test-with-oracle-all: ## Run all tests against a Oracle database container.
 test-with-oracle-all: test-launch-oracle
 	./manage.py test --mayan-apps --settings=mayan.settings.testing.docker.db_oracle --nomigrations
 	@docker rm -f test-oracle || true
@@ -106,59 +130,58 @@ test-with-oracle-all: test-launch-oracle
 
 # Documentation
 
-docs-serve: ## Run the livehtml documentation generator.
+docs-serve:
 	cd docs;make livehtml
 
-docs-spellcheck: ## Spellcheck the documentation.
+docs-spellcheck:
+
 	sphinx-build -b spelling -d docs/_build/ docs docs/_build/spelling
 
 # Translations
 
-translations-make: ## Refresh all translation files.
+translations-make:
 	contrib/scripts/process_messages.py -m
 
-translations-compile: ## Compile all translation files.
+translations-compile:
 	contrib/scripts/process_messages.py -c
 
-translations-push: ## Upload all translation files to Transifex.
+translations-push:
 	tx push -s
 
-translations-pull: ## Download all translation files from Transifex.
+translations-pull:
 	tx pull -f
 
-translations-all: ## Execute all translations targets.
-translations-all: translations-make translations-push translations-pull translations-compile
+
+generate-setup:
+	@./generate_setup.py
+	@echo "Complete."
 
 # Releases
 
-increase-version: ## Increase the version number of the entire project's files.
+increase-version:
 	@VERSION=`grep "__version__ =" mayan/__init__.py| cut -d\' -f 2|./increase_version.py - $(PART)`; \
 	BUILD=`echo $$VERSION|awk '{split($$VERSION,a,"."); printf("0x%02d%02d%02d\n", a[1],a[2], a[3])}'`; \
 	sed -i -e "s/__build__ = 0x[0-9]*/__build__ = $${BUILD}/g" mayan/__init__.py; \
 	sed -i -e "s/__version__ = '[0-9\.]*'/__version__ = '$${VERSION}'/g" mayan/__init__.py; \
-	echo $$VERSION > docker/rootfs/version
+	echo $$VERSION > docker/version
 	make generate-setup
 
-python-test-release: ## Package (sdist and wheel) and upload to the PyPI test server.
-python-test-release: clean wheel
+test-release: clean wheel
 	twine upload dist/* -r testpypi
 	@echo "Test with: pip install -i https://testpypi.python.org/pypi mayan-edms"
 
-python-release: ## Package (sdist and wheel) and upload a release.
-python-release: clean python-wheel
+release: clean wheel
 	twine upload dist/* -r pypi
 
-python-sdist: ## Build the source distribution package.
-python-sdist: clean
+sdist: clean
 	python setup.py sdist
 	ls -l dist
 
-python-wheel: ## Build the wheel distribution package.
-python-wheel: clean python-sdist
+wheel: clean sdist
 	pip wheel --no-index --no-deps --wheel-dir dist dist/*.tar.gz
 	ls -l dist
 
-python-release-test-via-docker-ubuntu: ## Package (sdist and wheel) and upload to the PyPI test server using an Ubuntu Docker builder.
+release-test-via-docker-ubuntu:
 	docker run --rm --name mayan_release -v $(HOME):/host_home:ro -v `pwd`:/host_source -w /source ubuntu:16.04 /bin/bash -c "\
 	echo "LC_ALL=\"en_US.UTF-8\"" >> /etc/default/locale && \
 	locale-gen en_US.UTF-8 && \
@@ -171,7 +194,7 @@ python-release-test-via-docker-ubuntu: ## Package (sdist and wheel) and upload t
 	cp -r /host_home/.pypirc ~/.pypirc && \
 	make test-release"
 
-python-release-via-docker-ubuntu: ## Package (sdist and wheel) and upload to PyPI using an Ubuntu Docker builder.
+release-via-docker-ubuntu:
 	docker run --rm --name mayan_release -v $(HOME):/host_home:ro -v `pwd`:/host_source -w /source ubuntu:16.04 /bin/bash -c "\
 	apt-get update && \
 	apt-get -y install locales && \
@@ -185,7 +208,7 @@ python-release-via-docker-ubuntu: ## Package (sdist and wheel) and upload to PyP
 	cp -r /host_home/.pypirc ~/.pypirc && \
 	make release"
 
-test-sdist-via-docker-ubuntu: ## Make an sdist package and test it using an Ubuntu Docker container.
+test-sdist-via-docker-ubuntu:
 	docker run --rm --name mayan_sdist_test -v $(HOME):/host_home:ro -v `pwd`:/host_source -w /source ubuntu:16.04 /bin/bash -c "\
 	cp -r /host_source/* . && \
 	echo "LC_ALL=\"en_US.UTF-8\"" >> /etc/default/locale && \
@@ -198,7 +221,7 @@ test-sdist-via-docker-ubuntu: ## Make an sdist package and test it using an Ubun
 	make sdist-test-suit \
 	"
 
-test-wheel-via-docker-ubuntu: ## Make a wheel package and test it using an Ubuntu Docker container.
+test-wheel-via-docker-ubuntu:
 	docker run --rm --name mayan_wheel_test -v $(HOME):/host_home:ro -v `pwd`:/host_source -w /source ubuntu:16.04 /bin/bash -c "\
 	cp -r /host_source/* . && \
 	echo "LC_ALL=\"en_US.UTF-8\"" >> /etc/default/locale && \
@@ -211,7 +234,7 @@ test-wheel-via-docker-ubuntu: ## Make a wheel package and test it using an Ubunt
 	make wheel-test-suit \
 	"
 
-python-sdist-test-suit: sdist
+sdist-test-suit: sdist
 	rm -f -R _virtualenv
 	virtualenv _virtualenv
 	sh -c '\
@@ -222,7 +245,7 @@ python-sdist-test-suit: sdist
 	_virtualenv/bin/mayan-edms.py test --mayan-apps \
 	'
 
-python-wheel-test-suit: wheel
+wheel-test-suit: wheel
 	rm -f -R _virtualenv
 	virtualenv _virtualenv
 	sh -c '\
@@ -233,107 +256,64 @@ python-wheel-test-suit: wheel
 	_virtualenv/bin/mayan-edms.py test --mayan-apps \
 	'
 
-generate-setup: ## Create and update the setup.py file.
-generate-setup: generate-requirements
-	@./generate_setup.py
-	@echo "Complete."
-
-generate-requirements: ## Generate all requirements files from the project depedency declarations.
-	@./manage.py generaterequirements build > requirements/build.txt
-	@./manage.py generaterequirements development > requirements/development.txt
-	@./manage.py generaterequirements documentation > requirements/documentation.txt
-	@./manage.py generaterequirements production --only=pathlib2 >> requirements/documentation.txt
-	@./manage.py generaterequirements testing > requirements/testing-base.txt
-	@./manage.py generaterequirements production --exclude=django > requirements/base.txt
-	@./manage.py generaterequirements production --only=django > requirements/common.txt
-
-gitlab-release-documentation: ## Trigger the documentation build and publication using GitLab CI
-gitlab-release-documentation:
-	git push
-	git push --tags
-	git push origin :releases/documentation || true
-	git push origin HEAD:releases/documentation
-
-gitlab-release-docker: ## Trigger the Docker image build and publication using GitLab CI
-gitlab-release-docker:
-	git push
-	git push --tags
-	git push origin :releases/docker || true
-	git push origin HEAD:releases/docker
-
-gitlab-release-python: ## Trigger the Python package build and publication using GitLab CI
-gitlab-release-python:
-	git push
-	git push --tags
-	git push origin :releases/python || true
-	git push origin HEAD:releases/python
-
-gitlab-release-all: ## Trigger the Python package, Docker image, and documentation build and publication using GitLab CI
-gitlab-release-all:
-	git push
-	git push --tags
-	git push origin :releases/all || true
-	git push origin HEAD:releases/all
-
 # Dev server
 
-runserver: ## Run the development server.
-	./manage.py runserver --nothreading --settings=mayan.settings.development $(ADDRPORT)
+runserver:
+	./manage.py runserver --settings=mayan.settings.development $(ADDRPORT)
 
-runserver_plus: ## Run the Django extension's development server.
-	./manage.py runserver_plus --nothreading --settings=mayan.settings.development $(ADDRPORT)
+runserver_plus:
+	./manage.py runserver_plus --settings=mayan.settings.development $(ADDRPORT)
 
-shell_plus: ## Run the shell_plus command.
+shell_plus:
 	./manage.py shell_plus --settings=mayan.settings.development
 
-test-with-docker-services-on: ## Launch and initialize production-like services using Docker (Postgres and Redis).
-	docker run -d --name redis -p 6379:6379 $(DOCKER_REDIS_IMAGE_VERSION)
-	docker run -d --name postgres -p 5432:5432 $(DOCKER_POSTGRES_IMAGE_VERSION)
+test-with-docker-services-on:
+	docker run -d --name redis -p 6379:6379 redis
+	docker run -d --name postgres -p 5432:5432 postgres
 	while ! nc -z 127.0.0.1 6379; do sleep 1; done
 	while ! nc -z 127.0.0.1 5432; do sleep 1; done
 	sleep 4
-	pip install psycopg2==$(PYTHON_PSYCOPG2_VERSION) redis==$(PYTHON_REDIS_VERSION)
 	./manage.py initialsetup --settings=mayan.settings.staging.docker
 
-test-with-docker-services-off: ## Stop and delete the Docker production-like services.
+test-with-docker-services-off:
 	docker stop postgres redis
 	docker rm postgres redis
 
-test-with-docker-frontend: ## Launch a front end instance that uses the production-like services.
+test-with-docker-frontend:
 	./manage.py runserver --settings=mayan.settings.staging.docker
 
-test-with-docker-worker: ## Launch a worker instance that uses the production-like services.
-	DJANGO_SETTINGS_MODULE=mayan.settings.staging.docker ./manage.py celery worker -A mayan -B -l INFO -O fair
+test-with-docker-worker:
+	./manage.py celery worker --settings=mayan.settings.staging.docker -B -l INFO
 
-docker-mysql-on: ## Launch and initialize a MySQL Docker container.
-	docker run -d --name mysql -p 3306:3306 -e MYSQL_ALLOW_EMPTY_PASSWORD=True -e MYSQL_DATABASE=mayan_edms $(DOCKER_MYSQL_IMAGE_VERSION)
+docker-mysql-on:
+	docker run -d --name mysql -p 3306:3306 -e MYSQL_ALLOW_EMPTY_PASSWORD=True -e MYSQL_DATABASE=mayan_edms mysql
 	while ! nc -z 127.0.0.1 3306; do sleep 1; done
 
-docker-mysql-off: ## Stop and delete the MySQL Docker container.
+docker-mysql-off:
 	docker stop mysql
 	docker rm mysql
 
-docker-postgres-on: ## Launch and initialize a PostgreSQL Docker container.
-	docker run -d --name postgres -p 5432:5432 $(DOCKER_POSTGRES_IMAGE_VERSION)
+docker-postgres-on:
+	docker run -d --name postgres -p 5432:5432 postgres
 	while ! nc -z 127.0.0.1 5432; do sleep 1; done
 
-docker-postgres-off: ## Stop and delete the PostgreSQL Docker container.
+docker-postgres-off:
 	docker stop postgres
 	docker rm postgres
 
 
 # Security
 
-safety-check: ## Run a package safety check.
+safety-check:
 	safety check
 
 
 # Other
-find-gitignores: ## Find stray .gitignore files.
+find-gitignores:
 	@export FIND_GITIGNORES=`find -name '.gitignore'| wc -l`; \
 	if [ $${FIND_GITIGNORES} -gt 1 ] ;then echo "More than one .gitignore found."; fi
 
-python-build:
+build:
 	docker rm -f mayan-edms-build || true && \
 	docker run --rm --name mayan-edms-build -v $(HOME):/host_home:ro -v `pwd`:/host_source -w /source python:2-slim sh -c "\
 	rm /host_source/dist -R || true && \
@@ -346,13 +326,11 @@ python-build:
 	make wheel && \
 	cp dist/* /host_source/dist/"
 
-check-readme: ## Checks validity of the README.rst file for PyPI publication.
+check-readme:
 	python setup.py check -r -s
 
-check-missing-migrations: ## Make sure all models have proper migrations.
+check-missing-migrations:
 	./manage.py makemigrations --dry-run --noinput --check
 
-setup-dev-environment: ## Bootstrap a virtualenv by install all dependencies to start developing.
-	pip install -r requirements.txt -r requirements/development.txt -r requirements/testing-base.txt -r requirements/documentation.txt -r requirements/build.txt
 
--include docker/Makefile
+include docker/Makefile

README.md

@@ -1,10 +1,11 @@
-[![Donation](https://img.shields.io/badge/donation-PayPal-brightgreen)](https://paypal.me/MayanEDMS)
 [![pypi][pypi]][pypi-url]
+[![builds][builds]][builds-url]
+[![coverage][cover]][cover-url]
 ![python][python]
 ![license][license]
 [![Docker pulls](https://img.shields.io/docker/pulls/mayanedms/mayanedms.svg?maxAge=3600)](https://hub.docker.com/r/mayanedms/mayanedms/)
 [![Docker Stars](https://img.shields.io/docker/stars/mayanedms/mayanedms.svg?maxAge=3600)](https://hub.docker.com/r/mayanedms/mayanedms/)
-
+[![Docker layers](https://images.microbadger.com/badges/image/mayanedms/mayanedms.svg)](https://microbadger.com/images/mayanedms/mayanedms)
 
 [pypi]: http://img.shields.io/pypi/v/mayan-edms.svg
 [pypi-url]: http://badge.fury.io/py/mayan-edms
@@ -42,18 +43,6 @@
 
 </div>
 
-<h2 align="center">Book</h2>
-
-The final version of the book "Exploring Mayan EDMS" available now!
-
-<p align="center">
-    <a href="https://sellfy.com/p/um2fkx/">
-        <img width="400" src="https://d12swbtw719y4s.cloudfront.net/images/v6RpxW40/aP0qKLjkPiAuXZhYuB45/wDAULAzFyx.jpeg?w=548">
-    </a>
-</p>
-
-The link is https://sellfy.com/p/um2fkx/
-
 <h2 align="center">Installation</h2>
 
 The easiest way to use Mayan EDMS is by using the official

README.rst

@@ -1,4 +1,4 @@
-|donation| |pypi| |coverage| |python| |license| |docker_pulls| |docker_stars| |docker_layers|
+|pypi| |builds| |coverage| |python| |license| |docker_pulls| |docker_stars| |docker_layers|
 
 
 .. image:: https://gitlab.com/mayan-edms/mayan-edms/raw/master/docs/_static/mayan_logo.png
@@ -25,15 +25,6 @@ at: https://docs.mayan-edms.com/topics/installation.html
 
 .. _Docker: https://www.docker.com/
 
-The final version of the book "Exploring Mayan EDMS" available now!
-
-.. image:: https://camo.githubusercontent.com/89d2fe787507c9247aa7bb406e2682b53eb7a5f9/68747470733a2f2f64313273776274773731397934732e636c6f756466726f6e742e6e65742f696d616765732f76365270785734302f615030714b4c6a6b50694175585a6859754234352f774441554c417a4679782e6a7065673f773d353438
-    :align: center
-    :width: 300
-    :target: https://sellfy.com/p/um2fkx/
-
-Click the image or visit: https://sellfy.com/p/um2fkx/
-
 Hardware requirements
 
 - 2 Gigabytes of RAM (1 Gigabyte if OCR is turned off).
@@ -53,8 +44,6 @@ Important links
 - `Videos <https://www.youtube.com/channel/UCJOOXHP1MJ9lVA7d8ZTlHPw>`__
 
 
-.. |donation| image:: https://img.shields.io/badge/donation-PayPal-brightgreen
-   :target: https://paypal.me/MayanEDMS
 .. |pypi| image:: http://img.shields.io/pypi/v/mayan-edms.svg
    :target: http://badge.fury.io/py/mayan-edms
 .. |builds| image:: https://gitlab.com/mayan-edms/mayan-edms/badges/master/build.svg

__init__.py.tmpl

@@ -9,6 +9,6 @@ __author__ = 'Roberto Rosario'
 __author_email__ = 'roberto.rosario@mayan-edms.com'
 __description__ = 'Free Open Source Electronic Document Management System'
 __license__ = 'Apache 2.0'
-__copyright_short__ = '2011 Roberto Rosario'
+__copyright_short__ = '2011-2018 Roberto Rosario'
 __copyright__ = '{} {}'.format('Copyright', __copyright_short__)
 __website__ = 'https://www.mayan-edms.com'

config.env

@@ -1,16 +0,0 @@
-DOCKER_MYSQL_IMAGE_VERSION=mysql:8.0
-DOCKER_ORACLE_IMAGE_VERSION=wnameless/oracle-xe-11g
-DOCKER_POSTGRES_IMAGE_VERSION=postgres:9.6-alpine
-DOCKER_RABBITMQ_IMAGE_VERSION=rabbitmq:3-alpine
-DOCKER_REDIS_IMAGE_VERSION=redis:5.0-alpine
-PYTHON_AMQP_VERSION=2.5.2
-PYTHON_FLOWER_VERSION=0.9.3
-PYTHON_LIBRABBITMQ_VERSION=2.0.0
-PYTHON_MYSQL_VERSION=1.4.4
-PYTHON_ORACLE_VERSION=7.2.3
-PYTHON_PSYCOPG2_VERSION=2.8.3
-PYTHON_PSUTIL_VERSION=5.6.2
-PYTHON_REDIS_VERSION=3.2.1
-SOURCE_CODE_REPOSITORY=https://gitlab.com/mayan-edms/mayan-edms/
-SOURCE_CODE_GIT=https://gitlab.com/mayan-edms/mayan-edms.git
-SOURCE_CODE_ISSUES=https://gitlab.com/mayan-edms/mayan-edms/issues/

contrib/scripts/docker/run-tests.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+apt-get update
+apt-get install -y --no-install-recommends tesseract-ocr-deu
+
+$MAYAN_PIP_BIN install -r $DOCKER_ROOT/requirements-testing.txt
+
+$MAYAN_BIN test --mayan-apps --settings=mayan.settings.testing

contrib/scripts/fix-permisssion.sh

@@ -0,0 +1,25 @@
+!/bin/bash
+Directories[0]="."
+
+# Setup find correctly.
+export IFS=$'\n'
+
+# Loop through our array.
+for x in ${Directories[@]}
+	do
+		# Find all directories & subdirectories
+		for i in $(find $x -type d)
+			do
+				# Fix Permissions
+				chmod -c 775 $i
+				chown -c www-data:www-data $i
+			done
+
+		# Find all Files
+		for i in $(find $x -type f)
+			do
+				# Fix Permissions
+				chmod -c 664 $i
+				chown -c www-data:www-data $i
+			done
+	done

contrib/scripts/install/development.sh

@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+
+INSTALLATION_DIRECTORY=/home/vagrant/mayan-edms/
+DB_NAME=mayan_edms
+DB_PASSWORD=test123
+
+cat << EOF | sudo tee -a /etc/motd.tail
+**********************************sudo apt
+
+Mayan EDMS Vagrant Development Box
+
+**********************************
+EOF
+
+# Update sources
+echo -e "\n -> Running apt-get update & upgrade \n"
+sudo apt-get -qq update
+sudo apt-get -y upgrade
+
+echo -e "\n -> Installing core binaries \n"
+sudo apt-get -y install exiftool git-core python-virtualenv gcc python-dev libjpeg-dev libpng-dev libtiff-dev tesseract-ocr poppler-utils libreoffice
+
+echo -e "\n -> Cloning development branch of repository \n"
+git clone /mayan-edms-repository/ $INSTALLATION_DIRECTORY
+cd $INSTALLATION_DIRECTORY
+git checkout development
+git reset HEAD --hard
+
+echo -e "\n -> Setting up virtual env \n"
+virtualenv venv
+source venv/bin/activate
+
+echo -e "\n -> Installing python dependencies \n"
+pip install -r requirements.txt
+
+echo -e "\n -> Running Mayan EDMS initial setup \n"
+./manage.py initialsetup
+
+echo -e "\n -> Installing Redis server \n"
+sudo apt-get install -y redis-server
+pip install redis
+
+echo -e "\n -> Installing testing software \n"
+pip install coverage
+
+echo -e "\n -> Installing MySQL \n"
+sudo debconf-set-selections <<< 'mysql-server mysql-server/root_password password '$DB_PASSWORD
+sudo debconf-set-selections <<< 'mysql-server mysql-server/root_password_again password '$DB_PASSWORD
+sudo apt-get install -y mysql-server libmysqlclient-dev
+# Create a passwordless root and travis users
+mysql -u root -p$DB_PASSWORD -e "SET PASSWORD = PASSWORD('');"
+mysql -u root -e "CREATE USER 'travis'@'localhost' IDENTIFIED BY '';GRANT ALL PRIVILEGES ON * . * TO 'travis'@'localhost';FLUSH PRIVILEGES;"
+mysql -u travis -e "CREATE DATABASE $DB_NAME;"
+pip install mysql-python
+
+echo -e "\n -> Installing PostgreSQL \n"
+sudo apt-get install -y postgresql postgresql-server-dev-all
+sudo -u postgres psql -c 'create database mayan_edms;' -U postgres
+sudo cat > /etc/postgresql/9.3/main/pg_hba.conf << EOF
+local   all             postgres                                trust
+
+# TYPE  DATABASE        USER            ADDRESS                 METHOD
+
+# "local" is for Unix domain socket connections only
+local   all             all                                     peer
+# IPv4 local connections:
+host    all             all             127.0.0.1/32            md5
+# IPv6 local connections:
+host    all             all             ::1/128                 md5
+EOF
+
+pip install -q psycopg2

contrib/scripts/install/docker.sh

@@ -5,30 +5,24 @@ set -e
 #   $ curl -fsSL get.mayan-edms.com -o get-mayan-edms.sh
 #   $ sh get-mayan-edms.sh
 #
-# NOTE: Before executing, make sure to verify the contents of the script
+# NOTE: Make sure to verify the contents of the script
 #       you downloaded matches the contents of docker.sh
 #       located at https://gitlab.com/mayan-edms/mayan-edms/blob/master/contrib/scripts/install/docker.sh
+#       before executing.
 
 : ${VERBOSE:=true}
 : ${INSTALL_DOCKER:=false}
 : ${DELETE_VOLUMES:=false}
-: ${USE_DOCKER_NETWORK:=true}
-: ${DOCKER_NETWORK_NAME:=mayan}
 : ${DATABASE_USER:=mayan}
 : ${DATABASE_NAME:=mayan}
 : ${DATABASE_PASSWORD:=mayanuserpass}
-: ${DOCKER_POSTGRES_IMAGE:=postgres:9.6-alpine}
+: ${DOCKER_POSTGRES_IMAGE:=postgres:9.5}
 : ${DOCKER_POSTGRES_CONTAINER:=mayan-edms-postgres}
 : ${DOCKER_POSTGRES_VOLUME:=/docker-volumes/mayan-edms/postgres}
 : ${DOCKER_POSTGRES_PORT:=5432}
-: ${DOCKER_POSTGRES_DELAY:=10}
-: ${DOCKER_REDIS_IMAGE:=redis:5.0-alpine}
-: ${DOCKER_REDIS_CONTAINER:=mayan-edms-redis}
-: ${DOCKER_REDIS_PORT:=6379}
 : ${DOCKER_MAYAN_IMAGE:=mayanedms/mayanedms:latest}
 : ${DOCKER_MAYAN_CONTAINER:=mayan-edms}
 : ${DOCKER_MAYAN_VOLUME:=/docker-volumes/mayan-edms/media}
-: ${DOCKER_MAYAN_PORT:=80}
 
 cat << EOF
 
@@ -50,8 +44,6 @@ echo "Variable values to be used:"
 echo "---------------------------"
 echo "INSTALL_DOCKER: $INSTALL_DOCKER"
 echo "DELETE_VOLUMES: $DELETE_VOLUMES"
-echo "USE_DOCKER_NETWORK: $USE_DOCKER_NETWORK"
-echo "DOCKER_NETWORK_NAME: $DOCKER_NETWORK_NAME"
 echo "DATABASE_USER: $DATABASE_USER"
 echo "DATABASE_NAME: $DATABASE_NAME"
 echo "DATABASE_PASSWORD: $DATABASE_PASSWORD"
@@ -59,19 +51,10 @@ echo "DOCKER_POSTGRES_IMAGE: $DOCKER_POSTGRES_IMAGE"
 echo "DOCKER_POSTGRES_CONTAINER: $DOCKER_POSTGRES_CONTAINER"
 echo "DOCKER_POSTGRES_VOLUME: $DOCKER_POSTGRES_VOLUME"
 echo "DOCKER_POSTGRES_PORT: $DOCKER_POSTGRES_PORT"
-echo "DOCKER_POSTGRES_DELAY: $DOCKER_POSTGRES_DELAY"
-echo "DOCKER_REDIS_IMAGE: $DOCKER_REDIS_IMAGE"
-echo "DOCKER_REDIS_CONTAINER: $DOCKER_REDIS_CONTAINER"
-echo "DOCKER_REDIS_PORT: $DOCKER_REDIS_PORT"
 echo "DOCKER_MAYAN_IMAGE: $DOCKER_MAYAN_IMAGE"
 echo "DOCKER_MAYAN_CONTAINER: $DOCKER_MAYAN_CONTAINER"
 echo "DOCKER_MAYAN_VOLUME: $DOCKER_MAYAN_VOLUME"
-echo "DOCKER_MAYAN_PORT: $DOCKER_MAYAN_PORT"
-echo
-echo "Override any of them by setting them before the script. "
-echo "Example: INSTALL_DOCKER=true sh get-mayan-edms.sh"
-
-echo "\nStarting in 10 seconds. Press CTRL+C to cancel."
+echo "\nStarting in 10 seconds."
 sleep 10
 fi
 
@@ -89,62 +72,33 @@ if [ -z `which docker`  ]; then
 fi
 
 echo -n "* Removing existing Mayan EDMS and PostgreSQL containers (no data will be lost)..."
-docker rm -f $DOCKER_REDIS_CONTAINER >/dev/null 2>&1  || true
-docker rm -f $DOCKER_POSTGRES_CONTAINER >/dev/null 2>&1  || true
-docker rm -f $DOCKER_MAYAN_CONTAINER >/dev/null 2>&1  || true
+true || docker stop $DOCKER_MAYAN_CONTAINER >/dev/null 2>&1
+true || docker rm $DOCKER_MAYAN_CONTAINER >/dev/null 2>&1
+true || docker stop $DOCKER_POSTGRES_CONTAINER >/dev/null 2>&1
+true || docker rm $DOCKER_POSTGRES_CONTAINER >/dev/null 2>&1
 echo "Done"
 
 if [ "$DELETE_VOLUMES" = true ]; then
-echo -n "* Deleting Docker volumes in 5 seconds (warning: this will delete all document data). Press CTRL+C to cancel..."
+echo -n "* Deleting Docker volumes in 5 seconds (warning: this delete all document data)..."
 sleep 5
-rm DOCKER_MAYAN_VOLUME -Rf || true
-rm DOCKER_POSTGRES_VOLUME -Rf || true
+true || rm DOCKER_MAYAN_VOLUME -Rf
+true || rm DOCKER_POSTGRES_VOLUME -Rf
 echo "Done"
 fi
 
-echo -n "* Pulling (downloading) the Redis Docker image..."
-docker pull $DOCKER_REDIS_IMAGE > /dev/null
+echo -n "* Pulling (downloading) the Mayan EDMS Docker image..."
+docker pull $DOCKER_MAYAN_IMAGE >/dev/null
 echo "Done"
 
 echo -n "* Pulling (downloading) the PostgreSQL Docker image..."
 docker pull $DOCKER_POSTGRES_IMAGE > /dev/null
 echo "Done"
 
-echo -n "* Pulling (downloading) the Mayan EDMS Docker image..."
-docker pull $DOCKER_MAYAN_IMAGE >/dev/null
-echo "Done"
-
-if [ "$USE_DOCKER_NETWORK" = true ]; then
-    echo -n "* Creating Docker network..."
-    docker network create $DOCKER_NETWORK_NAME 2> /dev/null || true
-    # Ignore error if the network already exists
-    echo "Done"
-fi
-
-if [ "$USE_DOCKER_NETWORK" = true ]; then
-    NETWORK_ARGUMENT="--network=$DOCKER_NETWORK_NAME"
-    POSTGRES_PORT_ARGUMENT=""
-    REDIS_PORT_ARGUMENT=""
-    MAYAN_DATABASE_PORT_ARGUMENT=""
-    MAYAN_DATABASE_HOST_ARGUMENT="-e MAYAN_DATABASE_HOST=$DOCKER_POSTGRES_CONTAINER"
-    MAYAN_CELERY_BROKER_URL_ARGUMENT="-e MAYAN_CELERY_BROKER_URL=redis://$DOCKER_REDIS_CONTAINER:6379/0"
-    MAYAN_CELERY_RESULT_BACKEND_ARGUMENT="-e MAYAN_CELERY_RESULT_BACKEND=redis://$DOCKER_REDIS_CONTAINER:6379/1"
-else
-    NETWORK_ARGUMENT=""
-    POSTGRES_PORT_ARGUMENT="-e $DOCKER_POSTGRES_PORT:5432"
-    REDIS_PORT_ARGUMENT="-e $DOCKER_REDIS_PORT:6379"
-    MAYAN_DATABASE_PORT_ARGUMENT="-e MAYAN_DATABASE_PORT=$DOCKER_POSTGRES_PORT"
-    MAYAN_DATABASE_HOST_ARGUMENT="-e MAYAN_DATABASE_HOST=172.17.0.1"
-    MAYAN_CELERY_BROKER_URL_ARGUMENT="-e MAYAN_CELERY_BROKER_URL=redis://172.17.0.1:6379/0"
-    MAYAN_CELERY_RESULT_BACKEND_ARGUMENT="-e MAYAN_CELERY_RESULT_BACKEND=redis://172.17.0.1:6379/1"
-fi
-
 echo -n "* Deploying the PostgreSQL container..."
 docker run -d \
 --name $DOCKER_POSTGRES_CONTAINER \
-$NETWORK_ARGUMENT \
 --restart=always \
-$POSTGRES_PORT_ARGUMENT \
+-p $DOCKER_POSTGRES_PORT:5432 \
 -e POSTGRES_USER=$DATABASE_USER \
 -e POSTGRES_DB=$DATABASE_NAME \
 -e POSTGRES_PASSWORD=$DATABASE_PASSWORD \
@@ -152,47 +106,26 @@ $POSTGRES_PORT_ARGUMENT \
 $DOCKER_POSTGRES_IMAGE >/dev/null
 echo "Done"
 
-echo -n "* Deploying the Redis container..."
-docker run -d \
---name $DOCKER_REDIS_CONTAINER \
-$NETWORK_ARGUMENT \
---restart=always \
-$REDIS_PORT_ARGUMENT \
-$DOCKER_REDIS_IMAGE \
-redis-server \
---appendonly no \
---databases 2 \
---maxmemory 100mb \
---maxmemory-policy allkeys-lru \
---maxclients 500 \
---save "" \
---tcp-backlog 256 \
->/dev/null
-echo "Done"
-
-echo -n "* Waiting for the PostgreSQL container to be ready (${DOCKER_POSTGRES_DELAY} seconds)..."
-sleep $DOCKER_POSTGRES_DELAY
+echo -n "* Waiting for the PostgreSQL container to be ready (10 seconds)..."
+sleep 10
 echo "Done"
 
 echo -n "* Deploying Mayan EDMS container..."
 docker run -d \
 --name $DOCKER_MAYAN_CONTAINER \
-$NETWORK_ARGUMENT \
 --restart=always \
--p $DOCKER_MAYAN_PORT:8000 \
+-p 80:8000 \
 -e MAYAN_DATABASE_ENGINE=django.db.backends.postgresql \
-$MAYAN_DATABASE_HOST_ARGUMENT \
-$MAYAN_DATABASE_PORT_ARGUMENT \
+-e MAYAN_DATABASE_HOST=172.17.0.1 \
 -e MAYAN_DATABASE_NAME=$DATABASE_NAME \
 -e MAYAN_DATABASE_PASSWORD=$DATABASE_PASSWORD \
 -e MAYAN_DATABASE_USER=$DATABASE_USER \
--e MAYAN_DATABASE_CONN_MAX_AGE=0 \
-$MAYAN_CELERY_BROKER_URL_ARGUMENT \
-$MAYAN_CELERY_RESULT_BACKEND_ARGUMENT \
+-e MAYAN_DATABASE_PORT=$DOCKER_POSTGRES_PORT \
+-e MAYAN_DATABASE_CONN_MAX_AGE=60 \
 -v $DOCKER_MAYAN_VOLUME:/var/lib/mayan \
 $DOCKER_MAYAN_IMAGE >/dev/null
 echo "Done"
 
 echo -n "* Waiting for the Mayan EDMS container to be ready (might take a few minutes)..."
-while ! curl --output /dev/null --silent --head --fail http://localhost:$DOCKER_MAYAN_PORT; do sleep 1 && echo -n .; done;
+while ! curl --output /dev/null --silent --head --fail http://localhost:80; do sleep 1 && echo -n .; done;
 echo "Done"

contrib/scripts/install/production.sh

@@ -0,0 +1,171 @@
+#!/usr/bin/env bash
+
+# ====== CONFIG ======
+INSTALLATION_DIRECTORY=/usr/share/mayan-edms/
+DB_NAME=mayan_edms
+DB_USERNAME=mayan
+DB_PASSWORD=test123
+# ==== END CONFIG ====
+
+cat << EOF | tee -a /etc/motd.tail
+**********************************
+
+Mayan EDMS Vagrant Production Box
+
+**********************************
+EOF
+
+echo -e "\n -> Running apt-get update & upgrade \n"
+apt-get -qq update
+apt-get -y upgrade
+
+echo -e "\n -> Installing core binaries \n"
+apt-get install exiftool nginx supervisor redis-server postgresql libpq-dev libjpeg-dev libmagic1 libpng-dev libreoffice libtiff-dev gcc ghostscript gpgv python-dev python-virtualenv tesseract-ocr poppler-utils -y
+
+echo -e "\n -> Setting up virtualenv \n"
+rm -f ${INSTALLATION_DIRECTORY}
+virtualenv ${INSTALLATION_DIRECTORY}
+source ${INSTALLATION_DIRECTORY}bin/activate
+
+echo -e "\n -> Installing Mayan EDMS from PyPI \n"
+pip install mayan-edms
+
+echo -e "\n -> Installing Python client for PostgreSQL, Redis, and uWSGI \n"
+pip install psycopg2 redis uwsgi
+
+echo -e "\n -> Creating the database for the installation \n"
+echo "CREATE USER mayan WITH PASSWORD '$DB_PASSWORD';" | sudo -u postgres psql
+sudo -u postgres createdb -O $DB_USERNAME $DB_NAME
+
+echo -e "\n -> Creating the directories for the logs \n"
+mkdir /var/log/mayan
+
+echo -e "\n -> Making a convenience symlink \n"
+cd ${INSTALLATION_DIRECTORY}
+ln -s lib/python2.7/site-packages/mayan .
+
+echo -e "\n -> Creating an initial settings file \n"
+mayan-edms.py createsettings
+
+echo -e "\n -> Updating the mayan/settings/local.py file \n"
+cat >> mayan/settings/local.py << EOF
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.postgresql_psycopg2',
+        'NAME': '$DB_NAME',
+        'USER': '$DB_USERNAME',
+        'PASSWORD': '$DB_PASSWORD',
+        'HOST': 'localhost',
+        'PORT': '5432',
+    }
+}
+
+BROKER_URL = 'redis://127.0.0.1:6379/0'
+CELERY_RESULT_BACKEND = 'redis://127.0.0.1:6379/0'
+EOF
+
+echo -e "\n -> Migrating the database or initialize the project \n"
+mayan-edms.py initialsetup
+
+echo -e "\n -> Disabling the default NGINX site \n"
+rm -f /etc/nginx/sites-enabled/default
+
+echo -e "\n -> Creating a uwsgi.ini file \n"
+cat > uwsgi.ini << EOF
+[uwsgi]
+chdir = ${INSTALLATION_DIRECTORY}lib/python2.7/site-packages/mayan
+chmod-socket = 664
+chown-socket = www-data:www-data
+env = DJANGO_SETTINGS_MODULE=mayan.settings.production
+gid = www-data
+logto = /var/log/uwsgi/%n.log
+pythonpath = ${INSTALLATION_DIRECTORY}lib/python2.7/site-packages
+master = True
+max-requests = 5000
+socket = ${INSTALLATION_DIRECTORY}uwsgi.sock
+uid = www-data
+vacuum = True
+wsgi-file = ${INSTALLATION_DIRECTORY}lib/python2.7/site-packages/mayan/wsgi.py
+EOF
+
+echo -e "\n -> Creating the directory for the uWSGI log files \n"
+mkdir -p /var/log/uwsgi
+
+echo -e "\n -> Creating the NGINX site file for Mayan EDMS, /etc/nginx/sites-available/mayan \n"
+cat > /etc/nginx/sites-available/mayan << EOF
+server {
+    listen 80;
+    server_name localhost;
+
+    location / {
+        include uwsgi_params;
+        uwsgi_pass unix:${INSTALLATION_DIRECTORY}uwsgi.sock;
+
+        client_max_body_size 30M;  # Increse if your plan to upload bigger documents
+        proxy_read_timeout 30s;  # Increase if your document uploads take more than 30 seconds
+    }
+
+    location /static {
+        alias ${INSTALLATION_DIRECTORY}mayan/media/static;
+        expires 1h;
+    }
+
+    location /favicon.ico {
+        alias ${INSTALLATION_DIRECTORY}mayan/media/static/appearance/images/favicon.ico;
+        expires 1h;
+    }
+}
+EOF
+
+echo -e "\n -> Enabling the NGINX site for Mayan EDMS \n"
+ln -s /etc/nginx/sites-available/mayan /etc/nginx/sites-enabled/
+
+echo -e "\n -> Creating the supervisor file for the uWSGI process, /etc/supervisor/conf.d/mayan-uwsgi.conf \n"
+cat > /etc/supervisor/conf.d/mayan-uwsgi.conf << EOF
+[program:mayan-uwsgi]
+command = ${INSTALLATION_DIRECTORY}bin/uwsgi --ini ${INSTALLATION_DIRECTORY}uwsgi.ini
+user = root
+autostart = true
+autorestart = true
+redirect_stderr = true
+EOF
+
+echo -e "\n -> Creating the supervisor file for the Celery worker, /etc/supervisor/conf.d/mayan-celery.conf \n"
+cat > /etc/supervisor/conf.d/mayan-celery.conf << EOF
+[program:mayan-worker]
+command = ${INSTALLATION_DIRECTORY}bin/python ${INSTALLATION_DIRECTORY}bin/mayan-edms.py celery --settings=mayan.settings.production worker -l ERROR
+directory = ${INSTALLATION_DIRECTORY}
+user = www-data
+stdout_logfile = /var/log/mayan/worker-stdout.log
+stderr_logfile = /var/log/mayan/worker-stderr.log
+autostart = true
+autorestart = true
+startsecs = 10
+stopwaitsecs = 10
+killasgroup = true
+priority = 998
+
+[program:mayan-beat]
+command = ${INSTALLATION_DIRECTORY}bin/python ${INSTALLATION_DIRECTORY}bin/mayan-edms.py celery --settings=mayan.settings.production beat -l ERROR
+directory = ${INSTALLATION_DIRECTORY}
+user = www-data
+numprocs = 1
+stdout_logfile = /var/log/mayan/beat-stdout.log
+stderr_logfile = /var/log/mayan/beat-stderr.log
+autostart = true
+autorestart = true
+startsecs = 10
+stopwaitsecs = 1
+killasgroup = true
+priority = 998
+EOF
+
+echo -e "\n -> Collecting the static files \n"
+mayan-edms.py collectstatic --noinput
+
+echo -e "\n -> Making the installation directory readable and writable by the webserver user \n"
+chown www-data:www-data ${INSTALLATION_DIRECTORY} -R
+
+echo -e "\n -> Restarting the services \n"
+/etc/init.d/nginx restart
+/etc/init.d/supervisor restart

contrib/scripts/process_messages.py

@@ -10,21 +10,19 @@ import sh
 
 APP_LIST = (
     'acls', 'appearance', 'authentication', 'autoadmin', 'cabinets',
-    'checkouts', 'common', 'converter', 'dashboards', 'dependencies',
-    'django_gpg', 'document_comments', 'document_indexing',
-    'document_parsing', 'document_signatures', 'document_states',
-    'documents', 'dynamic_search', 'events', 'file_caching',
-    'file_metadata', 'linking', 'lock_manager', 'mailer',
-    'mayan_statistics', 'metadata', 'mirroring', 'motd', 'navigation',
-    'ocr', 'permissions', 'platform', 'rest_api', 'smart_settings',
-    'sources', 'storage', 'tags', 'task_manager', 'templating',
-    'user_management', 'web_links'
+    'checkouts', 'common', 'converter', 'dashboards', 'django_gpg',
+    'document_comments', 'document_indexing', 'document_parsing',
+    'document_signatures', 'document_states', 'documents', 'dynamic_search',
+    'events', 'file_caching', 'linking', 'lock_manager', 'mayan_statistics',
+    'mailer', 'metadata', 'mirroring', 'motd', 'navigation', 'ocr',
+    'permissions', 'rest_api', 'smart_settings', 'sources', 'storage',
+    'tags', 'task_manager', 'user_management'
 )
 
 LANGUAGE_LIST = (
-    'ar', 'bg', 'bs_BA', 'cs', 'da_DK', 'de_DE', 'en', 'es', 'el', 'fa', 'fr',
-    'hu', 'id', 'it', 'lv', 'nl_NL', 'pl', 'pt', 'pt_BR', 'ro_RO', 'ru', 'sl_SI',
-    'tr_TR', 'vi_VN', 'zh',
+    'ar', 'bg', 'bs_BA', 'da', 'de_DE', 'en', 'es', 'fa', 'fr', 'hu', 'id',
+    'it', 'nl_NL', 'pl', 'pt', 'pt_BR', 'ro_RO', 'ru', 'sl_SI', 'tr_TR',
+    'vi_VN', 'zh_CN',
 )
 
 makemessages = sh.Command('django-admin.py')

contrib/scripts/start_gunicorn.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+NAME="mayan-edms"
+DJANGODIR=/usr/share/mayan-edms
+SOCKFILE=/var/tmp/filesystem.sock
+USER=www-data
+GROUP=www-data
+NUM_WORKERS=3
+DJANGO_SETTINGS_MODULE=mayan.settings.production
+DJANGO_WSGI_MODULE=mayan.wsgi
+TIMEOUT=600
+
+echo "Starting $NAME as `whoami`"
+
+# Activate the virtual environment
+cd $DJANGODIR
+source bin/activate
+export DJANGO_SETTINGS_MODULE=$DJANGO_SETTINGS_MODULE
+export PYTHONPATH=$DJANGODIR:$PYTHONPATH
+
+# Create the run directory if it doesn't exist
+RUNDIR=$(dirname $SOCKFILE)
+test -d $RUNDIR || mkdir -p $RUNDIR
+
+# Start your Django Unicorn
+# Programs meant to be run under supervisor should not daemonize themselves (do not use --daemon)
+exec bin/gunicorn ${DJANGO_WSGI_MODULE}:application \
+  --name $NAME \
+  --workers $NUM_WORKERS \
+  --user=$USER --group=$GROUP \
+  --log-level=debug \
+  --bind=unix:$SOCKFILE \
+  --timeout=$TIMEOUT
+
+

contrib/settings/ldap_connection_settings.py

@@ -1,139 +1,64 @@
 from __future__ import absolute_import
 
-# Install Python LDAP with:
-# $ pip install python-ldap
-# or if using Docker, pass the environment variable MAYAN_PIP_INSTALLS:
-# -e MAYAN_PIP_INSTALLS=python-ldap
 import ldap
+from django_auth_ldap.config import LDAPSearch
 
-from django_auth_ldap.config import (
-    LDAPSearch, LDAPSearchUnion, NestedActiveDirectoryGroupType
-)
+from .base import *  # NOQA
+from django.contrib.auth import get_user_model
 
-from mayan.settings.production import *
+SECRET_KEY = '<your secret key>'
 
-# Makes sure this works in Active Directory
-ldap.set_option(ldap.OPT_REFERRALS, False)
+# makes sure this works in Active Directory
+ldap.set_option(ldap.OPT_REFERRALS, 0)
 
-# Turn of debug output, turn this off when everything is working as expected
-ldap.set_option(ldap.OPT_DEBUG_LEVEL, 1)
-
-# Default: True
+# This is the default, but I like to be explicit.
 AUTH_LDAP_ALWAYS_UPDATE_USER = True
 
-# Use TLS to talk to the LDAP server
-# Requires acquiring the server's certificate
-# $ openssl s_client -connect <LDAP server>:636
-# Part of the output of this file will be the Base-64 encoded .cer file
-# that was presented for LDAPS. Cut and paste into a file beginning at
-# "-Begin Certificate" through "-End Certificate--" and save as a .crt, for
-# example: ldapserver.crt
-# $ CERT=ldapserver.crt
-# $ cp /root/$CERT /usr/share/ca-certificates/$CERT
-# # notice the + sign which tells to activate the certificate.
-# $ echo "+$CERT" >/etc/ca-certificates/update.d/activate_my_cert
-# $ dpkg-reconfigure ca-certificates;
-AUTH_LDAP_START_TLS = False
-
-LDAP_ADDITIONAL_USER_DN = 'dc=people'
-LDAP_ADMIN_DN = ''
-LDAP_BASE_DN = 'dc=<top level dc>,dc=co,dc=in'
-LDAP_PASSWORD = ''
-LDAP_USER_AUTO_CREATION = 'False'
-LDAP_URL = 'ldap://<LDAP server>:389/'
+LDAP_USER_AUTO_CREATION = "False"
+LDAP_URL = "ldap://<your ldap server IP>:389/"
+LDAP_BASE_DN = "dc=paramatrix,dc=co,dc=in"
+LDAP_ADDITIONAL_USER_DN = "dc=people"
+LDAP_ADMIN_DN = ""
+LDAP_PASSWORD = ""
 
+AUTH_LDAP_SERVER_URI = LDAP_URL
 AUTH_LDAP_BIND_DN = LDAP_ADMIN_DN
 AUTH_LDAP_BIND_PASSWORD = LDAP_PASSWORD
-AUTH_LDAP_SERVER_URI = LDAP_URL
 
-# Simple search
+
 AUTH_LDAP_USER_SEARCH = LDAPSearch(
     '%s,%s' % (LDAP_ADDITIONAL_USER_DN, LDAP_BASE_DN),
     ldap.SCOPE_SUBTREE, '(uid=%(user)s)'
 )
-
-# If you need to search in more than one place for a user, you can use
-# LDAPSearchUnion. This takes multiple LDAPSearch objects and returns the
-# union of the results. The precedence of the underlying searches is
-# unspecified.
-# https://django-auth-ldap.readthedocs.io/en/latest/authentication.html
-# AUTH_LDAP_USER_SEARCH = LDAPSearchUnion(
-#     LDAPSearch(
-#         'ou=Users,ou=Admin,dc=<top level DC>,dc=local', ldap.SCOPE_SUBTREE,
-#         '(samaccountname=%(user)s)'
-#     ),
-#     LDAPSearch(
-#         'ou=Users,ou=<second OU>,dc=<top level DC>,dc=local',
-#         ldap.SCOPE_SUBTREE, '(samaccountname=%(user)s)'
-#     ),
-#     LDAPSearch(
-#         'ou=Users,ou=<third OU>,dc=<top level DC>,dc=local',
-#         ldap.SCOPE_SUBTREE, '(samaccountname=%(user)s)'
-#     ),
-# )
-
-# User attributes to map from LDAP to Mayan's user model.
 AUTH_LDAP_USER_ATTR_MAP = {
     'first_name': 'cn',
     'last_name': 'sn',
     'email': 'mail'
 }
-# Another example map
-# AUTH_LDAP_USER_ATTR_MAP = {
-#     'username': 'sAMAccountName',
-#     'first_name': 'givenName',
-#     'last_name': 'sn',
-#     'email': 'mail'
-# }
-# Only string fields can be mapped to attributes. Boolean fields can be
-# defined by group membership:
-# AUTH_LDAP_USER_FLAGS_BY_GROUP = {
-#     'is_active': 'cn=active,ou=groups,dc=example,dc=com',
-#     'is_staff': (
-#         LDAPGroupQuery('cn=staff,ou=groups,dc=example,dc=com')
-#         | LDAPGroupQuery('cn=admin,ou=groups,dc=example,dc=com')
-#     ),
-#     'is_superuser': 'cn=superuser,ou=groups,dc=example,dc=com',
-# }
-
-# Simple group search
-# AUTH_LDAP_GROUP_SEARCH = LDAPSearch(
-#     'ou=groups,dc=example,dc=com', ldap.SCOPE_SUBTREE, '(objectClass=groupOfNames)'
-# )
-# AUTH_LDAP_GROUP_TYPE = GroupOfNamesType()
-
-# Advanced group search
-# AUTH_LDAP_GROUP_SEARCH = LDAPSearchUnion(
-#     LDAPSearch(
-#         'ou=Domain Global,OU=Security,OU=Groups,OU=<OU>,dc=<top level DC>,dc=local',
-#         ldap.SCOPE_SUBTREE,
-#         '(&(objectClass=group)(groupType:1.2.840.113556.1.4.803:=2147483648))'
-#     ),
-#     LDAPSearch(
-#         'ou=Domain Global,OU=Security,OU=Groups,OU=<OU>,dc=<top level DC>,dc=local',
-#         ldap.SCOPE_SUBTREE,
-#         '(&(objectClass=group)(groupType:1.2.840.113556.1.4.803:=2147483648))'
-#     ),
-# )
-# AUTH_LDAP_CACHE_GROUPS = True
-# AUTH_LDAP_FIND_GROUP_PERMS = False
-# AUTH_LDAP_GROUP_TYPE = NestedActiveDirectoryGroupType()
-# AUTH_LDAP_MIRROR_GROUPS = True
-
-# To minimize traffic to the LDAP server, LDAPBackend can make use of
-# Django’s cache framework to keep a copy of a user’s LDAP group memberships.
-# To enable this feature, set AUTH_LDAP_CACHE_TIMEOUT, which determines
-# the timeout of cache entries in seconds.
-# AUTH_LDAP_GROUP_CACHE_TIMEOUT = 3600
-
-# Limiting Access
-# The simplest use of groups is to limit the users who are allowed to log in.
-# If AUTH_LDAP_REQUIRE_GROUP is set, then only users who are members of that
-# group will successfully authenticate. AUTH_LDAP_DENY_GROUP is the reverse:
-# if given, members of this group will be rejected.
-# AUTH_LDAP_REQUIRE_GROUP = 'cn=enabled,ou=groups,dc=example,dc=com'
-# AUTH_LDAP_DENY_GROUP = 'cn=disabled,ou=groups,dc=example,dc=com'
-
 AUTHENTICATION_BACKENDS = (
     'django_auth_ldap.backend.LDAPBackend',
+    'mayan.settings.settings_local.EmailOrUsernameModelBackend',
 )
+
+
+class EmailOrUsernameModelBackend(object):
+    """
+    This is a ModelBacked that allows authentication with either a username or an email address.
+    """
+    def authenticate(self, username=None, password=None):
+        if '@' in username:
+            kwargs = {'email': username}
+        else:
+            kwargs = {'username': username}
+        try:
+            user = get_user_model().objects.get(**kwargs)
+            if user.check_password(password):
+                return user
+        except get_user_model().DoesNotExist:
+            return None
+
+    def get_user(self, username):
+        try:
+            return get_user_model().objects.get(pk=username)
+        except get_user_model().DoesNotExist:
+            return None

docker/Dockerfile

@@ -1,165 +1,165 @@
 # vim:set ft=dockerfile:
 
-####
-# BASE_IMAGE - Bare bones image with the base packages needed to run Mayan EDMS
-####
+####################
+# Base image start #
+####################
 
-FROM debian:10.0-slim as BASE_IMAGE
+FROM debian:9.4-slim as BASE_IMAGE
 
-LABEL maintainer="Roberto Rosario roberto.rosario@mayan-edms.com"
+MAINTAINER Roberto Rosario "roberto.rosario@mayan-edms.com"
 
-COPY config.env /config.env
+ENV DEBIAN_FRONTEND noninteractive
+ENV PYTHONUNBUFFERED 1
+ENV LC_ALL C.UTF-8
+ENV PROJECT_INSTALL_DIR=/opt/mayan-edms
+ENV PYTHON_PIP=${PROJECT_INSTALL_DIR}/bin/pip
 
-ENV PYTHONUNBUFFERED=1 \
-    LC_ALL=C.UTF-8 \
-    PROJECT_INSTALL_DIR=/opt/mayan-edms
-
-# Debian package caching
 ARG APT_PROXY
-RUN set -x \
-&& if [ "${APT_PROXY}" ]; \
-    then echo "Acquire::http { Proxy \"http://${APT_PROXY}\"; };" > /etc/apt/apt.conf.d/01proxy \
-; fi \
-# Install base OS packages to run Mayan EDMS
-&& DEBIAN_FRONTEND=noninteractive \
-apt-get update \
-&& apt-get install -y --no-install-recommends \
-    ca-certificates \
-    exiftool \
-    fonts-arphic-uming \
-    fonts-arphic-ukai \
-    ghostscript \
-    gpgv \
-    gnupg1 \
-    graphviz \
-    libfuse2 \
-    libmagic1 \
-    libmariadb3 \
-    libreoffice \
-    libpq5 \
-    poppler-utils \
-    python3-distutils \
-    sane-utils \
-    sudo \
-    supervisor \
-    tesseract-ocr tesseract-ocr-deu \
-# Remove make and build dependencies
-&& apt-get remove make libproxy-tools libreoffice-avmedia-backend-vlc libvlc-bin libvlc5 libvlccore9 adwaita-icon-theme gsettings-desktop-schemas libgstreamer-plugins-base1.0-0 -y \
-&& apt-get autoremove -y --purge \
-# Add mayan user
-&& adduser mayan --disabled-password --disabled-login --no-create-home --gecos "" \
+# Package caching
+RUN if [ "${APT_PROXY}" ]; then echo "Acquire::http { Proxy \"http://${APT_PROXY}\"; };" > /etc/apt/apt.conf.d/01proxy; fi
+
+# Install base Ubuntu libraries
+RUN apt-get update && \
+apt-get install -y --no-install-recommends \
+        exiftool \
+        g++ \
+        gcc \
+        ghostscript \
+        gpgv \
+        gnupg1 \
+        graphviz \
+        libffi-dev \
+        libfuse2 \
+        libjpeg-dev \
+        libmagic1 \
+        default-libmysqlclient-dev \
+        libpng-dev \
+        libpq-dev \
+        libreoffice \
+        libtiff-dev \
+        poppler-utils \
+        python-dev \
+        python-setuptools \
+        python-virtualenv \
+        python-wheel \
+        redis-server \
+        sane-utils \
+        sudo \
+        supervisor \
+        tesseract-ocr \
+        zlib1g-dev \
+&& \
+apt-get clean autoclean && \
+apt-get autoremove --purge -y && \
+rm -rf /var/lib/apt/lists/* && \
+rm -f /var/cache/apt/archives/*.deb
+
+# Install apt-get-install
+ADD https://raw.githubusercontent.com/guilhem/apt-get-install/master/apt-get-install /usr/bin/
+RUN chmod +x /usr/bin/apt-get-install
+
+RUN adduser mayan --disabled-password --disabled-login --no-create-home --gecos ""
+
 # Pillow can't find zlib or libjpeg on aarch64 (ODROID C2)
-&& if [ "$(uname -m)" = "aarch64" ]; then \
-    ln -s /usr/lib/aarch64-linux-gnu/libz.so /usr/lib/ \
-    && ln -s /usr/lib/aarch64-linux-gnu/libjpeg.so /usr/lib/ \
-; fi \
-# Pillow can't find zlib or libjpeg on armv7l (ODROID HC1)
-&& if [ "$(uname -m)" = "armv7l" ]; then \
-    ln -s /usr/lib/arm-linux-gnueabihf/libz.so /usr/lib/ \
-    && ln -s /usr/lib/arm-linux-gnueabihf/libjpeg.so /usr/lib/ \
+RUN if [ "$(uname -m)" = "aarch64" ]; then \
+ln -s /usr/lib/aarch64-linux-gnu/libz.so /usr/lib/ && \
+ln -s /usr/lib/aarch64-linux-gnu/libjpeg.so /usr/lib/ \
 ; fi
 
-####
-# BUILDER_IMAGE - This image builds the Python package and is discarded afterwards
-#                 only the build artifact is carried over to the next image.
-####
+# Pillow can't find zlib or libjpeg on armv7l (ODROID HC1)
+RUN if [ "$(uname -m)" = "armv7l" ]; then \
+apt-get install libssl-dev -y && \
+ln -s /usr/lib/arm-linux-gnueabihf/libz.so /usr/lib/ && \
+ln -s /usr/lib/arm-linux-gnueabihf/libjpeg.so /usr/lib/ \
+; fi
 
-# Reuse image
-FROM BASE_IMAGE as BUILDER_IMAGE
+# Discard data when Redis runs out of memory
+RUN echo "maxmemory-policy allkeys-lru" >> /etc/redis/redis.conf
 
-# Python libraries caching
-ARG PIP_INDEX_URL
-ARG PIP_TRUSTED_HOST
+# Disable saving the Redis database
+RUN echo "save \"\"" >> /etc/redis/redis.conf
 
-WORKDIR /src
+# Only provision 1 database
+RUN echo "databases 1" >> /etc/redis/redis.conf
 
-# Copy the source files needed to build the Python package
-COPY --chown=mayan:mayan requirements /src/requirements
-COPY --chown=mayan:mayan \
-    HISTORY.rst \
-    LICENSE \
-    MANIFEST.in \
-    README.md \
-    README.rst \
-    setup.py \
-    /src/
+#####################
+# Build image start #
+#####################
 
-COPY --chown=mayan:mayan mayan /src/mayan
+FROM debian:9.4-slim as BUILDER_IMAGE
 
-# Install development packages needed to build the Python packages
-RUN DEBIAN_FRONTEND=noninteractive \
-apt-get install -y --no-install-recommends \
-    default-libmysqlclient-dev \
-    libffi-dev \
-    libjpeg-dev \
-    libpng-dev \
-    libpq-dev \
-    libtiff-dev \
-    zlib1g-dev \
-    libssl-dev \
-    g++ \
-    gcc \
-    python3-dev \
-    python3-venv \
-&& mkdir -p "${PROJECT_INSTALL_DIR}" \
-&& chown -R mayan:mayan "${PROJECT_INSTALL_DIR}" \
-&& chown -R mayan:mayan /src
+ARG APT_PROXY
+# Package caching
+RUN if [ "${APT_PROXY}" ]; then echo "Acquire::http { Proxy \"http://${APT_PROXY}\"; };" > /etc/apt/apt.conf.d/01proxy; fi
 
-USER mayan
+WORKDIR /code
 
-RUN set -a \
-&& . /config.env \
-&& set +a \
-&& python3 -m venv "${PROJECT_INSTALL_DIR}" \
-&& . "${PROJECT_INSTALL_DIR}/bin/activate" \
-&& pip install --no-cache-dir \
-    amqp==$PYTHON_AMQP_VERSION \
-    mysqlclient==$PYTHON_MYSQL_VERSION \
-    psycopg2==$PYTHON_PSYCOPG2_VERSION \
-    redis==$PYTHON_REDIS_VERSION \
-    flower==$PYTHON_FLOWER_VERSION \
-# psutil is needed by ARM builds otherwise gevent and gunicorn fail to start
-&& UNAME=`uname -m` && if [ "${UNAME#*arm}" != $UNAME ]; then \
-    pip install --no-cache-dir \
-    psutil==$PYTHON_PSUTIL_VERSION \
-; fi \
-# Install the Python packages needed to build Mayan EDMS
-&& pip install --no-cache-dir -r /src/requirements/build.txt \
-# Build Mayan EDMS
-&& python3 setup.py sdist \
-# Install the built Mayan EDMS package
-&& pip install --no-cache-dir dist/mayan* \
-# Install the static content
-&& mayan-edms.py installdependencies \
-&& MAYAN_STATIC_ROOT=${PROJECT_INSTALL_DIR}/static mayan-edms.py preparestatic --link --noinput
+COPY . /code
 
-COPY --chown=mayan:mayan requirements/testing-base.txt "${PROJECT_INSTALL_DIR}"
+RUN apt-get update && apt-get install make python-dev python-pip -y
 
-####
-# Final image - BASE_IMAGE + BUILDER_IMAGE artifact (Mayan install directory)
-####
+RUN pip install -r requirements/build.txt
+
+ENV LC_ALL C.UTF-8
+
+RUN touch docker/Makefile
+
+RUN make wheel
+
+RUN chmod 777 dist -R
+
+#####################
+# Final image start #
+#####################
 
 FROM BASE_IMAGE
 
-COPY --from=BUILDER_IMAGE --chown=mayan:mayan "${PROJECT_INSTALL_DIR}/" "${PROJECT_INSTALL_DIR}/"
+RUN mkdir -p /opt
 
-USER root
+RUN python /usr/lib/python2.7/dist-packages/virtualenv.py $PROJECT_INSTALL_DIR
 
-COPY docker/rootfs /
+WORKDIR $PROJECT_INSTALL_DIR
 
+COPY --from=BUILDER_IMAGE /code/dist/*.whl .
+
+COPY --from=BUILDER_IMAGE /code/contrib/scripts/docker/run-tests.sh .
+
+COPY --from=BUILDER_IMAGE /code/requirements/testing-base.txt requirements-testing.txt
+
+COPY --from=BUILDER_IMAGE /code/docker/version .
+
+# Fix ownership
+RUN chown -R mayan:mayan $PROJECT_INSTALL_DIR
+
+# Install build Mayan EDMS
+RUN sudo -u mayan $PYTHON_PIP install --no-cache-dir *.whl && \
+rm *.whl
+
+# Install Python clients for librabbitmq, MySQL, PostgreSQL, REDIS
+RUN sudo -u mayan $PYTHON_PIP install --no-cache-dir librabbitmq==1.6.1 mysql-python==1.2.5 psycopg2==2.7.3.2 redis==2.10.6
+
+# Setup supervisor
+COPY docker/etc/supervisor/mayan.conf /etc/supervisor/conf.d
+
+RUN mkdir /var/lib/mayan
 VOLUME ["/var/lib/mayan"]
 
+COPY docker/entrypoint.sh /usr/local/bin/
+RUN ln -s usr/local/bin/entrypoint.sh / # backwards compat
 ENTRYPOINT ["entrypoint.sh"]
 
 EXPOSE 8000
-CMD ["run_all"]
+CMD ["mayan"]
 
-RUN ${PROJECT_INSTALL_DIR}/bin/mayan-edms.py platformtemplate supervisord_docker > /etc/supervisor/conf.d/mayan.conf \
-&& apt-get clean autoclean \
-&& apt-get autoremove --purge -y \
-&& rm -rf /var/lib/apt/lists/* \
-&& rm -f /var/cache/apt/archives/*.deb \
-# Remove temporary files owned by root from the platformtemplate step
-&& rm /tmp/* \
-# Keep displaying log messages to stdout
-&& find /var/log -type f | while read f; do echo -ne '' > $f; done;
+RUN rm /root/.cache -R
+RUN rm -rf /tmp/*
+
+RUN apt-get -y autoremove --purge && apt-get -y autoclean && apt-get -y clean
+
+RUN rm -rf /usr/share/man/*
+RUN rm -rf /usr/share/doc/*
+
+RUN find /var/lib/apt -type f | xargs rm -f
+RUN find /var/cache -type f -exec rm -rf {} \;
+
+RUN find /var/log -type f | while read f; do echo -ne '' > $f; done;

docker/Dockerfile.ubuntu

@@ -0,0 +1,150 @@
+# vim:set ft=dockerfile:
+
+####################
+# Base image start #
+####################
+
+FROM ubuntu:16.04 as BASE_IMAGE
+
+MAINTAINER Roberto Rosario "roberto.rosario@mayan-edms.com"
+
+ENV DEBIAN_FRONTEND noninteractive
+ENV PYTHONUNBUFFERED 1
+ENV LANG en_US.UTF-8
+ENV PROJECT_INSTALL_DIR=/usr/local/lib/python2.7/dist-packages/mayan
+
+ARG APT_PROXY
+# Package caching
+RUN if [ "${APT_PROXY}" ]; then echo "Acquire::http { Proxy \"http://${APT_PROXY}\"; };" > /etc/apt/apt.conf.d/01proxy; fi
+
+# Install base Ubuntu libraries
+RUN apt-get update && \
+apt-get install -y --no-install-recommends \
+        curl \
+        g++ \
+        gcc \
+        gettext-base \
+        ghostscript \
+        gpgv \
+        graphviz \
+        libffi-dev \
+        libjpeg-dev \
+        libmagic1 \
+        libmysqlclient-dev \
+        libpng-dev \
+        libpq-dev \
+        libreoffice \
+        libtiff-dev \
+        locales \
+        netcat-openbsd \
+        poppler-utils \
+        python-dev \
+        python-pip \
+        python-setuptools \
+        python-wheel \
+        redis-server \
+        supervisor \
+        tesseract-ocr \
+        zlib1g-dev \
+&& \
+apt-get clean autoclean && \
+apt-get autoremove --purge -y && \
+rm -rf /var/lib/apt/lists/* && \
+rm -f /var/cache/apt/archives/*.deb
+
+# Switch to UTF locale
+RUN echo "LC_ALL=\"en_US.UTF-8\"" >> /etc/default/locale && \
+locale-gen en_US.UTF-8 && \
+update-locale LANG=en_US.UTF-8 && \
+export LC_ALL=en_US.UTF-8
+
+# Install apt-get-install
+ADD https://raw.githubusercontent.com/guilhem/apt-get-install/master/apt-get-install /usr/bin/
+RUN chmod +x /usr/bin/apt-get-install
+
+# Install Python clients for PostgreSQL, REDIS, librabbitmq
+RUN pip install psycopg2==2.7.3.2 redis==2.10.6 mysql-python==1.2.5 librabbitmq==1.6.1
+
+RUN adduser mayan --disabled-password --disabled-login --no-create-home --gecos ""
+
+# Pillow can't find zlib or libjpeg on aarch64
+RUN if [ "$(uname -m)" = "aarch64" ]; then \
+ln -s /usr/lib/aarch64-linux-gnu/libz.so /usr/lib/ && \
+ln -s /usr/lib/aarch64-linux-gnu/libjpeg.so /usr/lib/ \
+; fi
+
+#####################
+# Build image start #
+#####################
+
+FROM python:2-alpine3.7 as BUILDER_IMAGE
+
+WORKDIR /code
+
+COPY . /code
+
+RUN apk update && \
+    apk add make
+
+RUN pip install -r requirements/build.txt
+
+RUN make wheel
+
+RUN chmod 777 dist -R
+
+#####################
+# Final image start #
+#####################
+
+FROM BASE_IMAGE
+
+WORKDIR /root/
+
+COPY --from=BUILDER_IMAGE /code/dist/*.whl .
+
+# Install build Mayan EDMS
+RUN pip install *.whl && \
+rm *.whl
+
+# Setup supervisor
+#RUN mkdir /etc/supervisor.d/
+COPY docker/etc/supervisor/beat.conf /etc/supervisor/conf.d
+COPY docker/etc/supervisor/gunicorn.conf /etc/supervisor/conf.d
+COPY docker/etc/supervisor/redis.conf /etc/supervisor/conf.d
+COPY docker/etc/supervisor/workers.conf /etc/supervisor/conf.d
+
+# Create the directory for the logs
+RUN mkdir /var/log/mayan
+
+# Fix ownership
+RUN chown -R mayan:mayan $PROJECT_INSTALL_DIR
+
+# Allow flanker to autogenerate its PLY files
+RUN chown -R mayan:mayan /usr/local/lib/python2.7/dist-packages/flanker/
+
+RUN mkdir /var/lib/mayan
+VOLUME ["/var/lib/mayan"]
+
+COPY docker/entrypoint.sh /usr/local/bin/
+RUN ln -s usr/local/bin/entrypoint.sh / # backwards compat
+ENTRYPOINT ["entrypoint.sh"]
+
+# Healthcheck setup
+HEALTHCHECK --interval=15s --timeout=1s --retries=20 \
+  CMD curl -s -f http://localhost/authentication/login/ | grep 'form' > /dev/null || exit 1
+
+EXPOSE 8000
+CMD ["mayan"]
+
+RUN rm /root/.cache -R
+RUN rm -rf /tmp/*
+
+RUN apt-get -y autoremove --purge && apt-get -y autoclean && apt-get -y clean
+
+RUN rm -rf /usr/share/man/*
+RUN rm -rf /usr/share/doc/*
+
+RUN find /var/lib/apt -type f | xargs rm -f
+RUN find /var/cache -type f -exec rm -rf {} \;
+
+RUN find /var/log -type f | while read f; do echo -ne '' > $f; done;

docker/Makefile

@@ -1,130 +1,33 @@
-#!make
-include config.env
-
-HOST_IP = `/sbin/ip route|awk '/docker0/ { print $$9 }'`
-APT_PROXY ?= $(HOST_IP):3142
+APT_PROXY ?= `/sbin/ip route|awk '/docker0/ { print $$9 }'`:3142
+IMAGE_VERSION ?= `cat docker/version`
 CONSOLE_COLUMNS ?= `echo $$(tput cols)`
 CONSOLE_LINES ?= `echo $$(tput lines)`
-IMAGE_VERSION ?= `cat docker/rootfs/version`
-PIP_INDEX_URL ?= http://$(HOST_IP):3141/root/pypi/+simple/
-PIP_TRUSTED_HOST ?= $(HOST_IP)
 
-docker-build: ## Build a new image locally.
+docker-help:
+	@echo
+	@echo "**** Docker makefile ****"
+	@echo "docker-build - Build a new image locally."
+	@echo "docker-build-with-proxy - Build a new image locally using an APT proxy."
+	@echo "docker-test-container - Build and run a test container."
+	@echo "docker-test-cleanup - Delete the test container and the test volume."
+	@echo "docker-test-all - Build and executed the test suite in a test container."
+	@echo "docker-shell - Launch a bash instance inside a running container. Pass the container name via DOCKER_CONTAINER."
+
+docker-build:
 	docker build -t mayanedms/mayanedms:$(IMAGE_VERSION) -f docker/Dockerfile .
 
-docker-build-with-proxy: ## Build a new image locally using an APT proxy as APT_PROXY.
-docker-build-with-proxy: devpi-start
-	docker build -t mayanedms/mayanedms:$(IMAGE_VERSION) -f docker/Dockerfile --build-arg APT_PROXY=$(APT_PROXY) --build-arg PIP_INDEX_URL=$(PIP_INDEX_URL) --build-arg PIP_TRUSTED_HOST=$(PIP_TRUSTED_HOST) --build-arg HTTP_PROXY=$(HTTP_PROXY) --build-arg HTTPS_PROXY=$(HTTPS_PROXY) .
-	$(MAKE) devpi-stop
+docker-build-with-proxy:
+	docker build -t mayanedms/mayanedms:$(IMAGE_VERSION) -f docker/Dockerfile --build-arg APT_PROXY=$(APT_PROXY) .
 
-docker-shell: ## Launch a bash instance inside a running container. Pass the container name via DOCKER_CONTAINER.
+docker-shell:
 	docker exec -e TERM=$(TERM) -e "COLUMNS=$(CONSOLE_COLUMNS)" -e "LINES=$(CONSOLE_LINES)" -it $(DOCKER_CONTAINER) /bin/bash
 
-docker-runtest-container: ## Run a test container.
-docker-runtest-container: docker-test-cleanup
-	docker run \
-	-d \
-	--name test-mayan-edms \
-	-p 80:8000 \
-	-v test-mayan_data:/var/lib/mayan \
-	mayanedms/mayanedms:$(IMAGE_VERSION)
+docker-test-container: docker-build-with-proxy docker-test-cleanup
+	docker run -d --name test-mayan-edms -p 80:8000 -v test-mayan_data:/var/lib/mayan mayanedms/mayanedms:$(DOCKER_VERSION)
 
-docker-runtest-cleanup: ## Delete the test container and the test volume.
+docker-test-cleanup:
 	@docker rm -f test-mayan-edms || true
 	@docker volume rm test-mayan_data || true
 
-docker-runtest-all: ## Executed the test suite in a test container.
-	docker run --rm mayanedms/mayanedms:$(IMAGE_VERSION) run_tests
-
-docker-compose-build:
-	docker-compose -f docker/docker-compose.yml -p mayan-edms build
-
-docker-compose-build-with-proxy: devpi-start
-	docker-compose -f docker/docker-compose.yml -p mayan-edms build --build-arg APT_PROXY=$(APT_PROXY) --build-arg PIP_INDEX_URL=$(PIP_INDEX_URL) --build-arg PIP_TRUSTED_HOST=$(PIP_TRUSTED_HOST) --build-arg HTTP_PROXY=$(HTTP_PROXY) --build-arg HTTPS_PROXY=$(HTTPS_PROXY)
-	$(MAKE) devpi-stop
-
-docker-compose-up:
-	docker-compose -f docker/docker-compose.yml -p mayan-edms up
-
-docker-staging-network-create:
-	@docker network rm mayan-staging || true
-	docker network create mayan-staging
-
-docker-staging-container-postgresql-start:
-	docker run \
-	-d \
-	--name mayan-staging-postgres \
-	--network=mayan-staging \
-	-e POSTGRES_USER=mayan \
-	-e POSTGRES_DB=mayan \
-	-e POSTGRES_PASSWORD=mayanuserpass \
-	-v mayan-staging-postgres:/var/lib/postgresql/data \
-	$(DOCKER_POSTGRES_IMAGE_VERSION)
-
-docker-staging-container-redis-start:
-	docker run \
-	-d \
-	--name mayan-staging-redis \
-	--network=mayan-staging \
-	-v mayan-staging-redis:/data \
-	$(DOCKER_REDIS_IMAGE_VERSION) \
-	redis-server \
-	--databases \
-	"2" \
-	--maxmemory-policy \
-	allkeys-lru \
-	--save \
-	""
-
-docker-staging-container-rabbitmq-start:
-	docker run \
-	-d \
-	--name mayan-staging-rabbitmq \
-	--network=mayan-staging \
-	-v mayan-staging-rabbitmq:/var/lib/rabbitmq \
-	$(DOCKER_RABBITMQ_IMAGE_VERSION) \
-
-docker-staging-container-mayan-start:
-	sleep 5 && docker run \
-	-d \
-	--name mayan-staging-app \
-	--network=mayan-staging \
-	-p 80:8000 \
-	-e MAYAN_DATABASE_ENGINE=django.db.backends.postgresql \
-	-e MAYAN_DATABASE_HOST=mayan-staging-postgres \
-	-e MAYAN_DATABASE_NAME=mayan \
-	-e MAYAN_DATABASE_PASSWORD=mayanuserpass \
-	-e MAYAN_DATABASE_USER=mayan \
-	-e MAYAN_CELERY_BROKER_URL=$(MAYAN_CELERY_BROKER_URL) \
-	-e MAYAN_CELERY_RESULT_BACKEND="redis://mayan-staging-redis:6379/1" \
-	-v mayan-staging-app:/var/lib/mayan \
-	mayanedms/mayanedms:$(IMAGE_VERSION)
-
-docker-staging-start-with-rabbitmq: MAYAN_CELERY_BROKER_URL="amqp://guest:guest@mayan-staging-rabbitmq:5672/"
-docker-staging-start-with-rabbitmq: docker-staging-start
-
-docker-staging-start-with-redis: MAYAN_CELERY_BROKER_URL="redis://mayan-staging-redis:6379/0"
-docker-staging-start-with-redis: docker-staging-start
-
-docker-staging-start: docker-staging-cleanup docker-staging-network-create docker-staging-container-postgresql-start docker-staging-container-rabbitmq-start docker-staging-container-redis-start docker-staging-container-mayan-start
-	docker logs -f mayan-staging-app
-
-docker-staging-cleanup: ## Delete the test container and the test volume.
-	@docker rm -f mayan-staging-app || true
-	@docker rm -f mayan-staging-redis || true
-	@docker rm -f mayan-staging-rabbitmq || true
-	@docker rm -f mayan-staging-postgres || true
-	@docker volume rm mayan-staging-app || true
-	@docker volume rm mayan-staging-postgres || true
-	@docker volume rm mayan-staging-rabbitmq || true
-	@docker volume rm mayan-staging-redis || true
-	@docker network rm mayan-staging || true
-
-devpi-init:
-	devpi-server --init || true
-
-devpi-start: devpi-init
-	devpi-server --start --host=0.0.0.0 || true
-
-devpi-stop:
-	devpi-server --stop || true
+docker-test-all: docker-build-with-proxy
+	docker run --rm run-tests

docker/README-short.txt

@@ -0,0 +1 @@
+Mayan EDMS is a free open source electronic document management system.

docker/docker-compose-development.yml

@@ -0,0 +1,72 @@
+version: '2.1'
+
+volumes:
+    broker:
+        driver: local
+    app:
+        driver: local
+    db:
+        driver: local
+    results:
+        driver: local
+
+services:
+  broker:
+    container_name: mayan-edms-broker
+    image: healthcheck/rabbitmq
+    environment:
+      RABBITMQ_DEFAULT_USER: mayan
+      RABBITMQ_DEFAULT_PASS: mayan
+      RABBITMQ_DEFAULT_VHOST: mayan
+    volumes:
+      - broker:/var/lib/rabbitmq
+  results:
+    container_name: mayan-edms-results
+    image: healthcheck/redis
+    volumes:
+      - results:/data
+  #db:
+  #  container_name: mayan-edms-db
+  #  image: healthcheck/mysql
+  #  environment:
+  #    MYSQL_DATABASE: mayan
+  #    MYSQL_PASSWORD: mayan-password
+  #    MYSQL_ROOT_PASSWORD: root-password
+  #    MYSQL_USER: mayan
+  #  volumes:
+  #    - db:/var/lib/mysql
+  db:
+    container_name: mayan-edms-db
+    image: healthcheck/postgres
+    environment:
+      POSTGRES_DB: mayan
+      POSTGRES_PASSWORD: mayan-password
+      POSTGRES_USER: mayan
+    volumes:
+      - db:/var/lib/postgresql/data
+  mayan-edms:
+    container_name: mayan-edms-app
+    image: mayan-edms/next
+    build:
+      context: ./
+      args:
+        - APT_PROXY=172.18.0.1:3142
+    depends_on:
+      broker:
+        condition: service_healthy
+      db:
+        condition: service_healthy
+      results:
+        condition: service_healthy
+    environment:
+      MAYAN_CELERY_BROKER_URL: amqp://mayan:mayan@broker:5672/mayan
+      MAYAN_CELERY_RESULT_BACKEND: redis://results:6379/0
+      MAYAN_DATABASE_ENGINE: django.db.backends.postgresql
+      MAYAN_DATABASE_HOST: db
+      MAYAN_DATABASE_NAME: mayan
+      MAYAN_DATABASE_PASSWORD: mayan-password
+      MAYAN_DATABASE_USER: mayan
+    ports:
+      - "80:80"
+    volumes:
+      - app:/var/lib/mayan

docker/docker-compose.yml

@@ -1,130 +1,58 @@
-version: '3.7'
+version: '2.1'
 
-networks:
-  mayan-bridge:
-    driver: bridge
+volumes:
+    broker:
+        driver: local
+    app:
+        driver: local
+    db:
+        driver: local
+    results:
+        driver: local
 
 services:
-  app:
-    build:
-      context: ..
-      dockerfile: ./docker/Dockerfile
-    depends_on:
-      - postgresql
-      - redis
-      # Enable to use RabbitMQ
-      #- rabbitmq
-    environment: &mayan_env
-      # Enable to use RabbitMQ
-      # MAYAN_CELERY_BROKER_URL: amqp://mayan:mayanrabbitpass@broker:5672/mayan
-      # Disable Redis Broker to use RabbitMQ as Broker
-      MAYAN_CELERY_BROKER_URL: redis://redis:6379/0
-      MAYAN_CELERY_RESULT_BACKEND: redis://redis:6379/1
-      MAYAN_DATABASES: "{'default':{'ENGINE':'django.db.backends.postgresql','NAME':'mayan','PASSWORD':'mayandbpass','USER':'mayan','HOST':'postgresql'}}"
-    image: mayanedms/mayanedms:3.3
-    networks:
-      - mayan-bridge
-    ports:
-      - "80:8000"
-    restart: unless-stopped
+  broker:
+    container_name: mayan-edms-broker
+    image: healthcheck/rabbitmq
+    environment:
+      RABBITMQ_DEFAULT_USER: mayan
+      RABBITMQ_DEFAULT_PASS: mayan
+      RABBITMQ_DEFAULT_VHOST: mayan
     volumes:
-      - /docker-volumes/mayan-edms/media:/var/lib/mayan
-
-  postgresql:
+      - broker:/var/lib/rabbitmq
+  results:
+    container_name: mayan-edms-results
+    image: healthcheck/redis
+    volumes:
+      - results:/data
+  db:
+    container_name: mayan-edms-db
+    image: healthcheck/postgres
     environment:
       POSTGRES_DB: mayan
-      POSTGRES_PASSWORD: mayandbpass
+      POSTGRES_PASSWORD: mayan-password
       POSTGRES_USER: mayan
-    image: postgres:9.6-alpine
-    networks:
-      - mayan-bridge
-    restart: unless-stopped
     volumes:
-      - /docker-volumes/mayan-edms/postgres:/var/lib/postgresql/data
-
-  redis:
-    command:
-      - redis-server
-      - --databases
-      - "2"
-      - --maxmemory-policy
-      - allkeys-lru
-      - --save
-      - ""
-    image: redis:5.0-alpine
-    networks:
-      - mayan-bridge
-    restart: unless-stopped
-
-  # Optional services
-
-  # celery_flower:
-  #   command:
-  #     - run_celery
-  #     - flower
-  #   depends_on:
-  #     - postgresql
-  #     - redis
-  #     # Enable to use RabbitMQ
-  #     # - rabbitmq
-  #   environment:
-  #     <<: *mayan_env
-  #   image: mayanedms/mayanedms:3.3
-  #   networks:
-  #     - mayan-bridge
-  #   ports:
-  #     - "5555:5555"
-  #   restart: unless-stopped
-
-  # Enable to use RabbitMQ
-  # rabbitmq:
-  #   container_name: mayan-edms-rabbitmq
-  #   image: healthcheck/rabbitmq
-  #   environment:
-  #     RABBITMQ_DEFAULT_USER: mayan
-  #     RABBITMQ_DEFAULT_PASS: mayanrabbitpass
-  #     RABBITMQ_DEFAULT_VHOST: mayan
-  #   networks:
-  #     - mayan-bridge
-  #   restart: unless-stopped
-  #   volumes:
-  #     - /docker-volumes/mayan-edms/rabbitmq:/var/lib/rabbitmq
-
-  # Enable to run standalone workers
-  # worker_fast:
-  #   command:
-  #     - run_worker
-  #     - fast
-  #   depends_on:
-  #     - postgresql
-  #     - redis
-  #     # Enable to use RabbitMQ
-  #     # - rabbitmq
-  #   environment:
-  #     <<: *mayan_env
-  #   image: mayanedms/mayanedms:3.3
-  #   networks:
-  #     - mayan-bridge
-  #   restart: unless-stopped
-  #   volumes:
-  #     - /docker-volumes/mayan-edms/media:/var/lib/mayan
-
-  # Enable to run frontend gunicorn
-  # frontend:
-  #   command:
-  #     - run_frontend
-  #   depends_on:
-  #     - postgresql
-  #     - redis
-  #     # Enable to use RabbitMQ
-  #     # - rabbitmq
-  #   environment:
-  #     <<: *mayan_env
-  #   image: mayanedms/mayanedms:3.3
-  #   networks:
-  #     - mayan-bridge
-  #   ports:
-  #     - "81:8000"
-  #   restart: unless-stopped
-  #   volumes:
-  #     - /docker-volumes/mayan-edms/media:/var/lib/mayan
+      - db:/var/lib/postgresql/data
+  mayan-edms:
+    container_name: mayan-edms-app
+    image: mayanedms/mayanedms:latest
+    depends_on:
+      broker:
+        condition: service_healthy
+      db:
+        condition: service_healthy
+      results:
+        condition: service_healthy
+    environment:
+      MAYAN_CELERY_BROKER_URL: amqp://mayan:mayan@broker:5672/mayan
+      MAYAN_CELERY_RESULT_BACKEND: redis://results:6379/0
+      MAYAN_DATABASE_ENGINE: django.db.backends.postgresql
+      MAYAN_DATABASE_HOST: db
+      MAYAN_DATABASE_NAME: mayan
+      MAYAN_DATABASE_PASSWORD: mayan-password
+      MAYAN_DATABASE_USER: mayan
+    ports:
+      - "80:8000"
+    volumes:
+      - app:/var/lib/mayan

docker/entrypoint.sh

@@ -0,0 +1,115 @@
+#!/bin/sh
+
+set -e
+echo "mayan: starting entrypoint.sh"
+INSTALL_FLAG=/var/lib/mayan/system/SECRET_KEY
+CONCURRENCY_ARGUMENT=--concurrency=
+export DOCKER_ROOT=/opt/mayan-edms
+
+export MAYAN_DEFAULT_CELERY_BROKER_URL=redis://127.0.0.1:6379/0
+export MAYAN_DEFAULT_CELERY_RESULT_BACKEND=redis://127.0.0.1:6379/0
+
+export MAYAN_ALLOWED_HOSTS='["*"]'
+export MAYAN_BIN=/opt/mayan-edms/bin/mayan-edms.py
+export MAYAN_CELERY_BROKER_URL=${MAYAN_CELERY_BROKER_URL:-${MAYAN_DEFAULT_CELERY_BROKER_URL}}
+export MAYAN_CELERY_RESULT_BACKEND=${MAYAN_CELERY_RESULT_BACKEND:-${MAYAN_DEFAULT_CELERY_RESULT_BACKEND}}
+export MAYAN_INSTALL_DIR=/opt/mayan-edms
+export MAYAN_PYTHON_BIN_DIR=/opt/mayan-edms/bin/
+export MAYAN_MEDIA_ROOT=/var/lib/mayan
+export MAYAN_SETTINGS_MODULE=${MAYAN_SETTINGS_MODULE:-mayan.settings.production}
+
+export MAYAN_GUNICORN_BIN=${MAYAN_PYTHON_BIN_DIR}gunicorn
+export MAYAN_GUNICORN_WORKERS=${MAYAN_GUNICORN_WORKERS:-2}
+export MAYAN_PIP_BIN=${MAYAN_PYTHON_BIN_DIR}pip
+
+MAYAN_WORKER_FAST_CONCURRENCY=${MAYAN_WORKER_FAST_CONCURRENCY:-1}
+MAYAN_WORKER_MEDIUM_CONCURRENCY=${MAYAN_WORKER_MEDIUM_CONCURRENCY:-1}
+MAYAN_WORKER_SLOW_CONCURRENCY=${MAYAN_WORKER_SLOW_CONCURRENCY:-1}
+
+if [ "$MAYAN_WORKER_FAST_CONCURRENCY" -eq 0 ]; then
+    MAYAN_WORKER_FAST_CONCURRENCY=
+else
+    MAYAN_WORKER_FAST_CONCURRENCY="${CONCURRENCY_ARGUMENT}${MAYAN_WORKER_FAST_CONCURRENCY}"
+fi
+export MAYAN_WORKER_FAST_CONCURRENCY
+
+if [ "$MAYAN_WORKER_MEDIUM_CONCURRENCY" -eq 0 ]; then
+    MAYAN_WORKER_MEDIUM_CONCURRENCY=
+else
+    MAYAN_WORKER_MEDIUM_CONCURRENCY="${CONCURRENCY_ARGUMENT}${MAYAN_WORKER_MEDIUM_CONCURRENCY}"
+fi
+export MAYAN_WORKER_MEDIUM_CONCURRENCY
+
+if [ "$MAYAN_WORKER_SLOW_CONCURRENCY" -eq 0 ]; then
+    MAYAN_WORKER_SLOW_CONCURRENCY=
+else
+    MAYAN_WORKER_SLOW_CONCURRENCY="${CONCURRENCY_ARGUMENT}${MAYAN_WORKER_SLOW_CONCURRENCY}"
+fi
+export MAYAN_WORKER_SLOW_CONCURRENCY
+
+export CELERY_ALWAYS_EAGER=False
+export PYTHONPATH=$PYTHONPATH:$MAYAN_MEDIA_ROOT
+
+chown mayan:mayan /var/lib/mayan -R
+
+initialize() {
+    echo "mayan: initialize()"
+    su mayan -c "${MAYAN_BIN} initialsetup --force"
+    su mayan -c "${MAYAN_BIN} collectstatic --noinput --clear"
+}
+
+upgrade() {
+    echo "mayan: upgrade()"
+    su mayan -c "${MAYAN_BIN} performupgrade"
+    su mayan -c "${MAYAN_BIN} collectstatic --noinput --clear"
+}
+
+start() {
+    echo "mayan: start()"
+    rm -rf /var/run/supervisor.sock
+    exec /usr/bin/supervisord -nc /etc/supervisor/supervisord.conf
+}
+
+os_package_installs() {
+    echo "mayan: os_package_installs()"
+    if [ "${MAYAN_APT_INSTALLS}" ]; then
+        apt-get-install $MAYAN_APT_INSTALLS
+    fi
+}
+
+pip_installs() {
+    echo "mayan: pip_installs()"
+    if [ "${MAYAN_PIP_INSTALLS}" ]; then
+        $MAYAN_PIP_BIN install $MAYAN_PIP_INSTALLS
+    fi
+}
+
+os_package_installs || true
+pip_installs || true
+
+case "$1" in
+
+mayan) # Check if this is a new install, otherwise try to upgrade the existing
+       # installation on subsequent starts
+       if [ ! -f $INSTALL_FLAG ]; then
+           initialize
+       else
+           upgrade
+       fi
+       start
+       ;;
+
+run-tests) # Check if this is a new install, otherwise try to upgrade the existing
+           # installation on subsequent starts
+           if [ ! -f $INSTALL_FLAG ]; then
+               initialize
+           else
+               upgrade
+           fi
+           $DOCKER_ROOT/run-tests.sh
+           ;;
+
+*) su mayan -c "$@";
+   ;;
+
+esac

docker/etc/supervisor/mayan.conf

@@ -0,0 +1,80 @@
+[program:mayan-gunicorn]
+autorestart = false
+autostart = true
+command = /bin/bash -c "${MAYAN_GUNICORN_BIN} -w ${MAYAN_GUNICORN_WORKERS} mayan.wsgi --max-requests 500 --max-requests-jitter 50 --worker-class gevent --bind 0.0.0.0:8000  --env DJANGO_SETTINGS_MODULE=${MAYAN_SETTINGS_MODULE}" --timeout 120
+redirect_stderr = true
+stderr_logfile = /dev/fd/2
+stderr_logfile_maxbytes = 0
+stdout_logfile = /dev/fd/1
+stdout_logfile_maxbytes = 0
+user = mayan
+
+[program:redis]
+autorestart = false
+autostart = true
+command = /bin/bash -c "if [ ${MAYAN_CELERY_BROKER_URL} == ${MAYAN_DEFAULT_BROKER_URL} ] && [ ${MAYAN_CELERY_RESULT_BACKEND} == ${MAYAN_DEFAULT_CELERY_RESULT_BACKEND} ];then /usr/bin/redis-server /etc/redis/;fi"
+stderr_logfile = /dev/fd/2
+stderr_logfile_maxbytes = 0
+stdout_logfile = /dev/fd/1
+stdout_logfile_maxbytes = 0
+user = root
+
+[program:mayan-worker-fast]
+autorestart = false
+autostart = true
+command = nice -n 1 /bin/bash -c "${MAYAN_BIN} celery --settings=${MAYAN_SETTINGS_MODULE} worker -l ERROR -Q converter,document_states_fast,sources_fast -n mayan-worker-fast.%%h ${MAYAN_WORKER_FAST_CONCURRENCY}"
+killasgroup = true
+numprocs = 1
+priority = 998
+startsecs = 10
+stderr_logfile = /dev/fd/2
+stderr_logfile_maxbytes = 0
+stdout_logfile = /dev/fd/1
+stdout_logfile_maxbytes = 0
+stopwaitsecs = 1
+user = mayan
+
+[program:mayan-worker-medium]
+autorestart = false
+autostart = true
+command = nice -n 18 /bin/bash -c "${MAYAN_BIN} celery --settings=${MAYAN_SETTINGS_MODULE} worker -l ERROR -Q checkouts_periodic,documents_periodic,indexing,metadata,sources,sources_periodic,uploads,documents -n mayan-worker-medium.%%h ${MAYAN_WORKER_MEDIUM_CONCURRENCY}"
+killasgroup = true
+numprocs = 1
+priority = 998
+startsecs = 10
+stderr_logfile = /dev/fd/2
+stderr_logfile_maxbytes = 0
+stdout_logfile = /dev/fd/1
+stdout_logfile_maxbytes = 0
+stopwaitsecs = 1
+user = mayan
+
+[program:mayan-worker-slow]
+autorestart = false
+autostart = true
+command = nice -n 19 /bin/bash -c "${MAYAN_BIN} celery --settings=${MAYAN_SETTINGS_MODULE} worker -l ERROR -Q mailing,tools,statistics,parsing,ocr -n mayan-worker-slow.%%h ${MAYAN_WORKER_SLOW_CONCURRENCY}"
+killasgroup = true
+numprocs = 1
+priority = 998
+startsecs = 10
+stderr_logfile = /dev/fd/2
+stderr_logfile_maxbytes = 0
+stdout_logfile = /dev/fd/1
+stdout_logfile_maxbytes = 0
+stopwaitsecs = 1
+user = mayan
+
+[program:mayan-celery-beat]
+autorestart = false
+autostart = true
+command = nice -n 1 /bin/bash -c "${MAYAN_BIN} celery --settings=${MAYAN_SETTINGS_MODULE} beat --pidfile= -l ERROR"
+killasgroup = true
+numprocs = 1
+priority = 998
+stderr_logfile = /dev/fd/2
+stderr_logfile_maxbytes = 0
+stdout_logfile = /dev/fd/1
+stdout_logfile_maxbytes = 0
+startsecs = 10
+stopwaitsecs = 1
+user = mayan

docker/rootfs/usr/local/bin/entrypoint.sh

@@ -1,169 +0,0 @@
-#!/bin/bash
-
-# Use bash and not sh to support argument slicing "${@:2}"
-# sh defaults to dash instead of bash.
-
-set -e
-echo "mayan: starting entrypoint.sh"
-INSTALL_FLAG=/var/lib/mayan/system/SECRET_KEY
-CONCURRENCY_ARGUMENT=--concurrency=
-
-DEFAULT_USER_UID=1000
-DEFAULT_USER_GID=1000
-
-MAYAN_USER_UID=${MAYAN_USER_UID:-${DEFAULT_USER_UID}}
-MAYAN_USER_GID=${MAYAN_USER_GID:-${DEFAULT_USER_GID}}
-
-export MAYAN_ALLOWED_HOSTS='["*"]'
-export MAYAN_BIN=/opt/mayan-edms/bin/mayan-edms.py
-export MAYAN_INSTALL_DIR=/opt/mayan-edms
-export MAYAN_PYTHON_BIN_DIR=/opt/mayan-edms/bin/
-export MAYAN_MEDIA_ROOT=/var/lib/mayan
-export MAYAN_SETTINGS_MODULE=${MAYAN_SETTINGS_MODULE:-mayan.settings.production}
-export DJANGO_SETTINGS_MODULE=${MAYAN_SETTINGS_MODULE}
-
-export MAYAN_GUNICORN_BIN=${MAYAN_PYTHON_BIN_DIR}gunicorn
-export MAYAN_GUNICORN_WORKERS=${MAYAN_GUNICORN_WORKERS:-2}
-export MAYAN_GUNICORN_TIMEOUT=${MAYAN_GUNICORN_TIMEOUT:-120}
-export MAYAN_PIP_BIN=${MAYAN_PYTHON_BIN_DIR}pip
-export MAYAN_STATIC_ROOT=${MAYAN_INSTALL_DIR}/static
-
-MAYAN_WORKER_FAST_CONCURRENCY=${MAYAN_WORKER_FAST_CONCURRENCY:-0}
-MAYAN_WORKER_MEDIUM_CONCURRENCY=${MAYAN_WORKER_MEDIUM_CONCURRENCY:-0}
-MAYAN_WORKER_SLOW_CONCURRENCY=${MAYAN_WORKER_SLOW_CONCURRENCY:-1}
-
-update_uid_gid() {
-    echo "mayan: update_uid_gid()"
-    groupmod mayan -g ${MAYAN_USER_GID} 2>/dev/null || true
-    usermod mayan -u ${MAYAN_USER_UID} -g ${MAYAN_USER_GID} 2>/dev/null
-
-    if [ ${MAYAN_USER_UID} -ne ${DEFAULT_USER_UID} ] || [ ${MAYAN_USER_GID} -ne ${DEFAULT_USER_GID} ]; then
-        echo "mayan: Updating file ownership. This might take a while if there are many documents."
-        chown -R mayan:mayan ${MAYAN_INSTALL_DIR} ${MAYAN_STATIC_ROOT} ${MAYAN_MEDIA_ROOT}
-    fi
-}
-
-update_uid_gid
-
-if [ "$MAYAN_WORKER_FAST_CONCURRENCY" -eq 0 ]; then
-    MAYAN_WORKER_FAST_CONCURRENCY=
-else
-    MAYAN_WORKER_FAST_CONCURRENCY="${CONCURRENCY_ARGUMENT}${MAYAN_WORKER_FAST_CONCURRENCY}"
-fi
-export MAYAN_WORKER_FAST_CONCURRENCY
-
-if [ "$MAYAN_WORKER_MEDIUM_CONCURRENCY" -eq 0 ]; then
-    MAYAN_WORKER_MEDIUM_CONCURRENCY=
-else
-    MAYAN_WORKER_MEDIUM_CONCURRENCY="${CONCURRENCY_ARGUMENT}${MAYAN_WORKER_MEDIUM_CONCURRENCY}"
-fi
-export MAYAN_WORKER_MEDIUM_CONCURRENCY
-
-if [ "$MAYAN_WORKER_SLOW_CONCURRENCY" -eq 0 ]; then
-    MAYAN_WORKER_SLOW_CONCURRENCY=
-else
-    MAYAN_WORKER_SLOW_CONCURRENCY="${CONCURRENCY_ARGUMENT}${MAYAN_WORKER_SLOW_CONCURRENCY}"
-fi
-export MAYAN_WORKER_SLOW_CONCURRENCY
-
-# Allow importing of user setting modules
-export PYTHONPATH=$PYTHONPATH:$MAYAN_MEDIA_ROOT
-
-apt_get_install() {
-    apt-get -q update
-    apt-get install -y --force-yes --no-install-recommends --auto-remove "$@"
-    apt-get -q clean
-    rm -rf /var/lib/apt/lists/*
-}
-
-initialsetup() {
-    echo "mayan: initialsetup()"
-    su mayan -c "${MAYAN_BIN} initialsetup --force --no-dependencies"
-}
-
-os_package_installs() {
-    echo "mayan: os_package_installs()"
-    if [ "${MAYAN_APT_INSTALLS}" ]; then
-        DEBIAN_FRONTEND=noninteractive apt_get_install $MAYAN_APT_INSTALLS
-    fi
-}
-
-pip_installs() {
-    echo "mayan: pip_installs()"
-    if [ "${MAYAN_PIP_INSTALLS}" ]; then
-        su mayan -c "${MAYAN_PIP_BIN} install $MAYAN_PIP_INSTALLS"
-    fi
-}
-
-run_all() {
-    echo "mayan: start()"
-    rm -rf /var/run/supervisor.sock
-    exec /usr/bin/supervisord -nc /etc/supervisor/supervisord.conf
-}
-
-performupgrade() {
-    echo "mayan: performupgrade()"
-    su mayan -c "${MAYAN_BIN} performupgrade --no-dependencies"
-}
-
-make_ready() {
-    # Check if this is a new install, otherwise try to upgrade the existing
-    # installation on subsequent starts
-    if [ ! -f $INSTALL_FLAG ]; then
-        initialsetup
-    else
-        performupgrade
-    fi
-}
-
-set_uid_guid() {
-    echo "mayan: changing uid/guid"
-    usermod mayan -u ${MAYAN_USER_UID:-${DEFAULT_USER_UID}}
-    groupmod mayan -g ${MAYAN_USER_GID:-${DEFAULT_USER_GID}}
-}
-
-os_package_installs || true
-pip_installs || true
-chown mayan:mayan /var/lib/mayan -R
-
-case "$1" in
-
-run_initialsetup)
-    initialsetup
-    ;;
-
-run_performupgrade)
-    performupgrade
-    ;;
-
-run_all)
-    make_ready
-    run_all
-    ;;
-
-run_celery)
-    run_celery.sh "${@:2}"
-    ;;
-
-run_command)
-    su mayan -c "${MAYAN_BIN} ${@:2}"
-    ;;
-
-run_frontend)
-    run_frontend.sh
-    ;;
-
-run_tests)
-    make_ready
-    run_tests.sh "${@:2}"
-    ;;
-
-run_worker)
-    run_worker.sh "${@:2}"
-    ;;
-
-*)
-    su mayan -c "$@"
-    ;;
-
-esac

docker/rootfs/usr/local/bin/run_celery.sh

@@ -1,5 +0,0 @@
-#!/bin/bash
-
-# Use -A and not --app. Both are the same but behave differently
-# -A can be located before the command while --app cannot.
-su mayan -c "${MAYAN_PYTHON_BIN_DIR}celery -A mayan $@"

docker/rootfs/usr/local/bin/run_frontend.sh

@@ -1,7 +0,0 @@
-#!/bin/bash
-
-MAYAN_GUNICORN_MAX_REQUESTS=${MAYAN_GUNICORN_MAX_REQUESTS:-500}
-MAYAN_GUNICORN_MAX_REQUESTS_JITTERS=${MAYAN_GUNICORN_MAX_REQUESTS_JITTERS:-50}
-MAYAN_GUNICORN_WORKER_CLASS=${MAYAN_GUNICORN_WORKER_CLASS:-sync}
-
-su mayan -c "${MAYAN_PYTHON_BIN_DIR}gunicorn -w ${MAYAN_GUNICORN_WORKERS} mayan.wsgi --max-requests ${MAYAN_GUNICORN_MAX_REQUESTS} --max-requests-jitter ${MAYAN_GUNICORN_MAX_REQUESTS_JITTERS} --worker-class ${MAYAN_GUNICORN_WORKER_CLASS} --bind 0.0.0.0:8000 --timeout ${MAYAN_GUNICORN_TIMEOUT}"

docker/rootfs/usr/local/bin/run_tests.sh

@@ -1,11 +0,0 @@
-#!/bin/bash
-
-export DEBIAN_FRONTEND=noninteractive
-TEST_ARGUMENT=${@:-"--mayan-apps"}
-
-apt-get update
-apt-get install -y --no-install-recommends gcc python3-dev tesseract-ocr-deu
-
-su mayan -c "${MAYAN_PIP_BIN} install -r ${MAYAN_INSTALL_DIR}/testing-base.txt"
-
-su mayan -c "${MAYAN_BIN} test ${TEST_ARGUMENT} --settings=mayan.settings.testing"

docker/rootfs/usr/local/bin/run_worker.sh

@@ -1,12 +0,0 @@
-#!/bin/bash
-
-export MAYAN_WORKER_NAME=${MAYAN_WORKER_NAME:-$1}
-
-QUEUE_LIST_DEFAULT=`su mayan -c "${MAYAN_PYTHON_BIN_DIR}mayan-edms.py platformtemplate worker_queues"`
-
-MAYAN_QUEUE_LIST=${MAYAN_QUEUE_LIST:-${QUEUE_LIST_DEFAULT}}
-
-# Use -A and not --app. Both are the same but behave differently
-# -A can be located before the command while --app cannot.
-# Pass ${@:2} to allow overriding the defaults arguments
-su mayan -c "${MAYAN_PYTHON_BIN_DIR}celery -A mayan worker -Ofair -l ERROR -Q ${MAYAN_QUEUE_LIST} ${@:2}"

docker/rootfs/version

@@ -1 +0,0 @@
-3.3.6

docker/stack.yml

@@ -9,32 +9,24 @@ volumes:
 services:
 
   db:
+    image: postgres
     environment:
       POSTGRES_DB: mayan
-      POSTGRES_PASSWORD: mayandbpass
+      POSTGRES_PASSWORD: mayan-password
       POSTGRES_USER: mayan
-    image: postgres
     volumes:
       - db:/var/lib/postgresql/data
 
   app:
-    environment:
-      MAYAN_CELERY_BROKER_URL: redis://redis:6379/1
-      MAYAN_CELERY_RESULT_BACKEND: redis://redis:6379/0
-      MAYAN_DATABASES: "{'default':{'ENGINE':'django.db.backends.postgresql','NAME':'mayan','PASSWORD':'mayandbpass','USER':'mayan','HOST':'db'}}"
     image: mayanedms/mayanedms:latest
     ports:
       - 80:8000
+    environment:
+      MAYAN_DATABASE_ENGINE: django.db.backends.postgresql
+      MAYAN_DATABASE_HOST: db
+      MAYAN_DATABASE_NAME: mayan
+      MAYAN_DATABASE_PASSWORD: mayan-password
+      MAYAN_DATABASE_USER: mayan
+      MAYAN_DATABASE_CONN_MAX_AGE: 60
     volumes:
       - app:/var/lib/mayan
-
-  redis:
-    command:
-      - redis-server
-      - --databases
-      - "2"
-      - --maxmemory-policy
-      - allkeys-lru
-      - --save
-      - ""
-    image: redis:5.0

docker/version

@@ -0,0 +1 @@
+3.1.9

docs/_ext/djangodocs.py

@@ -0,0 +1,229 @@
+"""
+Sphinx plugins for Django documentation.
+"""
+import os
+import re
+
+from docutils import nodes, transforms
+try:
+    import json
+except ImportError:
+    try:
+        import simplejson as json
+    except ImportError:
+        try:
+            from django.utils import simplejson as json
+        except ImportError:
+            json = None
+
+from sphinx import addnodes, roles, __version__ as sphinx_ver
+from sphinx.builders.html import StandaloneHTMLBuilder
+from sphinx.writers.html import SmartyPantsHTMLTranslator
+from sphinx.util.console import bold
+from sphinx.util.compat import Directive
+
+# RE for option descriptions without a '--' prefix
+simple_option_desc_re = re.compile(
+    r'([-_a-zA-Z0-9]+)(\s*.*?)(?=,\s+(?:/|-|--)|$)')
+
+def setup(app):
+    app.add_crossref_type(
+        directivename = "setting",
+        rolename      = "setting",
+        indextemplate = "pair: %s; setting",
+    )
+    app.add_crossref_type(
+        directivename = "templatetag",
+        rolename      = "ttag",
+        indextemplate = "pair: %s; template tag"
+    )
+    app.add_crossref_type(
+        directivename = "templatefilter",
+        rolename      = "tfilter",
+        indextemplate = "pair: %s; template filter"
+    )
+    app.add_crossref_type(
+        directivename = "fieldlookup",
+        rolename      = "lookup",
+        indextemplate = "pair: %s; field lookup type",
+    )
+    app.add_description_unit(
+        directivename = "django-admin",
+        rolename      = "djadmin",
+        indextemplate = "pair: %s; django-admin command",
+        parse_node    = parse_django_admin_node,
+    )
+    app.add_description_unit(
+        directivename = "django-admin-option",
+        rolename      = "djadminopt",
+        indextemplate = "pair: %s; django-admin command-line option",
+        parse_node    = parse_django_adminopt_node,
+    )
+    app.add_config_value('django_next_version', '0.0', True)
+    app.add_directive('versionadded', VersionDirective)
+    app.add_directive('versionchanged', VersionDirective)
+    app.add_builder(DjangoStandaloneHTMLBuilder)
+
+
+class VersionDirective(Directive):
+    has_content = True
+    required_arguments = 1
+    optional_arguments = 1
+    final_argument_whitespace = True
+    option_spec = {}
+
+    def run(self):
+        env = self.state.document.settings.env
+        arg0 = self.arguments[0]
+        is_nextversion = env.config.django_next_version == arg0
+        ret = []
+        node = addnodes.versionmodified()
+        ret.append(node)
+        if not is_nextversion:
+            if len(self.arguments) == 1:
+                linktext = 'Please see the release notes </releases/%s>' % (arg0)
+                xrefs = roles.XRefRole()('doc', linktext, linktext, self.lineno, self.state)
+                node.extend(xrefs[0])
+            node['version'] = arg0
+        else:
+            node['version'] = "Development version"
+        node['type'] = self.name
+        if len(self.arguments) == 2:
+            inodes, messages = self.state.inline_text(self.arguments[1], self.lineno+1)
+            node.extend(inodes)
+            if self.content:
+                self.state.nested_parse(self.content, self.content_offset, node)
+            ret = ret + messages
+        env.note_versionchange(node['type'], node['version'], node, self.lineno)
+        return ret
+
+
+class DjangoHTMLTranslator(SmartyPantsHTMLTranslator):
+    """
+    Django-specific reST to HTML tweaks.
+    """
+
+    # Don't use border=1, which docutils does by default.
+    def visit_table(self, node):
+        self._table_row_index = 0 # Needed by Sphinx
+        self.body.append(self.starttag(node, 'table', CLASS='docutils'))
+
+    # <big>? Really?
+    def visit_desc_parameterlist(self, node):
+        self.body.append('(')
+        self.first_param = 1
+        self.param_separator = node.child_text_separator
+
+    def depart_desc_parameterlist(self, node):
+        self.body.append(')')
+
+    if sphinx_ver < '1.0.8':
+        #
+        # Don't apply smartypants to literal blocks
+        #
+        def visit_literal_block(self, node):
+            self.no_smarty += 1
+            SmartyPantsHTMLTranslator.visit_literal_block(self, node)
+
+        def depart_literal_block(self, node):
+            SmartyPantsHTMLTranslator.depart_literal_block(self, node)
+            self.no_smarty -= 1
+
+    #
+    # Turn the "new in version" stuff (versionadded/versionchanged) into a
+    # better callout -- the Sphinx default is just a little span,
+    # which is a bit less obvious that I'd like.
+    #
+    # FIXME: these messages are all hardcoded in English. We need to change
+    # that to accomodate other language docs, but I can't work out how to make
+    # that work.
+    #
+    version_text = {
+        'deprecated':       'Deprecated in Django %s',
+        'versionchanged':   'Changed in Django %s',
+        'versionadded':     'New in Django %s',
+    }
+
+    def visit_versionmodified(self, node):
+        self.body.append(
+            self.starttag(node, 'div', CLASS=node['type'])
+        )
+        title = "%s%s" % (
+            self.version_text[node['type']] % node['version'],
+            len(node) and ":" or "."
+        )
+        self.body.append('<span class="title">%s</span> ' % title)
+
+    def depart_versionmodified(self, node):
+        self.body.append("</div>\n")
+
+    # Give each section a unique ID -- nice for custom CSS hooks
+    def visit_section(self, node):
+        old_ids = node.get('ids', [])
+        node['ids'] = ['s-' + i for i in old_ids]
+        node['ids'].extend(old_ids)
+        SmartyPantsHTMLTranslator.visit_section(self, node)
+        node['ids'] = old_ids
+
+def parse_django_admin_node(env, sig, signode):
+    command = sig.split(' ')[0]
+    env._django_curr_admin_command = command
+    title = "django-admin.py %s" % sig
+    signode += addnodes.desc_name(title, title)
+    return sig
+
+def parse_django_adminopt_node(env, sig, signode):
+    """A copy of sphinx.directives.CmdoptionDesc.parse_signature()"""
+    from sphinx.domains.std import option_desc_re
+    count = 0
+    firstname = ''
+    for m in option_desc_re.finditer(sig):
+        optname, args = m.groups()
+        if count:
+            signode += addnodes.desc_addname(', ', ', ')
+        signode += addnodes.desc_name(optname, optname)
+        signode += addnodes.desc_addname(args, args)
+        if not count:
+            firstname = optname
+        count += 1
+    if not count:
+        for m in simple_option_desc_re.finditer(sig):
+            optname, args = m.groups()
+            if count:
+                signode += addnodes.desc_addname(', ', ', ')
+            signode += addnodes.desc_name(optname, optname)
+            signode += addnodes.desc_addname(args, args)
+            if not count:
+                firstname = optname
+            count += 1
+    if not firstname:
+        raise ValueError
+    return firstname
+
+
+class DjangoStandaloneHTMLBuilder(StandaloneHTMLBuilder):
+    """
+    Subclass to add some extra things we need.
+    """
+
+    name = 'djangohtml'
+
+    def finish(self):
+        super(DjangoStandaloneHTMLBuilder, self).finish()
+        if json is None:
+            self.warn("cannot create templatebuiltins.js due to missing simplejson dependency")
+            return
+        self.info(bold("writing templatebuiltins.js..."))
+        xrefs = self.env.domaindata["std"]["objects"]
+        templatebuiltins = {
+            "ttags": [n for ((t, n), (l, a)) in xrefs.items()
+                        if t == "templatetag" and l == "ref/templates/builtins"],
+            "tfilters": [n for ((t, n), (l, a)) in xrefs.items()
+                        if t == "templatefilter" and l == "ref/templates/builtins"],
+        }
+        outfilename = os.path.join(self.outdir, "templatebuiltins.js")
+        f = open(outfilename, 'wb')
+        f.write('var django_template_builtins = ')
+        json.dump(templatebuiltins, f)
+        f.write(';\n')
+        f.close();

docs/_static/css/custom.css

@@ -1,17 +0,0 @@
-.menuselection {
-    color: white;
-    background: #2c3e50;
-    padding: 3px;
-}
-
-.document a.internal {
-    text-decoration: none;
-    border-bottom: 1px dotted #707070;
-}
-
-.document a.internal:visited {
-    color: #2980B9;
-    text-decoration: none;
-    border-bottom: 1px dotted #707070;
-}
-

docs/_templates/donate.html

@@ -0,0 +1,16 @@
+<h3>Support</h3>
+<p>
+    <a href='http://www.mayan-edms.com/providers/'>Consulting and support plans are available, click here</a>.
+</p>
+<hr />
+<p>
+    Or consider supporting Mayan EDMS by contributing to its development. (US tax payers, please note this contribution is not tax deductible).
+</p>
+<form action="https://www.paypal.com/cgi-bin/webscr" method="post" target="_top">
+    <input type="hidden" name="cmd" value="_s-xclick">
+    <input type="hidden" name="hosted_button_id" value="3PXN336XFXQNN">
+    <input type="image" src="https://www.paypalobjects.com/en_US/i/btn/btn_donateCC_LG.gif" border="0" name="submit" alt="PayPal - The safer, easier way to pay online!">
+    <img alt="" border="0" src="https://www.paypalobjects.com/en_US/i/scr/pixel.gif" width="1" height="1" style="display: none !important;">
+</form>
+
+<hr />

docs/_templates/message_area.html

@@ -1,41 +0,0 @@
-<style>
-.wy-body-for-nav #message-area {
-    color: #b3b3b3;
-    margin: 1em 2em 1em 1em;
-}
-#message-area {
-    border: 1px dotted #2980B9;
-    padding: .5em;
-    margin: 1em 0;
-    text-align: center;
-}
-</style>
-
-<div id="message-area">
-    <h3>Get the book!</h3>
-
-    <a href="https://www.mayan-edms.com/book/">
-        <img src="{{ pathto('_static/book_cover.jpg', 1) }}" />
-    </a>
-
-    <hr />
-
-    <p>
-        On-site consulting and support plans are available, <a href='https://www.mayan-edms.com/support/'>click here</a>.
-    </p>
-
-    <hr />
-
-    <p>
-        Or consider donating to support the continued development of the project.
-    </p>
-
-    <form action="https://www.paypal.com/cgi-bin/webscr" method="post" target="_top">
-        <input type="hidden" name="cmd" value="_s-xclick">
-        <input type="hidden" name="hosted_button_id" value="3PXN336XFXQNN">
-        <input type="image" src="https://www.paypalobjects.com/en_US/i/btn/btn_donateCC_LG.gif" border="0" name="submit" alt="PayPal - The safer, easier way to pay online!">
-        <img alt="" border="0" src="https://www.paypalobjects.com/en_US/i/scr/pixel.gif" width="1" height="1" style="display: none !important;">
-    </form>
-</div>
-
-

docs/callbacks.py

@@ -1,8 +0,0 @@
-from __future__ import unicode_literals
-
-
-def get_source_read_callback(substitutions):
-    def global_substitution_function(app, docname, source):
-        for old, new in substitutions:
-            source[0] = source[0].replace(old, new)
-    return global_substitution_function

docs/chapters/acls.rst

@@ -1,6 +1,6 @@
-====================
+********************
 Access control lists
-====================
+********************
 
 Besides the permissions system explained in :doc:`../chapters/permissions`,
 Mayan EDMS provides per object permission granting. This feature is used to

docs/chapters/api.rst

@@ -0,0 +1,28 @@
+***
+API
+***
+
+Mayan EDMS provides an HTTP REST Application Program Interface (or API). This
+API allows integration with 3rd party software using simple HTTP requests.
+
+Several API authentication methods are provides: **Session**, **Token**,
+and **HTTP Basic**.
+
+The URL for the API can be found via the :menuselection:`Tools --> REST API
+menu. The API is also self-documenting. The live API documentation can be
+found in the :menuselection:`Tools --> API Documentation (Swagger)` menu for
+the Swagger version and in the
+:menuselection:`Tools --> API Documentation (ReDoc)` menu for the ReDoc version.
+
+The are a few ways to structure REST APIs. In the case of Mayan EDMS, API
+endpoints are structured by resource type. Examples:
+
+* /cabinets - To view or create new cabinets
+* /cabinets/<id> - To view the details, edit, or delete an existing cabinet.
+* /cabinets/<id>/documents - To view, add, or remove documents from an existing
+  cabinet.
+* /cabinets/<id>/documents/<id> - To view, add, or remove one document from an
+  existing cabinet.
+
+The API supports the HTTP verbs: **GET**, **POST**, **PUT**, **PATCH**,
+and **DELETE**.

docs/chapters/app_creation.rst

@@ -0,0 +1,143 @@
+************
+App creation
+************
+
+Mayan EDMS apps are essentially Django app with some extra code to register
+navigation, permissions and other relationships.
+
+
+App modules
+===========
+
+- __init__.py
+
+  Should be empty if possible. No initialization code should be here, use the
+  ready() method of the MayanAppConfig class in the apps.py module.
+
+- admin.py
+
+  Standard Django app module to define how models are to be presented in the
+  admin interface.
+
+- api_views.py
+
+  REST API views go here. Mayan EDMS uses Django REST Framework API view
+  classes.
+
+- apps.py
+
+  Contains the MayanAppConfig subclass as required by Django 1.7 and up. This
+  is a place to define the app name and translatable verbose name as well as
+  code to be execute when the modules of the app are ready.
+
+- classes.py
+
+  Hold python classes to be used internally or externally. Any class defined by
+  the app that is not a model.
+
+- events.py
+
+  Define event class instances that are later committed to a log by custom
+  code.
+
+- exceptions.py
+
+  Custom exceptions defined by the app.
+
+- fields.py
+
+  Place any custom form field classed you define here.
+
+- forms.py
+
+  Standard Django app module that hold custom form classes.
+
+- handlers.py
+
+  Contains the signal handlers, functions that will process a given signal
+  emitted from this or other apps. Connect the handler functions to the
+  corresponding signal in the ready() method of the MayanAppConfig subclass in
+  apps.py
+
+- links.py
+
+  Defines the links to be used by the app. Import only from the navigation app
+  and the local permissions.py file.
+
+- literals.py
+
+  Stores magic numbers, module choices (if static), settings defaults, and
+  constants. Should contain all capital case variables. Must not import from
+  any other module.
+
+- managers.py
+
+  Standard Django app module that hold custom model managers. These act as
+  model class method to performs actions in a series of model instances or
+  utilitarian actions on external models instances.
+
+- models.py
+
+  Standard Django app module that defines ORM persistent data schema.
+
+- permissions.py
+
+  Defines the permissions to be used to validate user access by links and views.
+  Imports only from the permissions app. Link or view conditions such as
+  testing for is_staff or is_super_user flag are defined in this same module.
+
+- runtime.py
+
+  Use this module when you need the same instance of a class for the entire app.
+  This module acts as a shared memory space for the other modules of the app or
+  other apps.
+
+- serializers.py
+
+  Hold Django REST Framework serializers used by the api_views.py module.
+
+- settings.py
+
+  Define the configuration settings instances that the app will use.
+
+- signals.py
+
+  Any custom defined signal goes here.
+
+- statistics.py
+
+  Provides functions that will compute any sort of statistical information on
+  the app’s data.
+
+- tasks.py
+
+  Code to be execute in the background or as an out-of-process action.
+
+- tests/ directory
+
+  Hold test modules. There should be one test_*.py module for each aspect being
+  tested, examples: test_api.py, test_views.py, test_parsers.py, test_permissions.py
+  Any shared constant data used by the tests should be added to tests/literals.py
+
+- utils.py
+
+  Holds utilitarian code that doesn't fit on any other app module or that is
+  used by several modules in the app. Anything used internally by the app that
+  is not a class or a literal (should be as little as possible)
+
+- widgets.py
+
+  HTML widgets go here. This should be the only place with presentation
+  directives in the app (aside the templates).
+
+
+Views
+=====
+
+The module common.generics provides custom generic class based views to be used.
+The basic views used to create, edit, view and delete objects in Mayan EDMS
+are: ``SingleObjectCreateView``, ``SingleObjectDetailView``,
+``SingleObjectEditView``, and ``SingleObjectListView``.
+
+These views handle aspects relating to view permissions, object permissions,
+post action redirection and template context generation.

docs/chapters/backups.rst

@@ -0,0 +1,70 @@
+*******
+Backups
+*******
+
+To backup your install of Mayan EDMS just copy the actual document files and
+the database content. If you are using the default storage backend, the
+document files should be found in the ``media`` folder of your installation.
+
+To dump the content of your database manager refer to the documentation chapter
+regarding database data "dumping".
+
+Here is an example of how to perform a backup and a restore of a PostgreSQL
+database.
+
+To dump the database into an SQL text file::
+
+    pg_dump -h <host> -U <database user> -c <database name> -W > `date +%Y-%m-%d"_"%H-%M-%S`.sql
+
+Example::
+
+    pg_dump -h 127.0.0.1 -U mayan -c mayan -W > `date +%Y-%m-%d"_"%H-%M-%S`.sql
+
+To restore the database from the SQL text file::
+
+    psql -h <host> -U <database user> -d <database name> -W -f <sql dump file>
+
+Example::
+
+    psql -h 127.0.0.1 -U mayan -d mayan -W -f 2018-06-07_18-10-56.sql
+
+Here is an example of how to perform a backup and a restore of a PostgreSQL
+Docker container using a compressed dump file. A dump file is not compatible or
+can be used interchangeable with an SQL text file.
+
+To backup a PostgreSQL Docker container::
+
+    docker exec <container name> pg_dump -U <database user> -Fc -c <database name> > `date +%Y-%m-%d"_"%H-%M-%S`.dump
+
+Example::
+
+    docker exec mayan-edms-db pg_dump -U mayan -Fc -c mayan > `date +%Y-%m-%d"_"%H-%M-%S`.dump
+
+This will produce a compressed dump file with the current date and time as the filename.
+
+To restore a PostgreSQL Docker container::
+
+    docker exec -i <container name> pg_restore -U <database user> -d <database name> < <dump file>
+
+Since it is not possible to drop a currently open PostgreSQL database, this
+command must be used on a new and empty PostsgreSQL container.
+
+Example::
+
+    docker run -d \
+    --name mayan-edms-pg-new \
+    --restart=always \
+    -p 5432:5432 \
+    -e POSTGRES_USER=mayan \
+    -e POSTGRES_DB=mayan \
+    -e POSTGRES_PASSWORD=mayanuserpass \
+    -v /docker-volumes/mayan-edms/postgres-new:/var/lib/postgresql/data \
+    -d postgres:9.5
+
+    docker exec -i mayan-edms-pg-new pg_restore -U mayan -d mayan < 2018-06-07_17-09-34.dump
+
+More information at:
+
+ - PostgreSQL: https://www.postgresql.org/docs/current/static/backup.html
+ - MySQL: https://dev.mysql.com/doc/refman/5.7/en/mysqldump.html
+ - SQLite: Just copy the file ``mayan/media/db.sqlite3``

docs/chapters/backups.txt

@@ -1,18 +0,0 @@
-=======
-Backups
-=======
-
-To backup your install of Mayan EDMS just copy the ``media`` folder and
-the database content.
-
-If you are using the default storage backend, the ``media`` should be located
-at |MAYAN_MEDIA_ROOT|::
-
-    sudo tar -zcvf backup.tar.gz |MAYAN_MEDIA_ROOT|
-
-To dump the content of your database manager refer to the documentation chapter
-regarding database data "dumping".
-
-- PostgreSQL: https://www.postgresql.org/docs/current/backup-dump.html
-- MySQL: https://dev.mysql.com/doc/refman/5.7/en/mysqldump.html
-

docs/chapters/caa_entity.rst

@@ -1,6 +1,6 @@
-==================================================
+**************************************************
 Mayan EDMS Entity Contributor Assignment Agreement
-==================================================
+**************************************************
 
 Thank you for your interest in contributing to Mayan EDMS ("We" or "Us").
 
@@ -14,7 +14,6 @@ software project managed by Us.
 
 1. Definitions
 ==============
-
 "You" means any Legal Entity on behalf of whom a Contribution has been received
 by Us. "Legal Entity" means an entity which is not a natural person.
 "Affiliates" means other Legal Entities that control, are controlled by, or
@@ -56,7 +55,6 @@ first Submit a Contribution to Us, whichever is earlier.
 2. Grant of Rights
 ==================
 
-
 2.1 Copyright Assignment
 ------------------------
 
@@ -103,7 +101,6 @@ license the Contribution only under the terms of the license or licenses which
 We are using on the Submission Date for the Material (including any rights to
 adopt any future version of a license if permitted).
 
-
 2.4 Moral Rights
 ----------------
 
@@ -111,14 +108,12 @@ If moral rights apply to the Contribution, to the maximum extent permitted by
 law, You waive and agree not to assert such moral rights against Us or our
 successors in interest, or any of our licensees, either direct or indirect.
 
-
 2.5 Our Rights
 --------------
 
 You acknowledge that We are not obligated to use Your Contribution as part of
 the Material and may decide to include any Contribution We consider appropriate.
 
-
 2.6 Reservation of Rights
 -------------------------
 
@@ -164,7 +159,6 @@ OR OTHERWISE) UPON WHICH THE CLAIM IS BASED.
 6. Miscellaneous
 ================
 
-
 6.1 Jurisdiction
 ----------------
 
@@ -176,25 +170,19 @@ and the parties intend to avoid the application of the UN Convention to this
 Agreement and, thus, exclude the application of the UN Convention in its
 entirety to this Agreement.
 
-
 6.2 Acceptance
 --------------
-
 This Agreement sets out the entire agreement between You and Us for Your
 Contributions to Us and overrides all other agreements or understandings.
 
-
 6.3 Third parties
 -----------------
-
 If You or We assign the rights or obligations received through this Agreement
 to a third party, as a condition of the assignment, that third party must
 agree in writing to abide by all the rights and obligations in the Agreement.
 
-
 6.4 Unmet responsibilities
 --------------------------
-
 The failure of either party to require performance by the other party of any
 provision of this Agreement in one situation shall not affect the right of a
 party to require such performance at any time in the future. A waiver of
@@ -202,10 +190,8 @@ performance under a provision in one situation shall not be considered a
 waiver of the performance of the provision in the future or a waiver of the
 provision in its entirety.
 
-
 6.5 Continuation
 ----------------
-
 If any provision of this Agreement is found void and unenforceable, such
 provision will be replaced to the extent possible with a provision that comes
 closest to the meaning of the original provision and which is enforceable.

docs/chapters/caa_individual.rst

@@ -1,6 +1,6 @@
-======================================================
+******************************************************
 Mayan EDMS Individual Contributor Assignment Agreement
-======================================================
+******************************************************
 
 Thank you for your interest in contributing to Mayan EDMS ("We" or "Us").
 
@@ -11,7 +11,6 @@ caa@mayan-edms.com. This is a legally binding document, so please read it
 carefully before agreeing to it. The Agreement may cover more than one
 software project managed by Us.
 
-
 1. Definitions
 ==============
 
@@ -44,11 +43,9 @@ or otherwise designated in writing by You as "Not a Contribution."
 "Effective Date" means the date You execute this Agreement or the date You first
 Submit a Contribution to Us, whichever is earlier.
 
-
 2. Grant of Rights
 ==================
 
-
 2.1 Copyright Assignment
 ------------------------
 
@@ -90,7 +87,6 @@ and provided that this license is conditioned upon compliance with Section 2.3.
 
 2.3 Outbound License
 --------------------
-
 As a condition on the grant of rights in Sections 2.1 and 2.2, We agree to
 license the Contribution only under the terms of the license or licenses which
 We are using on the Submission Date for the Material (including any rights to
@@ -160,7 +156,6 @@ OR OTHERWISE) UPON WHICH THE CLAIM IS BASED.
 6. Miscellaneous
 ================
 
-
 6.1 Jurisdiction
 ----------------
 
@@ -172,14 +167,12 @@ and the parties intend to avoid the application of the UN Convention to this
 Agreement and, thus, exclude the application of the UN Convention in its
 entirety to this Agreement.
 
-
 6.2 Acceptance
 --------------
 
 This Agreement sets out the entire agreement between You and Us for Your
 Contributions to Us and overrides all other agreements or understandings.
 
-
 6.3 Third parties
 -----------------
 
@@ -187,7 +180,6 @@ If You or We assign the rights or obligations received through this Agreement
 to a third party, as a condition of the assignment, that third party must
 agree in writing to abide by all the rights and obligations in the Agreement.
 
-
 6.4 Unmet responsibilities
 --------------------------
 
@@ -198,7 +190,6 @@ performance under a provision in one situation shall not be considered a
 waiver of the performance of the provision in the future or a waiver of the
 provision in its entirety.
 
-
 6.5 Continuation
 ----------------
 

docs/chapters/cabinets.rst

@@ -1,11 +1,13 @@
-========
 Cabinets
 ========
 
 Cabinets are a multi-level method to organize documents. Each cabinet can
-contain documents as well as other sub level cabinets.
+contain documents as well as other sub level cabinets. Cabinets can
+be added from the :menuselection:`Cabinets --> Add new` menu.
 
 With a cabinet structure created, the next step is to add documents to them.
+To add documents to a cabinet, select the cabinet tab of a document, and
+select :menuselection:`Actions --> Add to cabinets` link.
 
 Documents can also be added in bulk to one or more cabinets from any view
 that lists documents.

docs/chapters/checkouts.rst

@@ -1,6 +1,6 @@
-=========
+*********
 Checkouts
-=========
+*********
 
 Checkouts are a way to block certain accesses or actions of a document for a
 period of time.

docs/chapters/database_conversion.rst

@@ -0,0 +1,103 @@
+*******************
+Database conversion
+*******************
+
+Version 3.1.x added a new management command to help convert data residing in
+an SQLite database to other database managers like PostgreSQL. Here is the
+conversion procedure.
+
+Direct install
+==============
+
+* Make a backup of your existing SQLite database and documents by copying the
+  ``/opt/mayan-edms/media`` folder.
+* :doc:`Upgrade to at least version 3.1.3. <../releases/3.1.3>`
+*  Migrate the existing SQLite database with the command ``performupgrade``::
+
+    sudo -u mayan MAYAN_MEDIA_ROOT=/opt/mayan-edms/media /opt/mayan-edms/bin/mayan-edms.py performupgrade
+
+* Install PostgreSQL::
+
+    sudo apt-get install postgresql libpq-dev
+
+* Provision a PostgreSQL database::
+
+    sudo -u postgres psql -c "CREATE USER mayan WITH password 'mayanuserpass';"
+    sudo -u postgres createdb -O mayan mayan
+
+* Install the Python client for PostgreSQL::
+
+    sudo -u mayan /opt/mayan-edms/bin/pip install --no-cache-dir psycopg2==2.7.3.2
+
+* Copy the newly created fallback config file::
+
+    cp /opt/mayan-edms/media/config_backup.yml /opt/mayan-edms/media/config.yml
+
+* Edit the configuration file to add the entry for the PostgreSQL database and
+  rename the SQLite database to 'old'::
+
+    # Before
+    DATABASES:
+      default:
+        ATOMIC_REQUESTS: false
+        AUTOCOMMIT: true
+        CONN_MAX_AGE: 0
+        ENGINE: django.db.backends.sqlite3
+        HOST: ''
+        NAME: /opt/mayan-edms/media/db.sqlite3
+        OPTIONS: {}
+        PASSWORD: ''
+        PORT: ''
+        TEST: {CHARSET: null, COLLATION: null, MIRROR: null, NAME: null}
+        TIME_ZONE: null
+        USER: ''
+
+    # After
+    DATABASES:
+      old:
+        ATOMIC_REQUESTS: false
+        AUTOCOMMIT: true
+        CONN_MAX_AGE: 0
+        ENGINE: django.db.backends.sqlite3
+        HOST: ''
+        NAME: /opt/mayan-edms/media/db.sqlite3
+        OPTIONS: {}
+        PASSWORD: ''
+        PORT: ''
+        TEST: {CHARSET: null, COLLATION: null, MIRROR: null, NAME: null}
+        TIME_ZONE: null
+        USER: ''
+      default:
+        ATOMIC_REQUESTS: false
+        AUTOCOMMIT: true
+        CONN_MAX_AGE: 0
+        ENGINE: django.db.backends.postgresql
+        HOST: '127.0.0.1'
+        NAME: /opt/mayan-edms/media/db.sqlite3
+        OPTIONS: {}
+        PASSWORD: 'mayanuserpass'
+        PORT: ''
+        TEST: {CHARSET: null, COLLATION: null, MIRROR: null, NAME: null}
+        TIME_ZONE: null
+        USER: 'mayan'
+
+* Migrate the new database to create the empty tables::
+
+    sudo -u mayan MAYAN_DATABASE_ENGINE=django.db.backends.postgresql MAYAN_DATABASE_NAME=mayan MAYAN_DATABASE_PASSWORD=mayanuserpass MAYAN_DATABASE_USER=mayan MAYAN_DATABASE_HOST=127.0.0.1 MAYAN_MEDIA_ROOT=/opt/mayan-edms/media /opt/mayan-edms/bin/mayan-edms.py migrate
+
+* Convert the data in the SQLite and store it in the PostgreSQL database::
+
+    sudo -u mayan MAYAN_DATABASE_ENGINE=django.db.backends.postgresql MAYAN_DATABASE_NAME=mayan MAYAN_DATABASE_PASSWORD=mayanuserpass MAYAN_DATABASE_USER=mayan MAYAN_DATABASE_HOST=127.0.0.1 MAYAN_MEDIA_ROOT=/opt/mayan-edms/media /opt/mayan-edms/bin/mayan-edms.py convertdb --from=old --to=default --force
+
+* Update the supervisor config file to have Mayan EDMS run from the PostgreSQL database::
+
+    [supervisord]
+    environment=
+        <...>
+        MAYAN_DATABASE_ENGINE=django.db.backends.postgresql,
+        MAYAN_DATABASE_HOST=127.0.0.1,
+        MAYAN_DATABASE_NAME=mayan,
+        MAYAN_DATABASE_PASSWORD=mayanuserpass,
+        MAYAN_DATABASE_USER=mayan,
+        MAYAN_DATABASE_CONN_MAX_AGE=360,
+        <...>

docs/chapters/deploying.rst

@@ -0,0 +1,375 @@
+******************
+Direct deployments
+******************
+
+Mayan EDMS should be deployed like any other Django_ project and
+preferably using virtualenv_. Below are some ways to deploy and use Mayan EDMS.
+Do not use more than one method.
+
+Being a Django_ and a Python_ project, familiarity with these technologies is
+recommended to better understand why Mayan EDMS does some of the things it
+does.
+
+
+Basic deployment
+================
+This setup uses less memory and CPU resources at the expense of some speed.
+
+Binary dependencies
+-------------------
+
+If using a Debian_ or Ubuntu_ based Linux distribution, get the executable
+requirements using::
+
+    sudo apt-get install exiftool g++ gcc ghostscript gnupg1 graphviz \
+    libfuse2 libjpeg-dev libmagic1 libpq-dev libpng-dev libreoffice \
+    libtiff-dev poppler-utils postgresql python-dev python-virtualenv \
+    redis-server sane-utils supervisor tesseract-ocr zlib1g-dev -y
+
+Create an user account for the installation:
+--------------------------------------------
+::
+
+    sudo adduser mayan --disabled-password --disabled-login --no-create-home --gecos ""
+
+Create the parent directory where the project will be deployed:
+---------------------------------------------------------------
+::
+
+    sudo mkdir /opt
+
+Create the Python virtual environment for the installation:
+-----------------------------------------------------------
+::
+
+    sudo virtualenv /opt/mayan-edms
+
+Make the mayan user the owner of the installation directory:
+------------------------------------------------------------
+::
+
+    sudo chown mayan:mayan /opt/mayan-edms -R
+
+Install Mayan EDMS from PyPI:
+-----------------------------
+::
+
+    sudo -u mayan /opt/mayan-edms/bin/pip install --no-cache-dir mayan-edms
+
+Install the Python client for PostgreSQL and Redis:
+---------------------------------------------------
+::
+
+    sudo -u mayan /opt/mayan-edms/bin/pip install --no-cache-dir psycopg2==2.7.3.2 redis==2.10.6
+
+Create the database for the installation:
+-----------------------------------------
+::
+
+    sudo -u postgres psql -c "CREATE USER mayan WITH password 'mayanuserpass';"
+    sudo -u postgres createdb -O mayan mayan
+
+Initialize the project:
+-----------------------
+::
+
+    sudo -u mayan MAYAN_DATABASES='{default: {ENGINE: django.db.backends.postgresql, NAME: mayan, PASSWORD: mayanuserpass, USER: mayan, HOST=127.0.0.1}}' \
+    MAYAN_MEDIA_ROOT=/opt/mayan-edms/media \
+    /opt/mayan-edms/bin/mayan-edms.py initialsetup
+
+Collect the static files:
+-------------------------
+::
+
+    sudo -u mayan MAYAN_MEDIA_ROOT=/opt/mayan-edms/media \
+    /opt/mayan-edms/bin/mayan-edms.py collectstatic --noinput
+
+Create the supervisor file at ``/etc/supervisor/conf.d/mayan.conf``:
+--------------------------------------------------------------------
+::
+
+    [supervisord]
+    environment=
+        MAYAN_ALLOWED_HOSTS='["*"]',  # Allow access to other network hosts other than localhost
+        MAYAN_CELERY_BROKER_URL="redis://127.0.0.1:6379/0",
+        MAYAN_CELERY_RESULT_BACKEND="redis://127.0.0.1:6379/0",
+        PYTHONPATH=/opt/mayan-edms/lib/python2.7/site-packages:/opt/mayan-edms/data,
+        MAYAN_MEDIA_ROOT=/opt/mayan-edms/media,
+        MAYAN_DATABASES='{default: {ENGINE: django.db.backends.postgresql, HOST: 127.0.0.1, NAME: mayan, PASSWORD: mayanuserpass, USER: mayan, CONN_MAX_AGE: 60}}',
+        DJANGO_SETTINGS_MODULE=mayan.settings.production
+
+    [program:mayan-gunicorn]
+    autorestart = true
+    autostart = true
+    command = /opt/mayan-edms/bin/gunicorn -w 2 mayan.wsgi --max-requests 500 --max-requests-jitter 50 --worker-class gevent --bind 0.0.0.0:8000 --timeout 120
+    user = mayan
+
+    [program:mayan-worker-fast]
+    autorestart = true
+    autostart = true
+    command = nice -n 1 /opt/mayan-edms/bin/mayan-edms.py celery worker -l ERROR -Q converter,document_states_fast,sources_fast -n mayan-worker-fast.%%h --concurrency=1
+    killasgroup = true
+    numprocs = 1
+    priority = 998
+    startsecs = 10
+    stopwaitsecs = 1
+    user = mayan
+
+    [program:mayan-worker-medium]
+    autorestart = true
+    autostart = true
+    command = nice -n 18 /opt/mayan-edms/bin/mayan-edms.py celery worker -l ERROR -Q checkouts_periodic,documents_periodic,indexing,metadata,sources,sources_periodic,uploads,documents -n mayan-worker-medium.%%h --concurrency=1
+    killasgroup = true
+    numprocs = 1
+    priority = 998
+    startsecs = 10
+    stopwaitsecs = 1
+    user = mayan
+
+    [program:mayan-worker-slow]
+    autorestart = true
+    autostart = true
+    command = nice -n 19 /opt/mayan-edms/bin/mayan-edms.py celery worker -l ERROR -Q mailing,tools,statistics,parsing,ocr -n mayan-worker-slow.%%h --concurrency=1
+    killasgroup = true
+    numprocs = 1
+    priority = 998
+    startsecs = 10
+    stopwaitsecs = 1
+    user = mayan
+
+    [program:mayan-celery-beat]
+    autorestart = true
+    autostart = true
+    command = nice -n 1 /opt/mayan-edms/bin/mayan-edms.py celery beat --pidfile= -l ERROR
+    killasgroup = true
+    numprocs = 1
+    priority = 998
+    startsecs = 10
+    stopwaitsecs = 1
+    user = mayan
+
+Configure Redis to discard data when it runs out of memory, not save its database and only keep 1 database:
+-----------------------------------------------------------------------------------------------------------
+::
+
+    echo "maxmemory-policy allkeys-lru" >> /etc/redis/redis.conf
+    echo "save \"\"" >> /etc/redis/redis.conf
+    echo "databases 1" >> /etc/redis/redis.conf
+    systemctl restart redis
+
+Enable and restart the services [1_]:
+-------------------------------------
+::
+
+    systemctl enable supervisor
+    systemctl restart supervisor
+
+
+.. _deployment_advanced:
+
+Advanced deployment
+===================
+
+This variation uses RabbitMQ as the message broker and removes the fast worker
+concurrency restriction. RabbitMQ consumes more memory but scales to thousands
+of messages. RabbitMQ messages are also persistent, this means that pending
+tasks are not lost in the case of a restart. The database connection lifetime
+is increased to 10 minutes. The Gunicorn workers are increased to 3.
+
+Binary dependencies
+-------------------
+
+If using a Debian_ or Ubuntu_ based Linux distribution, get the executable
+requirements using::
+
+    sudo apt-get install exiftool g++ gcc ghostscript gnupg1 graphviz \
+    libfuse2 libjpeg-dev libmagic1 libpq-dev libpng-dev libreoffice \
+    libtiff-dev poppler-utils postgresql python-dev python-virtualenv \
+    rabbitmq-server redis-server sane-utils supervisor tesseract-ocr \
+    zlib1g-dev -y
+
+Create an user account for the installation:
+--------------------------------------------
+::
+
+    sudo adduser mayan --disabled-password --disabled-login --no-create-home --gecos ""
+
+Create the parent directory where the project will be deployed:
+---------------------------------------------------------------
+::
+
+    sudo mkdir /opt
+
+Create the Python virtual environment for the installation:
+-----------------------------------------------------------
+::
+
+    sudo virtualenv /opt/mayan-edms
+
+Make the mayan user the owner of the installation directory:
+------------------------------------------------------------
+::
+
+    sudo chown mayan:mayan /opt/mayan-edms -R
+
+Install Mayan EDMS from PyPI:
+-----------------------------
+::
+
+    sudo -u mayan /opt/mayan-edms/bin/pip install --no-cache-dir mayan-edms
+
+Install the Python client for PostgreSQL and Redis:
+---------------------------------------------------
+::
+
+    sudo -u mayan /opt/mayan-edms/bin/pip install --no-cache-dir librabbitmq==2.0.0 psycopg2==2.7.3.2 redis==2.10.6
+
+Create the database for the installation:
+-----------------------------------------
+::
+
+    sudo -u postgres psql -c "CREATE USER mayan WITH password 'mayanuserpass';"
+    sudo -u postgres createdb -O mayan mayan
+
+Initialize the project:
+-----------------------
+::
+
+    sudo -u mayan MAYAN_DATABASES='{default: {ENGINE: django.db.backends.postgresql, NAME: mayan, PASSWORD: mayanuserpass, USER: mayan, HOST=127.0.0.1}}' \
+    MAYAN_MEDIA_ROOT=/opt/mayan-edms/media \
+    /opt/mayan-edms/bin/mayan-edms.py initialsetup
+
+Collect the static files:
+-------------------------
+::
+
+    sudo -u mayan MAYAN_MEDIA_ROOT=/opt/mayan-edms/media \
+    /opt/mayan-edms/bin/mayan-edms.py collectstatic --noinput
+
+Create the RabbitMQ user and vhost:
+-----------------------------------
+::
+
+    sudo rabbitmqctl add_user mayan mayanrabbitmqpassword
+    sudo rabbitmqctl add_vhost mayan
+    sudo rabbitmqctl set_permissions -p mayan mayan ".*" ".*" ".*"
+
+Create the supervisor file at ``/etc/supervisor/conf.d/mayan.conf``:
+--------------------------------------------------------------------
+::
+
+    [supervisord]
+    environment=
+        MAYAN_ALLOWED_HOSTS='["*"]',  # Allow access to other network hosts other than localhost
+        MAYAN_CELERY_BROKER_URL="amqp://mayan:mayanrabbitmqpassword@localhost:5672/mayan",
+        MAYAN_CELERY_RESULT_BACKEND="redis://127.0.0.1:6379/0",
+        PYTHONPATH=/opt/mayan-edms/lib/python2.7/site-packages:/opt/mayan-edms/data,
+        MAYAN_MEDIA_ROOT=/opt/mayan-edms/media,
+        MAYAN_DATABASES='{default: {ENGINE: django.db.backends.postgresql, HOST: 127.0.0.1, NAME: mayan, PASSWORD: mayanuserpass, USER: mayan, CONN_MAX_AGE: 60}}',
+        DJANGO_SETTINGS_MODULE=mayan.settings.production
+
+    [program:mayan-gunicorn]
+    autorestart = true
+    autostart = true
+    command = /opt/mayan-edms/bin/gunicorn -w 3 mayan.wsgi --max-requests 500 --max-requests-jitter 50 --worker-class gevent --bind 0.0.0.0:8000 --timeout 120
+    user = mayan
+
+    [program:mayan-worker-fast]
+    autorestart = true
+    autostart = true
+    command = nice -n 1 /opt/mayan-edms/bin/mayan-edms.py celery worker -l ERROR -Q converter,document_states_fast,sources_fast -n mayan-worker-fast.%%h
+    killasgroup = true
+    numprocs = 1
+    priority = 998
+    startsecs = 10
+    stopwaitsecs = 1
+    user = mayan
+
+    [program:mayan-worker-medium]
+    autorestart = true
+    autostart = true
+    command = nice -n 18 /opt/mayan-edms/bin/mayan-edms.py celery worker -l ERROR -Q checkouts_periodic,documents_periodic,indexing,metadata,sources,sources_periodic,uploads,documents -n mayan-worker-medium.%%h --concurrency=1
+    killasgroup = true
+    numprocs = 1
+    priority = 998
+    startsecs = 10
+    stopwaitsecs = 1
+    user = mayan
+
+    [program:mayan-worker-slow]
+    autorestart = true
+    autostart = true
+    command = nice -n 19 /opt/mayan-edms/bin/mayan-edms.py celery worker -l ERROR -Q mailing,tools,statistics,parsing,ocr -n mayan-worker-slow.%%h --concurrency=1
+    killasgroup = true
+    numprocs = 1
+    priority = 998
+    startsecs = 10
+    stopwaitsecs = 1
+    user = mayan
+
+    [program:mayan-celery-beat]
+    autorestart = true
+    autostart = true
+    command = nice -n 1 /opt/mayan-edms/bin/mayan-edms.py celery beat --pidfile= -l ERROR
+    killasgroup = true
+    numprocs = 1
+    priority = 998
+    startsecs = 10
+    stopwaitsecs = 1
+    user = mayan
+
+Configure Redis to discard data when it runs out of memory, not save its database and only keep 1 database:
+-----------------------------------------------------------------------------------------------------------
+::
+
+    echo "maxmemory-policy allkeys-lru" >> /etc/redis/redis.conf
+    echo "save \"\"" >> /etc/redis/redis.conf
+    echo "databases 1" >> /etc/redis/redis.conf
+    systemctl restart redis
+
+Enable and restart the services [1_]:
+-------------------------------------
+::
+
+    systemctl enable supervisor
+    systemctl restart supervisor
+
+
+Troubleshooting
+===============
+
+- Due to OS differences some binaries might reside in different locations.
+  Use environment variables or the configuration file to tell Mayan EDMS where
+  to file these binaries.
+
+  Example: OpenBSD. Add the following entries to supervisor configuration files.
+  ::
+
+      MAYAN_DOCUMENT_PARSING_PDFTOTEXT_PATH=/usr/local/bin/pdftotext,
+      MAYAN_SIGNATURES_GPG_PATH=/usr/local/bin/gpg,
+      MAYAN_SOURCES_SCANIMAGE_PATH: /usr/local/bin/scanimage,
+
+  Alternatively a symlink from the actual binary location to where Mayan
+  EDMS is expecting them to be found by default also works for some users::
+
+      ln -s /usr/local/bin/gpg /usr/bin/gpg1
+
+  Example 2: Ubuntu 16.04. Add the following entries to supervisor
+  configuration files.
+  ::
+
+      MAYAN_SIGNATURES_GPG_PATH=/usr/bin/gpg1,
+
+  Or add a symlink::
+
+      ln -s /usr/bin/gpg /usr/bin/gpg1
+
+[1]: https://bugs.launchpad.net/ubuntu/+source/supervisor/+bug/1594740
+
+.. _Debian: https://www.debian.org/
+.. _Django: https://www.djangoproject.com/
+.. _Python: https://www.python.org/
+.. _SQLite: https://www.sqlite.org/
+.. _Ubuntu: http://www.ubuntu.com/
+.. _virtualenv: http://www.virtualenv.org/en/latest/index.html
+.. _1: https://bugs.launchpad.net/ubuntu/+source/supervisor/+bug/1594740

docs/chapters/deploying.txt

@@ -1,273 +0,0 @@
-.. _deployment_basic:
-
-=================
-Direct deployment
-=================
-
-Mayan EDMS should be deployed like any other Django_ project and
-preferably using virtualenv_. Below are some ways to deploy and use Mayan EDMS.
-
-Being a Django_ and a Python_ project, familiarity with these technologies is
-recommended to better understand why Mayan EDMS does some of the things it
-does.
-
-Compilers and development libraries will be installed to compile runtime
-libraries. LibreOffice and Poppler utils will also be installed as they are
-used to convert document files. Supervisor (https://supervisord.org/), a
-Process Control System, will be used to monitor and keep all Mayan processes
-running.
-
-
-Basic deployment
-================
-This setup uses less memory and CPU resources at the expense of some speed.
-For another setup that offers more performance and scalability refer to the
-`Advanced deployment`_ below.
-
-#. Install binary dependencies:
-
-   If using a Debian_ or Ubuntu_ based Linux distribution, get the executable
-   requirements using:
-
-   .. code-block:: bash
-
-       sudo apt-get install exiftool g++ gcc ghostscript gnupg1 graphviz \
-       libfuse2 libjpeg-dev libmagic1 libpq-dev libpng-dev libreoffice \
-       libtiff-dev poppler-utils postgresql python3-dev python3-virtualenv
-       redis-server sane-utils supervisor tesseract-ocr zlib1g-dev -y
-
-   .. note::
-
-       Platforms with the ARM CPU might also need additional requirements:
-
-       .. code-block:: bash
-
-           sudo apt-get install libffi-dev libssl-dev -y
-
-
-#. Create the user account for the installation:
-
-   This will create an unprivileged user account that is also unable to login:
-
-   .. code-block:: bash
-
-       sudo adduser mayan --disabled-password --disabled-login --gecos ""
-
-
-#. Create the parent directory where the project will be deployed:
-
-   ``/opt/`` is a good choice as it is meant is for "software and add-on packages
-   that are not part of the default installation". (https://www.tldp.org/LDP/Linux-Filesystem-Hierarchy/html/opt.html).
-   Create the ``/opt`` directory if it doesn't already exists:
-
-   .. code-block:: bash
-
-       sudo mkdir /opt
-
-
-#. Create the Python virtual environment:
-
-   This will keep all the Python packages installed here isolated from the rest
-   of the Python packages in the system:
-
-   .. code-block:: bash
-
-       sudo virtualenv |MAYAN_INSTALLATION_DIRECTORY| -p /usr/bin/python3
-
-
-#. Make the mayan user the owner of the installation directory:
-
-   .. code-block:: bash
-
-       sudo chown mayan:mayan |MAYAN_INSTALLATION_DIRECTORY| -R
-
-
-#. Install Mayan EDMS from PyPI:
-
-   .. code-block:: bash
-
-       sudo -u mayan |MAYAN_PIP_BIN| install --no-use-pep517 mayan-edms
-
-
-#. Install the Python client for PostgreSQL and Redis:
-
-   .. code-block:: bash
-
-       sudo -u mayan |MAYAN_PIP_BIN| install --no-use-pep517 psycopg2==|PYTHON_PSYCOPG2_VERSION| redis==|PYTHON_REDIS_VERSION|
-
-   .. note::
-
-       Platforms with the ARM CPU might also need additional requirements:
-
-       .. code-block:: bash
-
-           sudo -u mayan |MAYAN_PIP_BIN| install --no-use-pep517 psutil==|PYTHON_PSUTIL_VERSION|
-
-
-#. Create the database for the installation:
-
-   .. code-block:: bash
-
-       sudo -u postgres psql -c "CREATE USER mayan WITH password 'mayanuserpass';"
-       sudo -u postgres createdb -O mayan mayan
-
-
-#. Initialize the project:
-
-   This step will create all the database structures, download static media files
-   like JavaScript libraries and HTML frameworks, and create and initial admin
-   account with a random password.
-
-   .. note::
-
-       For simplicity, the ``MAYAN_MEDIA_ROOT`` folder is set to be a subfolder
-       of the installation. If you want to keep your files separated from
-       the installation files, change the value of the ``MAYAN_MEDIA_ROOT``
-       variable in this and all subsequent steps. Be sure to first create the
-       folder and give ownership of it to the ``mayan`` user with the ``chown``
-       command.
-
-   .. warning::
-
-       If this step is interrupted, even if it is later resumed, will
-       cause the automatic admin user to not be created in some cases. Make
-       sure all environment variables and values are correct. If this
-       happens, refer to the troubleshooting chapters:
-       :ref:`troubleshooting-autoadmin-account` and
-       :ref:`troubleshooting-admin-password`.
-
-   .. code-block:: bash
-
-       sudo -u mayan MAYAN_DATABASES="{'default':{'ENGINE':'django.db.backends.postgresql','NAME':'mayan','PASSWORD':'mayanuserpass','USER':'mayan','HOST':'127.0.0.1'}}" \
-       MAYAN_MEDIA_ROOT=|MAYAN_MEDIA_ROOT| \
-       |MAYAN_BIN| initialsetup
-
-
-#. Collect the static files:
-
-   This step merges and compressed static media files so they can be served more
-   effectively.
-
-   .. code-block:: bash
-
-       sudo -u mayan MAYAN_MEDIA_ROOT=|MAYAN_MEDIA_ROOT| \
-       |MAYAN_BIN| preparestatic --noinput
-
-
-#. Create the supervisor file at ``|MAYAN_SUPERVISOR_CONF|``:
-
-   .. code-block:: bash
-
-       sudo -u mayan MAYAN_DATABASES="{'default':{'ENGINE':'django.db.backends.postgresql','NAME':'mayan','PASSWORD':'mayanuserpass','USER':'mayan','HOST':'127.0.0.1'}}" \
-       MAYAN_MEDIA_ROOT=|MAYAN_MEDIA_ROOT| \
-       |MAYAN_BIN| platformtemplate supervisord | sudo sh -c "cat > |MAYAN_SUPERVISOR_CONF|"
-
-
-#. Configure Redis:
-
-   Configure Redis to discard data when it runs out of memory, not save its
-   database and only keep 2 database:
-
-   .. code-block:: bash
-
-       sudo echo "maxmemory-policy allkeys-lru" >> /etc/redis/redis.conf
-       sudo echo "save \"\"" >> /etc/redis/redis.conf
-       sudo echo "databases 2" >> /etc/redis/redis.conf
-       sudo systemctl restart redis
-
-#. Enable and restart the services [1_]:
-
-   .. code-block:: bash
-
-       sudo systemctl enable supervisor
-       sudo systemctl restart supervisor
-
-
-#. Cleaning up:
-
-   The following operating system dependencies are only needed during
-   installation and can be removed.
-
-   .. code-block:: bash
-
-       sudo apt-get remove --purge libjpeg-dev libpq-dev libpng-dev libtiff-dev zlib1g-dev
-
-
-.. _deployment_advanced:
-
-Advanced deployment
-===================
-
-This variation uses RabbitMQ as the message broker. RabbitMQ consumes more
-memory but scales to thousands of messages per second. RabbitMQ messages are also
-persistent by default, this means that pending tasks are not lost in the case
-of a restart or power failure. The Gunicorn workers are increased to 3.
-
-
-#. Install RabbitMQ:
-
-   If using a Debian_ or Ubuntu_ based Linux distribution, get the executable
-   requirements using:
-
-   .. code-block:: bash
-
-       sudo apt-get install rabbitmq-server -y
-
-
-#. Install the Python client for RabbitMQ:
-
-   .. code-block:: bash
-
-       sudo -u mayan |MAYAN_PIP_BIN| install --no-use-pep517 amqp==|PYTHON_AMQP_VERSION|
-
-
-#. Create the RabbitMQ user and vhost:
-
-   .. code-block:: bash
-
-       sudo rabbitmqctl add_user mayan mayanrabbitmqpassword
-       sudo rabbitmqctl add_vhost mayan
-       sudo rabbitmqctl set_permissions -p mayan mayan ".*" ".*" ".*"
-
-
-#. Edit the supervisor file at ``|MAYAN_SUPERVISOR_CONF|``:
-
-   Replace (paying attention to the comma at the end):
-
-   .. code-block:: ini
-
-       MAYAN_CELERY_BROKER_URL="redis://127.0.0.1:6379/0",
-
-   with:
-
-   .. code-block:: ini
-
-       MAYAN_CELERY_BROKER_URL="amqp://mayan:mayanrabbitmqpassword@localhost:5672/mayan",
-
-   increase the number of Gunicorn workers to 3 in the line (``-w 2`` section):
-
-   .. code-block:: ini
-
-       command = |MAYAN_GUNICORN_BIN| -w 2 mayan.wsgi --max-requests 1000 --max-requests-jitter 50 --worker-class gevent --bind 0.0.0.0:8000 --timeout 120
-
-   remove the concurrency limit (or increase it) of the fast worker (remove ``--concurrency=1``).
-
-
-#. Restart the services:
-
-   .. code-block:: bash
-
-       sudo supervisorctl reread
-       sudo supervisorctl restart all
-
-
-
-[1]: https://bugs.launchpad.net/ubuntu/+source/supervisor/+bug/1594740
-
-.. _Debian: https://www.debian.org/
-.. _Django: https://www.djangoproject.com/
-.. _Python: https://www.python.org/
-.. _SQLite: https://www.sqlite.org/
-.. _Ubuntu: http://www.ubuntu.com/
-.. _virtualenv: http://www.virtualenv.org/en/latest/index.html
-.. _1: https://bugs.launchpad.net/ubuntu/+source/supervisor/+bug/1594740

docs/chapters/development/code_structure.txt

@@ -1,243 +0,0 @@
-==============
-Code structure
-==============
-
-
-Follow PEP8
------------
-
-Whenever possible, but don't obsess over things like line length:
-
-.. code-block:: bash
-
-    $ flake8 --ignore=E501,E128,E122 |less
-
-To perform automatic PEP8 checks, install flake8's git hook using:
-
-.. code-block:: bash
-
-    $ flake8 --install-hook git
-
-
-Imports
--------
-
-Import order should be:
-
-- Standard Python modules
-- Installed Python modules
-- Core Django modules
-- Installed Django modules
-- Mayan EDMS modules
-- Local imports
-
-Example:
-
-.. code-block:: bash
-
-    from __future__ import absolute_import
-
-    # Standard Python library
-    import base64
-
-    # 3rd party installed Python libraries
-    import requests
-
-    # Django core modules
-    from django.db.models import Q
-    from django.template.defaultfilters import slugify
-    from django.utils.translation import ugettext
-    from django.utils.translation import ugettext_lazy as _
-
-    # 3rd party installed Django libraries
-    from rest_framework import APIView
-
-    # Mayan apps
-    from metadata.classes import MetadataClass
-
-    # Local app imports (relative)
-    from .conf.settings import (
-        AVAILABLE_INDEXING_FUNCTIONS,
-        MAX_SUFFIX_COUNT, SLUGIFY_PATHS
-    )
-    from .exceptions import MaxSuffixCountReached
-    from .filesystem import (
-        fs_create_index_directory, fs_create_document_link,
-        fs_delete_document_link, fs_delete_index_directory,
-        assemble_suffixed_filename
-    )
-    from .models import Index, IndexInstanceNode, DocumentRenameCount
-
-All local app module imports are in relative form. Local app module name is to
-be referenced as little as possible, unless required by a specific feature,
-trick, restriction (e.g., Runtime modification of the module's attributes).
-
-Incorrect:
-
-.. code-block:: bash
-
-
-    # documents app views.py model
-    from documents.models import Document
-
-Correct:
-
-.. code-block:: bash
-
-    # documents app views.py model
-    from .models import Document
-
-
-Dependencies
-------------
-
-Mayan EDMS apps follow a hierarchical model of dependency. Apps import from
-their parents or siblings, never from their children. Think plugins. A parent
-app must never assume anything about a possible existing child app. The
-documents app and the Document model are the basic entities; they must never
-import anything else. The common and main apps are the base apps.
-
-
-Variables
----------
-
-Naming of variables should follow a Major to Minor convention, usually
-including the purpose of the variable as the first piece of the name, using
-underscores as spaces. camelCase is not used in Mayan EDMS.
-
-Examples:
-
-Links:
-
-.. code-block:: bash
-
-    link_document_page_transformation_list = ...
-    link_document_page_transformation_create = ...
-    link_document_page_transformation_edit = ...
-    link_document_page_transformation_delete = ...
-
-Constants:
-
-.. code-block:: bash
-
-    PERMISSION_SMART_LINK_VIEW = ...
-    PERMISSION_SMART_LINK_CREATE = ...
-    PERMISSION_SMART_LINK_DELETE = ...
-    PERMISSION_SMART_LINK_EDIT = ...
-
-Classes:
-
-.. code-block:: bash
-
-    class Document(models.Model):
-    class DocumentPage(models.Model):
-    class DocumentPageTransformation(models.Model):
-    class DocumentType(models.Model):
-    class DocumentTypeFilename(models.Model):
-
-
-Strings
--------
-
-Quotation character used in Mayan EDMS for strings is the single quote.
-Double quote is used for multiple line comments or HTML markup.
-
-
-Migrations
-----------
-
-Migrations should do only one thing (example: either create a table, move data
-to a new table or remove an old table) to aid retrying on failure.
-
-
-General
--------
-
-Code should appear in their modules in alphabetic order or in their order of
-importance if it makes more sense for the specific application. This makes
-visual scanning easier on modules with a large number of imports, views or
-classes. Class methods that return a value should be pretended with a
-``get_`` to differentiate from an object’s properties. When a variable refers
-to a file it should be named as follows:
-
-- filename:  The file’s name and extension only.
-- filepath:  The entire path to the file including the filename.
-- path:  A path to a directory.
-
-Flash messages should end with a period as applicable for the language.
-Only exception is when the tail of the message contains an exceptions message
-as passed directly from the exception object.
-
-
-Steps to deploy a development version
-=====================================
-
-.. code-block:: bash
-
-    $ git clone |SOURCE_CODE_GIT|
-    $ cd mayan-edms
-    $ git checkout <Corresponding branch>
-    $ virtualenv venv
-    $ source venv/bin/activate
-    $ pip install -r requirements.txt
-    $ ./manage.py initialsetup
-    $ ./manage.py preparestatic
-    $ ./manage.py runserver
-
-
-
-Debugging
-=========
-
-Mayan EDMS makes extensive use of Django's new
-:django-docs:`logging capabilities <topics/logging>`.
-
-By default debug logging for all apps is turned on. If you wish to customize
-how logging is managed turn off automatic logging by setting
-`COMMON_AUTO_LOGGING` to ``False`` and add the following lines to your
-``settings/local.py`` file::
-
-    LOGGING = {
-        'version': 1,
-        'disable_existing_loggers': True,
-        'formatters': {
-            'verbose': {
-                'format': '%(levelname)s %(asctime)s %(name)s %(process)d %(thread)d %(message)s'
-            },
-            'intermediate': {
-                'format': '%(name)s <%(process)d> [%(levelname)s] "%(funcName)s() %(message)s"'
-            },
-            'simple': {
-                'format': '%(levelname)s %(message)s'
-            },
-        },
-        'handlers': {
-            'console':{
-                'level':'DEBUG',
-                'class':'logging.StreamHandler',
-                'formatter': 'intermediate'
-            }
-        },
-        'loggers': {
-            'documents': {
-                'handlers':['console'],
-                'propagate': True,
-                'level':'DEBUG',
-            },
-            'common': {
-                'handlers':['console'],
-                'propagate': True,
-                'level':'DEBUG',
-            },
-        }
-    }
-
-
-Likewise, to see the debug output of the ``tags`` app, just add the following inside the ``loggers`` block::
-
-
-    'tags': {
-        'handlers':['console'],
-        'propagate': True,
-        'level':'DEBUG',
-    },

docs/chapters/development/contributing.txt

@@ -1,18 +0,0 @@
-============
-Contributing
-============
-
-Mayan EDMS is under active development, and contributions are welcome.
-
-If you have a feature request, suggestion or bug report, please open a new
-issue on the `GitLab issue tracker`_. To submit patches, please send a merge
-request on GitLab_.
-
-.. _GitLab: |SOURCE_CODE_REPOSITORY|
-.. _`GitLab issue tracker`: |SOURCE_CODE_ISSUES|
-
-
-Contributing changes
-====================
-Follow the latest contributing guidelines outlined here:
-|SOURCE_CODE_REPOSITORY|blob/master/CONTRIBUTING.md

docs/chapters/development/documentation.txt

@@ -1,24 +0,0 @@
-=============
-Documentation
-=============
-
-The documentation is written in `reStructured Text`_ format, processed with
-Sphinx_, and resides in the ``docs`` directory. In order to build it, you will
-first need to install the documentation editing dependencies with::
-
-    $ pip install -r requirements/documentation.txt
-
-Then, to build an HTML version of the documentation, run the following command
-from the **docs** directory::
-
-    $ make docs-serve
-
-The generated documentation can be viewed by browsing to http://127.0.0.1:8000
-or by browsing to the ``docs/_build/html`` directory.
-
-You can also generate the documentation in formats other than HTML. Consult the
-Sphinx_ documentation for more details.
-
-.. _`reStructured Text`: http://docutils.sourceforge.net/rst.html
-.. _Sphinx: http://sphinx.pocoo.org
-

docs/chapters/development/index.txt

@@ -1,29 +0,0 @@
-==============
-For developers
-==============
-
-- :doc:`philosophies`
-- :doc:`code_structure`
-- :doc:`documentation`
-- :doc:`source_control`
-- :doc:`contributing`
-- :doc:`releases`
-- :doc:`translations`
-- :doc:`../../mercs/index`
-- :doc:`caa_entity`
-- :doc:`caa_individual`
-
-
-.. toctree::
-    :hidden:
-
-    philosophies
-    code_structure
-    documentation
-    source_control
-    contributing
-    releases
-    translations
-    ../../mercs/index
-    caa_entity
-    caa_individual

docs/chapters/development/philosophies.txt

@@ -1,29 +0,0 @@
-====================
-Project philosophies
-====================
-
-How to think about Mayan EDMS when doing changes or adding new features;
-why things are the way they are in Mayan EDMS:
-
-- Functionality must be as market/sector independent as possible, code for the
-  95% of use cases.
-- Each user must be able to configure and customize it to their needs after
-  install.
-- Abstract as much as possible, each app must be an expert in just one thing,
-  for other things they should use the API/classes/functions of other apps.
-- Assume as little as possible about anything outside the project
-  (hardware, OS, storage).
-- Provide Python based abstraction so that a default install runs with a single
-  step.
-- No hard dependencies on binaries unless there is no other choice.
-- Provide “drivers” or switchable backends to allow users to fine tune the
-  installation.
-- Call to binaries only when there is no other choice or the Python choices are
-  not viable/mature/efficient.
-- Each app is as independent and self contained as possible. Exceptions, the
-  basic requirements: navigation, permissions, common, main.
-- If an app is meant to be used by more than one other app, it should be as
-  generic as possible in regard to the project and another app will bridge the functionality.
-
-  - Example: since indexing (document_indexing) only applies to documents, the
-    app is specialized and depends on the documents app.

docs/chapters/development/releases.txt

@@ -1,173 +0,0 @@
-===================
-Installable package
-===================
-
-Source file package
--------------------
-
-This is the sequence of step used to produce an installable package:
-
-1. Generate the packaged version (will produce dist/mayan-edms-x.y.z.tar.gz)::
-
-    $ make sdist
-
-2. Do a test install::
-
-    $ cd /tmp
-    $ virtualenv venv
-    $ source venv/bin/activate
-    $ pip install <path of the Git repository>/dist/mayan-edms-x.y.z.tar.gz
-    $ mayan-edms.py initialsetup
-    $ mayan-edms.py preparestatic
-    $ mayan-edms.py runserver
-
-
-Wheel package
--------------
-
-1. Install the development requirements::
-
-    $ pip install -r requirements/development.txt
-
-2. Create wheel package using the makefile::
-
-    $ make wheel
-
-3. Do a test install::
-
-    $ cd /tmp
-    $ virtualenv venv
-    $ source venv/bin/activate
-    $ pip install <path of the Git repository>/dist/mayan_edms-x.y.z-py2-none-any.whl
-    $ mayan-edms.py initialsetup
-    $ mayan-edms.py preparestatic
-    $ mayan-edms.py runserver
-
-
-Version numbering
-=================
-
-Mayan EDMS uses the Semantic Versioning (http://semver.org/) method to choose
-version numbers along with Python's PEP-0440 (https://www.python.org/dev/peps/pep-0440/)
-to format them.
-
-X.YaN   # Alpha release
-X.YbN   # Beta release
-X.YrcN  # Release Candidate
-X.Y     # Final release
-
-
-Release checklist
-=================
-
-#. Check for missing migrations::
-
-    make check-missing-migrations
-
-#. Synchronize translations::
-
-    make translations-pull
-
-#. Compile translations::
-
-    make translations-compile
-
-#. Update changelog.
-#. Write release notes.
-#. Scan the code with flake8 for simple style warnings.
-#. Check README.rst format with::
-
-    python setup.py check -r -s
-
-   or with::
-
-       make check-readme
-
-#. Bump version in ``mayan/__init__.py`` and ``docker/rootfs/version``::
-
-    make increase-version PART=<major, minor or micro>
-
-#. Update requirements version in ``setup.py`` using:
-   ::
-
-       make generate-setup
-
-#. Commit as version bump.
-#. Build source package and test:
-   ::
-
-       make test-sdist-via-docker-ubuntu
-
-#. Build wheel package and test:
-   ::
-
-       make test-wheel-via-docker-ubuntu
-
-#. Tag version:
-   ::
-
-       git tag -a vX.Y.Z -m "Version X.Y.Z"
-
-#. Generate set ``setup.py`` again to update the build number::
-
-    make generate-setup
-
-#. Commit the new ``setup.py`` file.
-
-#. Release the version using one of the two following methods: GitLab CI or
-   manual
-
-
-Release using GitLab CI using the makefile
-------------------------------------------
-
-#. Use of the GitLab makefile targets: ``gitlab-release-all``,
-   ``gitlab-release-docker``, ``gitlab-release-documentation``,
-   ``gitlab-release-python``.
-
-
-Release using GitLab CI manually
---------------------------------
-
-#. Push the current brach:
-   ::
-
-       git push
-
-
-#. Push the new tags:
-   ::
-
-       git push --tags
-
-
-#. Delete the corresponding ``releases/`` branch:
-   ::
-
-       git push origin :releases/<branch>
-
-#. Push the current branch to the corresponding origin ``releases/`` branch:
-   ::
-
-       git push origin <local branch>:releases/<branch>
-
-
-Manual release
---------------
-
-#. Build and upload a test release:
-   ::
-
-       make release-test-via-docker-ubuntu
-
-#. Build and upload a final release:
-   ::
-
-       make release-via-docker-ubuntu
-
-Other steps
------------
-
-#. Update the contrib/scripts/install/docker.sh values
-
-#. Upload contrib/scripts/install/docker.sh to https://get.mayan-edms.com

docs/chapters/development/source_control.txt

@@ -1,99 +0,0 @@
-==============
-Source Control
-==============
-
-Mayan EDMS source is controlled with Git_.
-
-The project is publicly accessible, hosted and can be cloned from **GitLab** using::
-
-    $ git clone |SOURCE_CODE_GIT|
-
-
-Git branch structure
---------------------
-
-Mayan EDMS follows a simplified model layout based on Vincent Driessen's
-`Successful Git Branching Model`_ blog post.
-
-``/versions/micro``
-    Working branch for the next bugfix release. Micro increment (third digit).
-    Only bug fixes, minor features, back-ported urgent features. This
-    branch is stable and safe for production.
-``/versions/minor``
-    Working branch for the next minor release (second digit). New features,
-    occasional breakage. Not for production but should run in test
-    environment most of the time. This is the branch you will want to
-    try out if you want to check out new features.
-``/versions/major``
-    Working branch for the next major release (first digit). New features,
-    incompatible changes to the user facing interfaces. Broken most of the
-    time, not for production and should only be cloned by developers
-    with experience with Mayan's development.
-``master``
-    Current production release (|version|). Points to the latest version of
-    the latest series. Production quality code.
-``features/``
-    Working branches for unfinished and unmerged feature. Likely unstable,
-    don't use in production. Once the feature is complete, it is merged
-    into one of the versions branches and deleted.
-
-Special branches:
-
-``releases/all``
-    Pushing code to this branch will trigger the build and release
-    a new Docker image, Documentation and Python package.
-``releases/docker``
-    Pushing code to this branch will trigger the build and release
-    of a new Docker image to Docker Hub.
-``releases/documentation``
-    Pushing code to this branch will trigger the build and release
-    of new documentation.
-``releases/python``
-    Pushing code to this branch will trigger the build and release
-    of a new Python package to PyPI.
-``nightly``
-    Pushing code to this branch will trigger the build and release
-    of a new Docker image based on development code to the GitLab image
-    repository only. The image will not be published to Docker Hub.
-
-Each release is tagged separately using annotated Git tags.
-
-When submitting patches, please place your code in its own ``feature/`` branch
-prior to opening a Merge Request on |SOURCE_CODE_ISSUES|.
-
-.. _Git: http://git-scm.org
-.. _`Successful Git Branching Model`: http://nvie.com/posts/a-successful-git-branching-model/
-
-
-Commit messages
----------------
-
-#. Use English as the language for the commit messages.
-#. Provide a subject line composed of a tag and a short explanation::
-
-    Indexing: Add document base property reindex
-
-#. Keep the subject line to 50 or less characters.
-#. Capitalize the subject line.
-#. Don't end the subject line with a period, leave like a phrase in English.
-#. Use active voice in the. Say what the commit will do when applied not what
-   you did::
-
-       Add new properties to the model.
-
-   Vs.
-   ::
-
-       Added new properties to the model.
-
-#. Limit the body of the commit to 72 characters.
-#. When a commit fixes or improves an issue add the issue number in the commit
-   message. Either in the subject or in the body.
-#. Sign commit messages.
-#. Use explicit language even for minor commits. Don't do::
-
-       Fix typo
-
-   Use::
-
-       Document: Fix typo in label description

docs/chapters/docker.rst

@@ -0,0 +1,291 @@
+************
+Docker image
+************
+
+
+Stopping and starting the container
+===================================
+
+To stop the container use::
+
+    docker stop mayan-edms
+
+
+To start the container again::
+
+    docker start mayan-edms
+
+
+.. _docker_environment_variables:
+
+Environment Variables
+=====================
+
+In addition to the all the environment variables supported by Mayan EDMS, the
+Mayan EDMS image provides some additional variables to configure the Docker
+specifics of the image.
+
+``MAYAN_DATABASE_ENGINE``
+
+Defaults to ``None``. This environment variable configures the database
+backend to use. If left unset, SQLite will be used. The database backends
+supported by this Docker image are:
+
+- ``'django.db.backends.postgresql'``
+- ``'django.db.backends.mysql'``
+- ``'django.db.backends.sqlite3'``
+
+When using the SQLite backend, the database file will be saved in the Docker
+volume. The SQLite database as used by Mayan EDMS is meant only for development
+or testing, never use it in production.
+
+``MAYAN_DATABASE_HOST``
+
+Defaults to `None`. This optional environment variable is used to set the
+hostname that will be used to connect to the database. This can be the
+hostname of another container or an IP address. For more information read
+the pertinent Django documentation page:
+:django-docs:`Settings, HOST <ref/settings/#host>`
+
+``MAYAN_DATABASE_PORT``
+
+Defaults to `None`. This optional environment variable is used to set the
+port number to use when connecting to the database. An empty string means
+the default port. Not used with SQLite. For more information read the
+pertinent Django documentation page:
+:django-docs:`Settings, PORT <ref/settings/#port>`
+
+``MAYAN_CELERY_BROKER_URL``
+
+This optional environment variable determines the broker that Celery will use
+to relay task messages between the frontend code and the background workers.
+For more information read the pertinent Celery Kombu documentation page: `Broker URL`_
+
+.. _Broker URL: http://kombu.readthedocs.io/en/latest/userguide/connections.html#connection-urls
+
+This Docker image supports using Redis and RabbitMQ as brokers.
+
+Caveat: If the `MAYAN_CELERY_BROKER_URL` and `MAYAN_CELERY_RESULT_BACKEND` environment
+variables are specified, the built-in Redis server inside the container will
+be disabled.
+
+``MAYAN_CELERY_RESULT_BACKEND``
+
+This optional environment variable determines the results backend that Celery
+will use to relay result messages from the background workers to the frontend
+code. For more information read the pertinent Celery Kombu documentation page:
+`Task result backend settings`_
+
+.. _Task result backend settings: http://docs.celeryproject.org/en/3.1/configuration.html#celery-result-backend
+
+This Docker image supports using Redis and RabbitMQ as result backends.
+
+Caveat: If the `MAYAN_CELERY_BROKER_URL` and `MAYAN_CELERY_RESULT_BACKEND` environment
+variables are specified, the built-in Redis server inside the container will
+be disabled.
+
+``MAYAN_SETTINGS_MODULE``
+
+Optional. Allows loading an alternate settings file.
+
+``MAYAN_GUNICORN_WORKERS``
+
+Optional. This environment variable controls the number of frontend workers
+that will be executed. If not specified the default is 2. For heavier loads,
+user a higher number. A formula recommended for this setting is the number
+of CPU cores + 1.
+
+``MAYAN_WORKER_FAST_CONCURRENCY``
+
+Optional. Changes the concurrency (number of child processes) of the Celery
+worker consuming the queues in the fast (low latency, short tasks) category.
+Default is 1. Use 0 to disable hardcoded concurrency and allow the Celery
+worker to launch its default number of child processes (equal to the number
+of CPUs detected).
+
+``MAYAN_WORKER_MEDIUM_CONCURRENCY``
+
+Optional. Changes the concurrency (number of child processes) of the Celery
+worker consuming the queues in the medium (medium latency, long running tasks)
+category. Default is 1. Use 0 to disable hardcoded concurrency and allow the
+Celery worker to launch its default number of child processes (equal to the
+number of CPUs detected).
+
+``MAYAN_WORKER_SLOW_CONCURRENCY``
+
+Optional. Changes the concurrency (number of child processes) of the Celery
+worker consuming the queues in the slow (high latency, very long running tasks)
+category. Default is 1. Use 0 to disable hardcoded concurrency and allow the
+Celery worker to launch its default number of child processes (equal to the
+number of CPUs detected).
+
+
+Accessing outside data
+======================
+
+To use Mayan EDMS's staging folders or watch folders from Docker, the data
+for these source must be made accessible to the container. This is done by
+mounting the folders in the host computer to folders inside the container.
+This is necessary because Docker containers do not have access to host data
+on purpose. For example, to make a folder in the host accessible as a watch
+folder, add the following to the Docker command line when starting the
+container::
+
+    -v /opt/scanned_files:/srv/watch_folder
+
+The command line would look like this::
+
+    docker run ... -v /opt/scanned_files:/srv/watch_folder mayanedms/mayanedms:latest
+
+Now create a watch folder in Mayan EDMS using the path ``/srv/watch_folder``
+and the documents from the host folder ``/opt/scanned_files`` will be
+automatically available. Use the same procedure to mount host folders to be
+used as staging folderes. In this example ``/srv/watch_folder`` was as the
+container directory, but any path can be used as long as it is not an
+already existing path or a path used by any other program.
+
+
+Performing backups
+==================
+
+To backup the existing data, stop the image and copy the content of the volume.
+For the example::
+
+    docker run -d --name mayan-edms --restart=always -p 80:8000 \
+    -v /docker-volumes/mayan:/var/lib/mayan \
+    -v /opt/scanned_files:/srv/watch_folder mayanedms/mayanedms:latest
+
+That would be the ``/docker-volumes/mayan folder``::
+
+    sudo tar -zcvf backup.tar.gz /docker-volumes/mayan
+    sudo chown `whoami` backup.tar.gz
+
+If using an external PostgreSQL or MySQL database or database containers, these
+too need to be backed up using their respective procedures. A simple solution
+is to copy the entire database container volume after the container has
+been stopped.
+
+
+Restoring from a backup
+=======================
+
+Uncompress the backup archive in the original docker volume using::
+
+    sudo tar -xvzf backup.tar.gz -C /
+
+
+Upgrading
+=========
+
+Upgrading a Mayan EDMS Docker container is actually a matter of stopping and
+deleting the container, downloading the most recent version of the image and
+starting a container again. The container will take care of updating the
+database structure to the newest version if necessary.
+
+**IMPORTANT!** Do not delete the volume storing the data, only the container.
+
+Stop the container to be upgraded::
+
+    docker stop mayan-edms
+
+
+Remove the container::
+
+    docker rm mayan-edms
+
+
+Pull the new image version::
+
+    docker pull mayanedms/mayanedms:latest
+
+
+Start the container again with the new image version::
+
+    docker run -d --name mayan-edms --restart=always -p 80:8000 -v /docker-volumes/mayan:/var/lib/mayan mayanedms/mayanedms:latest
+
+
+Building the image
+==================
+
+Clone the repository with::
+
+    git clone https://gitlab.com/mayan-edms/mayan-edms.git
+
+Change to the directory of the cloned repository::
+
+    cd mayan-edms
+
+Execute Docker's build command using the provided makefile::
+
+    make docker-build
+
+Or using an apt cacher to speed up the build::
+
+    make docker-build-with-proxy APT_PROXY=172.17.0.1:3142
+
+Replace the IP address `172.17.0.1` with the IP address of the computer
+running the APT proxy and caching service.
+
+
+Customizing the image
+=====================
+
+
+Simple method
+-------------
+
+If you just need to add a few Ubuntu or Python packages to your installation,
+you can use the following environment variables:
+
+``MAYAN_APT_INSTALLS``
+
+Specifies a list of Ubuntu .deb packages to be installed via APT when the
+container is first created. The installed packages are not lost when the image
+is stopped. Example: To install the Tesseract OCR language packs for German
+and Spanish add the following in your ``docker start`` command line::
+
+    -e MAYAN_APT_INSTALLS="tesseract-ocr-deu tesseract-ocr-spa"
+
+``MAYAN_PIP_INSTALLS``
+
+Specifies a list of Python packages to be installed via ``pip``. Packages will
+be downloaded from the Python Package Index (https://pypi.python.org) by
+default.
+
+
+Using Docker compose
+====================
+
+To deploy a complete production stack using the included Docker compose file
+execute::
+
+    docker-compose -f docker-compose.yml up -d
+
+This Docker compose file will provision four containers:
+
+- Postgres as the database
+- Redis as the Celery result storage
+- RabbitMQ as the Celery broker
+- Mayan EDMS using the above service containers
+
+To stop the stack use::
+
+    docker-compose -f docker-compose.yml stop
+
+The stack will also create four volumes to store the data of each container.
+These are:
+
+- mayan_app - The Mayan EDMS data container, normally called `mayan_data` when not using Docker compose.
+- mayan_broker - The broker volume, in this case RabbitMQ.
+- mayan_db - The database volume, in this case Postgres.
+- mayan_results - The celery result backend volume, in this case Redis.
+
+
+Nightly images
+==============
+The continuous integration pipeline used for testing development builds also
+produces a resulting Docker image. These are build automatically and their
+stability is not guaranteed. They should never be used in production.
+If you want to try out the Docker images the development uses or want a sneak
+peek at the new features being worked on checkout the container registry at:
+https://gitlab.com/mayan-edms/mayan-edms/container_registry

docs/chapters/docker/accessing_data.txt

@@ -1,30 +0,0 @@
-.. _docker-accessing-outside-data:
-
-
-======================
-Accessing outside data
-======================
-
-To use Mayan EDMS's staging folders or watch folders from Docker, the data
-for these source must be made accessible to the container. This is done by
-mounting the folders in the host computer to folders inside the container.
-This is necessary because Docker containers do not have access to host data
-on purpose. For example, to make a folder in the host accessible as a watch
-folder, add the following to the Docker command line when starting the
-container::
-
-    -v /opt/scanned_files:/scanned_files
-
-The command line would look like this::
-
-    docker run ... -v /opt/scanned_files:/scanned_files mayanedms/mayanedms:latest
-
-Now create a watch folder in Mayan EDMS using the path ``/scanned_files``
-and the documents from the host folder ``/opt/scanned_files`` will be
-automatically available. Use the same procedure to mount host folders to be
-used as staging folders. In this example ``/scanned_files`` was used as the
-container directory, but any path can be used as long as:
-
-- the path not an already existing path
-- the path is not used by any other program
-- the path is a single level path

docs/chapters/docker/backups.txt

@@ -1,47 +0,0 @@
-.. _docker_backups:
-
-==================
-Performing backups
-==================
-
-#. Document files
-
-   To backup the existing data, stop the image and copy the content of the volume.
-   For the example::
-
-       docker run -d --name mayan-edms --restart=always -p 80:8000 \
-       -v /docker-volumes/mayan:/var/lib/mayan \
-       -v /opt/scanned_files:/scanned_files mayanedms/mayanedms:|DOCKER_MAYAN_IMAGE_VERSION|
-
-   That would be the ``/docker-volumes/mayan folder``::
-
-       sudo tar -zcvf backup.tar.gz /docker-volumes/mayan
-       sudo chown `whoami` backup.tar.gz
-
-#. Database
-
-   If using an external PostgreSQL or MySQL database or database containers, these
-   too need to be backed up using their respective procedures. A simple solution
-   is to copy the entire database container volume after the container has
-   been stopped.
-
-   - PostgreSQL: https://www.postgresql.org/docs/current/backup-dump.html
-   - MySQL: https://dev.mysql.com/doc/refman/5.7/en/mysqldump.html
-
-
-=======================
-Restoring from a backup
-=======================
-
-
-#. Document files
-
-   Uncompress the backup archive in the original docker volume using::
-
-       sudo tar -xvzf backup.tar.gz -C /
-
-
-#. Database
-
-   - PostgreSQL: https://www.postgresql.org/docs/current/backup-dump.html
-   - MySQL: https://dev.mysql.com/doc/refman/5.7/en/mysqldump.html

docs/chapters/docker/building.txt

@@ -1,25 +0,0 @@
-.. _docker_building:
-
-==================
-Building the image
-==================
-
-Clone the repository with::
-
-    git clone |SOURCE_CODE_GIT|
-
-Change to the directory of the cloned repository::
-
-    cd mayan-edms
-
-Execute Docker's build command using the provided makefile::
-
-    make docker-build
-
-Or using an APT cache to speed up the build::
-
-    make docker-build-with-proxy APT_PROXY=172.17.0.1:3142
-
-Replace the IP address `172.17.0.1` with the IP address of the computer
-running the APT proxy and caching service.
-

docs/chapters/docker/commands.txt

@@ -1,10 +0,0 @@
-.. _docker_commands:
-
-=================================
-Executing commands on a container
-=================================
-
-To perform Mayan EDMS commands in a running container that will take all the
-settings from the container, use the following command::
-
-    docker exec -ti <container name> /usr/local/bin/entrypoint.sh "/opt/mayan-edms/bin/mayan-edms.py <command>"

docs/chapters/docker/customizing.txt

@@ -1,21 +0,0 @@
-=====================
-Customizing the image
-=====================
-
-If you just need to add a few Ubuntu or Python packages to your installation,
-you can use the following environment variables:
-
-``MAYAN_APT_INSTALLS``
-
-Specifies a list of Ubuntu .deb packages to be installed via APT when the
-container is first created. The installed packages are not lost when the image
-is stopped. Example: To install the Tesseract OCR language packs for German
-and Spanish add the following in your ``docker start`` command line::
-
-    -e MAYAN_APT_INSTALLS="tesseract-ocr-deu tesseract-ocr-spa"
-
-``MAYAN_PIP_INSTALLS``
-
-Specifies a list of Python packages to be installed via ``pip``. Packages will
-be downloaded from the Python Package Index (https://pypi.python.org) by
-default.

docs/chapters/docker/docker_compose.txt

@@ -1,30 +0,0 @@
-.. _docker_compose:
-
-====================
-Using Docker compose
-====================
-
-To deploy a complete production stack using the included Docker compose file
-execute::
-
-    docker-compose -f docker-compose.yml up -d
-
-This Docker compose file will provision four containers:
-
-- Postgres as the database
-- Redis as the Celery result storage
-- RabbitMQ as the Celery broker
-- Mayan EDMS using the above service containers
-
-To stop the stack use::
-
-    docker-compose -f docker-compose.yml stop
-
-The stack will also create four volumes to store the data of each container.
-These are:
-
-- mayan_app - The Mayan EDMS data container, normally called `mayan_data` when not using Docker compose.
-- mayan_broker - The broker volume, in this case RabbitMQ.
-- mayan_db - The database volume, in this case Postgres.
-- mayan_results - The celery result backend volume, in this case Redis.
-

docs/chapters/docker/drivers.txt

@@ -1,12 +0,0 @@
-.. _docker_image_drivers:
-
-================
-Included drivers
-================
-
-The Docker image supports using Redis and RabbitMQ as result backends. For
-databases, the image includes support for PostgreSQL and MySQL/MariaDB.
-Support for additional brokers or databases may be added using the
-``MAYAN_APT_INSTALL`` environment variable.
-
-

docs/chapters/docker/environment_variables.txt

@@ -1,63 +0,0 @@
-.. index:: Docker environment variables
-.. _docker_environment_variables:
-
-
-=====================
-Environment Variables
-=====================
-
-The common set of settings can also be modified via environment variables when
-using the Docker image. In addition to the common set of settings, some Docker
-image specific environment variables are available.
-
-``MAYAN_SETTINGS_MODULE``
-
-Optional. Allows loading an alternate settings file.
-
-
-``MAYAN_GUNICORN_TIMEOUT``
-
-Optional. Changes the amount of time the frontend worker will wait for a
-request to finish before raising a timeout error. The default is 120
-seconds.
-
-``MAYAN_GUNICORN_WORKERS``
-
-Optional. This environment variable controls the number of frontend workers
-that will be executed. If not specified the default is 2. For heavier loads,
-user a higher number. A formula recommended for this setting is the number
-of CPU cores + 1.
-
-``MAYAN_WORKER_FAST_CONCURRENCY``
-
-Optional. Changes the concurrency (number of child processes) of the Celery
-worker consuming the queues in the fast (low latency, short tasks) category.
-Default is 1. Use 0 to disable hardcoded concurrency and allow the Celery
-worker to launch its default number of child processes (equal to the number
-of CPUs detected).
-
-``MAYAN_WORKER_MEDIUM_CONCURRENCY``
-
-Optional. Changes the concurrency (number of child processes) of the Celery
-worker consuming the queues in the medium (medium latency, long running tasks)
-category. Default is 1. Use 0 to disable hardcoded concurrency and allow the
-Celery worker to launch its default number of child processes (equal to the
-number of CPUs detected).
-
-``MAYAN_WORKER_SLOW_CONCURRENCY``
-
-Optional. Changes the concurrency (number of child processes) of the Celery
-worker consuming the queues in the slow (high latency, very long running tasks)
-category. Default is 1. Use 0 to disable hardcoded concurrency and allow the
-Celery worker to launch its default number of child processes (equal to the
-number of CPUs detected).
-
-``MAYAN_USER_UID``
-
-Optional. Changes the UID of the ``mayan`` user internal to the Docker
-container. Defaults to 1000.
-
-``MAYAN_USER_GID``
-
-Optional. Changes the GID of the ``mayan`` user internal to the Docker
-container. Defaults to 1000.

docs/chapters/docker/index.txt

@@ -1,35 +0,0 @@
-.. index:: Docker
-
-======
-Docker
-======
-
-Docker is a container technology. Containers are a standard unit of software
-that packages up code and all its dependencies.
-
-
-- Installation: :doc:`install_simple` | :doc:`install_docker_network` | :doc:`docker_compose` | :doc:`start_stop`
-- Setup: :doc:`environment_variables` | :doc:`drivers`
-- Integration: :doc:`accessing_data`
-- Development: :doc:`building` | :doc:`customizing` | :doc:`nightly`
-- Administration: :doc:`backups` | :doc:`upgrading` | :doc:`commands`
-
-
-.. toctree::
-    :hidden:
-
-    install_simple
-    install_docker_network
-    start_stop
-    environment_variables
-    drivers
-    accessing_data
-    backups
-    upgrading
-    building
-    customizing
-    docker_compose
-    nightly
-    commands
-
-

docs/chapters/docker/install_docker_network.txt

@@ -1,67 +0,0 @@
-.. _docker_install_networked:
-
-================================
-Using a dedicated Docker network
-================================
-
-Use this method to avoid having to expose PostreSQL port to the host's network
-or if you have other PostgreSQL instances but still want to use the default
-port of 5432 for this installation.
-
-#. Create the network:
-
-   .. code-block:: bash
-
-       docker network create mayan
-
-#. Launch the PostgreSQL container with the network option and remove the
-   port binding (``-p 5432:5432``):
-
-   .. code-block:: bash
-
-       docker run \
-       -d \
-       --name mayan-edms-postgres \
-       --network=mayan \
-       --restart=always \
-       -e POSTGRES_USER=mayan \
-       -e POSTGRES_DB=mayan \
-       -e POSTGRES_PASSWORD=mayanuserpass \
-       -v /docker-volumes/mayan-edms/postgres:/var/lib/postgresql/data \
-       |DOCKER_POSTGRES_IMAGE_VERSION|
-
-#. Launch the Redis container with the network option and remove the port
-   binding (``-p 6379:6379``):
-
-   .. code-block:: bash
-
-       docker run \
-       -d \
-       --name mayan-edms-redis \
-       --network=mayan \
-       |DOCKER_REDIS_IMAGE_VERSION| \
-       redis-server \
-       --databases \
-       "2" \
-       --maxmemory-policy \
-       allkeys-lru \
-       --save \
-       ""
-
-#. Launch the Mayan EDMS container with the network option and change the
-   database hostname to the PostgreSQL container name (``mayan-edms-postgres``)
-   instead of the IP address of the Docker host (``172.17.0.1``):
-
-   .. code-block:: bash
-
-       docker run \
-       -d \
-       --name mayan-edms \
-       --network=mayan \
-       --restart=always \
-       -p 80:8000 \
-       -e MAYAN_DATABASES="{'default':{'ENGINE':'django.db.backends.postgresql','NAME':'mayan','PASSWORD':'mayanuserpass','USER':'mayan','HOST':'mayan-edms-postgres'}}" \
-       -e MAYAN_CELERY_BROKER_URL="redis://mayan-edms-redis:6379/0" \
-       -e MAYAN_CELERY_RESULT_BACKEND="redis://mayan-edms-redis:6379/1" \
-       -v /docker-volumes/mayan-edms/media:/var/lib/mayan \
-       mayanedms/mayanedms:|DOCKER_MAYAN_IMAGE_VERSION|

docs/chapters/docker/install_simple.txt

@@ -1,117 +0,0 @@
-.. _docker_install:
-
-
-==========================
-Simple Docker installation
-==========================
-
-#. Install Docker:
-
-   .. code-block:: bash
-
-       wget -qO- https://get.docker.com/ | sh
-
-   If you don't want run an automated script, follow the instructions outlined
-   in their documentation: https://docs.docker.com/install/
-
-   Once the Docker installation is finished, proceed to the link below to install
-   the Docker image for Mayan EDMS.
-
-
-#. Download the Mayan EDMS Docker image:
-
-   With Docker properly installed, proceed to download the Mayan EDMS Docker
-   image using the command:
-
-   .. code-block:: bash
-
-       docker pull mayanedms/mayanedms:|DOCKER_MAYAN_IMAGE_VERSION|
-
-
-#. Download the PostgreSQL Docker image:
-
-   .. code-block:: bash
-
-       docker pull |DOCKER_POSTGRES_IMAGE_VERSION|
-
-
-#. Download the Redis Docker image:
-
-   .. code-block:: bash
-
-       docker pull |DOCKER_REDIS_IMAGE_VERSION|
-
-
-#. Create and run a PostgreSQL container:
-
-   .. code-block:: bash
-
-       docker run \
-       -d \
-       --name mayan-edms-postgres \
-       --restart=always \
-       -p 5432:5432 \
-       -e POSTGRES_USER=mayan \
-       -e POSTGRES_DB=mayan \
-       -e POSTGRES_PASSWORD=mayanuserpass \
-       -v /docker-volumes/mayan-edms/postgres:/var/lib/postgresql/data \
-       |DOCKER_POSTGRES_IMAGE_VERSION|
-
-   The PostgreSQL container will have one database named ``mayan``, with an user
-   named ``mayan`` too, with a password of ``mayanuserpass``. The container will
-   expose its internal 5432 port (PostgreSQL's default port) via the host's
-   5432 port. The data of this container will reside on the host's
-   ``/docker-volumes/mayan-edms/postgres`` folder.
-
-
-#. Create and run a Redis container:
-
-   .. code-block:: bash
-
-       docker run \
-       -d \
-       --name mayan-edms-redis \
-       --restart=always \
-       -p 6379:6379 \
-       |DOCKER_REDIS_IMAGE_VERSION| \
-       redis-server \
-       --databases \
-       "2" \
-       --maxmemory-policy \
-       allkeys-lru \
-       --save \
-       ""
-
-   The Redis container will have two databases, one for background task messages,
-   and the other to hold the results of those background tasks. Redis is
-   configure to not save its content to disk and to automatically clear up
-   memory.
-
-#. Create and run a Mayan EDMS container:
-
-   .. code-block:: bash
-
-       docker run \
-       -d \
-       --name mayan-edms \
-       --restart=always \
-       -p 80:8000 \
-       -e MAYAN_DATABASES="{'default':{'ENGINE':'django.db.backends.postgresql','NAME':'mayan','PASSWORD':'mayanuserpass','USER':'mayan','HOST':'172.17.0.1'}}" \
-       -e MAYAN_CELERY_BROKER_URL="redis://172.17.0.1:6379/0" \
-       -e MAYAN_CELERY_RESULT_BACKEND="redis://172.17.0.1:6379/1" \
-       -v /docker-volumes/mayan-edms/media:/var/lib/mayan \
-       mayanedms/mayanedms:|DOCKER_MAYAN_IMAGE_VERSION|
-
-   The Mayan EDMS container will connect to the PostgreSQL container via the
-   ``172.17.0.1`` IP address (the Docker host's default IP address). It will
-   connect using the ``django.db.backends.postgresql`` database driver and
-   connect to the ``mayan`` database using the ``mayan`` user with the password
-   ``mayanuserpass``. The files of the container will be store in the
-   host's ``/docker-volumes/mayan-edms/media`` folder. The container will
-   expose its web service running on port 8000 on the host's port 80.
-
-   The container will be available by browsing to ``http://localhost`` or to
-   the IP address of the computer running the container.
-
-   If another web server is running on port 80 use a different port in the
-   ``-p`` option. For example: ``-p 81:8000``.

docs/chapters/docker/nightly.txt

@@ -1,12 +0,0 @@
-==============
-Nightly images
-==============
-
-The continuous integration pipeline used for testing development builds also
-produces a resulting Docker image. These are build automatically and their
-stability is not guaranteed. They should never be used in production.
-If you want to try out the Docker images the development uses or want a sneak
-peek at the new features being worked on checkout the container registry at:
-|SOURCE_CODE_REPOSITORY|container_registry
-
-

docs/chapters/docker/start_stop.txt

@@ -1,13 +0,0 @@
-===================================
-Stopping and starting the container
-===================================
-
-To stop the container use::
-
-    docker stop mayan-edms
-
-
-To start the container again::
-
-    docker start mayan-edms
-

docs/chapters/docker/upgrading.txt

@@ -1,30 +0,0 @@
-=========
-Upgrading
-=========
-
-Upgrading a Mayan EDMS Docker container is actually a matter of stopping and
-deleting the container, downloading the most recent version of the image and
-starting a container again. The container will take care of updating the
-database structure to the newest version if necessary.
-
-**IMPORTANT!** Do not delete the volume storing the data, only the container.
-
-Stop the container to be upgraded::
-
-    docker stop mayan-edms
-
-
-Remove the container::
-
-    docker rm mayan-edms
-
-
-Pull the new image version::
-
-    docker pull mayanedms/mayanedms:latest
-
-
-Start the container again with the new image version::
-
-    docker run -d --name mayan-edms --restart=always -p 80:8000 -v /docker-volumes/mayan:/var/lib/mayan mayanedms/mayanedms:latest
-

docs/chapters/docker_installation.rst

@@ -0,0 +1,109 @@
+*******************
+Docker installation
+*******************
+
+Docker is a system that allows running programs in isolated areas which
+have restricted access to resources, devices, and memory. Docker usage also
+distributing software as a single file.
+
+Make sure Docker is properly installed and working before attempting to install
+Mayan EDMS.
+
+Docker can be installed using their automated script::
+
+    wget -qO- https://get.docker.com/ | sh
+
+This installs the latest versions of Docker. If you don't want run an automated
+script follow the instructions outlined in their documentation:
+https://docs.docker.com/install/
+
+With Docker properly installed, proceed to download the Mayan EDMS image using the command::
+
+    docker pull mayanedms/mayanedms:<version>
+
+Then download version 9.5 of the Docker PostgreSQL image::
+
+    docker pull postgres:9.5
+
+Create and run a PostgreSQL container::
+
+    docker run -d \
+    --name mayan-edms-postgres \
+    --restart=always \
+    -p 5432:5432 \
+    -e POSTGRES_USER=mayan \
+    -e POSTGRES_DB=mayan \
+    -e POSTGRES_PASSWORD=mayanuserpass \
+    -v /docker-volumes/mayan-edms/postgres:/var/lib/postgresql/data \
+    -d postgres:9.5
+
+The PostgreSQL container will have one database named ``mayan``, with an user
+named ``mayan`` too, with a password of ``mayanuserpass``. The container will
+expose its internal 5432 port (PostgreSQL's default port) via the host's
+5432 port. The data of this container will reside on the host's
+``/docker-volumes/mayan-edms/postgres`` folder.
+
+Finally create and run a Mayan EDMS container. Change <version> with the
+latest version in numeric form (example: 2.7.3) or use the ``latest``
+identifier::
+
+    docker run -d \
+    --name mayan-edms \
+    --restart=always \
+    -p 80:8000 \
+    -e MAYAN_DATABASES='{default: {ENGINE: django.db.backends.postgresql, HOST: 172.17.0.1, NAME: mayan, PASSWORD: mayanuserpass, USER: mayan, CONN_MAX_AGE: 60}}' \
+    -v /docker-volumes/mayan-edms/media:/var/lib/mayan \
+    mayanedms/mayanedms:<version>
+
+The Mayan EDMS container will connect to the PostgreSQL container via the
+``172.17.0.1`` IP address (the Docker host's default IP address). It will
+connect using the ``django.db.backends.postgresql`` database driver and
+connect to the ``mayan`` database using the ``mayan`` user with the password
+``mayanuserpass``. The container will keep connections to the database
+for up to 60 seconds in an attempt to reuse them increasing response time
+and reducing memory usage. The files of the container will be store in the
+host's ``/docker-volumes/mayan-edms/media`` folder. The container will
+expose its web service running on port 8000 on the host's port 80.
+
+The container will be available by browsing to ``http://localhost`` or to
+the IP address of the computer running the container.
+
+If another web server is running on port 80 use a different port in the
+``-p`` option. For example: ``-p 81:8000``.
+
+
+Using a dedicated Docker network
+================================
+Use this method to avoid having to expose PostreSQL port to the host's network
+or if you have other PostgreSQL instances but still want to use the default
+port of 5432 for this installation.
+
+Create the network::
+
+    docker network create mayan
+
+Launch the PostgreSQL container with the network option and remove the port
+binding (``-p 5432:5432``)::
+
+    docker run -d \
+    --name mayan-edms-postgres \
+    --network=mayan \
+    --restart=always \
+    -e POSTGRES_USER=mayan \
+    -e POSTGRES_DB=mayan \
+    -e POSTGRES_PASSWORD=mayanuserpass \
+    -v /docker-volumes/mayan-edms/postgres:/var/lib/postgresql/data \
+    -d postgres:9.5
+
+Launch the Mayan EDMS container with the network option and change the
+database hostname to the PostgreSQL container name (``mayan-edms-postgres``)
+instead of the IP address of the Docker host (``172.17.0.1``)::
+
+    docker run -d \
+    --name mayan-edms \
+    --network=mayan \
+    --restart=always \
+    -p 80:8000 \
+    -e MAYAN_DATABASES='{default: {ENGINE: django.db.backends.postgresql, HOST: mayan-edms-postgres, NAME: mayan, PASSWORD: mayanuserpass, USER: mayan, CONN_MAX_AGE: 60}}' \
+    -v /docker-volumes/mayan-edms/media:/var/lib/mayan \
+    mayanedms/mayanedms:<version>

docs/chapters/document_types.rst

@@ -1,4 +1,3 @@
-==============
 Document types
 ==============
 
@@ -68,3 +67,6 @@ assigned when uploaded into Mayan EDMS. A document can only be of one
 type at a given moment, but if needed, the type of a document can be changed.
 Upon changing its type, the document will lose its previous settings and
 attributes, and will inherit the settings and attributes of its new type.
+
+Document types are create in the
+:menuselection:`System --> Setup --> Document types` menu.

docs/chapters/file_storage.rst

@@ -1,6 +1,6 @@
-============
+************
 File storage
-============
+************
 
 The files are stored and placed under Mayan EDMS "control" to avoid
 filename clashes each file gets renamed to its ``UUID`` (Universally Unique ID),

docs/chapters/index_examples.rst

@@ -1,36 +1,26 @@
-==============
+
 Index examples
-==============
+--------------
 
 Index of document types
-=======================
-
-.. admonition:: Permissions required
-    :class: warning
-
-    - The "Create new document indexes" permission is required for this action.
-    - The "Edit document indexes" permission is required for this action, globally
-      of via an ACL for a document index.
-    - The "Edit document types" permission is required for this action, globally
-      of via an ACL for a document type.
-
+^^^^^^^^^^^^^^^^^^^^^^^
 
 This index will create one level for each document type in the system and place
 links to the document of each respective type.
 
 #. Go to the :menuselection:`System --> Setup --> Indexes` menu.
-#. Create a new index using :guilabel:`Actions` > :guilabel:`Create new`.
+#. Create a new index using :menuselection:`Actions --> Create new`.
 #. Give it a label to describe it, and an internal name. The internal name is
    used when referencing this index in other parts of the system.
-#. Press the :guilabel:`Template` link of the newly created index.
-#. Select :guilabel:`New child node` to create a new level in which the
+#. Press the :menuselection:`Template` link of the newly created index.
+#. Select :menuselection:`New child node` to create a new level in which the
    following template code will be entered.
    ::
 
        {{ document.document_type }}
 
 #. Save the template.
-#. Click on :guilabel:`Document types` and associate this index with
+#. Go to :menuselection:`Document types` and associate this index with
    existing document types in the system.
 #. Finally go to :menuselection:`Tools --> Rebuild indexes` to execute the
    index template. The rebuild process is only necessary when changes are
@@ -56,7 +46,7 @@ of types "Invoice" and "Letter", these will now go into the level "Correspondenc
 
 
 Index document by department, taken from the first character of the invoice number metadata
-===========================================================================================
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 Requires one index node with the template::
 
@@ -66,7 +56,7 @@ Requires one index node with the template::
 
 
 Nested date index from a date contained in a metadata
-=====================================================
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 Assuming the metadata type is named **date_issued** with a date format
 of YYYY-MM-DD. The target is to have two levels: one for years and another
@@ -91,35 +81,35 @@ sub level for months.
 
 
 Index by OCR content
-====================
+^^^^^^^^^^^^^^^^^^^^
 
 This example indexes documents in a "quarterly report" level if they have the
 fragment “quarterly report” in the OCR text::
 
-    {% if "quarterly report" in document.latest_version.ocr_content|join:" "|lower %}Quarterly reports{% endif %}
+    {% if "quarterly report" in document.ocr_content.lower() %}Quarterly reports{% endif %}
 
 The same applies to text content extracted for the document::
 
-    {% if "quarterly report" in document.latest_version.content|join:" "|lower %}Quarterly reports{% endif %}
+    {% if "quarterly report" in document.content.lower() %}Quarterly reports{% endif %}
 
 
 
 Index documents not found in any cabinet
-========================================
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 ::
 
     {% if document.cabinets.count == 0 %}No Cabinets{% endif %}
 
 
-Index documents not tagged
-==========================
+Index untagged documents
+^^^^^^^^^^^^^^^^^^^^^^^^
 ::
 
     {% if document.tags.count == 0 %}No Tags{% endif %}
 
 
 Index documents specifically, by the year of a metadata field otherwise by their uploaded year
-==============================================================================================
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 ::
 
     {% for tag in document.tags.all %}{% if tag.label == "Taxes" %}{% if document.metadata_value_of.tax_year|length_is:"4" %}{{ document.metadata_value_of.tax_year }}{% else %}{{ document.date_added|date:"Y" }}{% endif %}{% endif %}{% endfor %}

docs/chapters/indexes.rst

@@ -1,11 +1,10 @@
-===============
-Index structure
-===============
+Indexes
+=======
 
 Indexes are an automatic method to hierarchically organize documents in
-relation to their properties (:doc:`../metadata`, label, MIME type,
+relation to their properties (:doc:`../chapters/metadata`, label, MIME type,
 etc). To use indexes you need to first create an index template. Once created,
-associate the index to one or more :doc:`../document_types`.
+associate the index to one or more :doc:`../chapters/document_types`.
 
 Index are hierarchical models so a tree template needs to be specified for them.
 This tree template will contain references to document metadata or properties
@@ -76,3 +75,41 @@ that will be generate based on the tree template would be as follows:
       year_1 -> document_3;
 
    }
+
+.. include:: ../chapters/index_examples.rst
+
+
+Mirroring
+---------
+
+Indexes can be exported as `FUSE <https://en.wikipedia.org/wiki/Filesystem_in_Userspace>`_
+filesystems. Using the management command ``mountindex`` we could export the
+previous example index as follows::
+
+    mkdir -p ~/indexes/products
+    mayan-edms.py mountindex product-sheets-per-year ~/indexes/products
+
+The ``~/indexes/products`` directory will now have a directory and files structure
+identical to that of the index. Once indexes are mounted with this command, they
+behave like any other filesystem directory and can even be further shared
+via the network with network file system software like
+`Samba <https://www.samba.org/>`_ or
+`NFS <https://en.wikipedia.org/wiki/Network_File_System>`_.
+
+.. blockdiag::
+
+    blockdiag {
+        orientation = portrait
+        span_width = 200;
+
+        index [ label = 'Product sheets per year', width=180 ];
+        block_device [ height = 100, label = "Block device\n(Hard drive)", shape = flowchart.database ];
+        network [ label = "Network", shape = cloud ];
+        user [ label = "Users", shape = actor ];
+
+        index -> block_device [ label = "mirroring", fontsize = 8 ];
+        block_device -> network -> user;
+   }
+
+Indexes and mirrored indexes are Read Only as they are generated as a result of
+prior activities like document uploads, metadata changes.

docs/chapters/indexes/index.txt

@@ -1,23 +0,0 @@
-.. index:: Indexes
-
-================
-Document indexes
-================
-
-Document indexes are a multi level structure similar to folder in a filesystem,
-however the hierarchy of the levels is automatically created, updated, or deleted
-based on rules set by the administrator.
-
-- :doc:`indexes`
-- :doc:`examples`
-- :doc:`mirroring`
-
-.. toctree::
-  :hidden:
-
-  indexes
-  mirroring
-  examples
-
-
-

docs/chapters/indexes/mirroring.txt

@@ -1,35 +0,0 @@
-=========
-Mirroring
-=========
-
-Indexes can be exported as `FUSE <https://en.wikipedia.org/wiki/Filesystem_in_Userspace>`_
-filesystems. Using the management command ``mountindex`` we could export the
-previous example index as follows::
-
-    mkdir -p ~/indexes/products
-    mayan-edms.py mountindex product-sheets-per-year ~/indexes/products
-
-The ``~/indexes/products`` directory will now have a directory and files structure
-identical to that of the index. Once indexes are mounted with this command, they
-behave like any other filesystem directory and can even be further shared
-via the network with network file system software like
-`Samba <https://www.samba.org/>`_ or
-`NFS <https://en.wikipedia.org/wiki/Network_File_System>`_.
-
-.. blockdiag::
-
-    blockdiag {
-        orientation = portrait
-        span_width = 200;
-
-        index [ label = 'Product sheets per year', width=180 ];
-        block_device [ height = 100, label = "Block device\n(Hard drive)", shape = flowchart.database ];
-        network [ label = "Network", shape = cloud ];
-        user [ label = "Users", shape = actor ];
-
-        index -> block_device [ label = "mirroring", fontsize = 8 ];
-        block_device -> network -> user;
-   }
-
-Indexes and mirrored indexes are Read Only as they are generated as a result of
-prior activities like document uploads, metadata changes.

docs/chapters/languages.rst

@@ -1,12 +1,12 @@
-=========
+*********
 Languages
-=========
+*********
 
 The list of languages choices in the language dropdown used for documents is
 based on the current ISO 639 list. This list can be quite extensive. To reduce
 the number of languages available use the setting ``DOCUMENTS_LANGUAGE_CODES``,
 and set it to a nested list of abbreviations. This setting can be found in the
-:menuselection:`System --> Setup --> Settings --> Common` menu.
+:menuselection:`System --> Setup -> Settings -> Common` menu.
 
 For example, to reduce the list to just English and Spanish use
 ::
@@ -31,6 +31,6 @@ Example::
 
 For more information check out the
 :ref:`environment variables <environment_variables>` chapter of the
-:doc:`../parts/settings` topic.
+:doc:`../topics/settings` topic.
 
 

docs/chapters/mailing.rst

@@ -1,12 +1,12 @@
-=================
+*****************
 Mailing documents
-=================
+*****************
 
 Sending emails in Mayan EDMS is controlled by two different system depending on
 the type of email being sent. These are administrative emails like password
 reset emails and user emails sent from the application. To configure
 administrative email for things like password reset check the topic:
-:doc:`../parts/administration`
+:doc:`../topics/administration`
 
 Application emails
 ==================
@@ -14,7 +14,8 @@ Application emails
 To allow users to send emails or documents from within the web interface,
 Mayan EDMS provides its our own email system called Mailing Profiles.
 Mailing Profiles support access control per user role and can use different
-email backends.
+email backends. Mailing Profiles are created from the
+:menuselection:`System --> Setup` menu.
 
 Once created mailing profiles allow users to send email messages from
 within the user interface containing either an URL link to the document or

docs/chapters/metadata.rst

@@ -1,6 +1,6 @@
-========
+********
 Metadata
-========
+********
 
 Metadata is the name of the attribute of a document. The concept of metadata is
 divided in two: **metadata types** (size, color, distance) and **metadata values** for

docs/chapters/object_storage.rst

@@ -0,0 +1,60 @@
+**************
+Object storage
+**************
+
+It is possible to use object storage instead of the default filesystem based
+storage. One such object storage system is Amazon S3 (Simple Storage Service)
+API compatible object storage. The following steps will configure Mayan EDMS
+to use a S3 style storage for documents.
+
+1. Install the django-storages and boto3 Python libraries.
+
+  * For the direct deployment method of installation use::
+
+        pip install django-storages boto3
+
+
+  * or if using the Docker image, add the following the command line that runs the container::
+
+    -e MAYAN_PIP_INSTALLS='django-storages boto3'
+
+
+2. From the web interface navigate to the :menuselection:`System --> Setup --> Setting --> Documents` menu.
+3. Locate the **DOCUMENTS_STORAGE_BACKEND** setting, press **Edit** and enter::
+
+    storages.backends.s3boto3.S3Boto3Storage
+
+4. Save and locate the setting **DOCUMENTS_STORAGE_BACKEND_ARGUMENTS**, press **Edit** and enter::
+
+    '{access_key: <your S3 access key>, secret_key: <your S3 secret key>, bucket_name: <S3 bucket name>}'
+
+5. Save and restart your Mayan EDMS installation for the setting to take effect.
+
+
+Storage
+=======
+Mayan EDMS stores documents in their original file format only changing the
+filename to avoid collision. For best input and output speed use a block
+based local filesystem for the ``/media`` sub folder of the path specified by
+the MEDIA_ROOT setting. For increased storage capacity use an object storage
+filesystem like S3.
+
+To use a S3 compatible object storage do the following:
+
+* Install the Python packages ``django-storages`` and ``boto3``:
+
+  * Using Python::
+
+      pip install django-storages boto3
+
+  * Using Docker::
+
+    -e MAYAN_PIP_INSTALLS='django-storages boto3'
+
+On the Mayan EDMS user interface, go to ``System``, ``Setup``, ``Settings``,
+``Documents`` and change the following setting:
+
+* ``DOCUMENTS_STORAGE_BACKEND`` to ``storages.backends.s3boto3.S3Boto3Storage``
+* ``DOCUMENTS_STORAGE_BACKEND_ARGUMENTS`` to ``'{access_key: <your access key>, secret_key: <your secret key>, bucket_name: <bucket name>}'``.
+
+Restart Mayan EDMS for the changes to take effect.

docs/chapters/ocr_backend.rst

@@ -1,10 +1,10 @@
-===========
+***********
 OCR backend
-===========
+***********
 
 Mayan EDMS ships an OCR backend that uses the FLOSS engine Tesseract
 (https://github.com/tesseract-ocr/tesseract/), but it can
-use other engines. To support other engines create a wrapper that subclasses the
+use other engines. To support other engines crate a wrapper that subclasses the
 ``OCRBackendBase`` class defined in mayan/apps/ocr/classes. This subclass should
 expose the ``execute`` method. For an example of how the Tesseract backend
 is implemented take a look at the file ``mayan/apps/ocr/backends/tesseract.py``

docs/chapters/password_reset.rst

@@ -1,6 +1,6 @@
-==============
+**************
 Password reset
-==============
+**************
 
 To use the password reset feature, administrative emails need to be configured.
 These are sent by the system itself and not by the users. Their usage and
@@ -15,6 +15,7 @@ via the :ref:`configuration file <configuration_file>`.
 
 Example::
 
+    DEFAULT_FROM_EMAIL: '<your administrator email>'
     EMAIL_BACKEND: django.core.mail.backends.smtp.EmailBackend
     EMAIL_HOST: '<your smtp ip address or hostname>'
     EMAIL_HOST_PASSWORD: '<your smtp password>'
@@ -27,7 +28,7 @@ Example::
 To change the reference URL in the password reset emails on in the
 default document mailing template modify the ``COMMON_PROJECT_URL`` setting.
 For information on the different ways to change a setting check the
-:doc:`../parts/settings` topic.
+:doc:`../topics/settings` topic.
 
 To test the email settings use the management command ``sendtestemail``.
 Example::

docs/chapters/password_validation.rst

@@ -0,0 +1,48 @@
+*******************
+Password validation
+*******************
+
+To help reduce the use of weak passwords, Mayan EDMS includes support for
+password validators. Password validator enforce policies by rejecting
+password that don't conform with the validator's logic.
+
+By default, Mayan EDMS sets this password validation setup:
+
+- That the password is not similar no any user attributes.
+- A minimum password size of 8 characters.
+- The password is not one of the 20,000 commonly used weak password.
+- That the password is not entirely numeric.
+
+This default is coded in the following manner by the default Python setup file::
+
+    AUTH_PASSWORD_VALIDATORS = [
+        {
+            'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
+        },
+        {
+            'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
+        },
+        {
+            'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
+        },
+        {
+            'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
+        },
+    ]
+
+If using the YAML configuration file the same setup would be coded in the
+following manner::
+
+    AUTH_PASSWORD_VALIDATORS:
+    - NAME: django.contrib.auth.password_validation.UserAttributeSimilarityValidator
+    - NAME: django.contrib.auth.password_validation.MinimumLengthValidator
+    - NAME: django.contrib.auth.password_validation.CommonPasswordValidator
+    - NAME: django.contrib.auth.password_validation.NumericPasswordValidator
+
+In addition to the password validators provided by Django
+:django-docs:`validators provided by Django <topics/auth/passwords/#included-validators>`,
+Mayan EDMS adds the following validators:
+
+.. autoclass:: mayan.apps.authentication.validators.MinimumCapitalLettersContentValidator
+
+.. autoclass:: mayan.apps.authentication.validators.MinimumNumberContentValidator