tesseract deu but restored else

Version 3.3.6

.drone.yml

@@ -0,0 +1,31 @@
+kind: pipeline
+name: default
+
+steps:
+  - name: docker  
+    image: plugins/docker
+    settings:
+      registry: registry.d1v3.de
+      repo: registry.d1v3.de/mayan-edms
+      username:
+        from_secret: docker_username
+      password:
+        from_secret: docker_password
+      dockerfile: docker/Dockerfile
+      tags: latest
+      auto_tag: true
+
+
+---
+kind: secret
+name: docker_username
+get:
+    path: kv/data/drone/docker
+    name: username
+---
+
+kind: secret
+name: docker_password
+get:
+    path: kv/data/drone/docker
+    name: token

.github/FUNDING.yml

@@ -0,0 +1 @@
+custom: https://www.mayan-edms.com/donation/

.gitignore

@@ -30,4 +30,8 @@ static_collected/
 /venv/
 /venv3/
 /whoosh_index/
+google_fonts/
 node_modules/
+docs/build/
+docs/_templates/layout.html
+docs/_templates/footer.html

.gitlab-ci.yml

@@ -17,9 +17,9 @@ job_docker_build:
     - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
   script:
     - docker build --pull -t "$CI_REGISTRY_IMAGE" -f docker/Dockerfile .
-    - VERSION=`cat docker/version`
+    - VERSION=`cat docker/rootfs/version`
     - docker tag "$CI_REGISTRY_IMAGE" "$CI_REGISTRY_IMAGE:$VERSION"
-    - docker run --rm "$CI_REGISTRY_IMAGE:$VERSION" run-tests
+    - docker run --rm "$CI_REGISTRY_IMAGE:$VERSION" run_tests
     - docker push "$CI_REGISTRY_IMAGE:$VERSION"
     - docker push "$CI_REGISTRY_IMAGE:latest"
     - docker tag "$CI_REGISTRY_IMAGE:$VERSION" registry-1.docker.io/mayanedms/mayanedms:"$VERSION"
@@ -37,7 +37,7 @@ job_docker_push:
   before_script:
     - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
   script:
-    - VERSION=`cat docker/version`
+    - VERSION=`cat docker/rootfs/version`
     - docker pull "$CI_REGISTRY_IMAGE:$VERSION"
     - docker tag "$CI_REGISTRY_IMAGE:$VERSION" registry-1.docker.io/mayanedms/mayanedms:"$VERSION"
     - docker tag "$CI_REGISTRY_IMAGE:$VERSION" registry-1.docker.io/mayanedms/mayanedms:latest
@@ -58,7 +58,7 @@ job_docker_nightly:
     - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
   script:
     - docker build --pull -t "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG" -f docker/Dockerfile .
-    - docker run --rm "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG" run-tests
+    - docker run --rm "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG" run_tests
     - docker push "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG"
   only:
     - nightly
@@ -152,13 +152,14 @@ job_push_python:
     - locale-gen en_US.UTF-8
     - update-locale LANG=en_US.UTF-8
     - export LC_ALL=en_US.UTF-8
-    - apt-get install -qq curl exiftool gcc ghostscript gnupg1 graphviz libfuse2 libjpeg-dev libmagic1 libpng-dev libtiff-dev poppler-utils libreoffice poppler-utils python-dev python-pip tesseract-ocr tesseract-ocr-deu
+    - apt-get install -qq curl exiftool firefox-geckodriver gcc ghostscript gnupg1 graphviz libfuse2 libjpeg-dev libmagic1 libpng-dev libtiff-dev poppler-utils libreoffice poppler-utils python-dev python-virtualenv python3-dev tesseract-ocr tesseract-ocr-deu
+    - virtualenv venv -p /usr/bin/python3
+    - . venv/bin/activate
     - pip install -r requirements.txt -r requirements/testing-base.txt
   only:
     - releases/all
     - releases/docker
     - releases/python
-    - master
     - staging
     - nightly
 
@@ -170,8 +171,10 @@ test-mysql:
   services:
     - mysql:8.0.3
   script:
-    - apt-get install -qq libmysqlclient-dev mysql-client
-    - pip install mysqlclient
+    - apt-get install -qq libmysqlclient-dev libssl-dev mysql-client
+    - . venv/bin/activate
+    - set -a && . ./config.env && set +a
+    - pip install mysqlclient==$PYTHON_MYSQL_VERSION
     - mysql -h"$MYSQL_PORT_3306_TCP_ADDR" -P"$MYSQL_PORT_3306_TCP_PORT" -uroot -p"$MYSQL_ENV_MYSQL_ROOT_PASSWORD" -e "set global character_set_server=utf8mb4;"
     - python manage.py test --mayan-apps --settings=mayan.settings.testing.gitlab-ci.db_mysql --nomigrations
   tags:
@@ -186,7 +189,9 @@ test-postgres:
     - postgres
   script:
     - apt-get install -qq libpq-dev
-    - pip install psycopg2
+    - . venv/bin/activate
+    - set -a && . ./config.env && set +a
+    - pip install psycopg2==$PYTHON_PSYCOPG2_VERSION
     - python manage.py test --mayan-apps --settings=mayan.settings.testing.gitlab-ci.db_postgres --nomigrations
   tags:
     - postgres
@@ -194,6 +199,7 @@ test-postgres:
 test-sqlite:
   <<: *test_base
   script:
+    - . venv/bin/activate
     - python manage.py test --mayan-apps --settings=mayan.settings.testing.gitlab-ci --nomigrations
 
 deploy_demo:

.tx/config

@@ -115,6 +115,12 @@ source_lang = en
 source_file = mayan/apps/events/locale/en/LC_MESSAGES/django.po
 type = PO
 
+[mayan-edms.file_caching-3-0]
+file_filter = mayan/apps/file_caching/locale/<lang>/LC_MESSAGES/django.po
+source_lang = en
+source_file = mayan/apps/file_caching/locale/en/LC_MESSAGES/django.po
+type = PO
+
 [mayan-edms.file_metadata-3-0]
 file_filter = mayan/apps/file_metadata/locale/<lang>/LC_MESSAGES/django.po
 source_lang = en
@@ -217,8 +223,21 @@ source_lang = en
 source_file = mayan/apps/task_manager/locale/en/LC_MESSAGES/django.po
 type = PO
 
+[mayan-edms.templating-3-0]
+file_filter = mayan/apps/templating/locale/<lang>/LC_MESSAGES/django.po
+source_lang = en
+source_file = mayan/apps/templating/locale/en/LC_MESSAGES/django.po
+type = PO
+
 [mayan-edms.user_management-2-0]
 file_filter = mayan/apps/user_management/locale/<lang>/LC_MESSAGES/django.po
 source_lang = en
 source_file = mayan/apps/user_management/locale/en/LC_MESSAGES/django.po
 type = PO
+
+[mayan-edms.web_link-3-0]
+file_filter = mayan/apps/web_links/locale/<lang>/LC_MESSAGES/django.po
+source_lang = en
+source_file = mayan/apps/web_links/locale/en/LC_MESSAGES/django.po
+type = PO
+

CONTRIBUTING.md

@@ -4,21 +4,25 @@ Contributing to Mayan EDMS
 Issues
 ------
 
-- Before submitting a new issue, check for **open issues** first and join the
-discussion.
+- The issue system is for reporting problems with the program code. For
+questions use the forum at: https://forum.mayan-edms.com/
+- Do not open issues asking for **support or consulting**. For paid support
+and consultation visit: https://www.mayan-edms.com/support/
+- The issue must be related to the code only, do not open issues for problems
+with databases, deployments, webservers, cloud providers, etc.
+- Before submitting a new issue, check for **existing issues** first and
+join the discussion.
 - If your issue is not attended in a while, **be patient**, the core team is
 small and the codebase big.
-- Try to reproduce the issue using a **separate, clean installation**, sometimes
-the issue can be caused by an error in a configuration file and not with the
-code itself.
-- **Do not upload** sensitive, private or classified information or files with
-your issue. If the issue is triggered by a user file, create a dummy file with the
-same properties that can trigger the issue and upload that file instead.
+- Try to reproduce the issue using a **separate, clean installation**,
+sometimes the issue can be caused by an error in a configuration file and
+not with the code itself.
+- **Do not upload** sensitive, private or classified information or files
+with your issue. If the issue is triggered by a user file, create a dummy
+file with the same properties that can trigger the issue and upload that
+file instead.
 - Add steps that trigger the issue in a **repeatable manner**.
 - **Screenshots** go a long way in helping understand problems.
-- The issue must be related to the code only, do not open issues for problems
-with deployments, webservers, cloud providers, etc.
-- Do not open issues asking for **support or consulting**.
 
 Code
 ----

LICENSE

@@ -1,4 +1,4 @@
-   Copyright 2011-2018 Roberto Rosario
+   Copyright 2011 Roberto Rosario
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

Makefile

@@ -1,8 +1,11 @@
-.PHONY: clean-pyc clean-build
+#!make
+include config.env
+
+.PHONY: clean clean-pyc clean-build test
 
 help:
 	@echo "Usage: make <target>\n"
-	@awk 'BEGIN {FS = ":.*##"} /^[a-zA-Z_-]+:.*?## / { printf "  * %-40s -%s\n", $$1, $$2 }' $(MAKEFILE_LIST)|sort
+	@awk 'BEGIN {FS = ":.*##"} /^[0-9a-zA-Z_-]+:.*?## / { printf "  * %-40s -%s\n", $$1, $$2 }' $(MAKEFILE_LIST)|sort
 
 # Cleaning
 
@@ -18,7 +21,7 @@ clean-pyc: ## Remove Python artifacts.
 	find . -name '*.pyc' -exec rm -f {} +
 	find . -name '*.pyo' -exec rm -f {} +
 	find . -name '*~' -exec rm -f {} +
-
+	find . -name '__pycache__' -exec rm -R -f {} +
 
 # Testing
 
@@ -26,17 +29,25 @@ test: clean-pyc
 test: ## MODULE=<python module name> - Run tests for a single app, module or test class.
 	./manage.py test $(MODULE) --settings=mayan.settings.testing.development --nomigrations $(ARGUMENTS)
 
+test-debug: clean-pyc
+test-debug: ## MODULE=<python module name> - Run tests for a single app, module or test class, in debug mode.
+	./manage.py test $(MODULE) --settings=mayan.settings.testing.development --nomigrations --debug-mode $(ARGUMENTS)
+
 test-all: ## Run all tests.
 test-all: clean-pyc
 	./manage.py test --mayan-apps --settings=mayan.settings.testing.development --nomigrations $(ARGUMENTS)
 
+test-all-debug: ## Run all tests in debug mode.
+test-all-debug: clean-pyc
+	./manage.py test --mayan-apps --settings=mayan.settings.testing.development --nomigrations --debug-mode $(ARGUMENTS)
+
 test-launch-postgres:
 	@docker rm -f test-postgres || true
 	@docker volume rm test-postgres || true
-	docker run -d --name test-postgres -p 5432:5432 -v test-postgres:/var/lib/postgresql/data healthcheck/postgres
+	docker run -d --name test-postgres -p 5432:5432 -v test-postgres:/var/lib/postgresql/data $(DOCKER_POSTGRES_IMAGE_VERSION)
 	sudo apt-get install -q libpq-dev
-	pip install psycopg2
-	while ! docker inspect --format='{{json .State.Health}}' test-postgres|grep 'Status":"healthy"'; do sleep 1; done
+	pip install psycopg2==$(PYTHON_PSYCOPG2_VERSION)
+	while ! nc -z 127.0.0.1 5432; do sleep 1; done
 
 test-with-postgres: ## MODULE=<python module name> - Run tests for a single app, module or test class against a Postgres database container.
 test-with-postgres: test-launch-postgres
@@ -53,10 +64,10 @@ test-with-postgres-all: test-launch-postgres
 test-launch-mysql:
 	@docker rm -f test-mysql || true
 	@docker volume rm test-mysql || true
-	docker run -d --name test-mysql -p 3306:3306 -e MYSQL_ALLOW_EMPTY_PASSWORD=True -e MYSQL_DATABASE=mayan -v test-mysql:/var/lib/mysql healthcheck/mysql
+	docker run -d --name test-mysql -p 3306:3306 -e MYSQL_ALLOW_EMPTY_PASSWORD=True -e MYSQL_DATABASE=mayan -v test-mysql:/var/lib/mysql $(DOCKER_MYSQL_IMAGE_VERSION)
 	sudo apt-get install -q libmysqlclient-dev mysql-client
-	pip install mysqlclient
-	while ! docker inspect --format='{{json .State.Health}}' test-mysql|grep 'Status":"healthy"'; do sleep 1; done
+	pip install mysqlclient==$(PYTHON_MYSQL_VERSION)
+	while ! nc -z 127.0.0.1 3306; do sleep 1; done
 	mysql -h 127.0.0.1 -P 3306 -uroot  -e "set global character_set_server=utf8mb4;"
 
 test-with-mysql: ## MODULE=<python module name> - Run tests for a single app, module or test class against a MySQL database container.
@@ -75,9 +86,9 @@ test-with-mysql-all: test-launch-mysql
 test-launch-oracle:
 	@docker rm -f test-oracle || true
 	@docker volume rm test-oracle || true
-	docker run -d --name test-oracle -p 49160:22 -p 49161:1521 -e ORACLE_ALLOW_REMOTE=true -v test-oracle:/u01/app/oracle wnameless/oracle-xe-11g
+	docker run -d --name test-oracle -p 49160:22 -p 49161:1521 -e ORACLE_ALLOW_REMOTE=true -v test-oracle:/u01/app/oracle $(DOCKER_ORACLE_IMAGE_VERSION)
 	# https://gist.github.com/kimus/10012910
-	pip install cx_Oracle
+	pip install cx_Oracle==$(PYTHON_ORACLE_VERSION)
 	while ! nc -z 127.0.0.1 49161; do sleep 1; done
 	sleep 10
 
@@ -115,6 +126,9 @@ translations-push: ## Upload all translation files to Transifex.
 translations-pull: ## Download all translation files from Transifex.
 	tx pull -f
 
+translations-all: ## Execute all translations targets.
+translations-all: translations-make translations-push translations-pull translations-compile
+
 # Releases
 
 increase-version: ## Increase the version number of the entire project's files.
@@ -227,27 +241,58 @@ generate-setup: generate-requirements
 generate-requirements: ## Generate all requirements files from the project depedency declarations.
 	@./manage.py generaterequirements build > requirements/build.txt
 	@./manage.py generaterequirements development > requirements/development.txt
+	@./manage.py generaterequirements documentation > requirements/documentation.txt
+	@./manage.py generaterequirements production --only=pathlib2 >> requirements/documentation.txt
 	@./manage.py generaterequirements testing > requirements/testing-base.txt
 	@./manage.py generaterequirements production --exclude=django > requirements/base.txt
 	@./manage.py generaterequirements production --only=django > requirements/common.txt
 
+gitlab-release-documentation: ## Trigger the documentation build and publication using GitLab CI
+gitlab-release-documentation:
+	git push
+	git push --tags
+	git push origin :releases/documentation || true
+	git push origin HEAD:releases/documentation
+
+gitlab-release-docker: ## Trigger the Docker image build and publication using GitLab CI
+gitlab-release-docker:
+	git push
+	git push --tags
+	git push origin :releases/docker || true
+	git push origin HEAD:releases/docker
+
+gitlab-release-python: ## Trigger the Python package build and publication using GitLab CI
+gitlab-release-python:
+	git push
+	git push --tags
+	git push origin :releases/python || true
+	git push origin HEAD:releases/python
+
+gitlab-release-all: ## Trigger the Python package, Docker image, and documentation build and publication using GitLab CI
+gitlab-release-all:
+	git push
+	git push --tags
+	git push origin :releases/all || true
+	git push origin HEAD:releases/all
+
 # Dev server
 
 runserver: ## Run the development server.
-	./manage.py runserver --settings=mayan.settings.development $(ADDRPORT)
+	./manage.py runserver --nothreading --settings=mayan.settings.development $(ADDRPORT)
 
 runserver_plus: ## Run the Django extension's development server.
-	./manage.py runserver_plus --settings=mayan.settings.development $(ADDRPORT)
+	./manage.py runserver_plus --nothreading --settings=mayan.settings.development $(ADDRPORT)
 
 shell_plus: ## Run the shell_plus command.
 	./manage.py shell_plus --settings=mayan.settings.development
 
 test-with-docker-services-on: ## Launch and initialize production-like services using Docker (Postgres and Redis).
-	docker run -d --name redis -p 6379:6379 redis
-	docker run -d --name postgres -p 5432:5432 postgres
+	docker run -d --name redis -p 6379:6379 $(DOCKER_REDIS_IMAGE_VERSION)
+	docker run -d --name postgres -p 5432:5432 $(DOCKER_POSTGRES_IMAGE_VERSION)
 	while ! nc -z 127.0.0.1 6379; do sleep 1; done
 	while ! nc -z 127.0.0.1 5432; do sleep 1; done
 	sleep 4
+	pip install psycopg2==$(PYTHON_PSYCOPG2_VERSION) redis==$(PYTHON_REDIS_VERSION)
 	./manage.py initialsetup --settings=mayan.settings.staging.docker
 
 test-with-docker-services-off: ## Stop and delete the Docker production-like services.
@@ -258,10 +303,10 @@ test-with-docker-frontend: ## Launch a front end instance that uses the producti
 	./manage.py runserver --settings=mayan.settings.staging.docker
 
 test-with-docker-worker: ## Launch a worker instance that uses the production-like services.
-	./manage.py celery worker --settings=mayan.settings.staging.docker -B -l INFO -O fair
+	DJANGO_SETTINGS_MODULE=mayan.settings.staging.docker ./manage.py celery worker -A mayan -B -l INFO -O fair
 
 docker-mysql-on: ## Launch and initialize a MySQL Docker container.
-	docker run -d --name mysql -p 3306:3306 -e MYSQL_ALLOW_EMPTY_PASSWORD=True -e MYSQL_DATABASE=mayan_edms mysql
+	docker run -d --name mysql -p 3306:3306 -e MYSQL_ALLOW_EMPTY_PASSWORD=True -e MYSQL_DATABASE=mayan_edms $(DOCKER_MYSQL_IMAGE_VERSION)
 	while ! nc -z 127.0.0.1 3306; do sleep 1; done
 
 docker-mysql-off: ## Stop and delete the MySQL Docker container.
@@ -269,7 +314,7 @@ docker-mysql-off: ## Stop and delete the MySQL Docker container.
 	docker rm mysql
 
 docker-postgres-on: ## Launch and initialize a PostgreSQL Docker container.
-	docker run -d --name postgres -p 5432:5432 postgres
+	docker run -d --name postgres -p 5432:5432 $(DOCKER_POSTGRES_IMAGE_VERSION)
 	while ! nc -z 127.0.0.1 5432; do sleep 1; done
 
 docker-postgres-off: ## Stop and delete the PostgreSQL Docker container.

README.md

@@ -1,11 +1,10 @@
+[![Donation](https://img.shields.io/badge/donation-PayPal-brightgreen)](https://paypal.me/MayanEDMS)
 [![pypi][pypi]][pypi-url]
-[![builds][builds]][builds-url]
-[![coverage][cover]][cover-url]
 ![python][python]
 ![license][license]
 [![Docker pulls](https://img.shields.io/docker/pulls/mayanedms/mayanedms.svg?maxAge=3600)](https://hub.docker.com/r/mayanedms/mayanedms/)
 [![Docker Stars](https://img.shields.io/docker/stars/mayanedms/mayanedms.svg?maxAge=3600)](https://hub.docker.com/r/mayanedms/mayanedms/)
-[![Docker layers](https://images.microbadger.com/badges/image/mayanedms/mayanedms.svg)](https://microbadger.com/images/mayanedms/mayanedms)
+
 
 [pypi]: http://img.shields.io/pypi/v/mayan-edms.svg
 [pypi-url]: http://badge.fury.io/py/mayan-edms
@@ -43,6 +42,18 @@
 
 </div>
 
+<h2 align="center">Book</h2>
+
+The final version of the book "Exploring Mayan EDMS" available now!
+
+<p align="center">
+    <a href="https://sellfy.com/p/um2fkx/">
+        <img width="400" src="https://d12swbtw719y4s.cloudfront.net/images/v6RpxW40/aP0qKLjkPiAuXZhYuB45/wDAULAzFyx.jpeg?w=548">
+    </a>
+</p>
+
+The link is https://sellfy.com/p/um2fkx/
+
 <h2 align="center">Installation</h2>
 
 The easiest way to use Mayan EDMS is by using the official

README.rst

@@ -1,4 +1,4 @@
-|pypi| |builds| |coverage| |python| |license| |docker_pulls| |docker_stars| |docker_layers|
+|donation| |pypi| |coverage| |python| |license| |docker_pulls| |docker_stars| |docker_layers|
 
 
 .. image:: https://gitlab.com/mayan-edms/mayan-edms/raw/master/docs/_static/mayan_logo.png
@@ -25,6 +25,15 @@ at: https://docs.mayan-edms.com/topics/installation.html
 
 .. _Docker: https://www.docker.com/
 
+The final version of the book "Exploring Mayan EDMS" available now!
+
+.. image:: https://camo.githubusercontent.com/89d2fe787507c9247aa7bb406e2682b53eb7a5f9/68747470733a2f2f64313273776274773731397934732e636c6f756466726f6e742e6e65742f696d616765732f76365270785734302f615030714b4c6a6b50694175585a6859754234352f774441554c417a4679782e6a7065673f773d353438
+    :align: center
+    :width: 300
+    :target: https://sellfy.com/p/um2fkx/
+
+Click the image or visit: https://sellfy.com/p/um2fkx/
+
 Hardware requirements
 
 - 2 Gigabytes of RAM (1 Gigabyte if OCR is turned off).
@@ -44,6 +53,8 @@ Important links
 - `Videos <https://www.youtube.com/channel/UCJOOXHP1MJ9lVA7d8ZTlHPw>`__
 
 
+.. |donation| image:: https://img.shields.io/badge/donation-PayPal-brightgreen
+   :target: https://paypal.me/MayanEDMS
 .. |pypi| image:: http://img.shields.io/pypi/v/mayan-edms.svg
    :target: http://badge.fury.io/py/mayan-edms
 .. |builds| image:: https://gitlab.com/mayan-edms/mayan-edms/badges/master/build.svg

__init__.py.tmpl

@@ -9,6 +9,6 @@ __author__ = 'Roberto Rosario'
 __author_email__ = 'roberto.rosario@mayan-edms.com'
 __description__ = 'Free Open Source Electronic Document Management System'
 __license__ = 'Apache 2.0'
-__copyright_short__ = '2011-2018 Roberto Rosario'
+__copyright_short__ = '2011 Roberto Rosario'
 __copyright__ = '{} {}'.format('Copyright', __copyright_short__)
 __website__ = 'https://www.mayan-edms.com'

config.env

@@ -0,0 +1,16 @@
+DOCKER_MYSQL_IMAGE_VERSION=mysql:8.0
+DOCKER_ORACLE_IMAGE_VERSION=wnameless/oracle-xe-11g
+DOCKER_POSTGRES_IMAGE_VERSION=postgres:9.6-alpine
+DOCKER_RABBITMQ_IMAGE_VERSION=rabbitmq:3-alpine
+DOCKER_REDIS_IMAGE_VERSION=redis:5.0-alpine
+PYTHON_AMQP_VERSION=2.5.2
+PYTHON_FLOWER_VERSION=0.9.3
+PYTHON_LIBRABBITMQ_VERSION=2.0.0
+PYTHON_MYSQL_VERSION=1.4.4
+PYTHON_ORACLE_VERSION=7.2.3
+PYTHON_PSYCOPG2_VERSION=2.8.3
+PYTHON_PSUTIL_VERSION=5.6.2
+PYTHON_REDIS_VERSION=3.2.1
+SOURCE_CODE_REPOSITORY=https://gitlab.com/mayan-edms/mayan-edms/
+SOURCE_CODE_GIT=https://gitlab.com/mayan-edms/mayan-edms.git
+SOURCE_CODE_ISSUES=https://gitlab.com/mayan-edms/mayan-edms/issues/

contrib/scripts/docker/run-tests.sh

@@ -1,10 +0,0 @@
-#!/bin/sh
-
-export DEBIAN_FRONTEND=noninteractive
-
-apt-get update
-apt-get install -y --no-install-recommends tesseract-ocr-deu
-
-$MAYAN_PIP_BIN install -r ${MAYAN_INSTALL_DIR}/requirements-testing.txt
-
-$MAYAN_BIN test --mayan-apps --settings=mayan.settings.testing

contrib/scripts/fix-permisssion.sh

@@ -1,25 +0,0 @@
-!/bin/bash
-Directories[0]="."
-
-# Setup find correctly.
-export IFS=$'\n'
-
-# Loop through our array.
-for x in ${Directories[@]}
-	do
-		# Find all directories & subdirectories
-		for i in $(find $x -type d)
-			do
-				# Fix Permissions
-				chmod -c 775 $i
-				chown -c www-data:www-data $i
-			done
-
-		# Find all Files
-		for i in $(find $x -type f)
-			do
-				# Fix Permissions
-				chmod -c 664 $i
-				chown -c www-data:www-data $i
-			done
-	done

contrib/scripts/install/development.sh

@@ -1,72 +0,0 @@
-#!/usr/bin/env bash
-
-INSTALLATION_DIRECTORY=/home/vagrant/mayan-edms/
-DB_NAME=mayan_edms
-DB_PASSWORD=test123
-
-cat << EOF | sudo tee -a /etc/motd.tail
-**********************************sudo apt
-
-Mayan EDMS Vagrant Development Box
-
-**********************************
-EOF
-
-# Update sources
-echo -e "\n -> Running apt-get update & upgrade \n"
-sudo apt-get -qq update
-sudo apt-get -y upgrade
-
-echo -e "\n -> Installing core binaries \n"
-sudo apt-get -y install git-core python-virtualenv gcc python-dev libjpeg-dev libpng-dev libtiff-dev tesseract-ocr poppler-utils libreoffice
-
-echo -e "\n -> Cloning development branch of repository \n"
-git clone /mayan-edms-repository/ $INSTALLATION_DIRECTORY
-cd $INSTALLATION_DIRECTORY
-git checkout development
-git reset HEAD --hard
-
-echo -e "\n -> Setting up virtual env \n"
-virtualenv venv
-source venv/bin/activate
-
-echo -e "\n -> Installing python dependencies \n"
-pip install -r requirements.txt
-
-echo -e "\n -> Running Mayan EDMS initial setup \n"
-./manage.py initialsetup
-
-echo -e "\n -> Installing Redis server \n"
-sudo apt-get install -y redis-server
-pip install redis
-
-echo -e "\n -> Installing testing software \n"
-pip install coverage
-
-echo -e "\n -> Installing MySQL \n"
-sudo debconf-set-selections <<< 'mysql-server mysql-server/root_password password '$DB_PASSWORD
-sudo debconf-set-selections <<< 'mysql-server mysql-server/root_password_again password '$DB_PASSWORD
-sudo apt-get install -y mysql-server libmysqlclient-dev
-# Create a passwordless root and travis users
-mysql -u root -p$DB_PASSWORD -e "SET PASSWORD = PASSWORD('');"
-mysql -u root -e "CREATE USER 'travis'@'localhost' IDENTIFIED BY '';GRANT ALL PRIVILEGES ON * . * TO 'travis'@'localhost';FLUSH PRIVILEGES;"
-mysql -u travis -e "CREATE DATABASE $DB_NAME;"
-pip install mysql-python
-
-echo -e "\n -> Installing PostgreSQL \n"
-sudo apt-get install -y postgresql postgresql-server-dev-all
-sudo -u postgres psql -c 'create database mayan_edms;' -U postgres
-sudo cat > /etc/postgresql/9.3/main/pg_hba.conf << EOF
-local   all             postgres                                trust
-
-# TYPE  DATABASE        USER            ADDRESS                 METHOD
-
-# "local" is for Unix domain socket connections only
-local   all             all                                     peer
-# IPv4 local connections:
-host    all             all             127.0.0.1/32            md5
-# IPv6 local connections:
-host    all             all             ::1/128                 md5
-EOF
-
-pip install -q psycopg2

contrib/scripts/install/docker.sh

@@ -5,24 +5,30 @@ set -e
 #   $ curl -fsSL get.mayan-edms.com -o get-mayan-edms.sh
 #   $ sh get-mayan-edms.sh
 #
-# NOTE: Make sure to verify the contents of the script
+# NOTE: Before executing, make sure to verify the contents of the script
 #       you downloaded matches the contents of docker.sh
 #       located at https://gitlab.com/mayan-edms/mayan-edms/blob/master/contrib/scripts/install/docker.sh
-#       before executing.
 
 : ${VERBOSE:=true}
 : ${INSTALL_DOCKER:=false}
 : ${DELETE_VOLUMES:=false}
+: ${USE_DOCKER_NETWORK:=true}
+: ${DOCKER_NETWORK_NAME:=mayan}
 : ${DATABASE_USER:=mayan}
 : ${DATABASE_NAME:=mayan}
 : ${DATABASE_PASSWORD:=mayanuserpass}
-: ${DOCKER_POSTGRES_IMAGE:=postgres:9.6}
+: ${DOCKER_POSTGRES_IMAGE:=postgres:9.6-alpine}
 : ${DOCKER_POSTGRES_CONTAINER:=mayan-edms-postgres}
 : ${DOCKER_POSTGRES_VOLUME:=/docker-volumes/mayan-edms/postgres}
 : ${DOCKER_POSTGRES_PORT:=5432}
+: ${DOCKER_POSTGRES_DELAY:=10}
+: ${DOCKER_REDIS_IMAGE:=redis:5.0-alpine}
+: ${DOCKER_REDIS_CONTAINER:=mayan-edms-redis}
+: ${DOCKER_REDIS_PORT:=6379}
 : ${DOCKER_MAYAN_IMAGE:=mayanedms/mayanedms:latest}
 : ${DOCKER_MAYAN_CONTAINER:=mayan-edms}
 : ${DOCKER_MAYAN_VOLUME:=/docker-volumes/mayan-edms/media}
+: ${DOCKER_MAYAN_PORT:=80}
 
 cat << EOF
 
@@ -44,6 +50,8 @@ echo "Variable values to be used:"
 echo "---------------------------"
 echo "INSTALL_DOCKER: $INSTALL_DOCKER"
 echo "DELETE_VOLUMES: $DELETE_VOLUMES"
+echo "USE_DOCKER_NETWORK: $USE_DOCKER_NETWORK"
+echo "DOCKER_NETWORK_NAME: $DOCKER_NETWORK_NAME"
 echo "DATABASE_USER: $DATABASE_USER"
 echo "DATABASE_NAME: $DATABASE_NAME"
 echo "DATABASE_PASSWORD: $DATABASE_PASSWORD"
@@ -51,10 +59,19 @@ echo "DOCKER_POSTGRES_IMAGE: $DOCKER_POSTGRES_IMAGE"
 echo "DOCKER_POSTGRES_CONTAINER: $DOCKER_POSTGRES_CONTAINER"
 echo "DOCKER_POSTGRES_VOLUME: $DOCKER_POSTGRES_VOLUME"
 echo "DOCKER_POSTGRES_PORT: $DOCKER_POSTGRES_PORT"
+echo "DOCKER_POSTGRES_DELAY: $DOCKER_POSTGRES_DELAY"
+echo "DOCKER_REDIS_IMAGE: $DOCKER_REDIS_IMAGE"
+echo "DOCKER_REDIS_CONTAINER: $DOCKER_REDIS_CONTAINER"
+echo "DOCKER_REDIS_PORT: $DOCKER_REDIS_PORT"
 echo "DOCKER_MAYAN_IMAGE: $DOCKER_MAYAN_IMAGE"
 echo "DOCKER_MAYAN_CONTAINER: $DOCKER_MAYAN_CONTAINER"
 echo "DOCKER_MAYAN_VOLUME: $DOCKER_MAYAN_VOLUME"
-echo "\nStarting in 10 seconds."
+echo "DOCKER_MAYAN_PORT: $DOCKER_MAYAN_PORT"
+echo
+echo "Override any of them by setting them before the script. "
+echo "Example: INSTALL_DOCKER=true sh get-mayan-edms.sh"
+
+echo "\nStarting in 10 seconds. Press CTRL+C to cancel."
 sleep 10
 fi
 
@@ -72,33 +89,62 @@ if [ -z `which docker`  ]; then
 fi
 
 echo -n "* Removing existing Mayan EDMS and PostgreSQL containers (no data will be lost)..."
-true || docker stop $DOCKER_MAYAN_CONTAINER >/dev/null 2>&1
-true || docker rm $DOCKER_MAYAN_CONTAINER >/dev/null 2>&1
-true || docker stop $DOCKER_POSTGRES_CONTAINER >/dev/null 2>&1
-true || docker rm $DOCKER_POSTGRES_CONTAINER >/dev/null 2>&1
+docker rm -f $DOCKER_REDIS_CONTAINER >/dev/null 2>&1  || true
+docker rm -f $DOCKER_POSTGRES_CONTAINER >/dev/null 2>&1  || true
+docker rm -f $DOCKER_MAYAN_CONTAINER >/dev/null 2>&1  || true
 echo "Done"
 
 if [ "$DELETE_VOLUMES" = true ]; then
-echo -n "* Deleting Docker volumes in 5 seconds (warning: this delete all document data)..."
+echo -n "* Deleting Docker volumes in 5 seconds (warning: this will delete all document data). Press CTRL+C to cancel..."
 sleep 5
-true || rm DOCKER_MAYAN_VOLUME -Rf
-true || rm DOCKER_POSTGRES_VOLUME -Rf
+rm DOCKER_MAYAN_VOLUME -Rf || true
+rm DOCKER_POSTGRES_VOLUME -Rf || true
 echo "Done"
 fi
 
-echo -n "* Pulling (downloading) the Mayan EDMS Docker image..."
-docker pull $DOCKER_MAYAN_IMAGE >/dev/null
+echo -n "* Pulling (downloading) the Redis Docker image..."
+docker pull $DOCKER_REDIS_IMAGE > /dev/null
 echo "Done"
 
 echo -n "* Pulling (downloading) the PostgreSQL Docker image..."
 docker pull $DOCKER_POSTGRES_IMAGE > /dev/null
 echo "Done"
 
+echo -n "* Pulling (downloading) the Mayan EDMS Docker image..."
+docker pull $DOCKER_MAYAN_IMAGE >/dev/null
+echo "Done"
+
+if [ "$USE_DOCKER_NETWORK" = true ]; then
+    echo -n "* Creating Docker network..."
+    docker network create $DOCKER_NETWORK_NAME 2> /dev/null || true
+    # Ignore error if the network already exists
+    echo "Done"
+fi
+
+if [ "$USE_DOCKER_NETWORK" = true ]; then
+    NETWORK_ARGUMENT="--network=$DOCKER_NETWORK_NAME"
+    POSTGRES_PORT_ARGUMENT=""
+    REDIS_PORT_ARGUMENT=""
+    MAYAN_DATABASE_PORT_ARGUMENT=""
+    MAYAN_DATABASE_HOST_ARGUMENT="-e MAYAN_DATABASE_HOST=$DOCKER_POSTGRES_CONTAINER"
+    MAYAN_CELERY_BROKER_URL_ARGUMENT="-e MAYAN_CELERY_BROKER_URL=redis://$DOCKER_REDIS_CONTAINER:6379/0"
+    MAYAN_CELERY_RESULT_BACKEND_ARGUMENT="-e MAYAN_CELERY_RESULT_BACKEND=redis://$DOCKER_REDIS_CONTAINER:6379/1"
+else
+    NETWORK_ARGUMENT=""
+    POSTGRES_PORT_ARGUMENT="-e $DOCKER_POSTGRES_PORT:5432"
+    REDIS_PORT_ARGUMENT="-e $DOCKER_REDIS_PORT:6379"
+    MAYAN_DATABASE_PORT_ARGUMENT="-e MAYAN_DATABASE_PORT=$DOCKER_POSTGRES_PORT"
+    MAYAN_DATABASE_HOST_ARGUMENT="-e MAYAN_DATABASE_HOST=172.17.0.1"
+    MAYAN_CELERY_BROKER_URL_ARGUMENT="-e MAYAN_CELERY_BROKER_URL=redis://172.17.0.1:6379/0"
+    MAYAN_CELERY_RESULT_BACKEND_ARGUMENT="-e MAYAN_CELERY_RESULT_BACKEND=redis://172.17.0.1:6379/1"
+fi
+
 echo -n "* Deploying the PostgreSQL container..."
 docker run -d \
 --name $DOCKER_POSTGRES_CONTAINER \
+$NETWORK_ARGUMENT \
 --restart=always \
--p $DOCKER_POSTGRES_PORT:5432 \
+$POSTGRES_PORT_ARGUMENT \
 -e POSTGRES_USER=$DATABASE_USER \
 -e POSTGRES_DB=$DATABASE_NAME \
 -e POSTGRES_PASSWORD=$DATABASE_PASSWORD \
@@ -106,26 +152,47 @@ docker run -d \
 $DOCKER_POSTGRES_IMAGE >/dev/null
 echo "Done"
 
-echo -n "* Waiting for the PostgreSQL container to be ready (10 seconds)..."
-sleep 10
+echo -n "* Deploying the Redis container..."
+docker run -d \
+--name $DOCKER_REDIS_CONTAINER \
+$NETWORK_ARGUMENT \
+--restart=always \
+$REDIS_PORT_ARGUMENT \
+$DOCKER_REDIS_IMAGE \
+redis-server \
+--appendonly no \
+--databases 2 \
+--maxmemory 100mb \
+--maxmemory-policy allkeys-lru \
+--maxclients 500 \
+--save "" \
+--tcp-backlog 256 \
+>/dev/null
+echo "Done"
+
+echo -n "* Waiting for the PostgreSQL container to be ready (${DOCKER_POSTGRES_DELAY} seconds)..."
+sleep $DOCKER_POSTGRES_DELAY
 echo "Done"
 
 echo -n "* Deploying Mayan EDMS container..."
 docker run -d \
 --name $DOCKER_MAYAN_CONTAINER \
+$NETWORK_ARGUMENT \
 --restart=always \
--p 80:8000 \
+-p $DOCKER_MAYAN_PORT:8000 \
 -e MAYAN_DATABASE_ENGINE=django.db.backends.postgresql \
--e MAYAN_DATABASE_HOST=172.17.0.1 \
+$MAYAN_DATABASE_HOST_ARGUMENT \
+$MAYAN_DATABASE_PORT_ARGUMENT \
 -e MAYAN_DATABASE_NAME=$DATABASE_NAME \
 -e MAYAN_DATABASE_PASSWORD=$DATABASE_PASSWORD \
 -e MAYAN_DATABASE_USER=$DATABASE_USER \
--e MAYAN_DATABASE_PORT=$DOCKER_POSTGRES_PORT \
 -e MAYAN_DATABASE_CONN_MAX_AGE=0 \
+$MAYAN_CELERY_BROKER_URL_ARGUMENT \
+$MAYAN_CELERY_RESULT_BACKEND_ARGUMENT \
 -v $DOCKER_MAYAN_VOLUME:/var/lib/mayan \
 $DOCKER_MAYAN_IMAGE >/dev/null
 echo "Done"
 
 echo -n "* Waiting for the Mayan EDMS container to be ready (might take a few minutes)..."
-while ! curl --output /dev/null --silent --head --fail http://localhost:80; do sleep 1 && echo -n .; done;
+while ! curl --output /dev/null --silent --head --fail http://localhost:$DOCKER_MAYAN_PORT; do sleep 1 && echo -n .; done;
 echo "Done"

contrib/scripts/install/production.sh

@@ -1,171 +0,0 @@
-#!/usr/bin/env bash
-
-# ====== CONFIG ======
-INSTALLATION_DIRECTORY=/usr/share/mayan-edms/
-DB_NAME=mayan_edms
-DB_USERNAME=mayan
-DB_PASSWORD=test123
-# ==== END CONFIG ====
-
-cat << EOF | tee -a /etc/motd.tail
-**********************************
-
-Mayan EDMS Vagrant Production Box
-
-**********************************
-EOF
-
-echo -e "\n -> Running apt-get update & upgrade \n"
-apt-get -qq update
-apt-get -y upgrade
-
-echo -e "\n -> Installing core binaries \n"
-apt-get install nginx supervisor redis-server postgresql libpq-dev libjpeg-dev libmagic1 libpng-dev libreoffice libtiff-dev gcc ghostscript gpgv python-dev python-virtualenv tesseract-ocr poppler-utils -y
-
-echo -e "\n -> Setting up virtualenv \n"
-rm -f ${INSTALLATION_DIRECTORY}
-virtualenv ${INSTALLATION_DIRECTORY}
-source ${INSTALLATION_DIRECTORY}bin/activate
-
-echo -e "\n -> Installing Mayan EDMS from PyPI \n"
-pip install mayan-edms
-
-echo -e "\n -> Installing Python client for PostgreSQL, Redis, and uWSGI \n"
-pip install psycopg2 redis uwsgi
-
-echo -e "\n -> Creating the database for the installation \n"
-echo "CREATE USER mayan WITH PASSWORD '$DB_PASSWORD';" | sudo -u postgres psql
-sudo -u postgres createdb -O $DB_USERNAME $DB_NAME
-
-echo -e "\n -> Creating the directories for the logs \n"
-mkdir /var/log/mayan
-
-echo -e "\n -> Making a convenience symlink \n"
-cd ${INSTALLATION_DIRECTORY}
-ln -s lib/python2.7/site-packages/mayan .
-
-echo -e "\n -> Creating an initial settings file \n"
-mayan-edms.py createsettings
-
-echo -e "\n -> Updating the mayan/settings/local.py file \n"
-cat >> mayan/settings/local.py << EOF
-DATABASES = {
-    'default': {
-        'ENGINE': 'django.db.backends.postgresql_psycopg2',
-        'NAME': '$DB_NAME',
-        'USER': '$DB_USERNAME',
-        'PASSWORD': '$DB_PASSWORD',
-        'HOST': 'localhost',
-        'PORT': '5432',
-    }
-}
-
-BROKER_URL = 'redis://127.0.0.1:6379/0'
-CELERY_RESULT_BACKEND = 'redis://127.0.0.1:6379/0'
-EOF
-
-echo -e "\n -> Migrating the database or initialize the project \n"
-mayan-edms.py initialsetup
-
-echo -e "\n -> Disabling the default NGINX site \n"
-rm -f /etc/nginx/sites-enabled/default
-
-echo -e "\n -> Creating a uwsgi.ini file \n"
-cat > uwsgi.ini << EOF
-[uwsgi]
-chdir = ${INSTALLATION_DIRECTORY}lib/python2.7/site-packages/mayan
-chmod-socket = 664
-chown-socket = www-data:www-data
-env = DJANGO_SETTINGS_MODULE=mayan.settings.production
-gid = www-data
-logto = /var/log/uwsgi/%n.log
-pythonpath = ${INSTALLATION_DIRECTORY}lib/python2.7/site-packages
-master = True
-max-requests = 5000
-socket = ${INSTALLATION_DIRECTORY}uwsgi.sock
-uid = www-data
-vacuum = True
-wsgi-file = ${INSTALLATION_DIRECTORY}lib/python2.7/site-packages/mayan/wsgi.py
-EOF
-
-echo -e "\n -> Creating the directory for the uWSGI log files \n"
-mkdir -p /var/log/uwsgi
-
-echo -e "\n -> Creating the NGINX site file for Mayan EDMS, /etc/nginx/sites-available/mayan \n"
-cat > /etc/nginx/sites-available/mayan << EOF
-server {
-    listen 80;
-    server_name localhost;
-
-    location / {
-        include uwsgi_params;
-        uwsgi_pass unix:${INSTALLATION_DIRECTORY}uwsgi.sock;
-
-        client_max_body_size 30M;  # Increse if your plan to upload bigger documents
-        proxy_read_timeout 30s;  # Increase if your document uploads take more than 30 seconds
-    }
-
-    location /static {
-        alias ${INSTALLATION_DIRECTORY}mayan/media/static;
-        expires 1h;
-    }
-
-    location /favicon.ico {
-        alias ${INSTALLATION_DIRECTORY}mayan/media/static/appearance/images/favicon.ico;
-        expires 1h;
-    }
-}
-EOF
-
-echo -e "\n -> Enabling the NGINX site for Mayan EDMS \n"
-ln -s /etc/nginx/sites-available/mayan /etc/nginx/sites-enabled/
-
-echo -e "\n -> Creating the supervisor file for the uWSGI process, /etc/supervisor/conf.d/mayan-uwsgi.conf \n"
-cat > /etc/supervisor/conf.d/mayan-uwsgi.conf << EOF
-[program:mayan-uwsgi]
-command = ${INSTALLATION_DIRECTORY}bin/uwsgi --ini ${INSTALLATION_DIRECTORY}uwsgi.ini
-user = root
-autostart = true
-autorestart = true
-redirect_stderr = true
-EOF
-
-echo -e "\n -> Creating the supervisor file for the Celery worker, /etc/supervisor/conf.d/mayan-celery.conf \n"
-cat > /etc/supervisor/conf.d/mayan-celery.conf << EOF
-[program:mayan-worker]
-command = ${INSTALLATION_DIRECTORY}bin/python ${INSTALLATION_DIRECTORY}bin/mayan-edms.py celery --settings=mayan.settings.production worker -Ofair -l ERROR
-directory = ${INSTALLATION_DIRECTORY}
-user = www-data
-stdout_logfile = /var/log/mayan/worker-stdout.log
-stderr_logfile = /var/log/mayan/worker-stderr.log
-autostart = true
-autorestart = true
-startsecs = 10
-stopwaitsecs = 10
-killasgroup = true
-priority = 998
-
-[program:mayan-beat]
-command = ${INSTALLATION_DIRECTORY}bin/python ${INSTALLATION_DIRECTORY}bin/mayan-edms.py celery --settings=mayan.settings.production beat -l ERROR
-directory = ${INSTALLATION_DIRECTORY}
-user = www-data
-numprocs = 1
-stdout_logfile = /var/log/mayan/beat-stdout.log
-stderr_logfile = /var/log/mayan/beat-stderr.log
-autostart = true
-autorestart = true
-startsecs = 10
-stopwaitsecs = 1
-killasgroup = true
-priority = 998
-EOF
-
-echo -e "\n -> Collecting the static files \n"
-mayan-edms.py collectstatic --noinput
-
-echo -e "\n -> Making the installation directory readable and writable by the webserver user \n"
-chown www-data:www-data ${INSTALLATION_DIRECTORY} -R
-
-echo -e "\n -> Restarting the services \n"
-/etc/init.d/nginx restart
-/etc/init.d/supervisor restart

contrib/scripts/process_messages.py

@@ -13,16 +13,17 @@ APP_LIST = (
     'checkouts', 'common', 'converter', 'dashboards', 'dependencies',
     'django_gpg', 'document_comments', 'document_indexing',
     'document_parsing', 'document_signatures', 'document_states',
-    'documents', 'dynamic_search', 'events', 'file_metadata', 'linking',
-    'lock_manager', 'mayan_statistics', 'mailer', 'metadata', 'mirroring',
-    'motd', 'navigation', 'ocr', 'permissions', 'platform', 'rest_api',
-    'smart_settings', 'sources', 'storage', 'tags', 'task_manager',
-    'user_management'
+    'documents', 'dynamic_search', 'events', 'file_caching',
+    'file_metadata', 'linking', 'lock_manager', 'mailer',
+    'mayan_statistics', 'metadata', 'mirroring', 'motd', 'navigation',
+    'ocr', 'permissions', 'platform', 'rest_api', 'smart_settings',
+    'sources', 'storage', 'tags', 'task_manager', 'templating',
+    'user_management', 'web_links'
 )
 
 LANGUAGE_LIST = (
     'ar', 'bg', 'bs_BA', 'cs', 'da_DK', 'de_DE', 'en', 'es', 'el', 'fa', 'fr',
-    'hu', 'id', 'it', 'nl_NL', 'pl', 'pt', 'pt_BR', 'ro_RO', 'ru', 'sl_SI',
+    'hu', 'id', 'it', 'lv', 'nl_NL', 'pl', 'pt', 'pt_BR', 'ro_RO', 'ru', 'sl_SI',
     'tr_TR', 'vi_VN', 'zh',
 )
 

contrib/scripts/start_gunicorn.sh

@@ -1,35 +0,0 @@
-#!/bin/bash
-
-NAME="mayan-edms"
-DJANGODIR=/usr/share/mayan-edms
-SOCKFILE=/var/tmp/filesystem.sock
-USER=www-data
-GROUP=www-data
-NUM_WORKERS=3
-DJANGO_SETTINGS_MODULE=mayan.settings.production
-DJANGO_WSGI_MODULE=mayan.wsgi
-TIMEOUT=600
-
-echo "Starting $NAME as `whoami`"
-
-# Activate the virtual environment
-cd $DJANGODIR
-source bin/activate
-export DJANGO_SETTINGS_MODULE=$DJANGO_SETTINGS_MODULE
-export PYTHONPATH=$DJANGODIR:$PYTHONPATH
-
-# Create the run directory if it doesn't exist
-RUNDIR=$(dirname $SOCKFILE)
-test -d $RUNDIR || mkdir -p $RUNDIR
-
-# Start your Django Unicorn
-# Programs meant to be run under supervisor should not daemonize themselves (do not use --daemon)
-exec bin/gunicorn ${DJANGO_WSGI_MODULE}:application \
-  --name $NAME \
-  --workers $NUM_WORKERS \
-  --user=$USER --group=$GROUP \
-  --log-level=debug \
-  --bind=unix:$SOCKFILE \
-  --timeout=$TIMEOUT
-
-

contrib/settings/ldap_connection_settings.py

@@ -1,64 +1,139 @@
 from __future__ import absolute_import
 
+# Install Python LDAP with:
+# $ pip install python-ldap
+# or if using Docker, pass the environment variable MAYAN_PIP_INSTALLS:
+# -e MAYAN_PIP_INSTALLS=python-ldap
 import ldap
-from django_auth_ldap.config import LDAPSearch
 
-from .base import *  # NOQA
-from django.contrib.auth import get_user_model
+from django_auth_ldap.config import (
+    LDAPSearch, LDAPSearchUnion, NestedActiveDirectoryGroupType
+)
 
-SECRET_KEY = '<your secret key>'
+from mayan.settings.production import *
 
-# makes sure this works in Active Directory
-ldap.set_option(ldap.OPT_REFERRALS, 0)
+# Makes sure this works in Active Directory
+ldap.set_option(ldap.OPT_REFERRALS, False)
 
-# This is the default, but I like to be explicit.
+# Turn of debug output, turn this off when everything is working as expected
+ldap.set_option(ldap.OPT_DEBUG_LEVEL, 1)
+
+# Default: True
 AUTH_LDAP_ALWAYS_UPDATE_USER = True
 
-LDAP_USER_AUTO_CREATION = "False"
-LDAP_URL = "ldap://<your ldap server IP>:389/"
-LDAP_BASE_DN = "dc=paramatrix,dc=co,dc=in"
-LDAP_ADDITIONAL_USER_DN = "dc=people"
-LDAP_ADMIN_DN = ""
-LDAP_PASSWORD = ""
+# Use TLS to talk to the LDAP server
+# Requires acquiring the server's certificate
+# $ openssl s_client -connect <LDAP server>:636
+# Part of the output of this file will be the Base-64 encoded .cer file
+# that was presented for LDAPS. Cut and paste into a file beginning at
+# "-Begin Certificate" through "-End Certificate--" and save as a .crt, for
+# example: ldapserver.crt
+# $ CERT=ldapserver.crt
+# $ cp /root/$CERT /usr/share/ca-certificates/$CERT
+# # notice the + sign which tells to activate the certificate.
+# $ echo "+$CERT" >/etc/ca-certificates/update.d/activate_my_cert
+# $ dpkg-reconfigure ca-certificates;
+AUTH_LDAP_START_TLS = False
+
+LDAP_ADDITIONAL_USER_DN = 'dc=people'
+LDAP_ADMIN_DN = ''
+LDAP_BASE_DN = 'dc=<top level dc>,dc=co,dc=in'
+LDAP_PASSWORD = ''
+LDAP_USER_AUTO_CREATION = 'False'
+LDAP_URL = 'ldap://<LDAP server>:389/'
 
-AUTH_LDAP_SERVER_URI = LDAP_URL
 AUTH_LDAP_BIND_DN = LDAP_ADMIN_DN
 AUTH_LDAP_BIND_PASSWORD = LDAP_PASSWORD
+AUTH_LDAP_SERVER_URI = LDAP_URL
 
-
+# Simple search
 AUTH_LDAP_USER_SEARCH = LDAPSearch(
     '%s,%s' % (LDAP_ADDITIONAL_USER_DN, LDAP_BASE_DN),
     ldap.SCOPE_SUBTREE, '(uid=%(user)s)'
 )
+
+# If you need to search in more than one place for a user, you can use
+# LDAPSearchUnion. This takes multiple LDAPSearch objects and returns the
+# union of the results. The precedence of the underlying searches is
+# unspecified.
+# https://django-auth-ldap.readthedocs.io/en/latest/authentication.html
+# AUTH_LDAP_USER_SEARCH = LDAPSearchUnion(
+#     LDAPSearch(
+#         'ou=Users,ou=Admin,dc=<top level DC>,dc=local', ldap.SCOPE_SUBTREE,
+#         '(samaccountname=%(user)s)'
+#     ),
+#     LDAPSearch(
+#         'ou=Users,ou=<second OU>,dc=<top level DC>,dc=local',
+#         ldap.SCOPE_SUBTREE, '(samaccountname=%(user)s)'
+#     ),
+#     LDAPSearch(
+#         'ou=Users,ou=<third OU>,dc=<top level DC>,dc=local',
+#         ldap.SCOPE_SUBTREE, '(samaccountname=%(user)s)'
+#     ),
+# )
+
+# User attributes to map from LDAP to Mayan's user model.
 AUTH_LDAP_USER_ATTR_MAP = {
     'first_name': 'cn',
     'last_name': 'sn',
     'email': 'mail'
 }
+# Another example map
+# AUTH_LDAP_USER_ATTR_MAP = {
+#     'username': 'sAMAccountName',
+#     'first_name': 'givenName',
+#     'last_name': 'sn',
+#     'email': 'mail'
+# }
+# Only string fields can be mapped to attributes. Boolean fields can be
+# defined by group membership:
+# AUTH_LDAP_USER_FLAGS_BY_GROUP = {
+#     'is_active': 'cn=active,ou=groups,dc=example,dc=com',
+#     'is_staff': (
+#         LDAPGroupQuery('cn=staff,ou=groups,dc=example,dc=com')
+#         | LDAPGroupQuery('cn=admin,ou=groups,dc=example,dc=com')
+#     ),
+#     'is_superuser': 'cn=superuser,ou=groups,dc=example,dc=com',
+# }
+
+# Simple group search
+# AUTH_LDAP_GROUP_SEARCH = LDAPSearch(
+#     'ou=groups,dc=example,dc=com', ldap.SCOPE_SUBTREE, '(objectClass=groupOfNames)'
+# )
+# AUTH_LDAP_GROUP_TYPE = GroupOfNamesType()
+
+# Advanced group search
+# AUTH_LDAP_GROUP_SEARCH = LDAPSearchUnion(
+#     LDAPSearch(
+#         'ou=Domain Global,OU=Security,OU=Groups,OU=<OU>,dc=<top level DC>,dc=local',
+#         ldap.SCOPE_SUBTREE,
+#         '(&(objectClass=group)(groupType:1.2.840.113556.1.4.803:=2147483648))'
+#     ),
+#     LDAPSearch(
+#         'ou=Domain Global,OU=Security,OU=Groups,OU=<OU>,dc=<top level DC>,dc=local',
+#         ldap.SCOPE_SUBTREE,
+#         '(&(objectClass=group)(groupType:1.2.840.113556.1.4.803:=2147483648))'
+#     ),
+# )
+# AUTH_LDAP_CACHE_GROUPS = True
+# AUTH_LDAP_FIND_GROUP_PERMS = False
+# AUTH_LDAP_GROUP_TYPE = NestedActiveDirectoryGroupType()
+# AUTH_LDAP_MIRROR_GROUPS = True
+
+# To minimize traffic to the LDAP server, LDAPBackend can make use of
+# Django’s cache framework to keep a copy of a user’s LDAP group memberships.
+# To enable this feature, set AUTH_LDAP_CACHE_TIMEOUT, which determines
+# the timeout of cache entries in seconds.
+# AUTH_LDAP_GROUP_CACHE_TIMEOUT = 3600
+
+# Limiting Access
+# The simplest use of groups is to limit the users who are allowed to log in.
+# If AUTH_LDAP_REQUIRE_GROUP is set, then only users who are members of that
+# group will successfully authenticate. AUTH_LDAP_DENY_GROUP is the reverse:
+# if given, members of this group will be rejected.
+# AUTH_LDAP_REQUIRE_GROUP = 'cn=enabled,ou=groups,dc=example,dc=com'
+# AUTH_LDAP_DENY_GROUP = 'cn=disabled,ou=groups,dc=example,dc=com'
+
 AUTHENTICATION_BACKENDS = (
     'django_auth_ldap.backend.LDAPBackend',
-    'mayan.settings.settings_local.EmailOrUsernameModelBackend',
 )
-
-
-class EmailOrUsernameModelBackend(object):
-    """
-    This is a ModelBacked that allows authentication with either a username or an email address.
-    """
-    def authenticate(self, username=None, password=None):
-        if '@' in username:
-            kwargs = {'email': username}
-        else:
-            kwargs = {'username': username}
-        try:
-            user = get_user_model().objects.get(**kwargs)
-            if user.check_password(password):
-                return user
-        except get_user_model().DoesNotExist:
-            return None
-
-    def get_user(self, username):
-        try:
-            return get_user_model().objects.get(pk=username)
-        except get_user_model().DoesNotExist:
-            return None

docker/Dockerfile

@@ -4,10 +4,12 @@
 # BASE_IMAGE - Bare bones image with the base packages needed to run Mayan EDMS
 ####
 
-FROM debian:9.8-slim as BASE_IMAGE
+FROM debian:10.0-slim as BASE_IMAGE
 
 LABEL maintainer="Roberto Rosario roberto.rosario@mayan-edms.com"
 
+COPY config.env /config.env
+
 ENV PYTHONUNBUFFERED=1 \
     LC_ALL=C.UTF-8 \
     PROJECT_INSTALL_DIR=/opt/mayan-edms
@@ -22,22 +24,25 @@ RUN set -x \
 && DEBIAN_FRONTEND=noninteractive \
 apt-get update \
 && apt-get install -y --no-install-recommends \
+    ca-certificates \
     exiftool \
+    fonts-arphic-uming \
+    fonts-arphic-ukai \
     ghostscript \
     gpgv \
     gnupg1 \
     graphviz \
     libfuse2 \
     libmagic1 \
-    libmariadbclient18 \
+    libmariadb3 \
     libreoffice \
     libpq5 \
     poppler-utils \
-    redis-server \
+    python3-distutils \
     sane-utils \
     sudo \
     supervisor \
-    tesseract-ocr \
+    tesseract-ocr tesseract-ocr-deu \
 # Remove make and build dependencies
 && apt-get remove make libproxy-tools libreoffice-avmedia-backend-vlc libvlc-bin libvlc5 libvlccore9 adwaita-icon-theme gsettings-desktop-schemas libgstreamer-plugins-base1.0-0 -y \
 && apt-get autoremove -y --purge \
@@ -52,22 +57,20 @@ apt-get update \
 && if [ "$(uname -m)" = "armv7l" ]; then \
     ln -s /usr/lib/arm-linux-gnueabihf/libz.so /usr/lib/ \
     && ln -s /usr/lib/arm-linux-gnueabihf/libjpeg.so /usr/lib/ \
-; fi \
-# Discard data when Redis runs out of memory
-&& echo "maxmemory-policy allkeys-lru" >> /etc/redis/redis.conf \
-# Disable saving the Redis database
-echo "save \"\"" >> /etc/redis/redis.conf \
-# Only provision 1 database
-&& echo "databases 1" >> /etc/redis/redis.conf
-
+; fi
 
 ####
-# BUILDER_IMAGE - This image buildS the Python package and is discarded afterwards
+# BUILDER_IMAGE - This image builds the Python package and is discarded afterwards
+#                 only the build artifact is carried over to the next image.
 ####
 
 # Reuse image
 FROM BASE_IMAGE as BUILDER_IMAGE
 
+# Python libraries caching
+ARG PIP_INDEX_URL
+ARG PIP_TRUSTED_HOST
+
 WORKDIR /src
 
 # Copy the source files needed to build the Python package
@@ -96,39 +99,44 @@ apt-get install -y --no-install-recommends \
     libssl-dev \
     g++ \
     gcc \
-    python-dev \
-    python-virtualenv \
+    python3-dev \
+    python3-venv \
 && mkdir -p "${PROJECT_INSTALL_DIR}" \
 && chown -R mayan:mayan "${PROJECT_INSTALL_DIR}" \
 && chown -R mayan:mayan /src
 
 USER mayan
-RUN python -m virtualenv "${PROJECT_INSTALL_DIR}" \
+
+RUN set -a \
+&& . /config.env \
+&& set +a \
+&& python3 -m venv "${PROJECT_INSTALL_DIR}" \
 && . "${PROJECT_INSTALL_DIR}/bin/activate" \
-&& pip install --no-cache-dir --no-use-pep517 \
-    librabbitmq==1.6.1 \
-    mysql-python==1.2.5 \
-    psycopg2==2.7.3.2 \
-    redis==2.10.6 \
+&& pip install --no-cache-dir \
+    amqp==$PYTHON_AMQP_VERSION \
+    mysqlclient==$PYTHON_MYSQL_VERSION \
+    psycopg2==$PYTHON_PSYCOPG2_VERSION \
+    redis==$PYTHON_REDIS_VERSION \
+    flower==$PYTHON_FLOWER_VERSION \
 # psutil is needed by ARM builds otherwise gevent and gunicorn fail to start
 && UNAME=`uname -m` && if [ "${UNAME#*arm}" != $UNAME ]; then \
-    pip install --no-cache-dir --no-use-pep517 \
-    psutil==5.6.2 \
+    pip install --no-cache-dir \
+    psutil==$PYTHON_PSUTIL_VERSION \
 ; fi \
 # Install the Python packages needed to build Mayan EDMS
-&& pip install --no-cache-dir --no-use-pep517 -r /src/requirements/build.txt \
+&& pip install --no-cache-dir -r /src/requirements/build.txt \
 # Build Mayan EDMS
-&& python setup.py sdist \
+&& python3 setup.py sdist \
 # Install the built Mayan EDMS package
-&& pip install --no-cache-dir --no-use-pep517 dist/mayan* \
+&& pip install --no-cache-dir dist/mayan* \
 # Install the static content
-&& mayan-edms.py installjavascript \
+&& mayan-edms.py installdependencies \
 && MAYAN_STATIC_ROOT=${PROJECT_INSTALL_DIR}/static mayan-edms.py preparestatic --link --noinput
 
 COPY --chown=mayan:mayan requirements/testing-base.txt "${PROJECT_INSTALL_DIR}"
 
 ####
-# Final image - BASE_IMAGE + Mayan install directory from the builder image
+# Final image - BASE_IMAGE + BUILDER_IMAGE artifact (Mayan install directory)
 ####
 
 FROM BASE_IMAGE
@@ -144,7 +152,7 @@ VOLUME ["/var/lib/mayan"]
 ENTRYPOINT ["entrypoint.sh"]
 
 EXPOSE 8000
-CMD ["mayan"]
+CMD ["run_all"]
 
 RUN ${PROJECT_INSTALL_DIR}/bin/mayan-edms.py platformtemplate supervisord_docker > /etc/supervisor/conf.d/mayan.conf \
 && apt-get clean autoclean \

docker/Makefile

@@ -1,25 +1,130 @@
-APT_PROXY ?= `/sbin/ip route|awk '/docker0/ { print $$9 }'`:3142
-IMAGE_VERSION ?= `cat docker/rootfs/version`
+#!make
+include config.env
+
+HOST_IP = `/sbin/ip route|awk '/docker0/ { print $$9 }'`
+APT_PROXY ?= $(HOST_IP):3142
 CONSOLE_COLUMNS ?= `echo $$(tput cols)`
 CONSOLE_LINES ?= `echo $$(tput lines)`
+IMAGE_VERSION ?= `cat docker/rootfs/version`
+PIP_INDEX_URL ?= http://$(HOST_IP):3141/root/pypi/+simple/
+PIP_TRUSTED_HOST ?= $(HOST_IP)
 
 docker-build: ## Build a new image locally.
 	docker build -t mayanedms/mayanedms:$(IMAGE_VERSION) -f docker/Dockerfile .
 
 docker-build-with-proxy: ## Build a new image locally using an APT proxy as APT_PROXY.
-	docker build -t mayanedms/mayanedms:$(IMAGE_VERSION) -f docker/Dockerfile --build-arg APT_PROXY=$(APT_PROXY) .
+docker-build-with-proxy: devpi-start
+	docker build -t mayanedms/mayanedms:$(IMAGE_VERSION) -f docker/Dockerfile --build-arg APT_PROXY=$(APT_PROXY) --build-arg PIP_INDEX_URL=$(PIP_INDEX_URL) --build-arg PIP_TRUSTED_HOST=$(PIP_TRUSTED_HOST) --build-arg HTTP_PROXY=$(HTTP_PROXY) --build-arg HTTPS_PROXY=$(HTTPS_PROXY) .
+	$(MAKE) devpi-stop
 
 docker-shell: ## Launch a bash instance inside a running container. Pass the container name via DOCKER_CONTAINER.
 	docker exec -e TERM=$(TERM) -e "COLUMNS=$(CONSOLE_COLUMNS)" -e "LINES=$(CONSOLE_LINES)" -it $(DOCKER_CONTAINER) /bin/bash
 
-docker-test-container: ## Build and run a test container.
-docker-test-container: docker-build-with-proxy docker-test-cleanup
-	docker run -d --name test-mayan-edms -p 80:8000 -v test-mayan_data:/var/lib/mayan mayanedms/mayanedms:$(DOCKER_VERSION)
+docker-runtest-container: ## Run a test container.
+docker-runtest-container: docker-test-cleanup
+	docker run \
+	-d \
+	--name test-mayan-edms \
+	-p 80:8000 \
+	-v test-mayan_data:/var/lib/mayan \
+	mayanedms/mayanedms:$(IMAGE_VERSION)
 
-docker-test-cleanup: ## Delete the test container and the test volume.
+docker-runtest-cleanup: ## Delete the test container and the test volume.
 	@docker rm -f test-mayan-edms || true
 	@docker volume rm test-mayan_data || true
 
-docker-test-all: ## Build and executed the test suite in a test container.
-docker-test-all: docker-build-with-proxy
-	docker run --rm run-tests
+docker-runtest-all: ## Executed the test suite in a test container.
+	docker run --rm mayanedms/mayanedms:$(IMAGE_VERSION) run_tests
+
+docker-compose-build:
+	docker-compose -f docker/docker-compose.yml -p mayan-edms build
+
+docker-compose-build-with-proxy: devpi-start
+	docker-compose -f docker/docker-compose.yml -p mayan-edms build --build-arg APT_PROXY=$(APT_PROXY) --build-arg PIP_INDEX_URL=$(PIP_INDEX_URL) --build-arg PIP_TRUSTED_HOST=$(PIP_TRUSTED_HOST) --build-arg HTTP_PROXY=$(HTTP_PROXY) --build-arg HTTPS_PROXY=$(HTTPS_PROXY)
+	$(MAKE) devpi-stop
+
+docker-compose-up:
+	docker-compose -f docker/docker-compose.yml -p mayan-edms up
+
+docker-staging-network-create:
+	@docker network rm mayan-staging || true
+	docker network create mayan-staging
+
+docker-staging-container-postgresql-start:
+	docker run \
+	-d \
+	--name mayan-staging-postgres \
+	--network=mayan-staging \
+	-e POSTGRES_USER=mayan \
+	-e POSTGRES_DB=mayan \
+	-e POSTGRES_PASSWORD=mayanuserpass \
+	-v mayan-staging-postgres:/var/lib/postgresql/data \
+	$(DOCKER_POSTGRES_IMAGE_VERSION)
+
+docker-staging-container-redis-start:
+	docker run \
+	-d \
+	--name mayan-staging-redis \
+	--network=mayan-staging \
+	-v mayan-staging-redis:/data \
+	$(DOCKER_REDIS_IMAGE_VERSION) \
+	redis-server \
+	--databases \
+	"2" \
+	--maxmemory-policy \
+	allkeys-lru \
+	--save \
+	""
+
+docker-staging-container-rabbitmq-start:
+	docker run \
+	-d \
+	--name mayan-staging-rabbitmq \
+	--network=mayan-staging \
+	-v mayan-staging-rabbitmq:/var/lib/rabbitmq \
+	$(DOCKER_RABBITMQ_IMAGE_VERSION) \
+
+docker-staging-container-mayan-start:
+	sleep 5 && docker run \
+	-d \
+	--name mayan-staging-app \
+	--network=mayan-staging \
+	-p 80:8000 \
+	-e MAYAN_DATABASE_ENGINE=django.db.backends.postgresql \
+	-e MAYAN_DATABASE_HOST=mayan-staging-postgres \
+	-e MAYAN_DATABASE_NAME=mayan \
+	-e MAYAN_DATABASE_PASSWORD=mayanuserpass \
+	-e MAYAN_DATABASE_USER=mayan \
+	-e MAYAN_CELERY_BROKER_URL=$(MAYAN_CELERY_BROKER_URL) \
+	-e MAYAN_CELERY_RESULT_BACKEND="redis://mayan-staging-redis:6379/1" \
+	-v mayan-staging-app:/var/lib/mayan \
+	mayanedms/mayanedms:$(IMAGE_VERSION)
+
+docker-staging-start-with-rabbitmq: MAYAN_CELERY_BROKER_URL="amqp://guest:guest@mayan-staging-rabbitmq:5672/"
+docker-staging-start-with-rabbitmq: docker-staging-start
+
+docker-staging-start-with-redis: MAYAN_CELERY_BROKER_URL="redis://mayan-staging-redis:6379/0"
+docker-staging-start-with-redis: docker-staging-start
+
+docker-staging-start: docker-staging-cleanup docker-staging-network-create docker-staging-container-postgresql-start docker-staging-container-rabbitmq-start docker-staging-container-redis-start docker-staging-container-mayan-start
+	docker logs -f mayan-staging-app
+
+docker-staging-cleanup: ## Delete the test container and the test volume.
+	@docker rm -f mayan-staging-app || true
+	@docker rm -f mayan-staging-redis || true
+	@docker rm -f mayan-staging-rabbitmq || true
+	@docker rm -f mayan-staging-postgres || true
+	@docker volume rm mayan-staging-app || true
+	@docker volume rm mayan-staging-postgres || true
+	@docker volume rm mayan-staging-rabbitmq || true
+	@docker volume rm mayan-staging-redis || true
+	@docker network rm mayan-staging || true
+
+devpi-init:
+	devpi-server --init || true
+
+devpi-start: devpi-init
+	devpi-server --start --host=0.0.0.0 || true
+
+devpi-stop:
+	devpi-server --stop || true

docker/README-short.txt

@@ -1 +0,0 @@
-Mayan EDMS is a free open source electronic document management system.

docker/docker-compose-development.yml

@@ -1,72 +0,0 @@
-version: '2.1'
-
-volumes:
-    broker:
-        driver: local
-    app:
-        driver: local
-    db:
-        driver: local
-    results:
-        driver: local
-
-services:
-  broker:
-    container_name: mayan-edms-broker
-    image: healthcheck/rabbitmq
-    environment:
-      RABBITMQ_DEFAULT_USER: mayan
-      RABBITMQ_DEFAULT_PASS: mayan
-      RABBITMQ_DEFAULT_VHOST: mayan
-    volumes:
-      - broker:/var/lib/rabbitmq
-  results:
-    container_name: mayan-edms-results
-    image: healthcheck/redis
-    volumes:
-      - results:/data
-  #db:
-  #  container_name: mayan-edms-db
-  #  image: healthcheck/mysql
-  #  environment:
-  #    MYSQL_DATABASE: mayan
-  #    MYSQL_PASSWORD: mayan-password
-  #    MYSQL_ROOT_PASSWORD: root-password
-  #    MYSQL_USER: mayan
-  #  volumes:
-  #    - db:/var/lib/mysql
-  db:
-    container_name: mayan-edms-db
-    image: healthcheck/postgres
-    environment:
-      POSTGRES_DB: mayan
-      POSTGRES_PASSWORD: mayan-password
-      POSTGRES_USER: mayan
-    volumes:
-      - db:/var/lib/postgresql/data
-  mayan-edms:
-    container_name: mayan-edms-app
-    image: mayan-edms/next
-    build:
-      context: ./
-      args:
-        - APT_PROXY=172.18.0.1:3142
-    depends_on:
-      broker:
-        condition: service_healthy
-      db:
-        condition: service_healthy
-      results:
-        condition: service_healthy
-    environment:
-      MAYAN_BROKER_URL: amqp://mayan:mayan@broker:5672/mayan
-      MAYAN_CELERY_RESULT_BACKEND: redis://results:6379/0
-      MAYAN_DATABASE_ENGINE: django.db.backends.postgresql
-      MAYAN_DATABASE_HOST: db
-      MAYAN_DATABASE_NAME: mayan
-      MAYAN_DATABASE_PASSWORD: mayan-password
-      MAYAN_DATABASE_USER: mayan
-    ports:
-      - "80:80"
-    volumes:
-      - app:/var/lib/mayan

docker/docker-compose.yml

@@ -1,58 +1,130 @@
-version: '2.1'
+version: '3.7'
 
-volumes:
-    broker:
-        driver: local
-    app:
-        driver: local
-    db:
-        driver: local
-    results:
-        driver: local
+networks:
+  mayan-bridge:
+    driver: bridge
 
 services:
-  broker:
-    container_name: mayan-edms-broker
-    image: healthcheck/rabbitmq
-    environment:
-      RABBITMQ_DEFAULT_USER: mayan
-      RABBITMQ_DEFAULT_PASS: mayan
-      RABBITMQ_DEFAULT_VHOST: mayan
-    volumes:
-      - broker:/var/lib/rabbitmq
-  results:
-    container_name: mayan-edms-results
-    image: healthcheck/redis
-    volumes:
-      - results:/data
-  db:
-    container_name: mayan-edms-db
-    image: healthcheck/postgres
-    environment:
-      POSTGRES_DB: mayan
-      POSTGRES_PASSWORD: mayan-password
-      POSTGRES_USER: mayan
-    volumes:
-      - db:/var/lib/postgresql/data
-  mayan-edms:
-    container_name: mayan-edms-app
-    image: mayanedms/mayanedms:latest
+  app:
+    build:
+      context: ..
+      dockerfile: ./docker/Dockerfile
     depends_on:
-      broker:
-        condition: service_healthy
-      db:
-        condition: service_healthy
-      results:
-        condition: service_healthy
-    environment:
-      MAYAN_BROKER_URL: amqp://mayan:mayan@broker:5672/mayan
-      MAYAN_CELERY_RESULT_BACKEND: redis://results:6379/0
-      MAYAN_DATABASE_ENGINE: django.db.backends.postgresql
-      MAYAN_DATABASE_HOST: db
-      MAYAN_DATABASE_NAME: mayan
-      MAYAN_DATABASE_PASSWORD: mayan-password
-      MAYAN_DATABASE_USER: mayan
+      - postgresql
+      - redis
+      # Enable to use RabbitMQ
+      #- rabbitmq
+    environment: &mayan_env
+      # Enable to use RabbitMQ
+      # MAYAN_CELERY_BROKER_URL: amqp://mayan:mayanrabbitpass@broker:5672/mayan
+      # Disable Redis Broker to use RabbitMQ as Broker
+      MAYAN_CELERY_BROKER_URL: redis://redis:6379/0
+      MAYAN_CELERY_RESULT_BACKEND: redis://redis:6379/1
+      MAYAN_DATABASES: "{'default':{'ENGINE':'django.db.backends.postgresql','NAME':'mayan','PASSWORD':'mayandbpass','USER':'mayan','HOST':'postgresql'}}"
+    image: mayanedms/mayanedms:3.3
+    networks:
+      - mayan-bridge
     ports:
       - "80:8000"
+    restart: unless-stopped
     volumes:
-      - app:/var/lib/mayan
+      - /docker-volumes/mayan-edms/media:/var/lib/mayan
+
+  postgresql:
+    environment:
+      POSTGRES_DB: mayan
+      POSTGRES_PASSWORD: mayandbpass
+      POSTGRES_USER: mayan
+    image: postgres:9.6-alpine
+    networks:
+      - mayan-bridge
+    restart: unless-stopped
+    volumes:
+      - /docker-volumes/mayan-edms/postgres:/var/lib/postgresql/data
+
+  redis:
+    command:
+      - redis-server
+      - --databases
+      - "2"
+      - --maxmemory-policy
+      - allkeys-lru
+      - --save
+      - ""
+    image: redis:5.0-alpine
+    networks:
+      - mayan-bridge
+    restart: unless-stopped
+
+  # Optional services
+
+  # celery_flower:
+  #   command:
+  #     - run_celery
+  #     - flower
+  #   depends_on:
+  #     - postgresql
+  #     - redis
+  #     # Enable to use RabbitMQ
+  #     # - rabbitmq
+  #   environment:
+  #     <<: *mayan_env
+  #   image: mayanedms/mayanedms:3.3
+  #   networks:
+  #     - mayan-bridge
+  #   ports:
+  #     - "5555:5555"
+  #   restart: unless-stopped
+
+  # Enable to use RabbitMQ
+  # rabbitmq:
+  #   container_name: mayan-edms-rabbitmq
+  #   image: healthcheck/rabbitmq
+  #   environment:
+  #     RABBITMQ_DEFAULT_USER: mayan
+  #     RABBITMQ_DEFAULT_PASS: mayanrabbitpass
+  #     RABBITMQ_DEFAULT_VHOST: mayan
+  #   networks:
+  #     - mayan-bridge
+  #   restart: unless-stopped
+  #   volumes:
+  #     - /docker-volumes/mayan-edms/rabbitmq:/var/lib/rabbitmq
+
+  # Enable to run standalone workers
+  # worker_fast:
+  #   command:
+  #     - run_worker
+  #     - fast
+  #   depends_on:
+  #     - postgresql
+  #     - redis
+  #     # Enable to use RabbitMQ
+  #     # - rabbitmq
+  #   environment:
+  #     <<: *mayan_env
+  #   image: mayanedms/mayanedms:3.3
+  #   networks:
+  #     - mayan-bridge
+  #   restart: unless-stopped
+  #   volumes:
+  #     - /docker-volumes/mayan-edms/media:/var/lib/mayan
+
+  # Enable to run frontend gunicorn
+  # frontend:
+  #   command:
+  #     - run_frontend
+  #   depends_on:
+  #     - postgresql
+  #     - redis
+  #     # Enable to use RabbitMQ
+  #     # - rabbitmq
+  #   environment:
+  #     <<: *mayan_env
+  #   image: mayanedms/mayanedms:3.3
+  #   networks:
+  #     - mayan-bridge
+  #   ports:
+  #     - "81:8000"
+  #   restart: unless-stopped
+  #   volumes:
+  #     - /docker-volumes/mayan-edms/media:/var/lib/mayan

docker/rootfs/usr/local/bin/entrypoint.sh

@@ -1,4 +1,7 @@
-#!/bin/sh
+#!/bin/bash
+
+# Use bash and not sh to support argument slicing "${@:2}"
+# sh defaults to dash instead of bash.
 
 set -e
 echo "mayan: starting entrypoint.sh"
@@ -6,32 +9,41 @@ INSTALL_FLAG=/var/lib/mayan/system/SECRET_KEY
 CONCURRENCY_ARGUMENT=--concurrency=
 
 DEFAULT_USER_UID=1000
-DEFAULT_USER_GUID=1000
+DEFAULT_USER_GID=1000
 
-export MAYAN_DEFAULT_BROKER_URL=redis://127.0.0.1:6379/0
-export MAYAN_DEFAULT_CELERY_RESULT_BACKEND=redis://127.0.0.1:6379/0
+MAYAN_USER_UID=${MAYAN_USER_UID:-${DEFAULT_USER_UID}}
+MAYAN_USER_GID=${MAYAN_USER_GID:-${DEFAULT_USER_GID}}
 
 export MAYAN_ALLOWED_HOSTS='["*"]'
 export MAYAN_BIN=/opt/mayan-edms/bin/mayan-edms.py
-export MAYAN_BROKER_URL=${MAYAN_BROKER_URL:-${MAYAN_DEFAULT_BROKER_URL}}
-export MAYAN_CELERY_RESULT_BACKEND=${MAYAN_CELERY_RESULT_BACKEND:-${MAYAN_DEFAULT_CELERY_RESULT_BACKEND}}
 export MAYAN_INSTALL_DIR=/opt/mayan-edms
 export MAYAN_PYTHON_BIN_DIR=/opt/mayan-edms/bin/
 export MAYAN_MEDIA_ROOT=/var/lib/mayan
 export MAYAN_SETTINGS_MODULE=${MAYAN_SETTINGS_MODULE:-mayan.settings.production}
+export DJANGO_SETTINGS_MODULE=${MAYAN_SETTINGS_MODULE}
 
 export MAYAN_GUNICORN_BIN=${MAYAN_PYTHON_BIN_DIR}gunicorn
 export MAYAN_GUNICORN_WORKERS=${MAYAN_GUNICORN_WORKERS:-2}
+export MAYAN_GUNICORN_TIMEOUT=${MAYAN_GUNICORN_TIMEOUT:-120}
 export MAYAN_PIP_BIN=${MAYAN_PYTHON_BIN_DIR}pip
 export MAYAN_STATIC_ROOT=${MAYAN_INSTALL_DIR}/static
 
-MAYAN_WORKER_FAST_CONCURRENCY=${MAYAN_WORKER_FAST_CONCURRENCY:-1}
-MAYAN_WORKER_MEDIUM_CONCURRENCY=${MAYAN_WORKER_MEDIUM_CONCURRENCY:-1}
+MAYAN_WORKER_FAST_CONCURRENCY=${MAYAN_WORKER_FAST_CONCURRENCY:-0}
+MAYAN_WORKER_MEDIUM_CONCURRENCY=${MAYAN_WORKER_MEDIUM_CONCURRENCY:-0}
 MAYAN_WORKER_SLOW_CONCURRENCY=${MAYAN_WORKER_SLOW_CONCURRENCY:-1}
 
-echo "mayan: changing uid/guid"
-usermod mayan -u ${MAYAN_USER_UID:-${DEFAULT_USER_UID}}
-groupmod mayan -g ${MAYAN_USER_GUID:-${DEFAULT_USER_GUID}}
+update_uid_gid() {
+    echo "mayan: update_uid_gid()"
+    groupmod mayan -g ${MAYAN_USER_GID} 2>/dev/null || true
+    usermod mayan -u ${MAYAN_USER_UID} -g ${MAYAN_USER_GID} 2>/dev/null
+
+    if [ ${MAYAN_USER_UID} -ne ${DEFAULT_USER_UID} ] || [ ${MAYAN_USER_GID} -ne ${DEFAULT_USER_GID} ]; then
+        echo "mayan: Updating file ownership. This might take a while if there are many documents."
+        chown -R mayan:mayan ${MAYAN_INSTALL_DIR} ${MAYAN_STATIC_ROOT} ${MAYAN_MEDIA_ROOT}
+    fi
+}
+
+update_uid_gid
 
 if [ "$MAYAN_WORKER_FAST_CONCURRENCY" -eq 0 ]; then
     MAYAN_WORKER_FAST_CONCURRENCY=
@@ -54,11 +66,9 @@ else
 fi
 export MAYAN_WORKER_SLOW_CONCURRENCY
 
-export CELERY_ALWAYS_EAGER=False
+# Allow importing of user setting modules
 export PYTHONPATH=$PYTHONPATH:$MAYAN_MEDIA_ROOT
 
-chown mayan:mayan /var/lib/mayan -R
-
 apt_get_install() {
     apt-get -q update
     apt-get install -y --force-yes --no-install-recommends --auto-remove "$@"
@@ -66,9 +76,9 @@ apt_get_install() {
     rm -rf /var/lib/apt/lists/*
 }
 
-initialize() {
-    echo "mayan: initialize()"
-    su mayan -c "${MAYAN_BIN} initialsetup --force --no-javascript"
+initialsetup() {
+    echo "mayan: initialsetup()"
+    su mayan -c "${MAYAN_BIN} initialsetup --force --no-dependencies"
 }
 
 os_package_installs() {
@@ -85,43 +95,75 @@ pip_installs() {
     fi
 }
 
-start() {
+run_all() {
     echo "mayan: start()"
     rm -rf /var/run/supervisor.sock
     exec /usr/bin/supervisord -nc /etc/supervisor/supervisord.conf
 }
 
-upgrade() {
-    echo "mayan: upgrade()"
-    su mayan -c "${MAYAN_BIN} performupgrade --no-javascript"
+performupgrade() {
+    echo "mayan: performupgrade()"
+    su mayan -c "${MAYAN_BIN} performupgrade --no-dependencies"
+}
+
+make_ready() {
+    # Check if this is a new install, otherwise try to upgrade the existing
+    # installation on subsequent starts
+    if [ ! -f $INSTALL_FLAG ]; then
+        initialsetup
+    else
+        performupgrade
+    fi
+}
+
+set_uid_guid() {
+    echo "mayan: changing uid/guid"
+    usermod mayan -u ${MAYAN_USER_UID:-${DEFAULT_USER_UID}}
+    groupmod mayan -g ${MAYAN_USER_GID:-${DEFAULT_USER_GID}}
 }
 
 os_package_installs || true
 pip_installs || true
+chown mayan:mayan /var/lib/mayan -R
 
 case "$1" in
 
-mayan) # Check if this is a new install, otherwise try to upgrade the existing
-       # installation on subsequent starts
-       if [ ! -f $INSTALL_FLAG ]; then
-           initialize
-       else
-           upgrade
-       fi
-       start
-       ;;
+run_initialsetup)
+    initialsetup
+    ;;
 
-run-tests) # Check if this is a new install, otherwise try to upgrade the existing
-           # installation on subsequent starts
-           if [ ! -f $INSTALL_FLAG ]; then
-               initialize
-           else
-               upgrade
-           fi
-           run-tests.sh
-           ;;
+run_performupgrade)
+    performupgrade
+    ;;
 
-*) su mayan -c "$@";
-   ;;
+run_all)
+    make_ready
+    run_all
+    ;;
+
+run_celery)
+    run_celery.sh "${@:2}"
+    ;;
+
+run_command)
+    su mayan -c "${MAYAN_BIN} ${@:2}"
+    ;;
+
+run_frontend)
+    run_frontend.sh
+    ;;
+
+run_tests)
+    make_ready
+    run_tests.sh "${@:2}"
+    ;;
+
+run_worker)
+    run_worker.sh "${@:2}"
+    ;;
+
+*)
+    su mayan -c "$@"
+    ;;
 
 esac

docker/rootfs/usr/local/bin/run-tests.sh

@@ -1,10 +0,0 @@
-#!/bin/sh
-
-export DEBIAN_FRONTEND=noninteractive
-
-apt-get update
-apt-get install -y --no-install-recommends gcc python-dev tesseract-ocr-deu
-
-su mayan -c "$MAYAN_PIP_BIN install -r ${MAYAN_INSTALL_DIR}/testing-base.txt"
-
-su mayan -c "$MAYAN_BIN test --mayan-apps --settings=mayan.settings.testing"

docker/rootfs/usr/local/bin/run_celery.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+# Use -A and not --app. Both are the same but behave differently
+# -A can be located before the command while --app cannot.
+su mayan -c "${MAYAN_PYTHON_BIN_DIR}celery -A mayan $@"

docker/rootfs/usr/local/bin/run_frontend.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+MAYAN_GUNICORN_MAX_REQUESTS=${MAYAN_GUNICORN_MAX_REQUESTS:-500}
+MAYAN_GUNICORN_MAX_REQUESTS_JITTERS=${MAYAN_GUNICORN_MAX_REQUESTS_JITTERS:-50}
+MAYAN_GUNICORN_WORKER_CLASS=${MAYAN_GUNICORN_WORKER_CLASS:-sync}
+
+su mayan -c "${MAYAN_PYTHON_BIN_DIR}gunicorn -w ${MAYAN_GUNICORN_WORKERS} mayan.wsgi --max-requests ${MAYAN_GUNICORN_MAX_REQUESTS} --max-requests-jitter ${MAYAN_GUNICORN_MAX_REQUESTS_JITTERS} --worker-class ${MAYAN_GUNICORN_WORKER_CLASS} --bind 0.0.0.0:8000 --timeout ${MAYAN_GUNICORN_TIMEOUT}"

docker/rootfs/usr/local/bin/run_tests.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+export DEBIAN_FRONTEND=noninteractive
+TEST_ARGUMENT=${@:-"--mayan-apps"}
+
+apt-get update
+apt-get install -y --no-install-recommends gcc python3-dev tesseract-ocr-deu
+
+su mayan -c "${MAYAN_PIP_BIN} install -r ${MAYAN_INSTALL_DIR}/testing-base.txt"
+
+su mayan -c "${MAYAN_BIN} test ${TEST_ARGUMENT} --settings=mayan.settings.testing"

docker/rootfs/usr/local/bin/run_worker.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+export MAYAN_WORKER_NAME=${MAYAN_WORKER_NAME:-$1}
+
+QUEUE_LIST_DEFAULT=`su mayan -c "${MAYAN_PYTHON_BIN_DIR}mayan-edms.py platformtemplate worker_queues"`
+
+MAYAN_QUEUE_LIST=${MAYAN_QUEUE_LIST:-${QUEUE_LIST_DEFAULT}}
+
+# Use -A and not --app. Both are the same but behave differently
+# -A can be located before the command while --app cannot.
+# Pass ${@:2} to allow overriding the defaults arguments
+su mayan -c "${MAYAN_PYTHON_BIN_DIR}celery -A mayan worker -Ofair -l ERROR -Q ${MAYAN_QUEUE_LIST} ${@:2}"

docker/rootfs/version

@@ -1 +1 @@
-3.2b1
+3.3.6

docker/stack.yml

@@ -9,24 +9,32 @@ volumes:
 services:
 
   db:
-    image: postgres
     environment:
       POSTGRES_DB: mayan
-      POSTGRES_PASSWORD: mayan-password
+      POSTGRES_PASSWORD: mayandbpass
       POSTGRES_USER: mayan
+    image: postgres
     volumes:
       - db:/var/lib/postgresql/data
 
   app:
+    environment:
+      MAYAN_CELERY_BROKER_URL: redis://redis:6379/1
+      MAYAN_CELERY_RESULT_BACKEND: redis://redis:6379/0
+      MAYAN_DATABASES: "{'default':{'ENGINE':'django.db.backends.postgresql','NAME':'mayan','PASSWORD':'mayandbpass','USER':'mayan','HOST':'db'}}"
     image: mayanedms/mayanedms:latest
     ports:
       - 80:8000
-    environment:
-      MAYAN_DATABASE_ENGINE: django.db.backends.postgresql
-      MAYAN_DATABASE_HOST: db
-      MAYAN_DATABASE_NAME: mayan
-      MAYAN_DATABASE_PASSWORD: mayan-password
-      MAYAN_DATABASE_USER: mayan
-      MAYAN_DATABASE_CONN_MAX_AGE: 0
     volumes:
       - app:/var/lib/mayan
+
+  redis:
+    command:
+      - redis-server
+      - --databases
+      - "2"
+      - --maxmemory-policy
+      - allkeys-lru
+      - --save
+      - ""
+    image: redis:5.0

docs/_ext/djangodocs.py

@@ -1,229 +0,0 @@
-"""
-Sphinx plugins for Django documentation.
-"""
-import os
-import re
-
-from docutils import nodes, transforms
-try:
-    import json
-except ImportError:
-    try:
-        import simplejson as json
-    except ImportError:
-        try:
-            from django.utils import simplejson as json
-        except ImportError:
-            json = None
-
-from sphinx import addnodes, roles, __version__ as sphinx_ver
-from sphinx.builders.html import StandaloneHTMLBuilder
-from sphinx.writers.html import SmartyPantsHTMLTranslator
-from sphinx.util.console import bold
-from sphinx.util.compat import Directive
-
-# RE for option descriptions without a '--' prefix
-simple_option_desc_re = re.compile(
-    r'([-_a-zA-Z0-9]+)(\s*.*?)(?=,\s+(?:/|-|--)|$)')
-
-def setup(app):
-    app.add_crossref_type(
-        directivename = "setting",
-        rolename      = "setting",
-        indextemplate = "pair: %s; setting",
-    )
-    app.add_crossref_type(
-        directivename = "templatetag",
-        rolename      = "ttag",
-        indextemplate = "pair: %s; template tag"
-    )
-    app.add_crossref_type(
-        directivename = "templatefilter",
-        rolename      = "tfilter",
-        indextemplate = "pair: %s; template filter"
-    )
-    app.add_crossref_type(
-        directivename = "fieldlookup",
-        rolename      = "lookup",
-        indextemplate = "pair: %s; field lookup type",
-    )
-    app.add_description_unit(
-        directivename = "django-admin",
-        rolename      = "djadmin",
-        indextemplate = "pair: %s; django-admin command",
-        parse_node    = parse_django_admin_node,
-    )
-    app.add_description_unit(
-        directivename = "django-admin-option",
-        rolename      = "djadminopt",
-        indextemplate = "pair: %s; django-admin command-line option",
-        parse_node    = parse_django_adminopt_node,
-    )
-    app.add_config_value('django_next_version', '0.0', True)
-    app.add_directive('versionadded', VersionDirective)
-    app.add_directive('versionchanged', VersionDirective)
-    app.add_builder(DjangoStandaloneHTMLBuilder)
-
-
-class VersionDirective(Directive):
-    has_content = True
-    required_arguments = 1
-    optional_arguments = 1
-    final_argument_whitespace = True
-    option_spec = {}
-
-    def run(self):
-        env = self.state.document.settings.env
-        arg0 = self.arguments[0]
-        is_nextversion = env.config.django_next_version == arg0
-        ret = []
-        node = addnodes.versionmodified()
-        ret.append(node)
-        if not is_nextversion:
-            if len(self.arguments) == 1:
-                linktext = 'Please see the release notes </releases/%s>' % (arg0)
-                xrefs = roles.XRefRole()('doc', linktext, linktext, self.lineno, self.state)
-                node.extend(xrefs[0])
-            node['version'] = arg0
-        else:
-            node['version'] = "Development version"
-        node['type'] = self.name
-        if len(self.arguments) == 2:
-            inodes, messages = self.state.inline_text(self.arguments[1], self.lineno+1)
-            node.extend(inodes)
-            if self.content:
-                self.state.nested_parse(self.content, self.content_offset, node)
-            ret = ret + messages
-        env.note_versionchange(node['type'], node['version'], node, self.lineno)
-        return ret
-
-
-class DjangoHTMLTranslator(SmartyPantsHTMLTranslator):
-    """
-    Django-specific reST to HTML tweaks.
-    """
-
-    # Don't use border=1, which docutils does by default.
-    def visit_table(self, node):
-        self._table_row_index = 0 # Needed by Sphinx
-        self.body.append(self.starttag(node, 'table', CLASS='docutils'))
-
-    # <big>? Really?
-    def visit_desc_parameterlist(self, node):
-        self.body.append('(')
-        self.first_param = 1
-        self.param_separator = node.child_text_separator
-
-    def depart_desc_parameterlist(self, node):
-        self.body.append(')')
-
-    if sphinx_ver < '1.0.8':
-        #
-        # Don't apply smartypants to literal blocks
-        #
-        def visit_literal_block(self, node):
-            self.no_smarty += 1
-            SmartyPantsHTMLTranslator.visit_literal_block(self, node)
-
-        def depart_literal_block(self, node):
-            SmartyPantsHTMLTranslator.depart_literal_block(self, node)
-            self.no_smarty -= 1
-
-    #
-    # Turn the "new in version" stuff (versionadded/versionchanged) into a
-    # better callout -- the Sphinx default is just a little span,
-    # which is a bit less obvious that I'd like.
-    #
-    # FIXME: these messages are all hardcoded in English. We need to change
-    # that to accomodate other language docs, but I can't work out how to make
-    # that work.
-    #
-    version_text = {
-        'deprecated':       'Deprecated in Django %s',
-        'versionchanged':   'Changed in Django %s',
-        'versionadded':     'New in Django %s',
-    }
-
-    def visit_versionmodified(self, node):
-        self.body.append(
-            self.starttag(node, 'div', CLASS=node['type'])
-        )
-        title = "%s%s" % (
-            self.version_text[node['type']] % node['version'],
-            len(node) and ":" or "."
-        )
-        self.body.append('<span class="title">%s</span> ' % title)
-
-    def depart_versionmodified(self, node):
-        self.body.append("</div>\n")
-
-    # Give each section a unique ID -- nice for custom CSS hooks
-    def visit_section(self, node):
-        old_ids = node.get('ids', [])
-        node['ids'] = ['s-' + i for i in old_ids]
-        node['ids'].extend(old_ids)
-        SmartyPantsHTMLTranslator.visit_section(self, node)
-        node['ids'] = old_ids
-
-def parse_django_admin_node(env, sig, signode):
-    command = sig.split(' ')[0]
-    env._django_curr_admin_command = command
-    title = "django-admin.py %s" % sig
-    signode += addnodes.desc_name(title, title)
-    return sig
-
-def parse_django_adminopt_node(env, sig, signode):
-    """A copy of sphinx.directives.CmdoptionDesc.parse_signature()"""
-    from sphinx.domains.std import option_desc_re
-    count = 0
-    firstname = ''
-    for m in option_desc_re.finditer(sig):
-        optname, args = m.groups()
-        if count:
-            signode += addnodes.desc_addname(', ', ', ')
-        signode += addnodes.desc_name(optname, optname)
-        signode += addnodes.desc_addname(args, args)
-        if not count:
-            firstname = optname
-        count += 1
-    if not count:
-        for m in simple_option_desc_re.finditer(sig):
-            optname, args = m.groups()
-            if count:
-                signode += addnodes.desc_addname(', ', ', ')
-            signode += addnodes.desc_name(optname, optname)
-            signode += addnodes.desc_addname(args, args)
-            if not count:
-                firstname = optname
-            count += 1
-    if not firstname:
-        raise ValueError
-    return firstname
-
-
-class DjangoStandaloneHTMLBuilder(StandaloneHTMLBuilder):
-    """
-    Subclass to add some extra things we need.
-    """
-
-    name = 'djangohtml'
-
-    def finish(self):
-        super(DjangoStandaloneHTMLBuilder, self).finish()
-        if json is None:
-            self.warn("cannot create templatebuiltins.js due to missing simplejson dependency")
-            return
-        self.info(bold("writing templatebuiltins.js..."))
-        xrefs = self.env.domaindata["std"]["objects"]
-        templatebuiltins = {
-            "ttags": [n for ((t, n), (l, a)) in xrefs.items()
-                        if t == "templatetag" and l == "ref/templates/builtins"],
-            "tfilters": [n for ((t, n), (l, a)) in xrefs.items()
-                        if t == "templatefilter" and l == "ref/templates/builtins"],
-        }
-        outfilename = os.path.join(self.outdir, "templatebuiltins.js")
-        f = open(outfilename, 'wb')
-        f.write('var django_template_builtins = ')
-        json.dump(templatebuiltins, f)
-        f.write(';\n')
-        f.close();

docs/_static/css/custom.css

@@ -3,3 +3,15 @@
     background: #2c3e50;
     padding: 3px;
 }
+
+.document a.internal {
+    text-decoration: none;
+    border-bottom: 1px dotted #707070;
+}
+
+.document a.internal:visited {
+    color: #2980B9;
+    text-decoration: none;
+    border-bottom: 1px dotted #707070;
+}
+

docs/_templates/donate.html

@@ -1,16 +0,0 @@
-<h3>Support</h3>
-<p>
-    <a href='http://www.mayan-edms.com/providers/'>Consulting and support plans are available, click here</a>.
-</p>
-<hr />
-<p>
-    Or consider supporting Mayan EDMS by contributing to its development. (US tax payers, please note this contribution is not tax deductible).
-</p>
-<form action="https://www.paypal.com/cgi-bin/webscr" method="post" target="_top">
-    <input type="hidden" name="cmd" value="_s-xclick">
-    <input type="hidden" name="hosted_button_id" value="3PXN336XFXQNN">
-    <input type="image" src="https://www.paypalobjects.com/en_US/i/btn/btn_donateCC_LG.gif" border="0" name="submit" alt="PayPal - The safer, easier way to pay online!">
-    <img alt="" border="0" src="https://www.paypalobjects.com/en_US/i/scr/pixel.gif" width="1" height="1" style="display: none !important;">
-</form>
-
-<hr />

docs/_templates/message_area.html

@@ -0,0 +1,41 @@
+<style>
+.wy-body-for-nav #message-area {
+    color: #b3b3b3;
+    margin: 1em 2em 1em 1em;
+}
+#message-area {
+    border: 1px dotted #2980B9;
+    padding: .5em;
+    margin: 1em 0;
+    text-align: center;
+}
+</style>
+
+<div id="message-area">
+    <h3>Get the book!</h3>
+
+    <a href="https://www.mayan-edms.com/book/">
+        <img src="{{ pathto('_static/book_cover.jpg', 1) }}" />
+    </a>
+
+    <hr />
+
+    <p>
+        On-site consulting and support plans are available, <a href='https://www.mayan-edms.com/support/'>click here</a>.
+    </p>
+
+    <hr />
+
+    <p>
+        Or consider donating to support the continued development of the project.
+    </p>
+
+    <form action="https://www.paypal.com/cgi-bin/webscr" method="post" target="_top">
+        <input type="hidden" name="cmd" value="_s-xclick">
+        <input type="hidden" name="hosted_button_id" value="3PXN336XFXQNN">
+        <input type="image" src="https://www.paypalobjects.com/en_US/i/btn/btn_donateCC_LG.gif" border="0" name="submit" alt="PayPal - The safer, easier way to pay online!">
+        <img alt="" border="0" src="https://www.paypalobjects.com/en_US/i/scr/pixel.gif" width="1" height="1" style="display: none !important;">
+    </form>
+</div>
+
+

docs/callbacks.py

@@ -0,0 +1,8 @@
+from __future__ import unicode_literals
+
+
+def get_source_read_callback(substitutions):
+    def global_substitution_function(app, docname, source):
+        for old, new in substitutions:
+            source[0] = source[0].replace(old, new)
+    return global_substitution_function

docs/chapters/acls.txt

@@ -1,6 +1,6 @@
-********************
+====================
 Access control lists
-********************
+====================
 
 Besides the permissions system explained in :doc:`../chapters/permissions`,
 Mayan EDMS provides per object permission granting. This feature is used to

docs/chapters/app_creation.rst

@@ -1,159 +0,0 @@
-************
-App creation
-************
-
-Mayan EDMS apps are essentially Django app with some extra code to register
-navigation, permissions and other relationships.
-
-
-App modules
-===========
-
-- __init__.py
-
-  Should be empty if possible. No initialization code should be here, use the
-  ready() method of the MayanAppConfig class in the apps.py module.
-
-- admin.py
-
-  Standard Django app module to define how models are to be presented in the
-  admin interface.
-
-- api_views.py
-
-  REST API views go here. Mayan EDMS uses Django REST Framework API view
-  classes.
-
-- apps.py
-
-  Contains the MayanAppConfig subclass as required by Django 1.7 and up. This
-  is a place to define the app name and translatable verbose name as well as
-  code to be execute when the modules of the app are ready.
-
-- classes.py
-
-  Hold python classes to be used internally or externally. Any class defined by
-  the app that is not a model.
-
-- events.py
-
-  Define event class instances that are later committed to a log by custom
-  code.
-
-- exceptions.py
-
-  Custom exceptions defined by the app.
-
-- fields.py
-
-  Place any custom form field classed you define here.
-
-- forms.py
-
-  Standard Django app module that hold custom form classes.
-
-- handlers.py
-
-  Contains the signal handlers, functions that will process a given signal
-  emitted from this or other apps. Connect the handler functions to the
-  corresponding signal in the ready() method of the MayanAppConfig subclass in
-  apps.py
-
-- html_widgets.py
-
-  Classes to render an HTML widget. HTML widget are not the same as Django's
-  native form widgets. Form widgets only work as part of a form field.
-  HTML widgets are for use outside of forms, such as in a table cell.
-
-- licenses.py
-
-  This module outlines the license text of the third party content used in 
-  the app. It could be other Python libraries, JavaScript libraries, etc.  
-
-- links.py
-
-  Defines the links to be used by the app. Import only from the navigation app
-  and the local permissions.py file.
-
-- literals.py
-
-  Stores magic numbers, module choices (if static), settings defaults, and
-  constants. Should contain all capital case variables. Must not import from
-  any other module.
-
-- managers.py
-
-  Standard Django app module that hold custom model managers. These act as
-  model class method to performs actions in a series of model instances or
-  utilitarian actions on external models instances.
-
-- models.py
-
-  Standard Django app module that defines ORM persistent data schema.
-
-- permissions.py
-
-  Defines the permissions to be used to validate user access by links and views.
-  Imports only from the permissions app. Link or view conditions such as
-  testing for is_staff or is_super_user flag are defined in this same module.
-
-- runtime.py
-
-  Use this module when you need the same instance of a class for the entire app.
-  This module acts as a shared memory space for the other modules of the app or
-  other apps.
-
-- search.py
-
-  Search model definitions. Define which field of the app's models are
-  searchable.
-
-- serializers.py
-
-  Hold Django REST Framework serializers used by the api_views.py module.
-
-- settings.py
-
-  Define the configuration settings instances that the app will use.
-
-- signals.py
-
-  Any custom defined signal goes here.
-
-- statistics.py
-
-  Provides functions that will compute any sort of statistical information on
-  the app’s data.
-
-- tasks.py
-
-  Code to be execute in the background or as an out-of-process action.
-
-- tests/ directory
-
-  Hold test modules. There should be one test_*.py module for each aspect being
-  tested, examples: test_api.py, test_views.py, test_parsers.py, test_permissions.py
-  Any shared constant data used by the tests should be added to tests/literals.py
-
-- utils.py
-
-  Holds utilitarian code that doesn't fit on any other app module or that is
-  used by several modules in the app. Anything used internally by the app that
-  is not a class or a literal (should be as little as possible)
-
-- widgets.py
-
-  Custom form widgets go here. This should be the only place with presentation
-  directives in the app (aside the templates).
-
-
-Views
-=====
-
-The module common.generics provides custom generic class based views to be used.
-The basic views used to create, edit, view and delete objects in Mayan EDMS
-are: SingleObjectCreateView, SingleObjectDetailView, SingleObjectEditView,
-and SingleObjectListView
-
-These views handle aspects relating to view permissions, object permissions,
-post action redirection and template context generation.

docs/chapters/backups.rst

@@ -1,70 +0,0 @@
-*******
-Backups
-*******
-
-To backup your install of Mayan EDMS just copy the actual document files and
-the database content. If you are using the default storage backend, the
-document files should be found in the ``media`` folder of your installation.
-
-To dump the content of your database manager refer to the documentation chapter
-regarding database data "dumping".
-
-Here is an example of how to perform a backup and a restore of a PostgreSQL
-database.
-
-To dump the database into an SQL text file::
-
-    pg_dump -h <host> -U <database user> -c <database name> -W > `date +%Y-%m-%d"_"%H-%M-%S`.sql
-
-Example::
-
-    pg_dump -h 127.0.0.1 -U mayan -c mayan -W > `date +%Y-%m-%d"_"%H-%M-%S`.sql
-
-To restore the database from the SQL text file::
-
-    psql -h <host> -U <database user> -d <database name> -W -f <sql dump file>
-
-Example::
-
-    psql -h 127.0.0.1 -U mayan -d mayan -W -f 2018-06-07_18-10-56.sql
-
-Here is an example of how to perform a backup and a restore of a PostgreSQL
-Docker container using a compressed dump file. A dump file is not compatible or
-can be used interchangeable with an SQL text file.
-
-To backup a PostgreSQL Docker container::
-
-    docker exec <container name> pg_dump -U <database user> -Fc -c <database name> > `date +%Y-%m-%d"_"%H-%M-%S`.dump
-
-Example::
-
-    docker exec mayan-edms-db pg_dump -U mayan -Fc -c mayan > `date +%Y-%m-%d"_"%H-%M-%S`.dump
-
-This will produce a compressed dump file with the current date and time as the filename.
-
-To restore a PostgreSQL Docker container::
-
-    docker exec -i <container name> pg_restore -U <database user> -d <database name> < <dump file>
-
-Since it is not possible to drop a currently open PostgreSQL database, this
-command must be used on a new and empty PostsgreSQL container.
-
-Example::
-
-    docker run -d \
-    --name mayan-edms-pg-new \
-    --restart=always \
-    -p 5432:5432 \
-    -e POSTGRES_USER=mayan \
-    -e POSTGRES_DB=mayan \
-    -e POSTGRES_PASSWORD=mayanuserpass \
-    -v /docker-volumes/mayan-edms/postgres-new:/var/lib/postgresql/data \
-    -d postgres:9.6
-
-    docker exec -i mayan-edms-pg-new pg_restore -U mayan -d mayan < 2018-06-07_17-09-34.dump
-
-More information at:
-
- - PostgreSQL: https://www.postgresql.org/docs/current/static/backup.html
- - MySQL: https://dev.mysql.com/doc/refman/5.7/en/mysqldump.html
- - SQLite: Just copy the file ``mayan/media/db.sqlite3``

docs/chapters/backups.txt

@@ -0,0 +1,18 @@
+=======
+Backups
+=======
+
+To backup your install of Mayan EDMS just copy the ``media`` folder and
+the database content.
+
+If you are using the default storage backend, the ``media`` should be located
+at |MAYAN_MEDIA_ROOT|::
+
+    sudo tar -zcvf backup.tar.gz |MAYAN_MEDIA_ROOT|
+
+To dump the content of your database manager refer to the documentation chapter
+regarding database data "dumping".
+
+- PostgreSQL: https://www.postgresql.org/docs/current/backup-dump.html
+- MySQL: https://dev.mysql.com/doc/refman/5.7/en/mysqldump.html
+

docs/chapters/cabinets.txt

@@ -1,13 +1,11 @@
+========
 Cabinets
 ========
 
 Cabinets are a multi-level method to organize documents. Each cabinet can
-contain documents as well as other sub level cabinets. Cabinets can
-be added from the :menuselection:`Cabinets --> Add new` menu.
+contain documents as well as other sub level cabinets.
 
 With a cabinet structure created, the next step is to add documents to them.
-To add documents to a cabinet, select the cabinet tab of a document, and
-select :menuselection:`Actions --> Add to cabinets` link.
 
 Documents can also be added in bulk to one or more cabinets from any view
 that lists documents.

docs/chapters/checkouts.txt

@@ -1,6 +1,6 @@
-*********
+=========
 Checkouts
-*********
+=========
 
 Checkouts are a way to block certain accesses or actions of a document for a
 period of time.

docs/chapters/database_conversion.rst

@@ -1,103 +0,0 @@
-*******************
-Database conversion
-*******************
-
-Version 3.1.x added a new management command to help convert data residing in
-an SQLite database to other database managers like PostgreSQL. Here is the
-conversion procedure.
-
-Direct install
-==============
-
-* Make a backup of your existing SQLite database and documents by copying the
-  ``/opt/mayan-edms/media`` folder.
-* :doc:`Upgrade to at least version 3.1.3. <../releases/3.1.3>`
-*  Migrate the existing SQLite database with the command ``performupgrade``::
-
-    sudo -u mayan MAYAN_MEDIA_ROOT=/opt/mayan-edms/media /opt/mayan-edms/bin/mayan-edms.py performupgrade
-
-* Install PostgreSQL::
-
-    sudo apt-get install postgresql libpq-dev
-
-* Provision a PostgreSQL database::
-
-    sudo -u postgres psql -c "CREATE USER mayan WITH password 'mayanuserpass';"
-    sudo -u postgres createdb -O mayan mayan
-
-* Install the Python client for PostgreSQL::
-
-    sudo -u mayan /opt/mayan-edms/bin/pip install --no-cache-dir --no-use-pep517 psycopg2==2.7.3.2
-
-* Copy the newly created fallback config file::
-
-    cp /opt/mayan-edms/media/config_backup.yml /opt/mayan-edms/media/config.yml
-
-* Edit the configuration file to add the entry for the PostgreSQL database and
-  rename the SQLite database to 'old'::
-
-    # Before
-    DATABASES:
-      default:
-        ATOMIC_REQUESTS: false
-        AUTOCOMMIT: true
-        CONN_MAX_AGE: 0
-        ENGINE: django.db.backends.sqlite3
-        HOST: ''
-        NAME: /opt/mayan-edms/media/db.sqlite3
-        OPTIONS: {}
-        PASSWORD: ''
-        PORT: ''
-        TEST: {CHARSET: null, COLLATION: null, MIRROR: null, NAME: null}
-        TIME_ZONE: null
-        USER: ''
-
-    # After
-    DATABASES:
-      old:
-        ATOMIC_REQUESTS: false
-        AUTOCOMMIT: true
-        CONN_MAX_AGE: 0
-        ENGINE: django.db.backends.sqlite3
-        HOST: ''
-        NAME: /opt/mayan-edms/media/db.sqlite3
-        OPTIONS: {}
-        PASSWORD: ''
-        PORT: ''
-        TEST: {CHARSET: null, COLLATION: null, MIRROR: null, NAME: null}
-        TIME_ZONE: null
-        USER: ''
-      default:
-        ATOMIC_REQUESTS: false
-        AUTOCOMMIT: true
-        CONN_MAX_AGE: 0
-        ENGINE: django.db.backends.postgresql
-        HOST: '127.0.0.1'
-        NAME: /opt/mayan-edms/media/db.sqlite3
-        OPTIONS: {}
-        PASSWORD: 'mayanuserpass'
-        PORT: ''
-        TEST: {CHARSET: null, COLLATION: null, MIRROR: null, NAME: null}
-        TIME_ZONE: null
-        USER: 'mayan'
-
-* Migrate the new database to create the empty tables::
-
-    sudo -u mayan MAYAN_DATABASE_ENGINE=django.db.backends.postgresql MAYAN_DATABASE_NAME=mayan MAYAN_DATABASE_PASSWORD=mayanuserpass MAYAN_DATABASE_USER=mayan MAYAN_DATABASE_HOST=127.0.0.1 MAYAN_MEDIA_ROOT=/opt/mayan-edms/media /opt/mayan-edms/bin/mayan-edms.py migrate
-
-* Convert the data in the SQLite and store it in the PostgreSQL database::
-
-    sudo -u mayan MAYAN_DATABASE_ENGINE=django.db.backends.postgresql MAYAN_DATABASE_NAME=mayan MAYAN_DATABASE_PASSWORD=mayanuserpass MAYAN_DATABASE_USER=mayan MAYAN_DATABASE_HOST=127.0.0.1 MAYAN_MEDIA_ROOT=/opt/mayan-edms/media /opt/mayan-edms/bin/mayan-edms.py convertdb --from=old --to=default --force
-
-* Update the supervisor config file to have Mayan EDMS run from the PostgreSQL database::
-
-    [supervisord]
-    environment=
-        <...>
-        MAYAN_DATABASE_ENGINE=django.db.backends.postgresql,
-        MAYAN_DATABASE_HOST=127.0.0.1,
-        MAYAN_DATABASE_NAME=mayan,
-        MAYAN_DATABASE_PASSWORD=mayanuserpass,
-        MAYAN_DATABASE_USER=mayan,
-        MAYAN_DATABASE_CONN_MAX_AGE=0,
-        <...>

docs/chapters/deploying.rst

@@ -1,256 +0,0 @@
-******************
-Direct deployments
-******************
-
-Mayan EDMS should be deployed like any other Django_ project and
-preferably using virtualenv_. Below are some ways to deploy and use Mayan EDMS.
-Do not use more than one method.
-
-Being a Django_ and a Python_ project, familiarity with these technologies is
-recommended to better understand why Mayan EDMS does some of the things it
-does.
-
-Compilers and development libraries will be installed to compile runtime
-libraries. LibreOffice and Poppler utils will also be installed as they are
-used to convert document files. Supervisor (http://supervisord.org/), a
-Process Control System, will be used to monitor and keep all Mayan processes
-running.
-
-
-Basic deployment
-================
-This setup uses less memory and CPU resources at the expense of some speed.
-For another setup that offers more performance and scalability refer to the
-`Advanced deployment`_ below.
-
-1. Install binary dependencies:
--------------------------------
-   If using a Debian_ or Ubuntu_ based Linux distribution, get the executable
-   requirements using::
-
-       sudo apt-get install g++ gcc ghostscript gnupg1 graphviz libfuse2 \
-       libjpeg-dev libmagic1 libpq-dev libpng-dev libreoffice libtiff-dev \
-       poppler-utils postgresql python-dev python-virtualenv redis-server \
-       sane-utils supervisor tesseract-ocr zlib1g-dev -y
-
-   .. note::
-
-       Platforms with the ARM CPU might also need additional requirements.
-       ::
-
-           apt-sudo get libffi-dev libssl-dev -y
-
-
-2. Create the user account for the installation:
-------------------------------------------------
-   This will create an unpriviledge user account that is also unable to login.
-   ::
-
-       sudo adduser mayan --disabled-password --disabled-login --no-create-home --gecos ""
-
-
-3. Create the parent directory where the project will be deployed:
-------------------------------------------------------------------
-   ``/opt/`` is a good choice as it is meant is for "software and add-on packages
-   that are not part of the default installation". (https://www.tldp.org/LDP/Linux-Filesystem-Hierarchy/html/opt.html)
-   ::
-
-       sudo mkdir /opt
-
-
-4. Create the Python virtual environment:
------------------------------------------
-   This will keep all the Python packages installed here isolated from the rest
-   of the Pythoon packages in the system.
-   ::
-
-       sudo virtualenv /opt/mayan-edms
-
-
-5. Make the mayan user the owner of the installation directory:
----------------------------------------------------------------
-   ::
-
-       sudo chown mayan:mayan /opt/mayan-edms -R
-
-
-6. Install Mayan EDMS from PyPI:
---------------------------------
-   ::
-
-       sudo -u mayan /opt/mayan-edms/bin/pip install --no-cache-dir --no-use-pep517 mayan-edms
-
-
-7. Install the Python client for PostgreSQL and Redis:
-------------------------------------------------------
-   ::
-
-       sudo -u mayan /opt/mayan-edms/bin/pip install --no-cache-dir --no-use-pep517 psycopg2==2.7.3.2 redis==2.10.6
-
-   .. note::
-
-       Platforms with the ARM CPU might also need additional requirements.
-       ::
-
-           sudo -u mayan /opt/mayan-edms/bin/pip install --no-cache-dir --no-use-pep517 psutil==5.6.2
-
-
-8. Create the database for the installation:
---------------------------------------------
-   ::
-
-       sudo -u postgres psql -c "CREATE USER mayan WITH password 'mayanuserpass';"
-       sudo -u postgres createdb -O mayan mayan
-
-
-9. Initialize the project:
---------------------------
-   This step will create all the database structures, download static media files
-   like JavaScript libraries and HTML frameworks, and create and initial admin
-   account with a random password.
-
-   .. note::
-
-       For simplicity, the ``MAYAN_MEDIA_ROOT`` folder is set to be a subfolder
-       of the installation. If you want to keep your files separated from
-       the installation files, change the value of the ``MAYAN_MEDIR_ROOT``
-       variable in this and all subsequent steps. Be sure to first create the
-       folder and give owership of it to the ``mayan`` user with the ``chown``
-       command.
-
-   .. warning::
-
-       If this step is interrupted, even if it is later resumed, will
-       cause the automatic admin user to no be created in some cases. Make sure all
-       environment variable and values are correct. If this happens, refer to the
-       troubleshooting chapters: :ref:`troubleshooting-autoadmin-account` and
-       :ref:`troubleshooting-admin-password`.
-
-   ::
-
-       sudo -u mayan MAYAN_DATABASE_ENGINE=django.db.backends.postgresql MAYAN_DATABASE_NAME=mayan \
-       MAYAN_DATABASE_PASSWORD=mayanuserpass MAYAN_DATABASE_USER=mayan \
-       MAYAN_DATABASE_HOST=127.0.0.1 MAYAN_MEDIA_ROOT=/opt/mayan-edms/media \
-       /opt/mayan-edms/bin/mayan-edms.py initialsetup
-
-
-10. Collect the static files:
------------------------------
-    This step merges and compressed static media files so they can be served more
-    effectively.
-
-    ::
-
-        sudo -u mayan MAYAN_MEDIA_ROOT=/opt/mayan-edms/media \
-        /opt/mayan-edms/bin/mayan-edms.py preparestatic --noinput
-
-
-11. Create the supervisor file at ``/etc/supervisor/conf.d/mayan.conf``:
-------------------------------------------------------------------------
-    ::
-
-        MAYAN_DATABASE_ENGINE=django.db.backends.postgresql MAYAN_DATABASE_NAME=mayan \
-        MAYAN_DATABASE_PASSWORD=mayanuserpass MAYAN_DATABASE_USER=mayan \
-        MAYAN_DATABASE_HOST=127.0.0.1 MAYAN_MEDIA_ROOT=/opt/mayan-edms/media \
-        /opt/mayan-edms/bin/mayan-edms.py platformtemplate supervisord > /etc/supervisor/conf.d/mayan.conf
-
-
-12. Configure Redis:
---------------------
-    Configure Redit to discard data when it runs out of memory, not save its
-    database and only keep 1 database:
-    ::
-
-        echo "maxmemory-policy allkeys-lru" >> /etc/redis/redis.conf
-        echo "save \"\"" >> /etc/redis/redis.conf
-        echo "databases 1" >> /etc/redis/redis.conf
-        systemctl restart redis
-
-13. Enable and restart the services [1_]:
------------------------------------------
-    ::
-
-        systemctl enable supervisor
-        systemctl restart supervisor
-
-
-14. Cleaning up:
-----------------
-    The following operating system dependencies are only needed during
-    installation and can be removed.
-    ::
-
-        apt-get remove --purge libjpeg-dev libpq-dev libpng-dev libtiff-dev zlib1g-dev
-
-
-.. _deployment_advanced:
-
-Advanced deployment
-===================
-
-This variation uses RabbitMQ as the message broker. RabbitMQ consumes more
-memory but scales to thousands of messages per second. RabbitMQ messages are also
-persistent by default, this means that pending tasks are not lost in the case
-of a restart or power failure. The Gunicorn workers are increased to 3.
-
-
-1. Install RabbitMQ:
---------------------
-
-   If using a Debian_ or Ubuntu_ based Linux distribution, get the executable
-   requirements using::
-
-       sudo apt-get install rabbitmq-server -y
-
-
-2. Install the Python client for RabbitMQ:
-------------------------------------------
-   ::
-
-       sudo -u mayan /opt/mayan-edms/bin/pip install --no-cache-dir --no-use-pep517 librabbitmq==2.0.0
-
-
-3. Create the RabbitMQ user and vhost:
---------------------------------------
-   ::
-
-       sudo rabbitmqctl add_user mayan mayanrabbitmqpassword
-       sudo rabbitmqctl add_vhost mayan
-       sudo rabbitmqctl set_permissions -p mayan mayan ".*" ".*" ".*"
-
-
-4. Edit the supervisor file at ``/etc/supervisor/conf.d/mayan.conf``:
----------------------------------------------------------------------
-   Replace (paying attention to the comma at the end)::
-
-       MAYAN_BROKER_URL="redis://127.0.0.1:6379/0",
-
-   with::
-
-       MAYAN_BROKER_URL="amqp://mayan:mayanuserpass@localhost:5672/mayan",
-
-   increase the number of Gunicorn workers to 3 in the line (``-w 2`` section)::
-
-       command = /opt/mayan-edms/bin/gunicorn -w 2 mayan.wsgi --max-requests 1000 --max-requests-jitter 50 --worker-class gevent --bind 0.0.0.0:8000 --timeout 120
-
-   remove the concurrency limit (or increase it) of the fast worker (remove ``--concurrency=1``).
-
-
-5. Restart the services:
-------------------------
-   ::
-
-       supervisorctl restart all
-
-
-
-
-[1]: https://bugs.launchpad.net/ubuntu/+source/supervisor/+bug/1594740
-
-.. _Debian: https://www.debian.org/
-.. _Django: https://www.djangoproject.com/
-.. _Python: https://www.python.org/
-.. _SQLite: https://www.sqlite.org/
-.. _Ubuntu: http://www.ubuntu.com/
-.. _virtualenv: http://www.virtualenv.org/en/latest/index.html
-.. _1: https://bugs.launchpad.net/ubuntu/+source/supervisor/+bug/1594740

docs/chapters/deploying.txt

@@ -0,0 +1,273 @@
+.. _deployment_basic:
+
+=================
+Direct deployment
+=================
+
+Mayan EDMS should be deployed like any other Django_ project and
+preferably using virtualenv_. Below are some ways to deploy and use Mayan EDMS.
+
+Being a Django_ and a Python_ project, familiarity with these technologies is
+recommended to better understand why Mayan EDMS does some of the things it
+does.
+
+Compilers and development libraries will be installed to compile runtime
+libraries. LibreOffice and Poppler utils will also be installed as they are
+used to convert document files. Supervisor (https://supervisord.org/), a
+Process Control System, will be used to monitor and keep all Mayan processes
+running.
+
+
+Basic deployment
+================
+This setup uses less memory and CPU resources at the expense of some speed.
+For another setup that offers more performance and scalability refer to the
+`Advanced deployment`_ below.
+
+#. Install binary dependencies:
+
+   If using a Debian_ or Ubuntu_ based Linux distribution, get the executable
+   requirements using:
+
+   .. code-block:: bash
+
+       sudo apt-get install exiftool g++ gcc ghostscript gnupg1 graphviz \
+       libfuse2 libjpeg-dev libmagic1 libpq-dev libpng-dev libreoffice \
+       libtiff-dev poppler-utils postgresql python3-dev python3-virtualenv
+       redis-server sane-utils supervisor tesseract-ocr zlib1g-dev -y
+
+   .. note::
+
+       Platforms with the ARM CPU might also need additional requirements:
+
+       .. code-block:: bash
+
+           sudo apt-get install libffi-dev libssl-dev -y
+
+
+#. Create the user account for the installation:
+
+   This will create an unprivileged user account that is also unable to login:
+
+   .. code-block:: bash
+
+       sudo adduser mayan --disabled-password --disabled-login --gecos ""
+
+
+#. Create the parent directory where the project will be deployed:
+
+   ``/opt/`` is a good choice as it is meant is for "software and add-on packages
+   that are not part of the default installation". (https://www.tldp.org/LDP/Linux-Filesystem-Hierarchy/html/opt.html).
+   Create the ``/opt`` directory if it doesn't already exists:
+
+   .. code-block:: bash
+
+       sudo mkdir /opt
+
+
+#. Create the Python virtual environment:
+
+   This will keep all the Python packages installed here isolated from the rest
+   of the Python packages in the system:
+
+   .. code-block:: bash
+
+       sudo virtualenv |MAYAN_INSTALLATION_DIRECTORY| -p /usr/bin/python3
+
+
+#. Make the mayan user the owner of the installation directory:
+
+   .. code-block:: bash
+
+       sudo chown mayan:mayan |MAYAN_INSTALLATION_DIRECTORY| -R
+
+
+#. Install Mayan EDMS from PyPI:
+
+   .. code-block:: bash
+
+       sudo -u mayan |MAYAN_PIP_BIN| install --no-use-pep517 mayan-edms
+
+
+#. Install the Python client for PostgreSQL and Redis:
+
+   .. code-block:: bash
+
+       sudo -u mayan |MAYAN_PIP_BIN| install --no-use-pep517 psycopg2==|PYTHON_PSYCOPG2_VERSION| redis==|PYTHON_REDIS_VERSION|
+
+   .. note::
+
+       Platforms with the ARM CPU might also need additional requirements:
+
+       .. code-block:: bash
+
+           sudo -u mayan |MAYAN_PIP_BIN| install --no-use-pep517 psutil==|PYTHON_PSUTIL_VERSION|
+
+
+#. Create the database for the installation:
+
+   .. code-block:: bash
+
+       sudo -u postgres psql -c "CREATE USER mayan WITH password 'mayanuserpass';"
+       sudo -u postgres createdb -O mayan mayan
+
+
+#. Initialize the project:
+
+   This step will create all the database structures, download static media files
+   like JavaScript libraries and HTML frameworks, and create and initial admin
+   account with a random password.
+
+   .. note::
+
+       For simplicity, the ``MAYAN_MEDIA_ROOT`` folder is set to be a subfolder
+       of the installation. If you want to keep your files separated from
+       the installation files, change the value of the ``MAYAN_MEDIA_ROOT``
+       variable in this and all subsequent steps. Be sure to first create the
+       folder and give ownership of it to the ``mayan`` user with the ``chown``
+       command.
+
+   .. warning::
+
+       If this step is interrupted, even if it is later resumed, will
+       cause the automatic admin user to not be created in some cases. Make
+       sure all environment variables and values are correct. If this
+       happens, refer to the troubleshooting chapters:
+       :ref:`troubleshooting-autoadmin-account` and
+       :ref:`troubleshooting-admin-password`.
+
+   .. code-block:: bash
+
+       sudo -u mayan MAYAN_DATABASES="{'default':{'ENGINE':'django.db.backends.postgresql','NAME':'mayan','PASSWORD':'mayanuserpass','USER':'mayan','HOST':'127.0.0.1'}}" \
+       MAYAN_MEDIA_ROOT=|MAYAN_MEDIA_ROOT| \
+       |MAYAN_BIN| initialsetup
+
+
+#. Collect the static files:
+
+   This step merges and compressed static media files so they can be served more
+   effectively.
+
+   .. code-block:: bash
+
+       sudo -u mayan MAYAN_MEDIA_ROOT=|MAYAN_MEDIA_ROOT| \
+       |MAYAN_BIN| preparestatic --noinput
+
+
+#. Create the supervisor file at ``|MAYAN_SUPERVISOR_CONF|``:
+
+   .. code-block:: bash
+
+       sudo -u mayan MAYAN_DATABASES="{'default':{'ENGINE':'django.db.backends.postgresql','NAME':'mayan','PASSWORD':'mayanuserpass','USER':'mayan','HOST':'127.0.0.1'}}" \
+       MAYAN_MEDIA_ROOT=|MAYAN_MEDIA_ROOT| \
+       |MAYAN_BIN| platformtemplate supervisord | sudo sh -c "cat > |MAYAN_SUPERVISOR_CONF|"
+
+
+#. Configure Redis:
+
+   Configure Redis to discard data when it runs out of memory, not save its
+   database and only keep 2 database:
+
+   .. code-block:: bash
+
+       sudo echo "maxmemory-policy allkeys-lru" >> /etc/redis/redis.conf
+       sudo echo "save \"\"" >> /etc/redis/redis.conf
+       sudo echo "databases 2" >> /etc/redis/redis.conf
+       sudo systemctl restart redis
+
+#. Enable and restart the services [1_]:
+
+   .. code-block:: bash
+
+       sudo systemctl enable supervisor
+       sudo systemctl restart supervisor
+
+
+#. Cleaning up:
+
+   The following operating system dependencies are only needed during
+   installation and can be removed.
+
+   .. code-block:: bash
+
+       sudo apt-get remove --purge libjpeg-dev libpq-dev libpng-dev libtiff-dev zlib1g-dev
+
+
+.. _deployment_advanced:
+
+Advanced deployment
+===================
+
+This variation uses RabbitMQ as the message broker. RabbitMQ consumes more
+memory but scales to thousands of messages per second. RabbitMQ messages are also
+persistent by default, this means that pending tasks are not lost in the case
+of a restart or power failure. The Gunicorn workers are increased to 3.
+
+
+#. Install RabbitMQ:
+
+   If using a Debian_ or Ubuntu_ based Linux distribution, get the executable
+   requirements using:
+
+   .. code-block:: bash
+
+       sudo apt-get install rabbitmq-server -y
+
+
+#. Install the Python client for RabbitMQ:
+
+   .. code-block:: bash
+
+       sudo -u mayan |MAYAN_PIP_BIN| install --no-use-pep517 amqp==|PYTHON_AMQP_VERSION|
+
+
+#. Create the RabbitMQ user and vhost:
+
+   .. code-block:: bash
+
+       sudo rabbitmqctl add_user mayan mayanrabbitmqpassword
+       sudo rabbitmqctl add_vhost mayan
+       sudo rabbitmqctl set_permissions -p mayan mayan ".*" ".*" ".*"
+
+
+#. Edit the supervisor file at ``|MAYAN_SUPERVISOR_CONF|``:
+
+   Replace (paying attention to the comma at the end):
+
+   .. code-block:: ini
+
+       MAYAN_CELERY_BROKER_URL="redis://127.0.0.1:6379/0",
+
+   with:
+
+   .. code-block:: ini
+
+       MAYAN_CELERY_BROKER_URL="amqp://mayan:mayanrabbitmqpassword@localhost:5672/mayan",
+
+   increase the number of Gunicorn workers to 3 in the line (``-w 2`` section):
+
+   .. code-block:: ini
+
+       command = |MAYAN_GUNICORN_BIN| -w 2 mayan.wsgi --max-requests 1000 --max-requests-jitter 50 --worker-class gevent --bind 0.0.0.0:8000 --timeout 120
+
+   remove the concurrency limit (or increase it) of the fast worker (remove ``--concurrency=1``).
+
+
+#. Restart the services:
+
+   .. code-block:: bash
+
+       sudo supervisorctl reread
+       sudo supervisorctl restart all
+
+
+
+[1]: https://bugs.launchpad.net/ubuntu/+source/supervisor/+bug/1594740
+
+.. _Debian: https://www.debian.org/
+.. _Django: https://www.djangoproject.com/
+.. _Python: https://www.python.org/
+.. _SQLite: https://www.sqlite.org/
+.. _Ubuntu: http://www.ubuntu.com/
+.. _virtualenv: http://www.virtualenv.org/en/latest/index.html
+.. _1: https://bugs.launchpad.net/ubuntu/+source/supervisor/+bug/1594740

docs/chapters/development/caa_entity.txt

@@ -1,6 +1,6 @@
-**************************************************
+==================================================
 Mayan EDMS Entity Contributor Assignment Agreement
-**************************************************
+==================================================
 
 Thank you for your interest in contributing to Mayan EDMS ("We" or "Us").
 
@@ -14,6 +14,7 @@ software project managed by Us.
 
 1. Definitions
 ==============
+
 "You" means any Legal Entity on behalf of whom a Contribution has been received
 by Us. "Legal Entity" means an entity which is not a natural person.
 "Affiliates" means other Legal Entities that control, are controlled by, or
@@ -55,6 +56,7 @@ first Submit a Contribution to Us, whichever is earlier.
 2. Grant of Rights
 ==================
 
+
 2.1 Copyright Assignment
 ------------------------
 
@@ -101,6 +103,7 @@ license the Contribution only under the terms of the license or licenses which
 We are using on the Submission Date for the Material (including any rights to
 adopt any future version of a license if permitted).
 
+
 2.4 Moral Rights
 ----------------
 
@@ -108,12 +111,14 @@ If moral rights apply to the Contribution, to the maximum extent permitted by
 law, You waive and agree not to assert such moral rights against Us or our
 successors in interest, or any of our licensees, either direct or indirect.
 
+
 2.5 Our Rights
 --------------
 
 You acknowledge that We are not obligated to use Your Contribution as part of
 the Material and may decide to include any Contribution We consider appropriate.
 
+
 2.6 Reservation of Rights
 -------------------------
 
@@ -159,6 +164,7 @@ OR OTHERWISE) UPON WHICH THE CLAIM IS BASED.
 6. Miscellaneous
 ================
 
+
 6.1 Jurisdiction
 ----------------
 
@@ -170,19 +176,25 @@ and the parties intend to avoid the application of the UN Convention to this
 Agreement and, thus, exclude the application of the UN Convention in its
 entirety to this Agreement.
 
+
 6.2 Acceptance
 --------------
+
 This Agreement sets out the entire agreement between You and Us for Your
 Contributions to Us and overrides all other agreements or understandings.
 
+
 6.3 Third parties
 -----------------
+
 If You or We assign the rights or obligations received through this Agreement
 to a third party, as a condition of the assignment, that third party must
 agree in writing to abide by all the rights and obligations in the Agreement.
 
+
 6.4 Unmet responsibilities
 --------------------------
+
 The failure of either party to require performance by the other party of any
 provision of this Agreement in one situation shall not affect the right of a
 party to require such performance at any time in the future. A waiver of
@@ -190,8 +202,10 @@ performance under a provision in one situation shall not be considered a
 waiver of the performance of the provision in the future or a waiver of the
 provision in its entirety.
 
+
 6.5 Continuation
 ----------------
+
 If any provision of this Agreement is found void and unenforceable, such
 provision will be replaced to the extent possible with a provision that comes
 closest to the meaning of the original provision and which is enforceable.

docs/chapters/development/caa_individual.txt

@@ -1,6 +1,6 @@
-******************************************************
+======================================================
 Mayan EDMS Individual Contributor Assignment Agreement
-******************************************************
+======================================================
 
 Thank you for your interest in contributing to Mayan EDMS ("We" or "Us").
 
@@ -11,6 +11,7 @@ caa@mayan-edms.com. This is a legally binding document, so please read it
 carefully before agreeing to it. The Agreement may cover more than one
 software project managed by Us.
 
+
 1. Definitions
 ==============
 
@@ -43,9 +44,11 @@ or otherwise designated in writing by You as "Not a Contribution."
 "Effective Date" means the date You execute this Agreement or the date You first
 Submit a Contribution to Us, whichever is earlier.
 
+
 2. Grant of Rights
 ==================
 
+
 2.1 Copyright Assignment
 ------------------------
 
@@ -87,6 +90,7 @@ and provided that this license is conditioned upon compliance with Section 2.3.
 
 2.3 Outbound License
 --------------------
+
 As a condition on the grant of rights in Sections 2.1 and 2.2, We agree to
 license the Contribution only under the terms of the license or licenses which
 We are using on the Submission Date for the Material (including any rights to
@@ -156,6 +160,7 @@ OR OTHERWISE) UPON WHICH THE CLAIM IS BASED.
 6. Miscellaneous
 ================
 
+
 6.1 Jurisdiction
 ----------------
 
@@ -167,12 +172,14 @@ and the parties intend to avoid the application of the UN Convention to this
 Agreement and, thus, exclude the application of the UN Convention in its
 entirety to this Agreement.
 
+
 6.2 Acceptance
 --------------
 
 This Agreement sets out the entire agreement between You and Us for Your
 Contributions to Us and overrides all other agreements or understandings.
 
+
 6.3 Third parties
 -----------------
 
@@ -180,6 +187,7 @@ If You or We assign the rights or obligations received through this Agreement
 to a third party, as a condition of the assignment, that third party must
 agree in writing to abide by all the rights and obligations in the Agreement.
 
+
 6.4 Unmet responsibilities
 --------------------------
 
@@ -190,6 +198,7 @@ performance under a provision in one situation shall not be considered a
 waiver of the performance of the provision in the future or a waiver of the
 provision in its entirety.
 
+
 6.5 Continuation
 ----------------
 

docs/chapters/development/code_structure.txt

@@ -0,0 +1,243 @@
+==============
+Code structure
+==============
+
+
+Follow PEP8
+-----------
+
+Whenever possible, but don't obsess over things like line length:
+
+.. code-block:: bash
+
+    $ flake8 --ignore=E501,E128,E122 |less
+
+To perform automatic PEP8 checks, install flake8's git hook using:
+
+.. code-block:: bash
+
+    $ flake8 --install-hook git
+
+
+Imports
+-------
+
+Import order should be:
+
+- Standard Python modules
+- Installed Python modules
+- Core Django modules
+- Installed Django modules
+- Mayan EDMS modules
+- Local imports
+
+Example:
+
+.. code-block:: bash
+
+    from __future__ import absolute_import
+
+    # Standard Python library
+    import base64
+
+    # 3rd party installed Python libraries
+    import requests
+
+    # Django core modules
+    from django.db.models import Q
+    from django.template.defaultfilters import slugify
+    from django.utils.translation import ugettext
+    from django.utils.translation import ugettext_lazy as _
+
+    # 3rd party installed Django libraries
+    from rest_framework import APIView
+
+    # Mayan apps
+    from metadata.classes import MetadataClass
+
+    # Local app imports (relative)
+    from .conf.settings import (
+        AVAILABLE_INDEXING_FUNCTIONS,
+        MAX_SUFFIX_COUNT, SLUGIFY_PATHS
+    )
+    from .exceptions import MaxSuffixCountReached
+    from .filesystem import (
+        fs_create_index_directory, fs_create_document_link,
+        fs_delete_document_link, fs_delete_index_directory,
+        assemble_suffixed_filename
+    )
+    from .models import Index, IndexInstanceNode, DocumentRenameCount
+
+All local app module imports are in relative form. Local app module name is to
+be referenced as little as possible, unless required by a specific feature,
+trick, restriction (e.g., Runtime modification of the module's attributes).
+
+Incorrect:
+
+.. code-block:: bash
+
+
+    # documents app views.py model
+    from documents.models import Document
+
+Correct:
+
+.. code-block:: bash
+
+    # documents app views.py model
+    from .models import Document
+
+
+Dependencies
+------------
+
+Mayan EDMS apps follow a hierarchical model of dependency. Apps import from
+their parents or siblings, never from their children. Think plugins. A parent
+app must never assume anything about a possible existing child app. The
+documents app and the Document model are the basic entities; they must never
+import anything else. The common and main apps are the base apps.
+
+
+Variables
+---------
+
+Naming of variables should follow a Major to Minor convention, usually
+including the purpose of the variable as the first piece of the name, using
+underscores as spaces. camelCase is not used in Mayan EDMS.
+
+Examples:
+
+Links:
+
+.. code-block:: bash
+
+    link_document_page_transformation_list = ...
+    link_document_page_transformation_create = ...
+    link_document_page_transformation_edit = ...
+    link_document_page_transformation_delete = ...
+
+Constants:
+
+.. code-block:: bash
+
+    PERMISSION_SMART_LINK_VIEW = ...
+    PERMISSION_SMART_LINK_CREATE = ...
+    PERMISSION_SMART_LINK_DELETE = ...
+    PERMISSION_SMART_LINK_EDIT = ...
+
+Classes:
+
+.. code-block:: bash
+
+    class Document(models.Model):
+    class DocumentPage(models.Model):
+    class DocumentPageTransformation(models.Model):
+    class DocumentType(models.Model):
+    class DocumentTypeFilename(models.Model):
+
+
+Strings
+-------
+
+Quotation character used in Mayan EDMS for strings is the single quote.
+Double quote is used for multiple line comments or HTML markup.
+
+
+Migrations
+----------
+
+Migrations should do only one thing (example: either create a table, move data
+to a new table or remove an old table) to aid retrying on failure.
+
+
+General
+-------
+
+Code should appear in their modules in alphabetic order or in their order of
+importance if it makes more sense for the specific application. This makes
+visual scanning easier on modules with a large number of imports, views or
+classes. Class methods that return a value should be pretended with a
+``get_`` to differentiate from an object’s properties. When a variable refers
+to a file it should be named as follows:
+
+- filename:  The file’s name and extension only.
+- filepath:  The entire path to the file including the filename.
+- path:  A path to a directory.
+
+Flash messages should end with a period as applicable for the language.
+Only exception is when the tail of the message contains an exceptions message
+as passed directly from the exception object.
+
+
+Steps to deploy a development version
+=====================================
+
+.. code-block:: bash
+
+    $ git clone |SOURCE_CODE_GIT|
+    $ cd mayan-edms
+    $ git checkout <Corresponding branch>
+    $ virtualenv venv
+    $ source venv/bin/activate
+    $ pip install -r requirements.txt
+    $ ./manage.py initialsetup
+    $ ./manage.py preparestatic
+    $ ./manage.py runserver
+
+
+
+Debugging
+=========
+
+Mayan EDMS makes extensive use of Django's new
+:django-docs:`logging capabilities <topics/logging>`.
+
+By default debug logging for all apps is turned on. If you wish to customize
+how logging is managed turn off automatic logging by setting
+`COMMON_AUTO_LOGGING` to ``False`` and add the following lines to your
+``settings/local.py`` file::
+
+    LOGGING = {
+        'version': 1,
+        'disable_existing_loggers': True,
+        'formatters': {
+            'verbose': {
+                'format': '%(levelname)s %(asctime)s %(name)s %(process)d %(thread)d %(message)s'
+            },
+            'intermediate': {
+                'format': '%(name)s <%(process)d> [%(levelname)s] "%(funcName)s() %(message)s"'
+            },
+            'simple': {
+                'format': '%(levelname)s %(message)s'
+            },
+        },
+        'handlers': {
+            'console':{
+                'level':'DEBUG',
+                'class':'logging.StreamHandler',
+                'formatter': 'intermediate'
+            }
+        },
+        'loggers': {
+            'documents': {
+                'handlers':['console'],
+                'propagate': True,
+                'level':'DEBUG',
+            },
+            'common': {
+                'handlers':['console'],
+                'propagate': True,
+                'level':'DEBUG',
+            },
+        }
+    }
+
+
+Likewise, to see the debug output of the ``tags`` app, just add the following inside the ``loggers`` block::
+
+
+    'tags': {
+        'handlers':['console'],
+        'propagate': True,
+        'level':'DEBUG',
+    },

docs/chapters/development/contributing.txt

@@ -0,0 +1,18 @@
+============
+Contributing
+============
+
+Mayan EDMS is under active development, and contributions are welcome.
+
+If you have a feature request, suggestion or bug report, please open a new
+issue on the `GitLab issue tracker`_. To submit patches, please send a merge
+request on GitLab_.
+
+.. _GitLab: |SOURCE_CODE_REPOSITORY|
+.. _`GitLab issue tracker`: |SOURCE_CODE_ISSUES|
+
+
+Contributing changes
+====================
+Follow the latest contributing guidelines outlined here:
+|SOURCE_CODE_REPOSITORY|blob/master/CONTRIBUTING.md

docs/chapters/development/documentation.txt

@@ -0,0 +1,24 @@
+=============
+Documentation
+=============
+
+The documentation is written in `reStructured Text`_ format, processed with
+Sphinx_, and resides in the ``docs`` directory. In order to build it, you will
+first need to install the documentation editing dependencies with::
+
+    $ pip install -r requirements/documentation.txt
+
+Then, to build an HTML version of the documentation, run the following command
+from the **docs** directory::
+
+    $ make docs-serve
+
+The generated documentation can be viewed by browsing to http://127.0.0.1:8000
+or by browsing to the ``docs/_build/html`` directory.
+
+You can also generate the documentation in formats other than HTML. Consult the
+Sphinx_ documentation for more details.
+
+.. _`reStructured Text`: http://docutils.sourceforge.net/rst.html
+.. _Sphinx: http://sphinx.pocoo.org
+

docs/chapters/development/index.txt

@@ -0,0 +1,29 @@
+==============
+For developers
+==============
+
+- :doc:`philosophies`
+- :doc:`code_structure`
+- :doc:`documentation`
+- :doc:`source_control`
+- :doc:`contributing`
+- :doc:`releases`
+- :doc:`translations`
+- :doc:`../../mercs/index`
+- :doc:`caa_entity`
+- :doc:`caa_individual`
+
+
+.. toctree::
+    :hidden:
+
+    philosophies
+    code_structure
+    documentation
+    source_control
+    contributing
+    releases
+    translations
+    ../../mercs/index
+    caa_entity
+    caa_individual

docs/chapters/development/philosophies.txt

@@ -0,0 +1,29 @@
+====================
+Project philosophies
+====================
+
+How to think about Mayan EDMS when doing changes or adding new features;
+why things are the way they are in Mayan EDMS:
+
+- Functionality must be as market/sector independent as possible, code for the
+  95% of use cases.
+- Each user must be able to configure and customize it to their needs after
+  install.
+- Abstract as much as possible, each app must be an expert in just one thing,
+  for other things they should use the API/classes/functions of other apps.
+- Assume as little as possible about anything outside the project
+  (hardware, OS, storage).
+- Provide Python based abstraction so that a default install runs with a single
+  step.
+- No hard dependencies on binaries unless there is no other choice.
+- Provide “drivers” or switchable backends to allow users to fine tune the
+  installation.
+- Call to binaries only when there is no other choice or the Python choices are
+  not viable/mature/efficient.
+- Each app is as independent and self contained as possible. Exceptions, the
+  basic requirements: navigation, permissions, common, main.
+- If an app is meant to be used by more than one other app, it should be as
+  generic as possible in regard to the project and another app will bridge the functionality.
+
+  - Example: since indexing (document_indexing) only applies to documents, the
+    app is specialized and depends on the documents app.

docs/chapters/development/releases.txt

@@ -0,0 +1,173 @@
+===================
+Installable package
+===================
+
+Source file package
+-------------------
+
+This is the sequence of step used to produce an installable package:
+
+1. Generate the packaged version (will produce dist/mayan-edms-x.y.z.tar.gz)::
+
+    $ make sdist
+
+2. Do a test install::
+
+    $ cd /tmp
+    $ virtualenv venv
+    $ source venv/bin/activate
+    $ pip install <path of the Git repository>/dist/mayan-edms-x.y.z.tar.gz
+    $ mayan-edms.py initialsetup
+    $ mayan-edms.py preparestatic
+    $ mayan-edms.py runserver
+
+
+Wheel package
+-------------
+
+1. Install the development requirements::
+
+    $ pip install -r requirements/development.txt
+
+2. Create wheel package using the makefile::
+
+    $ make wheel
+
+3. Do a test install::
+
+    $ cd /tmp
+    $ virtualenv venv
+    $ source venv/bin/activate
+    $ pip install <path of the Git repository>/dist/mayan_edms-x.y.z-py2-none-any.whl
+    $ mayan-edms.py initialsetup
+    $ mayan-edms.py preparestatic
+    $ mayan-edms.py runserver
+
+
+Version numbering
+=================
+
+Mayan EDMS uses the Semantic Versioning (http://semver.org/) method to choose
+version numbers along with Python's PEP-0440 (https://www.python.org/dev/peps/pep-0440/)
+to format them.
+
+X.YaN   # Alpha release
+X.YbN   # Beta release
+X.YrcN  # Release Candidate
+X.Y     # Final release
+
+
+Release checklist
+=================
+
+#. Check for missing migrations::
+
+    make check-missing-migrations
+
+#. Synchronize translations::
+
+    make translations-pull
+
+#. Compile translations::
+
+    make translations-compile
+
+#. Update changelog.
+#. Write release notes.
+#. Scan the code with flake8 for simple style warnings.
+#. Check README.rst format with::
+
+    python setup.py check -r -s
+
+   or with::
+
+       make check-readme
+
+#. Bump version in ``mayan/__init__.py`` and ``docker/rootfs/version``::
+
+    make increase-version PART=<major, minor or micro>
+
+#. Update requirements version in ``setup.py`` using:
+   ::
+
+       make generate-setup
+
+#. Commit as version bump.
+#. Build source package and test:
+   ::
+
+       make test-sdist-via-docker-ubuntu
+
+#. Build wheel package and test:
+   ::
+
+       make test-wheel-via-docker-ubuntu
+
+#. Tag version:
+   ::
+
+       git tag -a vX.Y.Z -m "Version X.Y.Z"
+
+#. Generate set ``setup.py`` again to update the build number::
+
+    make generate-setup
+
+#. Commit the new ``setup.py`` file.
+
+#. Release the version using one of the two following methods: GitLab CI or
+   manual
+
+
+Release using GitLab CI using the makefile
+------------------------------------------
+
+#. Use of the GitLab makefile targets: ``gitlab-release-all``,
+   ``gitlab-release-docker``, ``gitlab-release-documentation``,
+   ``gitlab-release-python``.
+
+
+Release using GitLab CI manually
+--------------------------------
+
+#. Push the current brach:
+   ::
+
+       git push
+
+
+#. Push the new tags:
+   ::
+
+       git push --tags
+
+
+#. Delete the corresponding ``releases/`` branch:
+   ::
+
+       git push origin :releases/<branch>
+
+#. Push the current branch to the corresponding origin ``releases/`` branch:
+   ::
+
+       git push origin <local branch>:releases/<branch>
+
+
+Manual release
+--------------
+
+#. Build and upload a test release:
+   ::
+
+       make release-test-via-docker-ubuntu
+
+#. Build and upload a final release:
+   ::
+
+       make release-via-docker-ubuntu
+
+Other steps
+-----------
+
+#. Update the contrib/scripts/install/docker.sh values
+
+#. Upload contrib/scripts/install/docker.sh to https://get.mayan-edms.com

docs/chapters/development/source_control.txt

@@ -0,0 +1,99 @@
+==============
+Source Control
+==============
+
+Mayan EDMS source is controlled with Git_.
+
+The project is publicly accessible, hosted and can be cloned from **GitLab** using::
+
+    $ git clone |SOURCE_CODE_GIT|
+
+
+Git branch structure
+--------------------
+
+Mayan EDMS follows a simplified model layout based on Vincent Driessen's
+`Successful Git Branching Model`_ blog post.
+
+``/versions/micro``
+    Working branch for the next bugfix release. Micro increment (third digit).
+    Only bug fixes, minor features, back-ported urgent features. This
+    branch is stable and safe for production.
+``/versions/minor``
+    Working branch for the next minor release (second digit). New features,
+    occasional breakage. Not for production but should run in test
+    environment most of the time. This is the branch you will want to
+    try out if you want to check out new features.
+``/versions/major``
+    Working branch for the next major release (first digit). New features,
+    incompatible changes to the user facing interfaces. Broken most of the
+    time, not for production and should only be cloned by developers
+    with experience with Mayan's development.
+``master``
+    Current production release (|version|). Points to the latest version of
+    the latest series. Production quality code.
+``features/``
+    Working branches for unfinished and unmerged feature. Likely unstable,
+    don't use in production. Once the feature is complete, it is merged
+    into one of the versions branches and deleted.
+
+Special branches:
+
+``releases/all``
+    Pushing code to this branch will trigger the build and release
+    a new Docker image, Documentation and Python package.
+``releases/docker``
+    Pushing code to this branch will trigger the build and release
+    of a new Docker image to Docker Hub.
+``releases/documentation``
+    Pushing code to this branch will trigger the build and release
+    of new documentation.
+``releases/python``
+    Pushing code to this branch will trigger the build and release
+    of a new Python package to PyPI.
+``nightly``
+    Pushing code to this branch will trigger the build and release
+    of a new Docker image based on development code to the GitLab image
+    repository only. The image will not be published to Docker Hub.
+
+Each release is tagged separately using annotated Git tags.
+
+When submitting patches, please place your code in its own ``feature/`` branch
+prior to opening a Merge Request on |SOURCE_CODE_ISSUES|.
+
+.. _Git: http://git-scm.org
+.. _`Successful Git Branching Model`: http://nvie.com/posts/a-successful-git-branching-model/
+
+
+Commit messages
+---------------
+
+#. Use English as the language for the commit messages.
+#. Provide a subject line composed of a tag and a short explanation::
+
+    Indexing: Add document base property reindex
+
+#. Keep the subject line to 50 or less characters.
+#. Capitalize the subject line.
+#. Don't end the subject line with a period, leave like a phrase in English.
+#. Use active voice in the. Say what the commit will do when applied not what
+   you did::
+
+       Add new properties to the model.
+
+   Vs.
+   ::
+
+       Added new properties to the model.
+
+#. Limit the body of the commit to 72 characters.
+#. When a commit fixes or improves an issue add the issue number in the commit
+   message. Either in the subject or in the body.
+#. Sign commit messages.
+#. Use explicit language even for minor commits. Don't do::
+
+       Fix typo
+
+   Use::
+
+       Document: Fix typo in label description

docs/chapters/development/translations.txt

@@ -1,6 +1,6 @@
-************
+============
 Translations
-************
+============
 
 Translations are handled online via the **Transifex** website:
 https://www.transifex.com/projects/p/mayan-edms/. To create a translation team

docs/chapters/docker.rst

@@ -1,447 +0,0 @@
-============
-Docker image
-============
-
-How to use this image
-=====================
-
-.. _docker_install:
-
-Start a Mayan EDMS Docker image
--------------------------------
-
-With Docker properly installed, proceed to download the Mayan EDMS image using
-the command::
-
-    docker pull mayanedms/mayanedms:<version>
-
-Instead of a specific version tag you may use then generic ``:latest`` tag
-to the get latest version available automatically. If you use the ``:latest``
-tag here, remember to do so in the next steps also.::
-
-    docker pull mayanedms/mayanedms:latest
-
-Then download version 9.6 of the Docker PostgreSQL image::
-
-    docker pull postgres:9.6
-
-Create and run a PostgreSQL container::
-
-    docker run -d \
-    --name mayan-edms-postgres \
-    --restart=always \
-    -p 5432:5432 \
-    -e POSTGRES_USER=mayan \
-    -e POSTGRES_DB=mayan \
-    -e POSTGRES_PASSWORD=mayanuserpass \
-    -v /docker-volumes/mayan-edms/postgres:/var/lib/postgresql/data \
-    -d postgres:9.6
-
-The PostgreSQL container will have one database named ``mayan``, with an user
-named ``mayan`` too, with a password of ``mayanuserpass``. The container will
-expose its internal 5432 port (PostgreSQL's default port) via the host's
-5432 port. The data of this container will reside on the host's
-``/docker-volumes/mayan-edms/postgres`` folder.
-
-Finally create and run a Mayan EDMS container.
-
-    docker run -d \
-    --name mayan-edms \
-    --restart=always \
-    -p 80:8000 \
-    -e MAYAN_DATABASE_ENGINE=django.db.backends.postgresql \
-    -e MAYAN_DATABASE_HOST=172.17.0.1 \
-    -e MAYAN_DATABASE_NAME=mayan \
-    -e MAYAN_DATABASE_PASSWORD=mayanuserpass \
-    -e MAYAN_DATABASE_USER=mayan \
-    -e MAYAN_DATABASE_CONN_MAX_AGE=0 \
-    -v /docker-volumes/mayan-edms/media:/var/lib/mayan \
-    mayanedms/mayanedms:<version>
-
-The Mayan EDMS container will connect to the PostgreSQL container via the
-``172.17.0.1`` IP address (the Docker host's default IP address). It will
-connect using the ``django.db.backends.postgresql`` database driver and
-connect to the ``mayan`` database using the ``mayan`` user with the password
-``mayanuserpass``. The container will keep connections to the database
-for up to 60 seconds in an attempt to reuse them increasing response time
-and reducing memory usage. The files of the container will be store in the
-host's ``/docker-volumes/mayan-edms/media`` folder. The container will
-expose its web service running on port 8000 on the host's port 80.
-
-The container will be available by browsing to ``http://localhost`` or to
-the IP address of the computer running the container.
-
-If another web server is running on port 80 use a different port in the
-``-p`` option. For example: ``-p 81:8000``.
-
-
-Using a dedicated Docker network
---------------------------------
-
-Use this method to avoid having to expose PostreSQL port to the host's network
-or if you have other PostgreSQL instances but still want to use the default
-port of 5432 for this installation.
-
-Create the network::
-
-    docker network create mayan
-
-Launch the PostgreSQL container with the network option and remove the port
-binding (``-p 5432:5432``)::
-
-    docker run -d \
-    --name mayan-edms-postgres \
-    --network=mayan \
-    --restart=always \
-    -e POSTGRES_USER=mayan \
-    -e POSTGRES_DB=mayan \
-    -e POSTGRES_PASSWORD=mayanuserpass \
-    -v /docker-volumes/mayan-edms/postgres:/var/lib/postgresql/data \
-    -d postgres:9.6
-
-Launch the Mayan EDMS container with the network option and change the
-database hostname to the PostgreSQL container name (``mayan-edms-postgres``)
-instead of the IP address of the Docker host (``172.17.0.1``)::
-
-    docker run -d \
-    --name mayan-edms \
-    --network=mayan \
-    --restart=always \
-    -p 80:8000 \
-    -e MAYAN_DATABASE_ENGINE=django.db.backends.postgresql \
-    -e MAYAN_DATABASE_HOST=mayan-edms-postgres \
-    -e MAYAN_DATABASE_NAME=mayan \
-    -e MAYAN_DATABASE_PASSWORD=mayanuserpass \
-    -e MAYAN_DATABASE_USER=mayan \
-    -e MAYAN_DATABASE_CONN_MAX_AGE=0 \
-    -v /docker-volumes/mayan-edms/media:/var/lib/mayan \
-    mayanedms/mayanedms:<version>
-
-
-Stopping and starting the container
------------------------------------
-
-To stop the container use::
-
-    docker stop mayan-edms
-
-
-To start the container again::
-
-    docker start mayan-edms
-
-
-.. _docker_environment_variables:
-
-
-Environment Variables
----------------------
-
-The Mayan EDMS image can be configure via environment variables.
-
-``MAYAN_DATABASE_ENGINE``
-
-Defaults to ``None``. This environment variable configures the database
-backend to use. If left unset, SQLite will be used. The database backends
-supported by this Docker image are:
-
-- ``'django.db.backends.postgresql'``
-- ``'django.db.backends.mysql'``
-- ``'django.db.backends.sqlite3'``
-
-When using the SQLite backend, the database file will be saved in the Docker
-volume. The SQLite database as used by Mayan EDMS is meant only for development
-or testing, never use it in production.
-
-``MAYAN_DATABASE_NAME``
-
-Defaults to 'mayan'. This optional environment variable can be used to define
-the database name that Mayan EDMS will connect to. For more information read
-the pertinent Django documentation page:
-:django-docs:`Connecting to the database <ref/databases/#connecting-to-the-database>`
-
-``MAYAN_DATABASE_USER``
-
-Defaults to 'mayan'. This optional environment variable is used to set the
-username that will be used to connect to the database. For more information
-read the pertinent Django documentation page:
-:django-docs:`Settings, USER <ref/settings/#user>`
-
-``MAYAN_DATABASE_PASSWORD``
-
-Defaults to ''. This optional environment variable is used to set the
-password that will be used to connect to the database. For more information
-read the pertinent Django documentation page:
-:django-docs:`Settings, PASSWORD <ref/settings/#password>`
-
-``MAYAN_DATABASE_HOST``
-
-Defaults to `None`. This optional environment variable is used to set the
-hostname that will be used to connect to the database. This can be the
-hostname of another container or an IP address. For more information read
-the pertinent Django documentation page:
-:django-docs:`Settings, HOST <ref/settings/#host>`
-
-``MAYAN_DATABASE_PORT``
-
-Defaults to `None`. This optional environment variable is used to set the
-port number to use when connecting to the database. An empty string means
-the default port. Not used with SQLite. For more information read the
-pertinent Django documentation page:
-:django-docs:`Settings, PORT <ref/settings/#port>`
-
-``MAYAN_BROKER_URL``
-
-This optional environment variable determines the broker that Celery will use
-to relay task messages between the frontend code and the background workers.
-For more information read the pertinent Celery Kombu documentation page: `Broker URL`_
-
-.. _Broker URL: http://kombu.readthedocs.io/en/latest/userguide/connections.html#connection-urls
-
-This Docker image supports using Redis and RabbitMQ as brokers.
-
-Caveat: If the `MAYAN_BROKER_URL` and `MAYAN_CELERY_RESULT_BACKEND` environment
-variables are specified, the built-in Redis server inside the container will
-be disabled.
-
-``MAYAN_CELERY_RESULT_BACKEND``
-
-This optional environment variable determines the results backend that Celery
-will use to relay result messages from the background workers to the frontend
-code. For more information read the pertinent Celery Kombu documentation page:
-`Task result backend settings`_
-
-.. _Task result backend settings: http://docs.celeryproject.org/en/3.1/configuration.html#celery-result-backend
-
-This Docker image supports using Redis and RabbitMQ as result backends.
-
-Caveat: If the `MAYAN_BROKER_URL` and `MAYAN_CELERY_RESULT_BACKEND` environment
-variables are specified, the built-in Redis server inside the container will
-be disabled.
-
-``MAYAN_SETTINGS_MODULE``
-
-Optional. Allows loading an alternate settings file.
-
-``MAYAN_DATABASE_CONN_MAX_AGE``
-
-Amount in seconds to keep a database connection alive. Allow reuse of database
-connections. For more information read the pertinent Django documentation
-page: :django-docs:`Settings, CONN_MAX_AGE <ref/settings/#conn-max-age>`
-According to new information Gunicorn's microthreads don't share connections
-and will exhaust the available Postgres connections available if a number
-other than 0 is used. Reference: https://serverfault.com/questions/635100/django-conn-max-age-persists-connections-but-doesnt-reuse-them-with-postgresq
-and https://github.com/benoitc/gunicorn/issues/996
-
-``MAYAN_GUNICORN_WORKERS``
-
-Optional. This environment variable controls the number of frontend workers
-that will be executed. If not specified the default is 2. For heavier loads,
-user a higher number. A formula recommended for this setting is the number
-of CPU cores + 1.
-
-``MAYAN_WORKER_FAST_CONCURRENCY``
-
-Optional. Changes the concurrency (number of child processes) of the Celery
-worker consuming the queues in the fast (low latency, short tasks) category.
-Default is 1. Use 0 to disable hardcoded concurrency and allow the Celery
-worker to launch its default number of child processes (equal to the number
-of CPUs detected).
-
-``MAYAN_WORKER_MEDIUM_CONCURRENCY``
-
-Optional. Changes the concurrency (number of child processes) of the Celery
-worker consuming the queues in the medium (medium latency, long running tasks)
-category. Default is 1. Use 0 to disable hardcoded concurrency and allow the
-Celery worker to launch its default number of child processes (equal to the
-number of CPUs detected).
-
-``MAYAN_WORKER_SLOW_CONCURRENCY``
-
-Optional. Changes the concurrency (number of child processes) of the Celery
-worker consuming the queues in the slow (high latency, very long running tasks)
-category. Default is 1. Use 0 to disable hardcoded concurrency and allow the
-Celery worker to launch its default number of child processes (equal to the
-number of CPUs detected).
-
-``MAYAN_USER_UID``
-
-Optional. Changes the UID of the ``mayan`` user internal to the Docker
-container. Defaults to 1000.
-
-``MAYAN_USER_GUID``
-
-Optional. Changes the GUID of the ``mayan`` user internal to the Docker
-container. Defaults to 1000.
-
-
-.. _docker-accessing-outside-data:
-
-Accessing outside data
-======================
-
-To use Mayan EDMS's staging folders or watch folders from Docker, the data
-for these source must be made accessible to the container. This is done by
-mounting the folders in the host computer to folders inside the container.
-This is necessary because Docker containers do not have access to host data
-on purpose. For example, to make a folder in the host accessible as a watch
-folder, add the following to the Docker command line when starting the
-container::
-
-    -v /opt/scanned_files:/srv/watch_folder
-
-The command line would look like this::
-
-    docker run ... -v /opt/scanned_files:/srv/watch_folder mayanedms/mayanedms:latest
-
-Now create a watch folder in Mayan EDMS using the path ``/srv/watch_folder``
-and the documents from the host folder ``/opt/scanned_files`` will be
-automatically available. Use the same procedure to mount host folders to be
-used as staging folderes. In this example ``/srv/watch_folder`` was as the
-container directory, but any path can be used as long as it is not an
-already existing path or a path used by any other program.
-
-
-Performing backups
-==================
-
-To backup the existing data, stop the image and copy the content of the volume.
-For the example::
-
-    docker run -d --name mayan-edms --restart=always -p 80:8000 \
-    -v /docker-volumes/mayan:/var/lib/mayan \
-    -v /opt/scanned_files:/srv/watch_folder mayanedms/mayanedms:latest
-
-That would be the ``/docker-volumes/mayan folder``::
-
-    sudo tar -zcvf backup.tar.gz /docker-volumes/mayan
-    sudo chown `whoami` backup.tar.gz
-
-If using an external PostgreSQL or MySQL database or database containers, these
-too need to be backed up using their respective procedures. A simple solution
-is to copy the entire database container volume after the container has
-been stopped.
-
-
-Restoring from a backup
-=======================
-
-Uncompress the backup archive in the original docker volume using::
-
-    sudo tar -xvzf backup.tar.gz -C /
-
-
-Upgrading
-=========
-
-Upgrading a Mayan EDMS Docker container is actually a matter of stopping and
-deleting the container, downloading the most recent version of the image and
-starting a container again. The container will take care of updating the
-database structure to the newest version if necessary.
-
-**IMPORTANT!** Do not delete the volume storing the data, only the container.
-
-Stop the container to be upgraded::
-
-    docker stop mayan-edms
-
-
-Remove the container::
-
-    docker rm mayan-edms
-
-
-Pull the new image version::
-
-    docker pull mayanedms/mayanedms:latest
-
-
-Start the container again with the new image version::
-
-    docker run -d --name mayan-edms --restart=always -p 80:8000 -v /docker-volumes/mayan:/var/lib/mayan mayanedms/mayanedms:latest
-
-
-Building the image
-==================
-
-Clone the repository with::
-
-    git clone https://gitlab.com/mayan-edms/mayan-edms.git
-
-Change to the directory of the cloned repository::
-
-    cd mayan-edms
-
-Execute Docker's build command using the provided makefile::
-
-    make docker-build
-
-Or using an apt cacher to speed up the build::
-
-    make docker-build-with-proxy APT_PROXY=172.17.0.1:3142
-
-Replace the IP address `172.17.0.1` with the IP address of the computer
-running the APT proxy and caching service.
-
-
-Customizing the image
-=====================
-
-Simple method
--------------
-
-If you just need to add a few Ubuntu or Python packages to your installation,
-you can use the following environment variables:
-
-``MAYAN_APT_INSTALLS``
-
-Specifies a list of Ubuntu .deb packages to be installed via APT when the
-container is first created. The installed packages are not lost when the image
-is stopped. Example: To install the Tesseract OCR language packs for German
-and Spanish add the following in your ``docker start`` command line::
-
-    -e MAYAN_APT_INSTALLS="tesseract-ocr-deu tesseract-ocr-spa"
-
-``MAYAN_PIP_INSTALLS``
-
-Specifies a list of Python packages to be installed via ``pip``. Packages will
-be downloaded from the Python Package Index (https://pypi.python.org) by
-default.
-
-
-Using Docker compose
-====================
-
-To deploy a complete production stack using the included Docker compose file
-execute::
-
-    docker-compose -f docker-compose.yml up -d
-
-This Docker compose file will provision four containers:
-
-- Postgres as the database
-- Redis as the Celery result storage
-- RabbitMQ as the Celery broker
-- Mayan EDMS using the above service containers
-
-To stop the stack use::
-
-    docker-compose -f docker-compose.yml stop
-
-The stack will also create four volumes to store the data of each container.
-These are:
-
-- mayan_app - The Mayan EDMS data container, normally called `mayan_data` when not using Docker compose.
-- mayan_broker - The broker volume, in this case RabbitMQ.
-- mayan_db - The database volume, in this case Postgres.
-- mayan_results - The celery result backend volume, in this case Redis.
-
-
-Nightly images
-==============
-The continuous integration pipeline used for testing development builds also
-produces a resulting Docker image. These are build automatically and their
-stability is not guaranteed. They should never be used in production.
-If you want to try out the Docker images the development uses or want a sneak
-peek at the new features being worked on checkout the container registry at:
-https://gitlab.com/mayan-edms/mayan-edms/container_registry

docs/chapters/docker/accessing_data.txt

@@ -0,0 +1,30 @@
+.. _docker-accessing-outside-data:
+
+
+======================
+Accessing outside data
+======================
+
+To use Mayan EDMS's staging folders or watch folders from Docker, the data
+for these source must be made accessible to the container. This is done by
+mounting the folders in the host computer to folders inside the container.
+This is necessary because Docker containers do not have access to host data
+on purpose. For example, to make a folder in the host accessible as a watch
+folder, add the following to the Docker command line when starting the
+container::
+
+    -v /opt/scanned_files:/scanned_files
+
+The command line would look like this::
+
+    docker run ... -v /opt/scanned_files:/scanned_files mayanedms/mayanedms:latest
+
+Now create a watch folder in Mayan EDMS using the path ``/scanned_files``
+and the documents from the host folder ``/opt/scanned_files`` will be
+automatically available. Use the same procedure to mount host folders to be
+used as staging folders. In this example ``/scanned_files`` was used as the
+container directory, but any path can be used as long as:
+
+- the path not an already existing path
+- the path is not used by any other program
+- the path is a single level path

docs/chapters/docker/backups.txt

@@ -0,0 +1,47 @@
+.. _docker_backups:
+
+==================
+Performing backups
+==================
+
+#. Document files
+
+   To backup the existing data, stop the image and copy the content of the volume.
+   For the example::
+
+       docker run -d --name mayan-edms --restart=always -p 80:8000 \
+       -v /docker-volumes/mayan:/var/lib/mayan \
+       -v /opt/scanned_files:/scanned_files mayanedms/mayanedms:|DOCKER_MAYAN_IMAGE_VERSION|
+
+   That would be the ``/docker-volumes/mayan folder``::
+
+       sudo tar -zcvf backup.tar.gz /docker-volumes/mayan
+       sudo chown `whoami` backup.tar.gz
+
+#. Database
+
+   If using an external PostgreSQL or MySQL database or database containers, these
+   too need to be backed up using their respective procedures. A simple solution
+   is to copy the entire database container volume after the container has
+   been stopped.
+
+   - PostgreSQL: https://www.postgresql.org/docs/current/backup-dump.html
+   - MySQL: https://dev.mysql.com/doc/refman/5.7/en/mysqldump.html
+
+
+=======================
+Restoring from a backup
+=======================
+
+
+#. Document files
+
+   Uncompress the backup archive in the original docker volume using::
+
+       sudo tar -xvzf backup.tar.gz -C /
+
+
+#. Database
+
+   - PostgreSQL: https://www.postgresql.org/docs/current/backup-dump.html
+   - MySQL: https://dev.mysql.com/doc/refman/5.7/en/mysqldump.html

docs/chapters/docker/building.txt

@@ -0,0 +1,25 @@
+.. _docker_building:
+
+==================
+Building the image
+==================
+
+Clone the repository with::
+
+    git clone |SOURCE_CODE_GIT|
+
+Change to the directory of the cloned repository::
+
+    cd mayan-edms
+
+Execute Docker's build command using the provided makefile::
+
+    make docker-build
+
+Or using an APT cache to speed up the build::
+
+    make docker-build-with-proxy APT_PROXY=172.17.0.1:3142
+
+Replace the IP address `172.17.0.1` with the IP address of the computer
+running the APT proxy and caching service.
+

docs/chapters/docker/commands.txt

@@ -0,0 +1,10 @@
+.. _docker_commands:
+
+=================================
+Executing commands on a container
+=================================
+
+To perform Mayan EDMS commands in a running container that will take all the
+settings from the container, use the following command::
+
+    docker exec -ti <container name> /usr/local/bin/entrypoint.sh "/opt/mayan-edms/bin/mayan-edms.py <command>"

docs/chapters/docker/customizing.txt

@@ -0,0 +1,21 @@
+=====================
+Customizing the image
+=====================
+
+If you just need to add a few Ubuntu or Python packages to your installation,
+you can use the following environment variables:
+
+``MAYAN_APT_INSTALLS``
+
+Specifies a list of Ubuntu .deb packages to be installed via APT when the
+container is first created. The installed packages are not lost when the image
+is stopped. Example: To install the Tesseract OCR language packs for German
+and Spanish add the following in your ``docker start`` command line::
+
+    -e MAYAN_APT_INSTALLS="tesseract-ocr-deu tesseract-ocr-spa"
+
+``MAYAN_PIP_INSTALLS``
+
+Specifies a list of Python packages to be installed via ``pip``. Packages will
+be downloaded from the Python Package Index (https://pypi.python.org) by
+default.

docs/chapters/docker/docker_compose.txt

@@ -0,0 +1,30 @@
+.. _docker_compose:
+
+====================
+Using Docker compose
+====================
+
+To deploy a complete production stack using the included Docker compose file
+execute::
+
+    docker-compose -f docker-compose.yml up -d
+
+This Docker compose file will provision four containers:
+
+- Postgres as the database
+- Redis as the Celery result storage
+- RabbitMQ as the Celery broker
+- Mayan EDMS using the above service containers
+
+To stop the stack use::
+
+    docker-compose -f docker-compose.yml stop
+
+The stack will also create four volumes to store the data of each container.
+These are:
+
+- mayan_app - The Mayan EDMS data container, normally called `mayan_data` when not using Docker compose.
+- mayan_broker - The broker volume, in this case RabbitMQ.
+- mayan_db - The database volume, in this case Postgres.
+- mayan_results - The celery result backend volume, in this case Redis.
+

docs/chapters/docker/drivers.txt

@@ -0,0 +1,12 @@
+.. _docker_image_drivers:
+
+================
+Included drivers
+================
+
+The Docker image supports using Redis and RabbitMQ as result backends. For
+databases, the image includes support for PostgreSQL and MySQL/MariaDB.
+Support for additional brokers or databases may be added using the
+``MAYAN_APT_INSTALL`` environment variable.
+
+

docs/chapters/docker/environment_variables.txt

@@ -0,0 +1,63 @@
+.. index:: Docker environment variables
+.. _docker_environment_variables:
+
+
+=====================
+Environment Variables
+=====================
+
+The common set of settings can also be modified via environment variables when
+using the Docker image. In addition to the common set of settings, some Docker
+image specific environment variables are available.
+
+``MAYAN_SETTINGS_MODULE``
+
+Optional. Allows loading an alternate settings file.
+
+
+``MAYAN_GUNICORN_TIMEOUT``
+
+Optional. Changes the amount of time the frontend worker will wait for a
+request to finish before raising a timeout error. The default is 120
+seconds.
+
+``MAYAN_GUNICORN_WORKERS``
+
+Optional. This environment variable controls the number of frontend workers
+that will be executed. If not specified the default is 2. For heavier loads,
+user a higher number. A formula recommended for this setting is the number
+of CPU cores + 1.
+
+``MAYAN_WORKER_FAST_CONCURRENCY``
+
+Optional. Changes the concurrency (number of child processes) of the Celery
+worker consuming the queues in the fast (low latency, short tasks) category.
+Default is 1. Use 0 to disable hardcoded concurrency and allow the Celery
+worker to launch its default number of child processes (equal to the number
+of CPUs detected).
+
+``MAYAN_WORKER_MEDIUM_CONCURRENCY``
+
+Optional. Changes the concurrency (number of child processes) of the Celery
+worker consuming the queues in the medium (medium latency, long running tasks)
+category. Default is 1. Use 0 to disable hardcoded concurrency and allow the
+Celery worker to launch its default number of child processes (equal to the
+number of CPUs detected).
+
+``MAYAN_WORKER_SLOW_CONCURRENCY``
+
+Optional. Changes the concurrency (number of child processes) of the Celery
+worker consuming the queues in the slow (high latency, very long running tasks)
+category. Default is 1. Use 0 to disable hardcoded concurrency and allow the
+Celery worker to launch its default number of child processes (equal to the
+number of CPUs detected).
+
+``MAYAN_USER_UID``
+
+Optional. Changes the UID of the ``mayan`` user internal to the Docker
+container. Defaults to 1000.
+
+``MAYAN_USER_GID``
+
+Optional. Changes the GID of the ``mayan`` user internal to the Docker
+container. Defaults to 1000.

docs/chapters/docker/index.txt

@@ -0,0 +1,35 @@
+.. index:: Docker
+
+======
+Docker
+======
+
+Docker is a container technology. Containers are a standard unit of software
+that packages up code and all its dependencies.
+
+
+- Installation: :doc:`install_simple` | :doc:`install_docker_network` | :doc:`docker_compose` | :doc:`start_stop`
+- Setup: :doc:`environment_variables` | :doc:`drivers`
+- Integration: :doc:`accessing_data`
+- Development: :doc:`building` | :doc:`customizing` | :doc:`nightly`
+- Administration: :doc:`backups` | :doc:`upgrading` | :doc:`commands`
+
+
+.. toctree::
+    :hidden:
+
+    install_simple
+    install_docker_network
+    start_stop
+    environment_variables
+    drivers
+    accessing_data
+    backups
+    upgrading
+    building
+    customizing
+    docker_compose
+    nightly
+    commands
+
+

docs/chapters/docker/install_docker_network.txt

@@ -0,0 +1,67 @@
+.. _docker_install_networked:
+
+================================
+Using a dedicated Docker network
+================================
+
+Use this method to avoid having to expose PostreSQL port to the host's network
+or if you have other PostgreSQL instances but still want to use the default
+port of 5432 for this installation.
+
+#. Create the network:
+
+   .. code-block:: bash
+
+       docker network create mayan
+
+#. Launch the PostgreSQL container with the network option and remove the
+   port binding (``-p 5432:5432``):
+
+   .. code-block:: bash
+
+       docker run \
+       -d \
+       --name mayan-edms-postgres \
+       --network=mayan \
+       --restart=always \
+       -e POSTGRES_USER=mayan \
+       -e POSTGRES_DB=mayan \
+       -e POSTGRES_PASSWORD=mayanuserpass \
+       -v /docker-volumes/mayan-edms/postgres:/var/lib/postgresql/data \
+       |DOCKER_POSTGRES_IMAGE_VERSION|
+
+#. Launch the Redis container with the network option and remove the port
+   binding (``-p 6379:6379``):
+
+   .. code-block:: bash
+
+       docker run \
+       -d \
+       --name mayan-edms-redis \
+       --network=mayan \
+       |DOCKER_REDIS_IMAGE_VERSION| \
+       redis-server \
+       --databases \
+       "2" \
+       --maxmemory-policy \
+       allkeys-lru \
+       --save \
+       ""
+
+#. Launch the Mayan EDMS container with the network option and change the
+   database hostname to the PostgreSQL container name (``mayan-edms-postgres``)
+   instead of the IP address of the Docker host (``172.17.0.1``):
+
+   .. code-block:: bash
+
+       docker run \
+       -d \
+       --name mayan-edms \
+       --network=mayan \
+       --restart=always \
+       -p 80:8000 \
+       -e MAYAN_DATABASES="{'default':{'ENGINE':'django.db.backends.postgresql','NAME':'mayan','PASSWORD':'mayanuserpass','USER':'mayan','HOST':'mayan-edms-postgres'}}" \
+       -e MAYAN_CELERY_BROKER_URL="redis://mayan-edms-redis:6379/0" \
+       -e MAYAN_CELERY_RESULT_BACKEND="redis://mayan-edms-redis:6379/1" \
+       -v /docker-volumes/mayan-edms/media:/var/lib/mayan \
+       mayanedms/mayanedms:|DOCKER_MAYAN_IMAGE_VERSION|

docs/chapters/docker/install_simple.txt

@@ -0,0 +1,117 @@
+.. _docker_install:
+
+
+==========================
+Simple Docker installation
+==========================
+
+#. Install Docker:
+
+   .. code-block:: bash
+
+       wget -qO- https://get.docker.com/ | sh
+
+   If you don't want run an automated script, follow the instructions outlined
+   in their documentation: https://docs.docker.com/install/
+
+   Once the Docker installation is finished, proceed to the link below to install
+   the Docker image for Mayan EDMS.
+
+
+#. Download the Mayan EDMS Docker image:
+
+   With Docker properly installed, proceed to download the Mayan EDMS Docker
+   image using the command:
+
+   .. code-block:: bash
+
+       docker pull mayanedms/mayanedms:|DOCKER_MAYAN_IMAGE_VERSION|
+
+
+#. Download the PostgreSQL Docker image:
+
+   .. code-block:: bash
+
+       docker pull |DOCKER_POSTGRES_IMAGE_VERSION|
+
+
+#. Download the Redis Docker image:
+
+   .. code-block:: bash
+
+       docker pull |DOCKER_REDIS_IMAGE_VERSION|
+
+
+#. Create and run a PostgreSQL container:
+
+   .. code-block:: bash
+
+       docker run \
+       -d \
+       --name mayan-edms-postgres \
+       --restart=always \
+       -p 5432:5432 \
+       -e POSTGRES_USER=mayan \
+       -e POSTGRES_DB=mayan \
+       -e POSTGRES_PASSWORD=mayanuserpass \
+       -v /docker-volumes/mayan-edms/postgres:/var/lib/postgresql/data \
+       |DOCKER_POSTGRES_IMAGE_VERSION|
+
+   The PostgreSQL container will have one database named ``mayan``, with an user
+   named ``mayan`` too, with a password of ``mayanuserpass``. The container will
+   expose its internal 5432 port (PostgreSQL's default port) via the host's
+   5432 port. The data of this container will reside on the host's
+   ``/docker-volumes/mayan-edms/postgres`` folder.
+
+
+#. Create and run a Redis container:
+
+   .. code-block:: bash
+
+       docker run \
+       -d \
+       --name mayan-edms-redis \
+       --restart=always \
+       -p 6379:6379 \
+       |DOCKER_REDIS_IMAGE_VERSION| \
+       redis-server \
+       --databases \
+       "2" \
+       --maxmemory-policy \
+       allkeys-lru \
+       --save \
+       ""
+
+   The Redis container will have two databases, one for background task messages,
+   and the other to hold the results of those background tasks. Redis is
+   configure to not save its content to disk and to automatically clear up
+   memory.
+
+#. Create and run a Mayan EDMS container:
+
+   .. code-block:: bash
+
+       docker run \
+       -d \
+       --name mayan-edms \
+       --restart=always \
+       -p 80:8000 \
+       -e MAYAN_DATABASES="{'default':{'ENGINE':'django.db.backends.postgresql','NAME':'mayan','PASSWORD':'mayanuserpass','USER':'mayan','HOST':'172.17.0.1'}}" \
+       -e MAYAN_CELERY_BROKER_URL="redis://172.17.0.1:6379/0" \
+       -e MAYAN_CELERY_RESULT_BACKEND="redis://172.17.0.1:6379/1" \
+       -v /docker-volumes/mayan-edms/media:/var/lib/mayan \
+       mayanedms/mayanedms:|DOCKER_MAYAN_IMAGE_VERSION|
+
+   The Mayan EDMS container will connect to the PostgreSQL container via the
+   ``172.17.0.1`` IP address (the Docker host's default IP address). It will
+   connect using the ``django.db.backends.postgresql`` database driver and
+   connect to the ``mayan`` database using the ``mayan`` user with the password
+   ``mayanuserpass``. The files of the container will be store in the
+   host's ``/docker-volumes/mayan-edms/media`` folder. The container will
+   expose its web service running on port 8000 on the host's port 80.
+
+   The container will be available by browsing to ``http://localhost`` or to
+   the IP address of the computer running the container.
+
+   If another web server is running on port 80 use a different port in the
+   ``-p`` option. For example: ``-p 81:8000``.

docs/chapters/docker/nightly.txt

@@ -0,0 +1,12 @@
+==============
+Nightly images
+==============
+
+The continuous integration pipeline used for testing development builds also
+produces a resulting Docker image. These are build automatically and their
+stability is not guaranteed. They should never be used in production.
+If you want to try out the Docker images the development uses or want a sneak
+peek at the new features being worked on checkout the container registry at:
+|SOURCE_CODE_REPOSITORY|container_registry
+
+

docs/chapters/docker/start_stop.txt

@@ -0,0 +1,13 @@
+===================================
+Stopping and starting the container
+===================================
+
+To stop the container use::
+
+    docker stop mayan-edms
+
+
+To start the container again::
+
+    docker start mayan-edms
+

docs/chapters/docker/upgrading.txt

@@ -0,0 +1,30 @@
+=========
+Upgrading
+=========
+
+Upgrading a Mayan EDMS Docker container is actually a matter of stopping and
+deleting the container, downloading the most recent version of the image and
+starting a container again. The container will take care of updating the
+database structure to the newest version if necessary.
+
+**IMPORTANT!** Do not delete the volume storing the data, only the container.
+
+Stop the container to be upgraded::
+
+    docker stop mayan-edms
+
+
+Remove the container::
+
+    docker rm mayan-edms
+
+
+Pull the new image version::
+
+    docker pull mayanedms/mayanedms:latest
+
+
+Start the container again with the new image version::
+
+    docker run -d --name mayan-edms --restart=always -p 80:8000 -v /docker-volumes/mayan:/var/lib/mayan mayanedms/mayanedms:latest
+

docs/chapters/document_types.txt

@@ -1,3 +1,4 @@
+==============
 Document types
 ==============
 
@@ -67,22 +68,3 @@ assigned when uploaded into Mayan EDMS. A document can only be of one
 type at a given moment, but if needed, the type of a document can be changed.
 Upon changing its type, the document will lose its previous settings and
 attributes, and will inherit the settings and attributes of its new type.
-
-
-Creating document types
-=======================
-
-.. admonition:: Permissions required
-    :class: warning
-
-    The "Create document types" permission is required for this action.
-
-
-#. Go to the :menuselection:`System --> Setup --> Document types` menu.
-#. From the :guilabel:`Actions` dropdown select :guilabel:`Create document type`.
-#. Enter a label to be shown to users when using this document type.
-#. Optional: Enter a period of time after which documents of this type will be moved to the trash.
-#. Optional: Select the unit of time for the period after which the document will be moved to the trash.
-#. Optional: Enter a period of time after which documents moved to the trash will be permanently deleted.
-#. Optional: Select the unit of time for the period of time after which the documents moved to the trash will be permanently deleted
-#. Press :guilabel:`Submit`.

docs/chapters/file_storage.txt

@@ -1,6 +1,6 @@
-************
+============
 File storage
-************
+============
 
 The files are stored and placed under Mayan EDMS "control" to avoid
 filename clashes each file gets renamed to its ``UUID`` (Universally Unique ID),

docs/chapters/indexes/examples.txt

@@ -1,9 +1,9 @@
-
+==============
 Index examples
---------------
+==============
 
 Index of document types
-^^^^^^^^^^^^^^^^^^^^^^^
+=======================
 
 .. admonition:: Permissions required
     :class: warning
@@ -56,7 +56,7 @@ of types "Invoice" and "Letter", these will now go into the level "Correspondenc
 
 
 Index document by department, taken from the first character of the invoice number metadata
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+===========================================================================================
 
 Requires one index node with the template::
 
@@ -66,7 +66,7 @@ Requires one index node with the template::
 
 
 Nested date index from a date contained in a metadata
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+=====================================================
 
 Assuming the metadata type is named **date_issued** with a date format
 of YYYY-MM-DD. The target is to have two levels: one for years and another
@@ -91,7 +91,7 @@ sub level for months.
 
 
 Index by OCR content
-^^^^^^^^^^^^^^^^^^^^
+====================
 
 This example indexes documents in a "quarterly report" level if they have the
 fragment “quarterly report” in the OCR text::
@@ -105,21 +105,21 @@ The same applies to text content extracted for the document::
 
 
 Index documents not found in any cabinet
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+========================================
 ::
 
     {% if document.cabinets.count == 0 %}No Cabinets{% endif %}
 
 
-Index untagged documents
-^^^^^^^^^^^^^^^^^^^^^^^^
+Index documents not tagged
+==========================
 ::
 
     {% if document.tags.count == 0 %}No Tags{% endif %}
 
 
 Index documents specifically, by the year of a metadata field otherwise by their uploaded year
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+==============================================================================================
 ::
 
     {% for tag in document.tags.all %}{% if tag.label == "Taxes" %}{% if document.metadata_value_of.tax_year|length_is:"4" %}{{ document.metadata_value_of.tax_year }}{% else %}{{ document.date_added|date:"Y" }}{% endif %}{% endif %}{% endfor %}

docs/chapters/indexes/index.txt

@@ -0,0 +1,23 @@
+.. index:: Indexes
+
+================
+Document indexes
+================
+
+Document indexes are a multi level structure similar to folder in a filesystem,
+however the hierarchy of the levels is automatically created, updated, or deleted
+based on rules set by the administrator.
+
+- :doc:`indexes`
+- :doc:`examples`
+- :doc:`mirroring`
+
+.. toctree::
+  :hidden:
+
+  indexes
+  mirroring
+  examples
+
+
+

docs/chapters/indexes/indexes.txt

@@ -1,10 +1,11 @@
-Indexes
-=======
+===============
+Index structure
+===============
 
 Indexes are an automatic method to hierarchically organize documents in
-relation to their properties (:doc:`../chapters/metadata`, label, MIME type,
+relation to their properties (:doc:`../metadata`, label, MIME type,
 etc). To use indexes you need to first create an index template. Once created,
-associate the index to one or more :doc:`../chapters/document_types`.
+associate the index to one or more :doc:`../document_types`.
 
 Index are hierarchical models so a tree template needs to be specified for them.
 This tree template will contain references to document metadata or properties
@@ -75,41 +76,3 @@ that will be generate based on the tree template would be as follows:
       year_1 -> document_3;
 
    }
-
-.. include:: ../chapters/index_examples.rst
-
-
-Mirroring
----------
-
-Indexes can be exported as `FUSE <https://en.wikipedia.org/wiki/Filesystem_in_Userspace>`_
-filesystems. Using the management command ``mountindex`` we could export the
-previous example index as follows::
-
-    mkdir -p ~/indexes/products
-    mayan-edms.py mountindex product-sheets-per-year ~/indexes/products
-
-The ``~/indexes/products`` directory will now have a directory and files structure
-identical to that of the index. Once indexes are mounted with this command, they
-behave like any other filesystem directory and can even be further shared
-via the network with network file system software like
-`Samba <https://www.samba.org/>`_ or
-`NFS <https://en.wikipedia.org/wiki/Network_File_System>`_.
-
-.. blockdiag::
-
-    blockdiag {
-        orientation = portrait
-        span_width = 200;
-
-        index [ label = 'Product sheets per year', width=180 ];
-        block_device [ height = 100, label = "Block device\n(Hard drive)", shape = flowchart.database ];
-        network [ label = "Network", shape = cloud ];
-        user [ label = "Users", shape = actor ];
-
-        index -> block_device [ label = "mirroring", fontsize = 8 ];
-        block_device -> network -> user;
-   }
-
-Indexes and mirrored indexes are Read Only as they are generated as a result of
-prior activities like document uploads, metadata changes.

docs/chapters/indexes/mirroring.txt

@@ -0,0 +1,35 @@
+=========
+Mirroring
+=========
+
+Indexes can be exported as `FUSE <https://en.wikipedia.org/wiki/Filesystem_in_Userspace>`_
+filesystems. Using the management command ``mountindex`` we could export the
+previous example index as follows::
+
+    mkdir -p ~/indexes/products
+    mayan-edms.py mountindex product-sheets-per-year ~/indexes/products
+
+The ``~/indexes/products`` directory will now have a directory and files structure
+identical to that of the index. Once indexes are mounted with this command, they
+behave like any other filesystem directory and can even be further shared
+via the network with network file system software like
+`Samba <https://www.samba.org/>`_ or
+`NFS <https://en.wikipedia.org/wiki/Network_File_System>`_.
+
+.. blockdiag::
+
+    blockdiag {
+        orientation = portrait
+        span_width = 200;
+
+        index [ label = 'Product sheets per year', width=180 ];
+        block_device [ height = 100, label = "Block device\n(Hard drive)", shape = flowchart.database ];
+        network [ label = "Network", shape = cloud ];
+        user [ label = "Users", shape = actor ];
+
+        index -> block_device [ label = "mirroring", fontsize = 8 ];
+        block_device -> network -> user;
+   }
+
+Indexes and mirrored indexes are Read Only as they are generated as a result of
+prior activities like document uploads, metadata changes.

docs/chapters/languages.txt

@@ -1,6 +1,6 @@
-*********
+=========
 Languages
-*********
+=========
 
 The list of languages choices in the language dropdown used for documents is
 based on the current ISO 639 list. This list can be quite extensive. To reduce
@@ -31,6 +31,6 @@ Example::
 
 For more information check out the
 :ref:`environment variables <environment_variables>` chapter of the
-:doc:`../topics/settings` topic.
+:doc:`../parts/settings` topic.
 
 

docs/chapters/mailing.txt

@@ -1,12 +1,12 @@
-*****************
+=================
 Mailing documents
-*****************
+=================
 
 Sending emails in Mayan EDMS is controlled by two different system depending on
 the type of email being sent. These are administrative emails like password
 reset emails and user emails sent from the application. To configure
 administrative email for things like password reset check the topic:
-:doc:`../topics/administration`
+:doc:`../parts/administration`
 
 Application emails
 ==================
@@ -14,8 +14,7 @@ Application emails
 To allow users to send emails or documents from within the web interface,
 Mayan EDMS provides its our own email system called Mailing Profiles.
 Mailing Profiles support access control per user role and can use different
-email backends. Mailing Profiles are created from the
-:menuselection:`System --> Setup` menu.
+email backends.
 
 Once created mailing profiles allow users to send email messages from
 within the user interface containing either an URL link to the document or

docs/chapters/metadata.rst

@@ -1,99 +0,0 @@
-********
-Metadata
-********
-
-Metadata is the name of the attribute of a document. The concept of metadata is
-divided in two: **metadata types** (size, color, distance) and **metadata values** for
-those types. Metadata types are defined in the setup menu and associated with
-document types. Then when a document is uploaded, a value for that metadata
-can be entered. There are two kinds of metadata type to document type relations:
-optional and required. When a metadata type is optional for a document type,
-it can be left blank for a document being uploaded and the upload will still
-be successful. On the other hand required metadata type must be given a value
-or it will not be possible to upload the document at hand.
-
-Examples of metadata type: Invoice number, color, employee id.
-
-The data entry of metadata types can be set to allow any value to be provided
-(the default) or a list of possible values can be entered in the ``Lookup``
-configuration option and users will be presented with a drop down list of options
-instead of the default text entry box.
-
-If metadata types are setup to allow any value to be entered a ``validation``
-option can be chosen to block the entry of invalid data. Metadata types also
-provide ``parsers`` which will not block the entry of data but are able to
-interpret and modify the value provided by the user to a conform to a specific
-format. An example of a provided parser is the date parser which will interpret
-and correct dates provided by users regardless of the format in which they are
-entered.
-
-
-Creating metadata types
-=======================
-
-.. admonition:: Permissions required
-    :class: warning
-
-    The "Create new metadata types" permission is required for this action.
-
-
-#. Go to the :menuselection:`System --> Setup --> Metadata types` menu.
-#. From the :guilabel:`Actions` dropdown select :guilabel:`Create new`.
-#. Provide a name to reference this metadata type in other parts of the system.
-#. Enter a label to be shown to users when using this metadata type.
-#. Optional: Enter a default value for the metadata type.
-#. Optional: Provide a comma separated list of options to restrict the data entry
-   when using this metadata type.
-#. Optional: Select a validator and a parser to validate and cleanup the data
-   entry when not using a predetermined list of values.
-#. Press :guilabel:`Submit`.
-
-
-Assigning a metadata type to a document type
-============================================
-
-.. admonition:: Permissions required
-    :class: warning
-
-    - The "Edit metadata types" permission is required for this action, globally or
-      via an ACL for a metadata type.
-    - Also the "Edit document type" permission
-      is required, globally or via an ACL for a document type.
-
-
-This action can be performed in two ways.
-
-Option 1: Via the metadata type view
-------------------------------------
-
-#. Go to the :menuselection:`System --> Setup --> Metadata types` menu.
-#. Click on the button :guilabel:`Document types` of the metadata type you which
-   to associate.
-#. From the list of existing document types press either:
-
-   - :guilabel:`None` if this metadata type will not be available for documents
-     of the type.
-   - :guilabel:`Optional` if this metadata type will be available and is
-     optional to provide a value for documents of the type.
-   - :guilabel:`Required` if this metadata type will be available and is
-     required to provide a value for documents of the type.
-
-#. Press :guilabel:`Save`.
-
-
-Option 2: Via the document type view
-------------------------------------
-
-#. Go to the :menuselection:`System --> Setup --> Document types` menu.
-#. Click on the button :guilabel:`Metadata types` of the metadata type you which
-   to associate.
-#. From the list of existing metadata types press either:
-
-   - :guilabel:`None` if this metadata type will not be available for documents
-     of the type.
-   - :guilabel:`Optional` if this metadata type will be available and is
-     optional to provide a value for documents of the type.
-   - :guilabel:`Required` if this metadata type will be available and is
-     required to provide a value for documents of the type.
-
-#. Press :guilabel:`Save`.

docs/chapters/metadata.txt

@@ -0,0 +1,28 @@
+========
+Metadata
+========
+
+Metadata is the name of the attribute of a document. The concept of metadata is
+divided in two: **metadata types** (size, color, distance) and **metadata values** for
+those types. Metadata types are defined in the setup menu and associated with
+document types. Then when a document is uploaded, a value for that metadata
+can be entered. There are two kinds of metadata type to document type relations:
+optional and required. When a metadata type is optional for a document type,
+it can be left blank for a document being uploaded and the upload will still
+be successful. On the other hand required metadata type must be given a value
+or it will not be possible to upload the document at hand.
+
+Examples of metadata type: Invoice number, color, employee id.
+
+The data entry of metadata types can be set to allow any value to be provided
+(the default) or a list of possible values can be entered in the ``Lookup``
+configuration option and users will be presented with a drop down list of options
+instead of the default text entry box.
+
+If metadata types are setup to allow any value to be entered a ``validation``
+option can be chosen to block the entry of invalid data. Metadata types also
+provide ``parsers`` which will not block the entry of data but are able to
+interpret and modify the value provided by the user to a conform to a specific
+format. An example of a provided parser is the date parser which will interpret
+and correct dates provided by users regardless of the format in which they are
+entered.

docs/chapters/object_storage.rst

@@ -1,31 +0,0 @@
-**************
-Object storage
-**************
-
-It is possible to use object storage instead of the default filesystem based
-storage. One such object storage system is Amazon S3 (Simple Storage Service)
-API compatible object storage. The following steps will configure Mayan EDMS
-to use a S3 style storage for documents.
-
-1. Install the django-storages and boto3 Python libraries.
-
-  * For the direct deployment method of installation use::
-
-        pip install django-storages boto3
-
-
-  * or if using the Docker image, add the following the command line that runs the container::
-
-    -e MAYAN_PIP_INSTALLS='django-storages boto3'
-
-
-2. From the web interface navigate to the :menuselection:`System --> Setup --> Setting --> Documents` menu.
-3. Locate the **DOCUMENTS_STORAGE_BACKEND** setting, press **Edit** and enter::
-
-    storages.backends.s3boto3.S3Boto3Storage
-
-4. Save and locate the setting **DOCUMENTS_STORAGE_BACKEND_ARGUMENTS**, press **Edit** and enter::
-
-    '{access_key: <your S3 access key>, secret_key: <your S3 secret key>, bucket_name: <S3 bucket name>}'
-
-5. Save and restart your Mayan EDMS installation for the setting to take effect.

docs/chapters/ocr_backend.txt

@@ -1,10 +1,10 @@
-***********
+===========
 OCR backend
-***********
+===========
 
 Mayan EDMS ships an OCR backend that uses the FLOSS engine Tesseract
 (https://github.com/tesseract-ocr/tesseract/), but it can
-use other engines. To support other engines crate a wrapper that subclasses the
+use other engines. To support other engines create a wrapper that subclasses the
 ``OCRBackendBase`` class defined in mayan/apps/ocr/classes. This subclass should
 expose the ``execute`` method. For an example of how the Tesseract backend
 is implemented take a look at the file ``mayan/apps/ocr/backends/tesseract.py``

docs/chapters/password_reset.txt

@@ -1,6 +1,6 @@
-**************
+==============
 Password reset
-**************
+==============
 
 To use the password reset feature, administrative emails need to be configured.
 These are sent by the system itself and not by the users. Their usage and
@@ -27,7 +27,7 @@ Example::
 To change the reference URL in the password reset emails on in the
 default document mailing template modify the ``COMMON_PROJECT_URL`` setting.
 For information on the different ways to change a setting check the
-:doc:`../topics/settings` topic.
+:doc:`../parts/settings` topic.
 
 To test the email settings use the management command ``sendtestemail``.
 Example::

docs/chapters/permissions.txt

@@ -1,6 +1,6 @@
-***********
+===========
 Permissions
-***********
+===========
 
 Mayan EDMS provides very fine control over which actions users can
 perform. Action control works by allowing ``roles``, that are composed of

docs/chapters/quick_labels.rst

@@ -1,70 +0,0 @@
-************
-Quick labels
-************
-
-Quick labels are predetermined filenames that allow the quick renaming of
-documents as they are uploaded or after they have been uploaded.
-
-Quick labels are added and associated to a document type.
-
-Example of quick label: Invoice, Receipt from X store, Purchase order.
-
-It is possible to preserve the file extension when using quick labels.
-Extensions are required for some operating system to be able to detect the
-correct file type to access the content.
-
-For example if a document file is named "file0001.pdf" and the quick label
-"Receipt from X store" is applied, the resulting document label will be
-"Receipt from X store.pdf".
-
-
-Creating quick labels
-=====================
-
-.. admonition:: Permissions required
-    :class: warning
-
-    The "Edit document types" permission is required for this action, either
-    globally or via an ACL for a document type.
-
-
-Since quick labels are associated with document types, creating quick labels
-must be done from the document type view.
-
-#. Go to the :menuselection:`System --> Setup --> Document types` menu.
-#. In the document type list, click on the :guilabel:`Quick labels` button of
-   the document type for which you wish to create a quick label.
-#. In the view titled "Quick labels for document type: <your document type>",
-   from the :guilabel:`Actions` dropdown select :guilabel:`Add quick label to document type`.
-#. At the quick label creation form enter the desired label and press :guilabel:`Save`.
-
-
-Using quick labels during upload
-================================
-
-#. Use the new document upload wizard from :menuselection:`Documents --> New document`.
-#. Select a document type and navigate to the penultimate step, where you have
-   the option to drag and drop files to upload.
-#. Select a an option from the :guilabel:`Quick document rename` dropdown.
-#. Optionally select the :guilabel:`Preserve extension` checkbox to keep the file
-   extension.
-#. Upload your documents.
-
-
-Using quick labels for existing documents
-=========================================
-
-.. admonition:: Permissions required
-    :class: warning
-
-    The "Edit document properties" permission is required for this action, either
-    globally or via an ACL for a document or document type.
-
-
-#. Navigate to the document preview view of the document to rename. Make sure
-   quick labels for the document type of the document select have been created.
-#. From the :guilabel:`Actions` dropdown select :guilabel:`Edit Properties`.
-#. Select a an option from the :guilabel:`Quick document rename` dropdown.
-#. Optionally select the :guilabel:`Preserve extension` checkbox to keep the file
-   extension.
-#. Press :guilabel:`Save` to rename the document.

docs/chapters/quick_labels.txt

@@ -0,0 +1,18 @@
+============
+Quick labels
+============
+
+Quick labels are predetermined filenames that allow the quick renaming of
+documents as they are uploaded or after they have been uploaded.
+
+Quick labels are added and associated to a document type.
+
+Example of quick label: Invoice, Receipt from X store, Purchase order.
+
+It is possible to preserve the file extension when using quick labels.
+Extensions are required for some operating system to be able to detect the
+correct file type to access the content.
+
+For example if a document file is named "file0001.pdf" and the quick label
+"Receipt from X store" is applied, the resulting document label will be
+"Receipt from X store.pdf".

docs/chapters/rest_api.txt

@@ -1,6 +1,6 @@
-***
-API
-***
+========
+REST API
+========
 
 Mayan EDMS provides an HTTP REST Application Program Interface (or API). This
 API allows integration with 3rd party software using simple HTTP requests.
@@ -8,7 +8,7 @@ API allows integration with 3rd party software using simple HTTP requests.
 Several API authentication methods are provides: **Session**, **Token**,
 and **HTTP Basic**.
 
-The URL for the API can be found via the :menuselection:`Tools --> REST API
+The URL for the API can be found via the :menuselection:`Tools --> REST API`
 menu. The API is also self-documenting. The live API documentation can be
 found in the :menuselection:`Tools --> API Documentation (Swagger)` menu for
 the Swagger version and in the
@@ -57,7 +57,7 @@ Get a list of document types::
 
 Upload a new document::
 
-    with open('test_document.pdf', mode='rb') as
+    with open('test_document.pdf', mode='rb') as file_object:
         requests.post('http://127.0.0.1:8000/api/documents/', auth=('username', 'password'), files={'file': file_object}, data={'document_type': 1}).json()
 
     {u'description': u'',

docs/chapters/scaling_up.rst

@@ -1,155 +0,0 @@
-**********
-Scaling up
-**********
-
-The default installation method fits most use cases. If you use case requires
-more speed or capacity here are some suggestion that can help you improve the
-performance of your installation.
-
-
-Change the database manager
-===========================
-
-Use PostgreSQL or MySQL as the database manager.
-Tweak the memory setting of the database manager to increase memory allocation.
-More PostgreSQL specific examples are available in their wiki page:
-https://wiki.postgresql.org/wiki/Performance_Optimization
-
-
-Increase the number of Gunicorn workers
-=======================================
-
-The Gunicorn workers process HTTP requests and affect the speed at which the
-website responds.
-
-If you are using the Docker image, change the value of the
-``MAYAN_GUNICORN_WORKERS`` environment variable (check the Docker image chapter:
-:ref:`docker_environment_variables`). Normally this variable defaults to 2.
-Increase this number to match the number of CPU cores + 1.
-
-If you are using the direct deployment methods, change the line that reads::
-
-    command = /opt/mayan-edms/bin/gunicorn -w 2 mayan.wsgi --max-requests 500 --max-requests-jitter 50 --worker-class gevent --bind 0.0.0.0:8000 --timeout 120
-
-And increase the value of the ``-w 2`` argument. This line is found in the
-``[program:mayan-gunicorn]`` section of the supervisor configuration file.
-
-
-Background task processing
-==========================
-
-The Celery workers are system processes that take care of the background
-tasks requested by the frontend interactions like document image rendering
-and periodic tasks like OCR. There are several dozen tasks defined in the code.
-These tasks are divided into queues based on the app of the relationship
-between the tasks. The queues by default are divided into three groups
-based on the speed at which they need to be processed. The document page
-image rendering for example is categorized as a high volume, short duration
-task. The OCR is a high volume, long duration task. Email checking is a
-low volume, medium duration tasks. It is not advisable to have the same
-worker processing OCR to process image rendering too. If the worker is
-processing several OCR tasks it will not be able to provide fast images
-when an user is browsing the user interface. This is why by default the
-queues are split into 3 workers: fast, medium, and slow. Each worker will handle
-queues based on the latency required by each queue group.
-
-
-Optimizations
--------------
-
-* Increase the number of workers and redistribute the queues among them
-  (only possible with direct deployments).
-* Launch more workers to service a queue. For example for faster document
-  image generation launch 2 workers to process the converter queue only
-  possible with direct deployments).
-* By default each worker process uses 1 thread. You can increase the thread
-  count of each worker process with the Docker environment options:
-
-  * ``MAYAN_WORKER_FAST_CONCURRENCY``
-  * ``MAYAN_WORKER_MEDIUM_CONCURRENCY``
-  * ``MAYAN_WORKER_SLOW_CONCURRENCY``
-
-* If using direct deployment, increase the value of the ``--concurrency=1``
-  argument of each worker in the supervisor file. You can also remove this
-  argument and let the Celery algorithm choose the number of threads to
-  launch. Usually this defaults to the number of CPU cores + 1.
-
-
-Change the message broker
-=========================
-Messages are the method of communication between front end interactive code
-and background tasks. In this regard messages can be thought as homologous
-to tasks requests. Improving how many messages can be sent, stored and
-sorted will impact the number of tasks the system can handle. To save on
-memory, the basic deployment method and the Docker image default to using
-Redis as a message broker. To increase capacity and reduce volatility of
-messages (pending tasks are not lost during shutdown) use RabbitMQ to
-shuffle messages.
-
-For direct installs refer to the :ref:`deployment_advanced` documentation
-section for the required changes.
-
-For the Docker image, launch a separate RabbitMQ container
-(https://hub.docker.com/_/rabbitmq/)::
-
-    docker run -d --name mayan-edms-rabbitmq -e RABBITMQ_DEFAULT_USER=mayan -e RABBITMQ_DEFAULT_PASS=mayanrabbitmqpassword -e RABBITMQ_DEFAULT_VHOST=mayan rabbitmq:3
-
-Pass the MAYAN_BROKER_URL environment variable (https://kombu.readthedocs.io/en/latest/userguide/connections.html#connection-urls)
-to the Mayan EDMS container so that it uses the RabbitMQ container the
-message broker::
-
-    -e MAYAN_BROKER_URL="amqp://mayan:mayanrabbitmqpassword@localhost:5672/mayan",
-
-When tasks finish, they leave behind a return status or the result of a
-calculation, these are stored for a while so that whoever requested the
-background task, is able retrieve the result. These results are stored in the
-result storage. By default a Redis server is launched inside the Mayan EDMS
-container. You can launch a separate Docker Redis container and tell the Mayan
-EDMS container to use this via the ``MAYAN_CELERY_RESULT_BACKEND`` environment
-variable. The format of this variable is explained here: http://docs.celeryproject.org/en/3.1/configuration.html#celery-result-backend
-
-
-Deployment type
-===============
-
-Docker provides a faster deployment and the overhead is not high on modern
-systems. It is however memory and CPU limited by default and you need to
-increase this limits. The settings to change the container resource limits
-are here: https://docs.docker.com/config/containers/resource_constraints/#limit-a-containers-access-to-memory
-
-For the best performance possible use the advanced deployment method on a
-host dedicated to serving only Mayan EDMS.
-
-
-Storage
-=======
-
-For best input and output speed use a block based local filesystem on an
-SSD drive for the ``/media`` sub folder. The location of the ``/media`` folder
-will be specified by the ``MEDIA_ROOT`` setting.
-
-If capacity is your bottom line, switch to an
-:doc:`object storage <../chapters/object_storage>` system.
-
-
-Use additional hosts
-====================
-
-When one host is not enough you can use multiple hosts and share the load.
-Make sure that all hosts share the ``/media`` folder as specified by the
-``MEDIA_ROOT`` setting, also the database, the broker, and the result storage.
-One setting that needs to be changed in this configuration is the lock
-manager backend.
-
-Resource locking is a technique to avoid two processes or tasks to modify
-the same resource at the same time causing a race condition. Mayan EDMS uses
-its own lock manager. By default the lock manager with use a simple file
-based lock backend ideal for single host installations. For multiple hosts
-installation the database backend must be used in other to coordinate the
-resource locks between the different hosts over a share data medium. This is
-accomplished by modifying the environment variable ``LOCK_MANAGER_BACKEND`` in
-both the direct deployment or the Docker image. Use the value
-``lock_manager.backends.model_lock.ModelLock`` to switch to the database
-resource lock backend. If you can also write your own lock manager backend
-for other data sharing mediums with better performance than a relational
-database like Redis, Memcached, Zoo Keeper.

docs/chapters/settings.txt

@@ -1,6 +1,6 @@
-********
+========
 Settings
-********
+========
 
 Mayan EDMS can be configure via environment variables or by setting files.
 
@@ -14,7 +14,7 @@ To use environment variables, lookup the name of the setting you want to
 override in the "Settings" menu. The "Settings" menu is located inside the
 "Setup" main menu. To pass a value via an environment variable append
 ``"MAYAN_"`` to the name of the settings option. For example, to change
-the number of documents displayed per page (COMMON_PAGINATE_BY, by default 40),
+the number of documents displayed per page (``COMMON_PAGINATE_BY``, by default 40),
 use::
 
     export MAYAN_COMMON_PAGINATE_BY=10
@@ -32,8 +32,8 @@ Via YAML configuration file
 .. versionadded:: 3.1
 
 It is possible to modify the different settings by creating or editing the
-``media/config.yml`` file. This file is formatted in the YAML markup language (
-http://yaml.org/). Here is an example of what the looks like::
+``media/config.yml`` file. This file is formatted in the YAML markup language
+(http://yaml.org/). Here is an example of what the looks like::
 
     DOCUMENT_PARSING_AUTO_PARSING: true
     DOCUMENT_PARSING_PDFTOTEXT_PATH: /usr/bin/pdftotext
@@ -58,16 +58,18 @@ to be valid. You can revert manually by copy the file or by using the
 Via Python settings files
 =========================
 
-Another way to configure Mayan EDMS is via Python-style, settings files.
+Another way to configure Mayan EDMS and the one required when more extensive
+setup is required, such as when using external Python libraries, is via
+Python-style, settings files.
 If Mayan EDMS was installed using the Python package a ``mayan_settings``
 folder will created for this purpose. If you installed Mayan EDMS
 according to the :doc:`../chapters/deploying` instructions provided in this
 documentation your ``mayan_settings`` folder should be located in the directory:
-``/usr/share/mayan-edms/mayan/media/mayan_settings``.
+``|MAYAN_MEDIA_ROOT|/mayan_settings/``.
 
 If Mayan EDMS was installed using Docker, the ``mayan_settings`` folder
 will be found inside the install Docker volume. If you installed Mayan EDMS
-according to the :doc:`../chapters/docker` instructions provided in this
+according to the :doc:`../chapters/docker/index` instructions provided in this
 documentation your ``mayan_settings`` folder should be located in the directory:
 ``/docker-volumes/mayan/mayan_settings``.
 
@@ -80,7 +82,7 @@ Create a file with any valid name and a ``.py`` extension in the
 Now add the corresponding lines to override the default settings.
 In the settings file, it is not necessary to prepend the string ``MAYAN_`` to
 the setting option. For example, to change the number of documents displayed
-per page (COMMON_PAGINATE_BY, by default 40),
+per page (``COMMON_PAGINATE_BY``, by default 40),
 use::
 
     COMMON_PAGINATE_BY=10
@@ -100,13 +102,24 @@ For the :doc:`../chapters/deploying` method, the full import path will be
 ``mayan.media.mayan_settings.mysettings`` and can be passed via the
 ``--settings`` command line argument like this::
 
-    python manage.py runserver --settings=mayan.media.mayan_settings.mysettings
+    |MAYAN_BIN| runserver --settings=mayan.media.mayan_settings.mysettings
 
 or via the ``DJANGO_SETTINGS_MODULE`` environment variable like this::
 
     export DJANGO_SETTINGS_MODULE=mayan.media.mayan_settings.mysettings
 
-For the :doc:`../chapters/docker` installation method, the full import path will be
+To make the use of the custom Python setting file permanent, update the ``|MAYAN_SUPERVISOR_CONF|``
+file. In the ``[supervisord]`` section, update the ``environment=`` value of
+``DJANGO_SETTINGS_MODULE`` from  the default ``mayan.settings.production`` to
+``mayan.media.mayan_settings.mysettings``. Pay attention to the indentation of
+the ``environment=`` entries.
+
+Force supervisor to read the changes and restart using::
+
+    sudo supervisorctl reread
+    sudo supervisorctl restart all
+
+For the :doc:`../chapters/docker/index` installation method, the full import path will be
 ``mayan_settings.mysettings`` and can only be passed via the
 ``MAYAN_SETTINGS_MODULE`` environment variable like this::
 

docs/chapters/signatures.txt

@@ -1,6 +1,6 @@
-*******************
+===================
 Document signatures
-*******************
+===================
 
 Mayan EDMS supports two types of document signatures: embedded and
 detached signatures. When a document with an embedded signature is

docs/chapters/smart_links.txt

@@ -1,3 +1,4 @@
+===========
 Smart links
 ===========
 
@@ -6,7 +7,7 @@ in their respective indexes. Smart links are useful when two documents are
 related somehow but are of different type or different hierarchical units.
 
 Example: A patient record can be related to a prescription drug information
-document, but they each belong to their own :doc:`../chapters/indexes`.
+document, but they each belong to their own :doc:`../chapters/indexes/index`.
 
 Smart links are rule based, but don't create any organizational structure.
 Smart links just show the documents that match the rules as evaluated against