tesseract deu but restored else

Version 3.3.6

.dockerignore

@@ -4,8 +4,7 @@ HISTORY.md
 mayan/media
 db.sqlite*
 docker
-!docker/etc
-!docker/entrypoint.sh
-!docker/version
+!docker/rootfs
 ./.*
 docs
+

.drone.yml

@@ -0,0 +1,31 @@
+kind: pipeline
+name: default
+
+steps:
+  - name: docker  
+    image: plugins/docker
+    settings:
+      registry: registry.d1v3.de
+      repo: registry.d1v3.de/mayan-edms
+      username:
+        from_secret: docker_username
+      password:
+        from_secret: docker_password
+      dockerfile: docker/Dockerfile
+      tags: latest
+      auto_tag: true
+
+
+---
+kind: secret
+name: docker_username
+get:
+    path: kv/data/drone/docker
+    name: username
+---
+
+kind: secret
+name: docker_password
+get:
+    path: kv/data/drone/docker
+    name: token

.flake8

@@ -1,2 +1,3 @@
 [flake8]
-ignore = E501
+exclude = node_modules,static,templates
+ignore = E501, N801, N802, N804, N805, N806

.github/FUNDING.yml

@@ -0,0 +1 @@
+custom: https://www.mayan-edms.com/donation/

.gitignore

@@ -7,6 +7,7 @@
 .coverage
 .coverage.tox*
 .idea/
+*.swp
 .tox/
 .vagrant
 _build/
@@ -29,4 +30,8 @@ static_collected/
 /venv/
 /venv3/
 /whoosh_index/
+google_fonts/
 node_modules/
+docs/build/
+docs/_templates/layout.html
+docs/_templates/footer.html

.gitlab-ci.yml

@@ -17,9 +17,9 @@ job_docker_build:
     - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
   script:
     - docker build --pull -t "$CI_REGISTRY_IMAGE" -f docker/Dockerfile .
-    - VERSION=`cat docker/version`
+    - VERSION=`cat docker/rootfs/version`
     - docker tag "$CI_REGISTRY_IMAGE" "$CI_REGISTRY_IMAGE:$VERSION"
-    - docker run --rm "$CI_REGISTRY_IMAGE:$VERSION" run-tests
+    - docker run --rm "$CI_REGISTRY_IMAGE:$VERSION" run_tests
     - docker push "$CI_REGISTRY_IMAGE:$VERSION"
     - docker push "$CI_REGISTRY_IMAGE:latest"
     - docker tag "$CI_REGISTRY_IMAGE:$VERSION" registry-1.docker.io/mayanedms/mayanedms:"$VERSION"
@@ -37,7 +37,7 @@ job_docker_push:
   before_script:
     - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
   script:
-    - VERSION=`cat docker/version`
+    - VERSION=`cat docker/rootfs/version`
     - docker pull "$CI_REGISTRY_IMAGE:$VERSION"
     - docker tag "$CI_REGISTRY_IMAGE:$VERSION" registry-1.docker.io/mayanedms/mayanedms:"$VERSION"
     - docker tag "$CI_REGISTRY_IMAGE:$VERSION" registry-1.docker.io/mayanedms/mayanedms:latest
@@ -58,7 +58,7 @@ job_docker_nightly:
     - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
   script:
     - docker build --pull -t "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG" -f docker/Dockerfile .
-    - docker run --rm "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG" run-tests
+    - docker run --rm "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG" run_tests
     - docker push "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG"
   only:
     - nightly
@@ -119,7 +119,7 @@ job_build_python:
     - update-locale LANG=en_US.UTF-8
     - export LC_ALL=en_US.UTF-8
     - pip install -r requirements/build.txt
-    - make wheel
+    - make python-wheel
   only:
     - releases/all
     - releases/python
@@ -140,7 +140,7 @@ job_push_python:
 
 .test_base: &test_base
   stage: test
-  image: ubuntu:16.04
+  image: ubuntu:18.04
   cache:
     paths:
       - ~/.cache/pip/
@@ -152,13 +152,14 @@ job_push_python:
     - locale-gen en_US.UTF-8
     - update-locale LANG=en_US.UTF-8
     - export LC_ALL=en_US.UTF-8
-    - apt-get install -qq curl gcc ghostscript gpgv gnupg graphviz libfuse2 libjpeg-dev libmagic1 libpng-dev libtiff-dev poppler-utils libreoffice poppler-utils python-dev python-pip tesseract-ocr tesseract-ocr-deu
-    - pip install -r requirements/testing.txt
+    - apt-get install -qq curl exiftool firefox-geckodriver gcc ghostscript gnupg1 graphviz libfuse2 libjpeg-dev libmagic1 libpng-dev libtiff-dev poppler-utils libreoffice poppler-utils python-dev python-virtualenv python3-dev tesseract-ocr tesseract-ocr-deu
+    - virtualenv venv -p /usr/bin/python3
+    - . venv/bin/activate
+    - pip install -r requirements.txt -r requirements/testing-base.txt
   only:
     - releases/all
     - releases/docker
     - releases/python
-    - master
     - staging
     - nightly
 
@@ -170,8 +171,10 @@ test-mysql:
   services:
     - mysql:8.0.3
   script:
-    - apt-get install -qq libmysqlclient-dev mysql-client
-    - pip install mysql-python
+    - apt-get install -qq libmysqlclient-dev libssl-dev mysql-client
+    - . venv/bin/activate
+    - set -a && . ./config.env && set +a
+    - pip install mysqlclient==$PYTHON_MYSQL_VERSION
     - mysql -h"$MYSQL_PORT_3306_TCP_ADDR" -P"$MYSQL_PORT_3306_TCP_PORT" -uroot -p"$MYSQL_ENV_MYSQL_ROOT_PASSWORD" -e "set global character_set_server=utf8mb4;"
     - python manage.py test --mayan-apps --settings=mayan.settings.testing.gitlab-ci.db_mysql --nomigrations
   tags:
@@ -186,7 +189,9 @@ test-postgres:
     - postgres
   script:
     - apt-get install -qq libpq-dev
-    - pip install psycopg2
+    - . venv/bin/activate
+    - set -a && . ./config.env && set +a
+    - pip install psycopg2==$PYTHON_PSYCOPG2_VERSION
     - python manage.py test --mayan-apps --settings=mayan.settings.testing.gitlab-ci.db_postgres --nomigrations
   tags:
     - postgres
@@ -194,6 +199,7 @@ test-postgres:
 test-sqlite:
   <<: *test_base
   script:
+    - . venv/bin/activate
     - python manage.py test --mayan-apps --settings=mayan.settings.testing.gitlab-ci --nomigrations
 
 deploy_demo:

.tx/config

@@ -19,6 +19,12 @@ source_lang = en
 source_file = mayan/apps/authentication/locale/en/LC_MESSAGES/django.po
 type = PO
 
+[mayan-edms.autoadmin-2-0]
+file_filter = mayan/apps/autoadmin/locale/<lang>/LC_MESSAGES/django.po
+source_lang = en
+source_file = mayan/apps/autoadmin/locale/en/LC_MESSAGES/django.po
+type = PO
+
 [mayan-edms.cabinets-2-0]
 file_filter = mayan/apps/cabinets/locale/<lang>/LC_MESSAGES/django.po
 source_lang = en
@@ -43,18 +49,24 @@ source_lang = en
 source_file = mayan/apps/converter/locale/en/LC_MESSAGES/django.po
 type = PO
 
+[mayan-edms.dashboards-2-0]
+file_filter = mayan/apps/dashboards/locale/<lang>/LC_MESSAGES/django.po
+source_lang = en
+source_file = mayan/apps/dashboards/locale/en/LC_MESSAGES/django.po
+type = PO
+
+[mayan-edms.dependencies-3-0]
+file_filter = mayan/apps/dependencies/locale/<lang>/LC_MESSAGES/django.po
+source_lang = en
+source_file = mayan/apps/dependencies/locale/en/LC_MESSAGES/django.po
+type = PO
+
 [mayan-edms.django_gpg-2-0]
 file_filter = mayan/apps/django_gpg/locale/<lang>/LC_MESSAGES/django.po
 source_lang = en
 source_file = mayan/apps/django_gpg/locale/en/LC_MESSAGES/django.po
 type = PO
 
-[mayan-edms.documents-2-0]
-file_filter = mayan/apps/documents/locale/<lang>/LC_MESSAGES/django.po
-source_lang = en
-source_file = mayan/apps/documents/locale/en/LC_MESSAGES/django.po
-type = PO
-
 [mayan-edms.document_comments-2-0]
 file_filter = mayan/apps/document_comments/locale/<lang>/LC_MESSAGES/django.po
 source_lang = en
@@ -85,6 +97,12 @@ source_lang = en
 source_file = mayan/apps/document_states/locale/en/LC_MESSAGES/django.po
 type = PO
 
+[mayan-edms.documents-2-0]
+file_filter = mayan/apps/documents/locale/<lang>/LC_MESSAGES/django.po
+source_lang = en
+source_file = mayan/apps/documents/locale/en/LC_MESSAGES/django.po
+type = PO
+
 [mayan-edms.dynamic_search-2-0]
 file_filter = mayan/apps/dynamic_search/locale/<lang>/LC_MESSAGES/django.po
 source_lang = en
@@ -97,6 +115,18 @@ source_lang = en
 source_file = mayan/apps/events/locale/en/LC_MESSAGES/django.po
 type = PO
 
+[mayan-edms.file_caching-3-0]
+file_filter = mayan/apps/file_caching/locale/<lang>/LC_MESSAGES/django.po
+source_lang = en
+source_file = mayan/apps/file_caching/locale/en/LC_MESSAGES/django.po
+type = PO
+
+[mayan-edms.file_metadata-3-0]
+file_filter = mayan/apps/file_metadata/locale/<lang>/LC_MESSAGES/django.po
+source_lang = en
+source_file = mayan/apps/file_metadata/locale/en/LC_MESSAGES/django.po
+type = PO
+
 [mayan-edms.linking-2-0]
 file_filter = mayan/apps/linking/locale/<lang>/LC_MESSAGES/django.po
 source_lang = en
@@ -145,6 +175,12 @@ source_lang = en
 source_file = mayan/apps/permissions/locale/en/LC_MESSAGES/django.po
 type = PO
 
+[mayan-edms.platform-2-0]
+file_filter = mayan/apps/platform/locale/<lang>/LC_MESSAGES/django.po
+source_lang = en
+source_file = mayan/apps/platform/locale/en/LC_MESSAGES/django.po
+type = PO
+
 [mayan-edms.rest_api-2-0]
 file_filter = mayan/apps/rest_api/locale/<lang>/LC_MESSAGES/django.po
 source_lang = en
@@ -187,8 +223,21 @@ source_lang = en
 source_file = mayan/apps/task_manager/locale/en/LC_MESSAGES/django.po
 type = PO
 
+[mayan-edms.templating-3-0]
+file_filter = mayan/apps/templating/locale/<lang>/LC_MESSAGES/django.po
+source_lang = en
+source_file = mayan/apps/templating/locale/en/LC_MESSAGES/django.po
+type = PO
+
 [mayan-edms.user_management-2-0]
 file_filter = mayan/apps/user_management/locale/<lang>/LC_MESSAGES/django.po
 source_lang = en
 source_file = mayan/apps/user_management/locale/en/LC_MESSAGES/django.po
 type = PO
+
+[mayan-edms.web_link-3-0]
+file_filter = mayan/apps/web_links/locale/<lang>/LC_MESSAGES/django.po
+source_lang = en
+source_file = mayan/apps/web_links/locale/en/LC_MESSAGES/django.po
+type = PO
+

CONTRIBUTING.md

@@ -4,26 +4,30 @@ Contributing to Mayan EDMS
 Issues
 ------
 
-- Before submitting a new issue, check for **open issues** first and join the
-discussion.
+- The issue system is for reporting problems with the program code. For
+questions use the forum at: https://forum.mayan-edms.com/
+- Do not open issues asking for **support or consulting**. For paid support
+and consultation visit: https://www.mayan-edms.com/support/
+- The issue must be related to the code only, do not open issues for problems
+with databases, deployments, webservers, cloud providers, etc.
+- Before submitting a new issue, check for **existing issues** first and
+join the discussion.
 - If your issue is not attended in a while, **be patient**, the core team is
 small and the codebase big.
-- Try to reproduce the issue using a **separate, clean installation**, sometimes
-the issue can be caused by an error in a configuration file and not with the
-code itself.
-- **Do not upload** sensitive, private or classified information or files with
-your issue. If the issue is triggered by a user file, create a dummy file with the
-same properties that can trigger the issue and upload that file instead.
+- Try to reproduce the issue using a **separate, clean installation**,
+sometimes the issue can be caused by an error in a configuration file and
+not with the code itself.
+- **Do not upload** sensitive, private or classified information or files
+with your issue. If the issue is triggered by a user file, create a dummy
+file with the same properties that can trigger the issue and upload that
+file instead.
 - Add steps that trigger the issue in a **repeatable manner**.
 - **Screenshots** go a long way in helping understand problems.
-- The issue must be related to the code only, do not open issues for problems
-with deployments, webservers, cloud providers, etc.
-- Do not open issues asking for **support or consulting**.
 
 Code
 ----
 
-1. Complete and mail, or scan and email the corresponding Contributor Assignment Agreement: [Mayan EDMS Individual Contributor Assignment Agreement](https://docs.mayan-edms.com/topics/caa_individual.rst) or [Mayan EDMS Entity Contributor Assignment Agreement](https://docs.mayan-edms.com/topics/topics/caa_entity.rst).
+1. Complete and mail, or scan and email the corresponding Contributor Assignment Agreement: [Mayan EDMS Individual Contributor Assignment Agreement](https://gitlab.com/mayan-edms/mayan-edms/blob/master/docs/chapters/caa_individual.rst) or [Mayan EDMS Entity Contributor Assignment Agreement](https://gitlab.com/mayan-edms/mayan-edms/blob/master/docs/chapters/caa_entity.rst).
 1. Fork [the repository](http://gitlab.com/mayan-edms/mayan-edms).
 1. Choose the version for which you want to develop. The code is divided in the
 following branches:

LICENSE

@@ -1,4 +1,4 @@
-   Copyright 2011-2018 Roberto Rosario
+   Copyright 2011 Roberto Rosario
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

MANIFEST.in

@@ -1,3 +1,6 @@
 include README.md LICENSE HISTORY.rst mayan/LICENSE
-recursive-include mayan/apps *.txt *.html *.css *.ico *.png *.jpg *.js *.po *.mo *.ttf *.woff *.woff2 *.gif *.eot *.svg *.doc *.pdf *.tiff *.sig *.asc *.gpg *.zip *.tar *.gz *.bz2 package.json
-global-exclude mayan/settings/local.py mayan/settings/travis/* mayan/media/*
+recursive-include mayan/apps *.txt *.html *.css *.ico *.png *.jpg *.js *.mo *.ttf *.woff *.woff2 *.gif *.eot *.svg *.doc *.pdf *.tiff *.sig *.asc *.gpg *.zip *.tar *.gz *.bz2 *.tmpl
+global-exclude mayan/settings/local.py *.po
+prune mayan/apps/*/static/*/node_modules/*
+prune mayan/settings/travis/*
+prune mayan/media/*

Makefile

@@ -1,90 +1,61 @@
-.PHONY: clean-pyc clean-build
+#!make
+include config.env
 
-help: docker-help
-	@echo
-	@echo "**** Main makefile ****"
-	@echo "clean-build - Remove build artifacts."
-	@echo "clean-pyc - Remove Python artifacts."
-	@echo "clean - Remove Python and build artifacts."
-	@echo "generate-setup - Create and updated setup.py"
-	@echo "check-readme - Checks validity of the README.rst file for PyPI publication."
-	@echo "check-missing_migrations - Make sure all models have proper migrations."
+.PHONY: clean clean-pyc clean-build test
 
-	@echo "test-all - Run all tests."
-	@echo "test MODULE=<python module name> - Run tests for a single app, module or test class."
-	@echo "test-with-postgres-all - Run all tests against a Postgres database container."
-	@echo "test-postgres MODULE=<python module name> - Run tests for a single app, module or test class against a Postgres database container."
-	@echo "test-with-mysql-all - Run all tests against a MySQL database container."
-	@echo "test-mysql MODULE=<python module name> - Run tests for a single app, module or test class against a MySQL database container."
-	@echo "test-with-oracle-all - Run all tests against a Oracle database container."
-	@echo "test-oracle MODULE=<python module name> - Run tests for a single app, module or test class against a Oracle database container."
-
-	@echo "docs-serve - Run the livehtml documentation generator."
-
-	@echo "translations-make - Refresh all translation files."
-	@echo "translations-compile - Compile all translation files."
-	@echo "translations-push - Upload all translation files to Transifex."
-	@echo "translations-pull - Download all translation files from Transifex."
-
-	@echo "sdist - Build the source distribution package."
-	@echo "wheel - Build the wheel distribution package."
-	@echo "release - Package (sdist and wheel) and upload a release."
-	@echo "test-release - Package (sdist and wheel) and upload to the PyPI test server."
-	@echo "release-test-via-docker-ubuntu - Package (sdist and wheel) and upload to the PyPI test server using an Ubuntu Docker builder."
-	@echo "release-via-docker-ubuntu - Package (sdist and wheel) and upload to PyPI using an Ubuntu Docker builder."
-	@echo "test-sdist-via-docker-ubuntu - Make an sdist packange and test it using an Ubuntu Docker container."
-	@echo "test-wheel-via-docker-ubuntu - Make a wheel package and test it using an Ubuntu Docker container."
-	@echo "runserver - Run the development server."
-	@echo "runserver_plus - Run the Django extension's development server."
-	@echo "shell_plus - Run the shell_plus command."
-
-	@echo "test-with-docker-services-on - Launch and initialize production-like services using Docker (Postgres and Redis)."
-	@echo "test-with-docker-services-off - Stop and delete the Docker production-like services."
-	@echo "test-with-docker-frontend - Launch a front end instance that uses the production-like services."
-	@echo "test-with-docker-worker - Launch a worker instance that uses the production-like services."
-	@echo "docker-mysql-on - Launch and initialize a MySQL Docker container."
-	@echo "docker-mysql-off - Stop and delete the MySQL Docker container."
-	@echo "docker-postgres-on - Launch and initialize a PostgreSQL Docker container."
-	@echo "docker-postgres-off - Stop and delete the PostgreSQL Docker container."
-
-	@echo "safety-check - Run a package safety check."
+help:
+	@echo "Usage: make <target>\n"
+	@awk 'BEGIN {FS = ":.*##"} /^[0-9a-zA-Z_-]+:.*?## / { printf "  * %-40s -%s\n", $$1, $$2 }' $(MAKEFILE_LIST)|sort
 
 # Cleaning
 
+clean: ## Remove Python and build artifacts.
 clean: clean-build clean-pyc
 
-clean-build:
+clean-build: ## Remove build artifacts.
 	rm -fr build/
 	rm -fr dist/
 	rm -fr *.egg-info
 
-clean-pyc:
+clean-pyc: ## Remove Python artifacts.
 	find . -name '*.pyc' -exec rm -f {} +
 	find . -name '*.pyo' -exec rm -f {} +
 	find . -name '*~' -exec rm -f {} +
-
+	find . -name '__pycache__' -exec rm -R -f {} +
 
 # Testing
 
-test:
+test: clean-pyc
+test: ## MODULE=<python module name> - Run tests for a single app, module or test class.
 	./manage.py test $(MODULE) --settings=mayan.settings.testing.development --nomigrations $(ARGUMENTS)
 
-test-all:
+test-debug: clean-pyc
+test-debug: ## MODULE=<python module name> - Run tests for a single app, module or test class, in debug mode.
+	./manage.py test $(MODULE) --settings=mayan.settings.testing.development --nomigrations --debug-mode $(ARGUMENTS)
+
+test-all: ## Run all tests.
+test-all: clean-pyc
 	./manage.py test --mayan-apps --settings=mayan.settings.testing.development --nomigrations $(ARGUMENTS)
 
+test-all-debug: ## Run all tests in debug mode.
+test-all-debug: clean-pyc
+	./manage.py test --mayan-apps --settings=mayan.settings.testing.development --nomigrations --debug-mode $(ARGUMENTS)
+
 test-launch-postgres:
 	@docker rm -f test-postgres || true
 	@docker volume rm test-postgres || true
-	docker run -d --name test-postgres -p 5432:5432 -v test-postgres:/var/lib/postgresql/data healthcheck/postgres
-	sudo apt-get install -qq libpq-dev
-	pip install psycopg2
-	while ! docker inspect --format='{{json .State.Health}}' test-postgres|grep 'Status":"healthy"'; do sleep 1; done
+	docker run -d --name test-postgres -p 5432:5432 -v test-postgres:/var/lib/postgresql/data $(DOCKER_POSTGRES_IMAGE_VERSION)
+	sudo apt-get install -q libpq-dev
+	pip install psycopg2==$(PYTHON_PSYCOPG2_VERSION)
+	while ! nc -z 127.0.0.1 5432; do sleep 1; done
 
+test-with-postgres: ## MODULE=<python module name> - Run tests for a single app, module or test class against a Postgres database container.
 test-with-postgres: test-launch-postgres
 	./manage.py test $(MODULE) --settings=mayan.settings.testing.docker.db_postgres --nomigrations
 	@docker rm -f test-postgres || true
 	@docker volume rm test-postgres || true
 
+test-with-postgres-all: ## Run all tests against a Postgres database container.
 test-with-postgres-all: test-launch-postgres
 	./manage.py test --mayan-apps --settings=mayan.settings.testing.docker.db_postgres --nomigrations
 	@docker rm -f test-postgres || true
@@ -93,17 +64,20 @@ test-with-postgres-all: test-launch-postgres
 test-launch-mysql:
 	@docker rm -f test-mysql || true
 	@docker volume rm test-mysql || true
-	docker run -d --name test-mysql -p 3306:3306 -e MYSQL_ALLOW_EMPTY_PASSWORD=True -e MYSQL_DATABASE=mayan -v test-mysql:/var/lib/mysql healthcheck/mysql
-	sudo apt-get install -qq libmysqlclient-dev mysql-client
-	pip install mysql-python
-	while ! docker inspect --format='{{json .State.Health}}' test-mysql|grep 'Status":"healthy"'; do sleep 1; done
+	docker run -d --name test-mysql -p 3306:3306 -e MYSQL_ALLOW_EMPTY_PASSWORD=True -e MYSQL_DATABASE=mayan -v test-mysql:/var/lib/mysql $(DOCKER_MYSQL_IMAGE_VERSION)
+	sudo apt-get install -q libmysqlclient-dev mysql-client
+	pip install mysqlclient==$(PYTHON_MYSQL_VERSION)
+	while ! nc -z 127.0.0.1 3306; do sleep 1; done
 	mysql -h 127.0.0.1 -P 3306 -uroot  -e "set global character_set_server=utf8mb4;"
 
+test-with-mysql: ## MODULE=<python module name> - Run tests for a single app, module or test class against a MySQL database container.
 test-with-mysql: test-launch-mysql
 	./manage.py test $(MODULE) --settings=mayan.settings.testing.docker.db_mysql --nomigrations
 	@docker rm -f test-mysql || true
 	@docker volume rm test-mysql || true
 
+
+test-with-mysql-all: ## Run all tests against a MySQL database container.
 test-with-mysql-all: test-launch-mysql
 	./manage.py test --mayan-apps --settings=mayan.settings.testing.docker.db_mysql --nomigrations
 	@docker rm -f test-mysql || true
@@ -112,17 +86,19 @@ test-with-mysql-all: test-launch-mysql
 test-launch-oracle:
 	@docker rm -f test-oracle || true
 	@docker volume rm test-oracle || true
-	docker run -d --name test-oracle -p 49160:22 -p 49161:1521 -e ORACLE_ALLOW_REMOTE=true -v test-oracle:/u01/app/oracle wnameless/oracle-xe-11g
+	docker run -d --name test-oracle -p 49160:22 -p 49161:1521 -e ORACLE_ALLOW_REMOTE=true -v test-oracle:/u01/app/oracle $(DOCKER_ORACLE_IMAGE_VERSION)
 	# https://gist.github.com/kimus/10012910
-	pip install cx_Oracle
+	pip install cx_Oracle==$(PYTHON_ORACLE_VERSION)
 	while ! nc -z 127.0.0.1 49161; do sleep 1; done
 	sleep 10
 
+test-with-oracle: ## MODULE=<python module name> - Run tests for a single app, module or test class against a Oracle database container.
 test-with-oracle: test-launch-oracle
 	./manage.py test $(MODULE) --settings=mayan.settings.testing.docker.db_oracle --nomigrations
 	@docker rm -f test-oracle || true
 	@docker volume rm test-oracle || true
 
+test-with-oracle-all: ## Run all tests against a Oracle database container.
 test-with-oracle-all: test-launch-oracle
 	./manage.py test --mayan-apps --settings=mayan.settings.testing.docker.db_oracle --nomigrations
 	@docker rm -f test-oracle || true
@@ -130,58 +106,59 @@ test-with-oracle-all: test-launch-oracle
 
 # Documentation
 
-docs-serve:
+docs-serve: ## Run the livehtml documentation generator.
 	cd docs;make livehtml
 
-docs-spellcheck:
-
+docs-spellcheck: ## Spellcheck the documentation.
 	sphinx-build -b spelling -d docs/_build/ docs docs/_build/spelling
 
 # Translations
 
-translations-make:
+translations-make: ## Refresh all translation files.
 	contrib/scripts/process_messages.py -m
 
-translations-compile:
+translations-compile: ## Compile all translation files.
 	contrib/scripts/process_messages.py -c
 
-translations-push:
+translations-push: ## Upload all translation files to Transifex.
 	tx push -s
 
-translations-pull:
+translations-pull: ## Download all translation files from Transifex.
 	tx pull -f
 
-
-generate-setup:
-	@./generate_setup.py
-	@echo "Complete."
+translations-all: ## Execute all translations targets.
+translations-all: translations-make translations-push translations-pull translations-compile
 
 # Releases
 
-increase-version:
+increase-version: ## Increase the version number of the entire project's files.
 	@VERSION=`grep "__version__ =" mayan/__init__.py| cut -d\' -f 2|./increase_version.py - $(PART)`; \
 	BUILD=`echo $$VERSION|awk '{split($$VERSION,a,"."); printf("0x%02d%02d%02d\n", a[1],a[2], a[3])}'`; \
 	sed -i -e "s/__build__ = 0x[0-9]*/__build__ = $${BUILD}/g" mayan/__init__.py; \
 	sed -i -e "s/__version__ = '[0-9\.]*'/__version__ = '$${VERSION}'/g" mayan/__init__.py; \
-	echo $$VERSION > docker/version
+	echo $$VERSION > docker/rootfs/version
 	make generate-setup
 
-test-release: clean wheel
+python-test-release: ## Package (sdist and wheel) and upload to the PyPI test server.
+python-test-release: clean wheel
 	twine upload dist/* -r testpypi
 	@echo "Test with: pip install -i https://testpypi.python.org/pypi mayan-edms"
 
-release: clean wheel
+python-release: ## Package (sdist and wheel) and upload a release.
+python-release: clean python-wheel
 	twine upload dist/* -r pypi
 
-sdist: clean
+python-sdist: ## Build the source distribution package.
+python-sdist: clean
 	python setup.py sdist
 	ls -l dist
 
-wheel: clean sdist
+python-wheel: ## Build the wheel distribution package.
+python-wheel: clean python-sdist
 	pip wheel --no-index --no-deps --wheel-dir dist dist/*.tar.gz
 	ls -l dist
 
-release-test-via-docker-ubuntu:
+python-release-test-via-docker-ubuntu: ## Package (sdist and wheel) and upload to the PyPI test server using an Ubuntu Docker builder.
 	docker run --rm --name mayan_release -v $(HOME):/host_home:ro -v `pwd`:/host_source -w /source ubuntu:16.04 /bin/bash -c "\
 	echo "LC_ALL=\"en_US.UTF-8\"" >> /etc/default/locale && \
 	locale-gen en_US.UTF-8 && \
@@ -194,7 +171,7 @@ release-test-via-docker-ubuntu:
 	cp -r /host_home/.pypirc ~/.pypirc && \
 	make test-release"
 
-release-via-docker-ubuntu:
+python-release-via-docker-ubuntu: ## Package (sdist and wheel) and upload to PyPI using an Ubuntu Docker builder.
 	docker run --rm --name mayan_release -v $(HOME):/host_home:ro -v `pwd`:/host_source -w /source ubuntu:16.04 /bin/bash -c "\
 	apt-get update && \
 	apt-get -y install locales && \
@@ -208,7 +185,7 @@ release-via-docker-ubuntu:
 	cp -r /host_home/.pypirc ~/.pypirc && \
 	make release"
 
-test-sdist-via-docker-ubuntu:
+test-sdist-via-docker-ubuntu: ## Make an sdist package and test it using an Ubuntu Docker container.
 	docker run --rm --name mayan_sdist_test -v $(HOME):/host_home:ro -v `pwd`:/host_source -w /source ubuntu:16.04 /bin/bash -c "\
 	cp -r /host_source/* . && \
 	echo "LC_ALL=\"en_US.UTF-8\"" >> /etc/default/locale && \
@@ -221,7 +198,7 @@ test-sdist-via-docker-ubuntu:
 	make sdist-test-suit \
 	"
 
-test-wheel-via-docker-ubuntu:
+test-wheel-via-docker-ubuntu: ## Make a wheel package and test it using an Ubuntu Docker container.
 	docker run --rm --name mayan_wheel_test -v $(HOME):/host_home:ro -v `pwd`:/host_source -w /source ubuntu:16.04 /bin/bash -c "\
 	cp -r /host_source/* . && \
 	echo "LC_ALL=\"en_US.UTF-8\"" >> /etc/default/locale && \
@@ -234,7 +211,7 @@ test-wheel-via-docker-ubuntu:
 	make wheel-test-suit \
 	"
 
-sdist-test-suit: sdist
+python-sdist-test-suit: sdist
 	rm -f -R _virtualenv
 	virtualenv _virtualenv
 	sh -c '\
@@ -245,7 +222,7 @@ sdist-test-suit: sdist
 	_virtualenv/bin/mayan-edms.py test --mayan-apps \
 	'
 
-wheel-test-suit: wheel
+python-wheel-test-suit: wheel
 	rm -f -R _virtualenv
 	virtualenv _virtualenv
 	sh -c '\
@@ -256,64 +233,107 @@ wheel-test-suit: wheel
 	_virtualenv/bin/mayan-edms.py test --mayan-apps \
 	'
 
+generate-setup: ## Create and update the setup.py file.
+generate-setup: generate-requirements
+	@./generate_setup.py
+	@echo "Complete."
+
+generate-requirements: ## Generate all requirements files from the project depedency declarations.
+	@./manage.py generaterequirements build > requirements/build.txt
+	@./manage.py generaterequirements development > requirements/development.txt
+	@./manage.py generaterequirements documentation > requirements/documentation.txt
+	@./manage.py generaterequirements production --only=pathlib2 >> requirements/documentation.txt
+	@./manage.py generaterequirements testing > requirements/testing-base.txt
+	@./manage.py generaterequirements production --exclude=django > requirements/base.txt
+	@./manage.py generaterequirements production --only=django > requirements/common.txt
+
+gitlab-release-documentation: ## Trigger the documentation build and publication using GitLab CI
+gitlab-release-documentation:
+	git push
+	git push --tags
+	git push origin :releases/documentation || true
+	git push origin HEAD:releases/documentation
+
+gitlab-release-docker: ## Trigger the Docker image build and publication using GitLab CI
+gitlab-release-docker:
+	git push
+	git push --tags
+	git push origin :releases/docker || true
+	git push origin HEAD:releases/docker
+
+gitlab-release-python: ## Trigger the Python package build and publication using GitLab CI
+gitlab-release-python:
+	git push
+	git push --tags
+	git push origin :releases/python || true
+	git push origin HEAD:releases/python
+
+gitlab-release-all: ## Trigger the Python package, Docker image, and documentation build and publication using GitLab CI
+gitlab-release-all:
+	git push
+	git push --tags
+	git push origin :releases/all || true
+	git push origin HEAD:releases/all
+
 # Dev server
 
-runserver:
-	./manage.py runserver --settings=mayan.settings.development $(ADDRPORT)
+runserver: ## Run the development server.
+	./manage.py runserver --nothreading --settings=mayan.settings.development $(ADDRPORT)
 
-runserver_plus:
-	./manage.py runserver_plus --settings=mayan.settings.development $(ADDRPORT)
+runserver_plus: ## Run the Django extension's development server.
+	./manage.py runserver_plus --nothreading --settings=mayan.settings.development $(ADDRPORT)
 
-shell_plus:
+shell_plus: ## Run the shell_plus command.
 	./manage.py shell_plus --settings=mayan.settings.development
 
-test-with-docker-services-on:
-	docker run -d --name redis -p 6379:6379 redis
-	docker run -d --name postgres -p 5432:5432 postgres
+test-with-docker-services-on: ## Launch and initialize production-like services using Docker (Postgres and Redis).
+	docker run -d --name redis -p 6379:6379 $(DOCKER_REDIS_IMAGE_VERSION)
+	docker run -d --name postgres -p 5432:5432 $(DOCKER_POSTGRES_IMAGE_VERSION)
 	while ! nc -z 127.0.0.1 6379; do sleep 1; done
 	while ! nc -z 127.0.0.1 5432; do sleep 1; done
 	sleep 4
+	pip install psycopg2==$(PYTHON_PSYCOPG2_VERSION) redis==$(PYTHON_REDIS_VERSION)
 	./manage.py initialsetup --settings=mayan.settings.staging.docker
 
-test-with-docker-services-off:
+test-with-docker-services-off: ## Stop and delete the Docker production-like services.
 	docker stop postgres redis
 	docker rm postgres redis
 
-test-with-docker-frontend:
+test-with-docker-frontend: ## Launch a front end instance that uses the production-like services.
 	./manage.py runserver --settings=mayan.settings.staging.docker
 
-test-with-docker-worker:
-	./manage.py celery worker --settings=mayan.settings.staging.docker -B -l INFO -O fair
+test-with-docker-worker: ## Launch a worker instance that uses the production-like services.
+	DJANGO_SETTINGS_MODULE=mayan.settings.staging.docker ./manage.py celery worker -A mayan -B -l INFO -O fair
 
-docker-mysql-on:
-	docker run -d --name mysql -p 3306:3306 -e MYSQL_ALLOW_EMPTY_PASSWORD=True -e MYSQL_DATABASE=mayan_edms mysql
+docker-mysql-on: ## Launch and initialize a MySQL Docker container.
+	docker run -d --name mysql -p 3306:3306 -e MYSQL_ALLOW_EMPTY_PASSWORD=True -e MYSQL_DATABASE=mayan_edms $(DOCKER_MYSQL_IMAGE_VERSION)
 	while ! nc -z 127.0.0.1 3306; do sleep 1; done
 
-docker-mysql-off:
+docker-mysql-off: ## Stop and delete the MySQL Docker container.
 	docker stop mysql
 	docker rm mysql
 
-docker-postgres-on:
-	docker run -d --name postgres -p 5432:5432 postgres
+docker-postgres-on: ## Launch and initialize a PostgreSQL Docker container.
+	docker run -d --name postgres -p 5432:5432 $(DOCKER_POSTGRES_IMAGE_VERSION)
 	while ! nc -z 127.0.0.1 5432; do sleep 1; done
 
-docker-postgres-off:
+docker-postgres-off: ## Stop and delete the PostgreSQL Docker container.
 	docker stop postgres
 	docker rm postgres
 
 
 # Security
 
-safety-check:
+safety-check: ## Run a package safety check.
 	safety check
 
 
 # Other
-find-gitignores:
+find-gitignores: ## Find stray .gitignore files.
 	@export FIND_GITIGNORES=`find -name '.gitignore'| wc -l`; \
 	if [ $${FIND_GITIGNORES} -gt 1 ] ;then echo "More than one .gitignore found."; fi
 
-build:
+python-build:
 	docker rm -f mayan-edms-build || true && \
 	docker run --rm --name mayan-edms-build -v $(HOME):/host_home:ro -v `pwd`:/host_source -w /source python:2-slim sh -c "\
 	rm /host_source/dist -R || true && \
@@ -326,11 +346,13 @@ build:
 	make wheel && \
 	cp dist/* /host_source/dist/"
 
-check-readme:
+check-readme: ## Checks validity of the README.rst file for PyPI publication.
 	python setup.py check -r -s
 
-check-missing-migrations:
+check-missing-migrations: ## Make sure all models have proper migrations.
 	./manage.py makemigrations --dry-run --noinput --check
 
+setup-dev-environment: ## Bootstrap a virtualenv by install all dependencies to start developing.
+	pip install -r requirements.txt -r requirements/development.txt -r requirements/testing-base.txt -r requirements/documentation.txt -r requirements/build.txt
 
-include docker/Makefile
+-include docker/Makefile

README.md

@@ -1,11 +1,10 @@
+[![Donation](https://img.shields.io/badge/donation-PayPal-brightgreen)](https://paypal.me/MayanEDMS)
 [![pypi][pypi]][pypi-url]
-[![builds][builds]][builds-url]
-[![coverage][cover]][cover-url]
 ![python][python]
 ![license][license]
 [![Docker pulls](https://img.shields.io/docker/pulls/mayanedms/mayanedms.svg?maxAge=3600)](https://hub.docker.com/r/mayanedms/mayanedms/)
 [![Docker Stars](https://img.shields.io/docker/stars/mayanedms/mayanedms.svg?maxAge=3600)](https://hub.docker.com/r/mayanedms/mayanedms/)
-[![Docker layers](https://images.microbadger.com/badges/image/mayanedms/mayanedms.svg)](https://microbadger.com/images/mayanedms/mayanedms)
+
 
 [pypi]: http://img.shields.io/pypi/v/mayan-edms.svg
 [pypi-url]: http://badge.fury.io/py/mayan-edms
@@ -43,6 +42,18 @@
 
 </div>
 
+<h2 align="center">Book</h2>
+
+The final version of the book "Exploring Mayan EDMS" available now!
+
+<p align="center">
+    <a href="https://sellfy.com/p/um2fkx/">
+        <img width="400" src="https://d12swbtw719y4s.cloudfront.net/images/v6RpxW40/aP0qKLjkPiAuXZhYuB45/wDAULAzFyx.jpeg?w=548">
+    </a>
+</p>
+
+The link is https://sellfy.com/p/um2fkx/
+
 <h2 align="center">Installation</h2>
 
 The easiest way to use Mayan EDMS is by using the official

README.rst

@@ -1,4 +1,4 @@
-|pypi| |builds| |coverage| |python| |license| |docker_pulls| |docker_stars| |docker_layers|
+|donation| |pypi| |coverage| |python| |license| |docker_pulls| |docker_stars| |docker_layers|
 
 
 .. image:: https://gitlab.com/mayan-edms/mayan-edms/raw/master/docs/_static/mayan_logo.png
@@ -25,6 +25,15 @@ at: https://docs.mayan-edms.com/topics/installation.html
 
 .. _Docker: https://www.docker.com/
 
+The final version of the book "Exploring Mayan EDMS" available now!
+
+.. image:: https://camo.githubusercontent.com/89d2fe787507c9247aa7bb406e2682b53eb7a5f9/68747470733a2f2f64313273776274773731397934732e636c6f756466726f6e742e6e65742f696d616765732f76365270785734302f615030714b4c6a6b50694175585a6859754234352f774441554c417a4679782e6a7065673f773d353438
+    :align: center
+    :width: 300
+    :target: https://sellfy.com/p/um2fkx/
+
+Click the image or visit: https://sellfy.com/p/um2fkx/
+
 Hardware requirements
 
 - 2 Gigabytes of RAM (1 Gigabyte if OCR is turned off).
@@ -44,6 +53,8 @@ Important links
 - `Videos <https://www.youtube.com/channel/UCJOOXHP1MJ9lVA7d8ZTlHPw>`__
 
 
+.. |donation| image:: https://img.shields.io/badge/donation-PayPal-brightgreen
+   :target: https://paypal.me/MayanEDMS
 .. |pypi| image:: http://img.shields.io/pypi/v/mayan-edms.svg
    :target: http://badge.fury.io/py/mayan-edms
 .. |builds| image:: https://gitlab.com/mayan-edms/mayan-edms/badges/master/build.svg

__init__.py.tmpl

@@ -9,6 +9,6 @@ __author__ = 'Roberto Rosario'
 __author_email__ = 'roberto.rosario@mayan-edms.com'
 __description__ = 'Free Open Source Electronic Document Management System'
 __license__ = 'Apache 2.0'
-__copyright_short__ = '2011-2018 Roberto Rosario'
+__copyright_short__ = '2011 Roberto Rosario'
 __copyright__ = '{} {}'.format('Copyright', __copyright_short__)
 __website__ = 'https://www.mayan-edms.com'

config.env

@@ -0,0 +1,16 @@
+DOCKER_MYSQL_IMAGE_VERSION=mysql:8.0
+DOCKER_ORACLE_IMAGE_VERSION=wnameless/oracle-xe-11g
+DOCKER_POSTGRES_IMAGE_VERSION=postgres:9.6-alpine
+DOCKER_RABBITMQ_IMAGE_VERSION=rabbitmq:3-alpine
+DOCKER_REDIS_IMAGE_VERSION=redis:5.0-alpine
+PYTHON_AMQP_VERSION=2.5.2
+PYTHON_FLOWER_VERSION=0.9.3
+PYTHON_LIBRABBITMQ_VERSION=2.0.0
+PYTHON_MYSQL_VERSION=1.4.4
+PYTHON_ORACLE_VERSION=7.2.3
+PYTHON_PSYCOPG2_VERSION=2.8.3
+PYTHON_PSUTIL_VERSION=5.6.2
+PYTHON_REDIS_VERSION=3.2.1
+SOURCE_CODE_REPOSITORY=https://gitlab.com/mayan-edms/mayan-edms/
+SOURCE_CODE_GIT=https://gitlab.com/mayan-edms/mayan-edms.git
+SOURCE_CODE_ISSUES=https://gitlab.com/mayan-edms/mayan-edms/issues/

contrib/scripts/docker/run-tests.sh

@@ -1,8 +0,0 @@
-#!/bin/sh
-
-apt-get update
-apt-get install -y --no-install-recommends tesseract-ocr-deu
-
-$MAYAN_PIP_BIN install -r $DOCKER_ROOT/requirements-testing.txt
-
-$MAYAN_BIN test --mayan-apps --settings=mayan.settings.testing

contrib/scripts/fix-permisssion.sh

@@ -1,25 +0,0 @@
-!/bin/bash
-Directories[0]="."
-
-# Setup find correctly.
-export IFS=$'\n'
-
-# Loop through our array.
-for x in ${Directories[@]}
-	do
-		# Find all directories & subdirectories
-		for i in $(find $x -type d)
-			do
-				# Fix Permissions
-				chmod -c 775 $i
-				chown -c www-data:www-data $i
-			done
-
-		# Find all Files
-		for i in $(find $x -type f)
-			do
-				# Fix Permissions
-				chmod -c 664 $i
-				chown -c www-data:www-data $i
-			done
-	done

contrib/scripts/install/development.sh

@@ -1,72 +0,0 @@
-#!/usr/bin/env bash
-
-INSTALLATION_DIRECTORY=/home/vagrant/mayan-edms/
-DB_NAME=mayan_edms
-DB_PASSWORD=test123
-
-cat << EOF | sudo tee -a /etc/motd.tail
-**********************************sudo apt
-
-Mayan EDMS Vagrant Development Box
-
-**********************************
-EOF
-
-# Update sources
-echo -e "\n -> Running apt-get update & upgrade \n"
-sudo apt-get -qq update
-sudo apt-get -y upgrade
-
-echo -e "\n -> Installing core binaries \n"
-sudo apt-get -y install git-core python-virtualenv gcc python-dev libjpeg-dev libpng-dev libtiff-dev tesseract-ocr poppler-utils libreoffice
-
-echo -e "\n -> Cloning development branch of repository \n"
-git clone /mayan-edms-repository/ $INSTALLATION_DIRECTORY
-cd $INSTALLATION_DIRECTORY
-git checkout development
-git reset HEAD --hard
-
-echo -e "\n -> Setting up virtual env \n"
-virtualenv venv
-source venv/bin/activate
-
-echo -e "\n -> Installing python dependencies \n"
-pip install -r requirements.txt
-
-echo -e "\n -> Running Mayan EDMS initial setup \n"
-./manage.py initialsetup
-
-echo -e "\n -> Installing Redis server \n"
-sudo apt-get install -y redis-server
-pip install redis
-
-echo -e "\n -> Installing testing software \n"
-pip install coverage
-
-echo -e "\n -> Installing MySQL \n"
-sudo debconf-set-selections <<< 'mysql-server mysql-server/root_password password '$DB_PASSWORD
-sudo debconf-set-selections <<< 'mysql-server mysql-server/root_password_again password '$DB_PASSWORD
-sudo apt-get install -y mysql-server libmysqlclient-dev
-# Create a passwordless root and travis users
-mysql -u root -p$DB_PASSWORD -e "SET PASSWORD = PASSWORD('');"
-mysql -u root -e "CREATE USER 'travis'@'localhost' IDENTIFIED BY '';GRANT ALL PRIVILEGES ON * . * TO 'travis'@'localhost';FLUSH PRIVILEGES;"
-mysql -u travis -e "CREATE DATABASE $DB_NAME;"
-pip install mysql-python
-
-echo -e "\n -> Installing PostgreSQL \n"
-sudo apt-get install -y postgresql postgresql-server-dev-all
-sudo -u postgres psql -c 'create database mayan_edms;' -U postgres
-sudo cat > /etc/postgresql/9.3/main/pg_hba.conf << EOF
-local   all             postgres                                trust
-
-# TYPE  DATABASE        USER            ADDRESS                 METHOD
-
-# "local" is for Unix domain socket connections only
-local   all             all                                     peer
-# IPv4 local connections:
-host    all             all             127.0.0.1/32            md5
-# IPv6 local connections:
-host    all             all             ::1/128                 md5
-EOF
-
-pip install -q psycopg2

contrib/scripts/install/docker.sh

@@ -5,24 +5,30 @@ set -e
 #   $ curl -fsSL get.mayan-edms.com -o get-mayan-edms.sh
 #   $ sh get-mayan-edms.sh
 #
-# NOTE: Make sure to verify the contents of the script
+# NOTE: Before executing, make sure to verify the contents of the script
 #       you downloaded matches the contents of docker.sh
 #       located at https://gitlab.com/mayan-edms/mayan-edms/blob/master/contrib/scripts/install/docker.sh
-#       before executing.
 
 : ${VERBOSE:=true}
 : ${INSTALL_DOCKER:=false}
 : ${DELETE_VOLUMES:=false}
+: ${USE_DOCKER_NETWORK:=true}
+: ${DOCKER_NETWORK_NAME:=mayan}
 : ${DATABASE_USER:=mayan}
 : ${DATABASE_NAME:=mayan}
 : ${DATABASE_PASSWORD:=mayanuserpass}
-: ${DOCKER_POSTGRES_IMAGE:=postgres:9.5}
+: ${DOCKER_POSTGRES_IMAGE:=postgres:9.6-alpine}
 : ${DOCKER_POSTGRES_CONTAINER:=mayan-edms-postgres}
 : ${DOCKER_POSTGRES_VOLUME:=/docker-volumes/mayan-edms/postgres}
 : ${DOCKER_POSTGRES_PORT:=5432}
+: ${DOCKER_POSTGRES_DELAY:=10}
+: ${DOCKER_REDIS_IMAGE:=redis:5.0-alpine}
+: ${DOCKER_REDIS_CONTAINER:=mayan-edms-redis}
+: ${DOCKER_REDIS_PORT:=6379}
 : ${DOCKER_MAYAN_IMAGE:=mayanedms/mayanedms:latest}
 : ${DOCKER_MAYAN_CONTAINER:=mayan-edms}
 : ${DOCKER_MAYAN_VOLUME:=/docker-volumes/mayan-edms/media}
+: ${DOCKER_MAYAN_PORT:=80}
 
 cat << EOF
 
@@ -44,6 +50,8 @@ echo "Variable values to be used:"
 echo "---------------------------"
 echo "INSTALL_DOCKER: $INSTALL_DOCKER"
 echo "DELETE_VOLUMES: $DELETE_VOLUMES"
+echo "USE_DOCKER_NETWORK: $USE_DOCKER_NETWORK"
+echo "DOCKER_NETWORK_NAME: $DOCKER_NETWORK_NAME"
 echo "DATABASE_USER: $DATABASE_USER"
 echo "DATABASE_NAME: $DATABASE_NAME"
 echo "DATABASE_PASSWORD: $DATABASE_PASSWORD"
@@ -51,10 +59,19 @@ echo "DOCKER_POSTGRES_IMAGE: $DOCKER_POSTGRES_IMAGE"
 echo "DOCKER_POSTGRES_CONTAINER: $DOCKER_POSTGRES_CONTAINER"
 echo "DOCKER_POSTGRES_VOLUME: $DOCKER_POSTGRES_VOLUME"
 echo "DOCKER_POSTGRES_PORT: $DOCKER_POSTGRES_PORT"
+echo "DOCKER_POSTGRES_DELAY: $DOCKER_POSTGRES_DELAY"
+echo "DOCKER_REDIS_IMAGE: $DOCKER_REDIS_IMAGE"
+echo "DOCKER_REDIS_CONTAINER: $DOCKER_REDIS_CONTAINER"
+echo "DOCKER_REDIS_PORT: $DOCKER_REDIS_PORT"
 echo "DOCKER_MAYAN_IMAGE: $DOCKER_MAYAN_IMAGE"
 echo "DOCKER_MAYAN_CONTAINER: $DOCKER_MAYAN_CONTAINER"
 echo "DOCKER_MAYAN_VOLUME: $DOCKER_MAYAN_VOLUME"
-echo "\nStarting in 10 seconds."
+echo "DOCKER_MAYAN_PORT: $DOCKER_MAYAN_PORT"
+echo
+echo "Override any of them by setting them before the script. "
+echo "Example: INSTALL_DOCKER=true sh get-mayan-edms.sh"
+
+echo "\nStarting in 10 seconds. Press CTRL+C to cancel."
 sleep 10
 fi
 
@@ -72,33 +89,62 @@ if [ -z `which docker`  ]; then
 fi
 
 echo -n "* Removing existing Mayan EDMS and PostgreSQL containers (no data will be lost)..."
-true || docker stop $DOCKER_MAYAN_CONTAINER >/dev/null 2>&1
-true || docker rm $DOCKER_MAYAN_CONTAINER >/dev/null 2>&1
-true || docker stop $DOCKER_POSTGRES_CONTAINER >/dev/null 2>&1
-true || docker rm $DOCKER_POSTGRES_CONTAINER >/dev/null 2>&1
+docker rm -f $DOCKER_REDIS_CONTAINER >/dev/null 2>&1  || true
+docker rm -f $DOCKER_POSTGRES_CONTAINER >/dev/null 2>&1  || true
+docker rm -f $DOCKER_MAYAN_CONTAINER >/dev/null 2>&1  || true
 echo "Done"
 
 if [ "$DELETE_VOLUMES" = true ]; then
-echo -n "* Deleting Docker volumes in 5 seconds (warning: this delete all document data)..."
+echo -n "* Deleting Docker volumes in 5 seconds (warning: this will delete all document data). Press CTRL+C to cancel..."
 sleep 5
-true || rm DOCKER_MAYAN_VOLUME -Rf
-true || rm DOCKER_POSTGRES_VOLUME -Rf
+rm DOCKER_MAYAN_VOLUME -Rf || true
+rm DOCKER_POSTGRES_VOLUME -Rf || true
 echo "Done"
 fi
 
-echo -n "* Pulling (downloading) the Mayan EDMS Docker image..."
-docker pull $DOCKER_MAYAN_IMAGE >/dev/null
+echo -n "* Pulling (downloading) the Redis Docker image..."
+docker pull $DOCKER_REDIS_IMAGE > /dev/null
 echo "Done"
 
 echo -n "* Pulling (downloading) the PostgreSQL Docker image..."
 docker pull $DOCKER_POSTGRES_IMAGE > /dev/null
 echo "Done"
 
+echo -n "* Pulling (downloading) the Mayan EDMS Docker image..."
+docker pull $DOCKER_MAYAN_IMAGE >/dev/null
+echo "Done"
+
+if [ "$USE_DOCKER_NETWORK" = true ]; then
+    echo -n "* Creating Docker network..."
+    docker network create $DOCKER_NETWORK_NAME 2> /dev/null || true
+    # Ignore error if the network already exists
+    echo "Done"
+fi
+
+if [ "$USE_DOCKER_NETWORK" = true ]; then
+    NETWORK_ARGUMENT="--network=$DOCKER_NETWORK_NAME"
+    POSTGRES_PORT_ARGUMENT=""
+    REDIS_PORT_ARGUMENT=""
+    MAYAN_DATABASE_PORT_ARGUMENT=""
+    MAYAN_DATABASE_HOST_ARGUMENT="-e MAYAN_DATABASE_HOST=$DOCKER_POSTGRES_CONTAINER"
+    MAYAN_CELERY_BROKER_URL_ARGUMENT="-e MAYAN_CELERY_BROKER_URL=redis://$DOCKER_REDIS_CONTAINER:6379/0"
+    MAYAN_CELERY_RESULT_BACKEND_ARGUMENT="-e MAYAN_CELERY_RESULT_BACKEND=redis://$DOCKER_REDIS_CONTAINER:6379/1"
+else
+    NETWORK_ARGUMENT=""
+    POSTGRES_PORT_ARGUMENT="-e $DOCKER_POSTGRES_PORT:5432"
+    REDIS_PORT_ARGUMENT="-e $DOCKER_REDIS_PORT:6379"
+    MAYAN_DATABASE_PORT_ARGUMENT="-e MAYAN_DATABASE_PORT=$DOCKER_POSTGRES_PORT"
+    MAYAN_DATABASE_HOST_ARGUMENT="-e MAYAN_DATABASE_HOST=172.17.0.1"
+    MAYAN_CELERY_BROKER_URL_ARGUMENT="-e MAYAN_CELERY_BROKER_URL=redis://172.17.0.1:6379/0"
+    MAYAN_CELERY_RESULT_BACKEND_ARGUMENT="-e MAYAN_CELERY_RESULT_BACKEND=redis://172.17.0.1:6379/1"
+fi
+
 echo -n "* Deploying the PostgreSQL container..."
 docker run -d \
 --name $DOCKER_POSTGRES_CONTAINER \
+$NETWORK_ARGUMENT \
 --restart=always \
--p $DOCKER_POSTGRES_PORT:5432 \
+$POSTGRES_PORT_ARGUMENT \
 -e POSTGRES_USER=$DATABASE_USER \
 -e POSTGRES_DB=$DATABASE_NAME \
 -e POSTGRES_PASSWORD=$DATABASE_PASSWORD \
@@ -106,26 +152,47 @@ docker run -d \
 $DOCKER_POSTGRES_IMAGE >/dev/null
 echo "Done"
 
-echo -n "* Waiting for the PostgreSQL container to be ready (10 seconds)..."
-sleep 10
+echo -n "* Deploying the Redis container..."
+docker run -d \
+--name $DOCKER_REDIS_CONTAINER \
+$NETWORK_ARGUMENT \
+--restart=always \
+$REDIS_PORT_ARGUMENT \
+$DOCKER_REDIS_IMAGE \
+redis-server \
+--appendonly no \
+--databases 2 \
+--maxmemory 100mb \
+--maxmemory-policy allkeys-lru \
+--maxclients 500 \
+--save "" \
+--tcp-backlog 256 \
+>/dev/null
+echo "Done"
+
+echo -n "* Waiting for the PostgreSQL container to be ready (${DOCKER_POSTGRES_DELAY} seconds)..."
+sleep $DOCKER_POSTGRES_DELAY
 echo "Done"
 
 echo -n "* Deploying Mayan EDMS container..."
 docker run -d \
 --name $DOCKER_MAYAN_CONTAINER \
+$NETWORK_ARGUMENT \
 --restart=always \
--p 80:8000 \
+-p $DOCKER_MAYAN_PORT:8000 \
 -e MAYAN_DATABASE_ENGINE=django.db.backends.postgresql \
--e MAYAN_DATABASE_HOST=172.17.0.1 \
+$MAYAN_DATABASE_HOST_ARGUMENT \
+$MAYAN_DATABASE_PORT_ARGUMENT \
 -e MAYAN_DATABASE_NAME=$DATABASE_NAME \
 -e MAYAN_DATABASE_PASSWORD=$DATABASE_PASSWORD \
 -e MAYAN_DATABASE_USER=$DATABASE_USER \
--e MAYAN_DATABASE_PORT=$DOCKER_POSTGRES_PORT \
--e MAYAN_DATABASE_CONN_MAX_AGE=60 \
+-e MAYAN_DATABASE_CONN_MAX_AGE=0 \
+$MAYAN_CELERY_BROKER_URL_ARGUMENT \
+$MAYAN_CELERY_RESULT_BACKEND_ARGUMENT \
 -v $DOCKER_MAYAN_VOLUME:/var/lib/mayan \
 $DOCKER_MAYAN_IMAGE >/dev/null
 echo "Done"
 
 echo -n "* Waiting for the Mayan EDMS container to be ready (might take a few minutes)..."
-while ! curl --output /dev/null --silent --head --fail http://localhost:80; do sleep 1 && echo -n .; done;
+while ! curl --output /dev/null --silent --head --fail http://localhost:$DOCKER_MAYAN_PORT; do sleep 1 && echo -n .; done;
 echo "Done"

contrib/scripts/install/production.sh

@@ -1,171 +0,0 @@
-#!/usr/bin/env bash
-
-# ====== CONFIG ======
-INSTALLATION_DIRECTORY=/usr/share/mayan-edms/
-DB_NAME=mayan_edms
-DB_USERNAME=mayan
-DB_PASSWORD=test123
-# ==== END CONFIG ====
-
-cat << EOF | tee -a /etc/motd.tail
-**********************************
-
-Mayan EDMS Vagrant Production Box
-
-**********************************
-EOF
-
-echo -e "\n -> Running apt-get update & upgrade \n"
-apt-get -qq update
-apt-get -y upgrade
-
-echo -e "\n -> Installing core binaries \n"
-apt-get install nginx supervisor redis-server postgresql libpq-dev libjpeg-dev libmagic1 libpng-dev libreoffice libtiff-dev gcc ghostscript gpgv python-dev python-virtualenv tesseract-ocr poppler-utils -y
-
-echo -e "\n -> Setting up virtualenv \n"
-rm -f ${INSTALLATION_DIRECTORY}
-virtualenv ${INSTALLATION_DIRECTORY}
-source ${INSTALLATION_DIRECTORY}bin/activate
-
-echo -e "\n -> Installing Mayan EDMS from PyPI \n"
-pip install mayan-edms
-
-echo -e "\n -> Installing Python client for PostgreSQL, Redis, and uWSGI \n"
-pip install psycopg2 redis uwsgi
-
-echo -e "\n -> Creating the database for the installation \n"
-echo "CREATE USER mayan WITH PASSWORD '$DB_PASSWORD';" | sudo -u postgres psql
-sudo -u postgres createdb -O $DB_USERNAME $DB_NAME
-
-echo -e "\n -> Creating the directories for the logs \n"
-mkdir /var/log/mayan
-
-echo -e "\n -> Making a convenience symlink \n"
-cd ${INSTALLATION_DIRECTORY}
-ln -s lib/python2.7/site-packages/mayan .
-
-echo -e "\n -> Creating an initial settings file \n"
-mayan-edms.py createsettings
-
-echo -e "\n -> Updating the mayan/settings/local.py file \n"
-cat >> mayan/settings/local.py << EOF
-DATABASES = {
-    'default': {
-        'ENGINE': 'django.db.backends.postgresql_psycopg2',
-        'NAME': '$DB_NAME',
-        'USER': '$DB_USERNAME',
-        'PASSWORD': '$DB_PASSWORD',
-        'HOST': 'localhost',
-        'PORT': '5432',
-    }
-}
-
-BROKER_URL = 'redis://127.0.0.1:6379/0'
-CELERY_RESULT_BACKEND = 'redis://127.0.0.1:6379/0'
-EOF
-
-echo -e "\n -> Migrating the database or initialize the project \n"
-mayan-edms.py initialsetup
-
-echo -e "\n -> Disabling the default NGINX site \n"
-rm -f /etc/nginx/sites-enabled/default
-
-echo -e "\n -> Creating a uwsgi.ini file \n"
-cat > uwsgi.ini << EOF
-[uwsgi]
-chdir = ${INSTALLATION_DIRECTORY}lib/python2.7/site-packages/mayan
-chmod-socket = 664
-chown-socket = www-data:www-data
-env = DJANGO_SETTINGS_MODULE=mayan.settings.production
-gid = www-data
-logto = /var/log/uwsgi/%n.log
-pythonpath = ${INSTALLATION_DIRECTORY}lib/python2.7/site-packages
-master = True
-max-requests = 5000
-socket = ${INSTALLATION_DIRECTORY}uwsgi.sock
-uid = www-data
-vacuum = True
-wsgi-file = ${INSTALLATION_DIRECTORY}lib/python2.7/site-packages/mayan/wsgi.py
-EOF
-
-echo -e "\n -> Creating the directory for the uWSGI log files \n"
-mkdir -p /var/log/uwsgi
-
-echo -e "\n -> Creating the NGINX site file for Mayan EDMS, /etc/nginx/sites-available/mayan \n"
-cat > /etc/nginx/sites-available/mayan << EOF
-server {
-    listen 80;
-    server_name localhost;
-
-    location / {
-        include uwsgi_params;
-        uwsgi_pass unix:${INSTALLATION_DIRECTORY}uwsgi.sock;
-
-        client_max_body_size 30M;  # Increse if your plan to upload bigger documents
-        proxy_read_timeout 30s;  # Increase if your document uploads take more than 30 seconds
-    }
-
-    location /static {
-        alias ${INSTALLATION_DIRECTORY}mayan/media/static;
-        expires 1h;
-    }
-
-    location /favicon.ico {
-        alias ${INSTALLATION_DIRECTORY}mayan/media/static/appearance/images/favicon.ico;
-        expires 1h;
-    }
-}
-EOF
-
-echo -e "\n -> Enabling the NGINX site for Mayan EDMS \n"
-ln -s /etc/nginx/sites-available/mayan /etc/nginx/sites-enabled/
-
-echo -e "\n -> Creating the supervisor file for the uWSGI process, /etc/supervisor/conf.d/mayan-uwsgi.conf \n"
-cat > /etc/supervisor/conf.d/mayan-uwsgi.conf << EOF
-[program:mayan-uwsgi]
-command = ${INSTALLATION_DIRECTORY}bin/uwsgi --ini ${INSTALLATION_DIRECTORY}uwsgi.ini
-user = root
-autostart = true
-autorestart = true
-redirect_stderr = true
-EOF
-
-echo -e "\n -> Creating the supervisor file for the Celery worker, /etc/supervisor/conf.d/mayan-celery.conf \n"
-cat > /etc/supervisor/conf.d/mayan-celery.conf << EOF
-[program:mayan-worker]
-command = ${INSTALLATION_DIRECTORY}bin/python ${INSTALLATION_DIRECTORY}bin/mayan-edms.py celery --settings=mayan.settings.production worker -Ofair -l ERROR
-directory = ${INSTALLATION_DIRECTORY}
-user = www-data
-stdout_logfile = /var/log/mayan/worker-stdout.log
-stderr_logfile = /var/log/mayan/worker-stderr.log
-autostart = true
-autorestart = true
-startsecs = 10
-stopwaitsecs = 10
-killasgroup = true
-priority = 998
-
-[program:mayan-beat]
-command = ${INSTALLATION_DIRECTORY}bin/python ${INSTALLATION_DIRECTORY}bin/mayan-edms.py celery --settings=mayan.settings.production beat -l ERROR
-directory = ${INSTALLATION_DIRECTORY}
-user = www-data
-numprocs = 1
-stdout_logfile = /var/log/mayan/beat-stdout.log
-stderr_logfile = /var/log/mayan/beat-stderr.log
-autostart = true
-autorestart = true
-startsecs = 10
-stopwaitsecs = 1
-killasgroup = true
-priority = 998
-EOF
-
-echo -e "\n -> Collecting the static files \n"
-mayan-edms.py collectstatic --noinput
-
-echo -e "\n -> Making the installation directory readable and writable by the webserver user \n"
-chown www-data:www-data ${INSTALLATION_DIRECTORY} -R
-
-echo -e "\n -> Restarting the services \n"
-/etc/init.d/nginx restart
-/etc/init.d/supervisor restart

contrib/scripts/process_messages.py

@@ -9,18 +9,21 @@ import sh
 
 
 APP_LIST = (
-    'acls', 'appearance', 'authentication', 'cabinets', 'checkouts', 'common',
-    'converter', 'django_gpg', 'document_comments', 'document_indexing',
-    'document_parsing', 'document_signatures', 'document_states', 'documents',
-    'dynamic_search', 'events', 'linking', 'lock_manager', 'mayan_statistics',
-    'mailer', 'metadata', 'mirroring', 'motd', 'navigation', 'ocr', 'permissions',
-    'rest_api', 'smart_settings', 'sources', 'storage', 'tags', 'task_manager',
-    'user_management'
+    'acls', 'appearance', 'authentication', 'autoadmin', 'cabinets',
+    'checkouts', 'common', 'converter', 'dashboards', 'dependencies',
+    'django_gpg', 'document_comments', 'document_indexing',
+    'document_parsing', 'document_signatures', 'document_states',
+    'documents', 'dynamic_search', 'events', 'file_caching',
+    'file_metadata', 'linking', 'lock_manager', 'mailer',
+    'mayan_statistics', 'metadata', 'mirroring', 'motd', 'navigation',
+    'ocr', 'permissions', 'platform', 'rest_api', 'smart_settings',
+    'sources', 'storage', 'tags', 'task_manager', 'templating',
+    'user_management', 'web_links'
 )
 
 LANGUAGE_LIST = (
     'ar', 'bg', 'bs_BA', 'cs', 'da_DK', 'de_DE', 'en', 'es', 'el', 'fa', 'fr',
-    'hu', 'id', 'it', 'nl_NL', 'pl', 'pt', 'pt_BR', 'ro_RO', 'ru', 'sl_SI',
+    'hu', 'id', 'it', 'lv', 'nl_NL', 'pl', 'pt', 'pt_BR', 'ro_RO', 'ru', 'sl_SI',
     'tr_TR', 'vi_VN', 'zh',
 )
 

contrib/scripts/start_gunicorn.sh

@@ -1,35 +0,0 @@
-#!/bin/bash
-
-NAME="mayan-edms"
-DJANGODIR=/usr/share/mayan-edms
-SOCKFILE=/var/tmp/filesystem.sock
-USER=www-data
-GROUP=www-data
-NUM_WORKERS=3
-DJANGO_SETTINGS_MODULE=mayan.settings.production
-DJANGO_WSGI_MODULE=mayan.wsgi
-TIMEOUT=600
-
-echo "Starting $NAME as `whoami`"
-
-# Activate the virtual environment
-cd $DJANGODIR
-source bin/activate
-export DJANGO_SETTINGS_MODULE=$DJANGO_SETTINGS_MODULE
-export PYTHONPATH=$DJANGODIR:$PYTHONPATH
-
-# Create the run directory if it doesn't exist
-RUNDIR=$(dirname $SOCKFILE)
-test -d $RUNDIR || mkdir -p $RUNDIR
-
-# Start your Django Unicorn
-# Programs meant to be run under supervisor should not daemonize themselves (do not use --daemon)
-exec bin/gunicorn ${DJANGO_WSGI_MODULE}:application \
-  --name $NAME \
-  --workers $NUM_WORKERS \
-  --user=$USER --group=$GROUP \
-  --log-level=debug \
-  --bind=unix:$SOCKFILE \
-  --timeout=$TIMEOUT
-
-

contrib/settings/ldap_connection_settings.py

@@ -1,64 +1,139 @@
 from __future__ import absolute_import
 
+# Install Python LDAP with:
+# $ pip install python-ldap
+# or if using Docker, pass the environment variable MAYAN_PIP_INSTALLS:
+# -e MAYAN_PIP_INSTALLS=python-ldap
 import ldap
-from django_auth_ldap.config import LDAPSearch
 
-from .base import *  # NOQA
-from django.contrib.auth import get_user_model
+from django_auth_ldap.config import (
+    LDAPSearch, LDAPSearchUnion, NestedActiveDirectoryGroupType
+)
 
-SECRET_KEY = '<your secret key>'
+from mayan.settings.production import *
 
-# makes sure this works in Active Directory
-ldap.set_option(ldap.OPT_REFERRALS, 0)
+# Makes sure this works in Active Directory
+ldap.set_option(ldap.OPT_REFERRALS, False)
 
-# This is the default, but I like to be explicit.
+# Turn of debug output, turn this off when everything is working as expected
+ldap.set_option(ldap.OPT_DEBUG_LEVEL, 1)
+
+# Default: True
 AUTH_LDAP_ALWAYS_UPDATE_USER = True
 
-LDAP_USER_AUTO_CREATION = "False"
-LDAP_URL = "ldap://<your ldap server IP>:389/"
-LDAP_BASE_DN = "dc=paramatrix,dc=co,dc=in"
-LDAP_ADDITIONAL_USER_DN = "dc=people"
-LDAP_ADMIN_DN = ""
-LDAP_PASSWORD = ""
+# Use TLS to talk to the LDAP server
+# Requires acquiring the server's certificate
+# $ openssl s_client -connect <LDAP server>:636
+# Part of the output of this file will be the Base-64 encoded .cer file
+# that was presented for LDAPS. Cut and paste into a file beginning at
+# "-Begin Certificate" through "-End Certificate--" and save as a .crt, for
+# example: ldapserver.crt
+# $ CERT=ldapserver.crt
+# $ cp /root/$CERT /usr/share/ca-certificates/$CERT
+# # notice the + sign which tells to activate the certificate.
+# $ echo "+$CERT" >/etc/ca-certificates/update.d/activate_my_cert
+# $ dpkg-reconfigure ca-certificates;
+AUTH_LDAP_START_TLS = False
+
+LDAP_ADDITIONAL_USER_DN = 'dc=people'
+LDAP_ADMIN_DN = ''
+LDAP_BASE_DN = 'dc=<top level dc>,dc=co,dc=in'
+LDAP_PASSWORD = ''
+LDAP_USER_AUTO_CREATION = 'False'
+LDAP_URL = 'ldap://<LDAP server>:389/'
 
-AUTH_LDAP_SERVER_URI = LDAP_URL
 AUTH_LDAP_BIND_DN = LDAP_ADMIN_DN
 AUTH_LDAP_BIND_PASSWORD = LDAP_PASSWORD
+AUTH_LDAP_SERVER_URI = LDAP_URL
 
-
+# Simple search
 AUTH_LDAP_USER_SEARCH = LDAPSearch(
     '%s,%s' % (LDAP_ADDITIONAL_USER_DN, LDAP_BASE_DN),
     ldap.SCOPE_SUBTREE, '(uid=%(user)s)'
 )
+
+# If you need to search in more than one place for a user, you can use
+# LDAPSearchUnion. This takes multiple LDAPSearch objects and returns the
+# union of the results. The precedence of the underlying searches is
+# unspecified.
+# https://django-auth-ldap.readthedocs.io/en/latest/authentication.html
+# AUTH_LDAP_USER_SEARCH = LDAPSearchUnion(
+#     LDAPSearch(
+#         'ou=Users,ou=Admin,dc=<top level DC>,dc=local', ldap.SCOPE_SUBTREE,
+#         '(samaccountname=%(user)s)'
+#     ),
+#     LDAPSearch(
+#         'ou=Users,ou=<second OU>,dc=<top level DC>,dc=local',
+#         ldap.SCOPE_SUBTREE, '(samaccountname=%(user)s)'
+#     ),
+#     LDAPSearch(
+#         'ou=Users,ou=<third OU>,dc=<top level DC>,dc=local',
+#         ldap.SCOPE_SUBTREE, '(samaccountname=%(user)s)'
+#     ),
+# )
+
+# User attributes to map from LDAP to Mayan's user model.
 AUTH_LDAP_USER_ATTR_MAP = {
     'first_name': 'cn',
     'last_name': 'sn',
     'email': 'mail'
 }
+# Another example map
+# AUTH_LDAP_USER_ATTR_MAP = {
+#     'username': 'sAMAccountName',
+#     'first_name': 'givenName',
+#     'last_name': 'sn',
+#     'email': 'mail'
+# }
+# Only string fields can be mapped to attributes. Boolean fields can be
+# defined by group membership:
+# AUTH_LDAP_USER_FLAGS_BY_GROUP = {
+#     'is_active': 'cn=active,ou=groups,dc=example,dc=com',
+#     'is_staff': (
+#         LDAPGroupQuery('cn=staff,ou=groups,dc=example,dc=com')
+#         | LDAPGroupQuery('cn=admin,ou=groups,dc=example,dc=com')
+#     ),
+#     'is_superuser': 'cn=superuser,ou=groups,dc=example,dc=com',
+# }
+
+# Simple group search
+# AUTH_LDAP_GROUP_SEARCH = LDAPSearch(
+#     'ou=groups,dc=example,dc=com', ldap.SCOPE_SUBTREE, '(objectClass=groupOfNames)'
+# )
+# AUTH_LDAP_GROUP_TYPE = GroupOfNamesType()
+
+# Advanced group search
+# AUTH_LDAP_GROUP_SEARCH = LDAPSearchUnion(
+#     LDAPSearch(
+#         'ou=Domain Global,OU=Security,OU=Groups,OU=<OU>,dc=<top level DC>,dc=local',
+#         ldap.SCOPE_SUBTREE,
+#         '(&(objectClass=group)(groupType:1.2.840.113556.1.4.803:=2147483648))'
+#     ),
+#     LDAPSearch(
+#         'ou=Domain Global,OU=Security,OU=Groups,OU=<OU>,dc=<top level DC>,dc=local',
+#         ldap.SCOPE_SUBTREE,
+#         '(&(objectClass=group)(groupType:1.2.840.113556.1.4.803:=2147483648))'
+#     ),
+# )
+# AUTH_LDAP_CACHE_GROUPS = True
+# AUTH_LDAP_FIND_GROUP_PERMS = False
+# AUTH_LDAP_GROUP_TYPE = NestedActiveDirectoryGroupType()
+# AUTH_LDAP_MIRROR_GROUPS = True
+
+# To minimize traffic to the LDAP server, LDAPBackend can make use of
+# Django’s cache framework to keep a copy of a user’s LDAP group memberships.
+# To enable this feature, set AUTH_LDAP_CACHE_TIMEOUT, which determines
+# the timeout of cache entries in seconds.
+# AUTH_LDAP_GROUP_CACHE_TIMEOUT = 3600
+
+# Limiting Access
+# The simplest use of groups is to limit the users who are allowed to log in.
+# If AUTH_LDAP_REQUIRE_GROUP is set, then only users who are members of that
+# group will successfully authenticate. AUTH_LDAP_DENY_GROUP is the reverse:
+# if given, members of this group will be rejected.
+# AUTH_LDAP_REQUIRE_GROUP = 'cn=enabled,ou=groups,dc=example,dc=com'
+# AUTH_LDAP_DENY_GROUP = 'cn=disabled,ou=groups,dc=example,dc=com'
+
 AUTHENTICATION_BACKENDS = (
     'django_auth_ldap.backend.LDAPBackend',
-    'mayan.settings.settings_local.EmailOrUsernameModelBackend',
 )
-
-
-class EmailOrUsernameModelBackend(object):
-    """
-    This is a ModelBacked that allows authentication with either a username or an email address.
-    """
-    def authenticate(self, username=None, password=None):
-        if '@' in username:
-            kwargs = {'email': username}
-        else:
-            kwargs = {'username': username}
-        try:
-            user = get_user_model().objects.get(**kwargs)
-            if user.check_password(password):
-                return user
-        except get_user_model().DoesNotExist:
-            return None
-
-    def get_user(self, username):
-        try:
-            return get_user_model().objects.get(pk=username)
-        except get_user_model().DoesNotExist:
-            return None

docker/Dockerfile

@@ -1,164 +1,165 @@
 # vim:set ft=dockerfile:
 
-####################
-# Base image start #
-####################
+####
+# BASE_IMAGE - Bare bones image with the base packages needed to run Mayan EDMS
+####
 
-FROM debian:9.4-slim as BASE_IMAGE
+FROM debian:10.0-slim as BASE_IMAGE
 
-MAINTAINER Roberto Rosario "roberto.rosario@mayan-edms.com"
+LABEL maintainer="Roberto Rosario roberto.rosario@mayan-edms.com"
 
-ENV DEBIAN_FRONTEND noninteractive
-ENV PYTHONUNBUFFERED 1
-ENV LC_ALL C.UTF-8
-ENV PROJECT_INSTALL_DIR=/opt/mayan-edms
-ENV PYTHON_PIP=${PROJECT_INSTALL_DIR}/bin/pip
+COPY config.env /config.env
 
+ENV PYTHONUNBUFFERED=1 \
+    LC_ALL=C.UTF-8 \
+    PROJECT_INSTALL_DIR=/opt/mayan-edms
+
+# Debian package caching
 ARG APT_PROXY
-# Package caching
-RUN if [ "${APT_PROXY}" ]; then echo "Acquire::http { Proxy \"http://${APT_PROXY}\"; };" > /etc/apt/apt.conf.d/01proxy; fi
-
-# Install base Ubuntu libraries
-RUN apt-get update && \
-apt-get install -y --no-install-recommends \
-        g++ \
-        gcc \
+RUN set -x \
+&& if [ "${APT_PROXY}" ]; \
+    then echo "Acquire::http { Proxy \"http://${APT_PROXY}\"; };" > /etc/apt/apt.conf.d/01proxy \
+; fi \
+# Install base OS packages to run Mayan EDMS
+&& DEBIAN_FRONTEND=noninteractive \
+apt-get update \
+&& apt-get install -y --no-install-recommends \
+    ca-certificates \
+    exiftool \
+    fonts-arphic-uming \
+    fonts-arphic-ukai \
     ghostscript \
     gpgv \
     gnupg1 \
     graphviz \
-        libffi-dev \
     libfuse2 \
-        libjpeg-dev \
     libmagic1 \
-        default-libmysqlclient-dev \
-        libpng-dev \
-        libpq-dev \
+    libmariadb3 \
     libreoffice \
-        libtiff-dev \
+    libpq5 \
     poppler-utils \
-        python-dev \
-        python-setuptools \
-        python-virtualenv \
-        python-wheel \
-        redis-server \
+    python3-distutils \
     sane-utils \
     sudo \
     supervisor \
-        tesseract-ocr \
+    tesseract-ocr tesseract-ocr-deu \
+# Remove make and build dependencies
+&& apt-get remove make libproxy-tools libreoffice-avmedia-backend-vlc libvlc-bin libvlc5 libvlccore9 adwaita-icon-theme gsettings-desktop-schemas libgstreamer-plugins-base1.0-0 -y \
+&& apt-get autoremove -y --purge \
+# Add mayan user
+&& adduser mayan --disabled-password --disabled-login --no-create-home --gecos "" \
+# Pillow can't find zlib or libjpeg on aarch64 (ODROID C2)
+&& if [ "$(uname -m)" = "aarch64" ]; then \
+    ln -s /usr/lib/aarch64-linux-gnu/libz.so /usr/lib/ \
+    && ln -s /usr/lib/aarch64-linux-gnu/libjpeg.so /usr/lib/ \
+; fi \
+# Pillow can't find zlib or libjpeg on armv7l (ODROID HC1)
+&& if [ "$(uname -m)" = "armv7l" ]; then \
+    ln -s /usr/lib/arm-linux-gnueabihf/libz.so /usr/lib/ \
+    && ln -s /usr/lib/arm-linux-gnueabihf/libjpeg.so /usr/lib/ \
+; fi
+
+####
+# BUILDER_IMAGE - This image builds the Python package and is discarded afterwards
+#                 only the build artifact is carried over to the next image.
+####
+
+# Reuse image
+FROM BASE_IMAGE as BUILDER_IMAGE
+
+# Python libraries caching
+ARG PIP_INDEX_URL
+ARG PIP_TRUSTED_HOST
+
+WORKDIR /src
+
+# Copy the source files needed to build the Python package
+COPY --chown=mayan:mayan requirements /src/requirements
+COPY --chown=mayan:mayan \
+    HISTORY.rst \
+    LICENSE \
+    MANIFEST.in \
+    README.md \
+    README.rst \
+    setup.py \
+    /src/
+
+COPY --chown=mayan:mayan mayan /src/mayan
+
+# Install development packages needed to build the Python packages
+RUN DEBIAN_FRONTEND=noninteractive \
+apt-get install -y --no-install-recommends \
+    default-libmysqlclient-dev \
+    libffi-dev \
+    libjpeg-dev \
+    libpng-dev \
+    libpq-dev \
+    libtiff-dev \
     zlib1g-dev \
     libssl-dev \
-&& \
-apt-get clean autoclean && \
-apt-get autoremove --purge -y && \
-rm -rf /var/lib/apt/lists/* && \
-rm -f /var/cache/apt/archives/*.deb
+    g++ \
+    gcc \
+    python3-dev \
+    python3-venv \
+&& mkdir -p "${PROJECT_INSTALL_DIR}" \
+&& chown -R mayan:mayan "${PROJECT_INSTALL_DIR}" \
+&& chown -R mayan:mayan /src
 
-# Install apt-get-install
-ADD https://raw.githubusercontent.com/guilhem/apt-get-install/master/apt-get-install /usr/bin/
-RUN chmod +x /usr/bin/apt-get-install
+USER mayan
 
-RUN adduser mayan --disabled-password --disabled-login --no-create-home --gecos ""
+RUN set -a \
+&& . /config.env \
+&& set +a \
+&& python3 -m venv "${PROJECT_INSTALL_DIR}" \
+&& . "${PROJECT_INSTALL_DIR}/bin/activate" \
+&& pip install --no-cache-dir \
+    amqp==$PYTHON_AMQP_VERSION \
+    mysqlclient==$PYTHON_MYSQL_VERSION \
+    psycopg2==$PYTHON_PSYCOPG2_VERSION \
+    redis==$PYTHON_REDIS_VERSION \
+    flower==$PYTHON_FLOWER_VERSION \
+# psutil is needed by ARM builds otherwise gevent and gunicorn fail to start
+&& UNAME=`uname -m` && if [ "${UNAME#*arm}" != $UNAME ]; then \
+    pip install --no-cache-dir \
+    psutil==$PYTHON_PSUTIL_VERSION \
+; fi \
+# Install the Python packages needed to build Mayan EDMS
+&& pip install --no-cache-dir -r /src/requirements/build.txt \
+# Build Mayan EDMS
+&& python3 setup.py sdist \
+# Install the built Mayan EDMS package
+&& pip install --no-cache-dir dist/mayan* \
+# Install the static content
+&& mayan-edms.py installdependencies \
+&& MAYAN_STATIC_ROOT=${PROJECT_INSTALL_DIR}/static mayan-edms.py preparestatic --link --noinput
 
-# Pillow can't find zlib or libjpeg on aarch64 (ODROID C2)
-RUN if [ "$(uname -m)" = "aarch64" ]; then \
-ln -s /usr/lib/aarch64-linux-gnu/libz.so /usr/lib/ && \
-ln -s /usr/lib/aarch64-linux-gnu/libjpeg.so /usr/lib/ \
-; fi
+COPY --chown=mayan:mayan requirements/testing-base.txt "${PROJECT_INSTALL_DIR}"
 
-# Pillow can't find zlib or libjpeg on armv7l (ODROID HC1)
-RUN if [ "$(uname -m)" = "armv7l" ]; then \
-ln -s /usr/lib/arm-linux-gnueabihf/libz.so /usr/lib/ && \
-ln -s /usr/lib/arm-linux-gnueabihf/libjpeg.so /usr/lib/ \
-; fi
-
-# Discard data when Redis runs out of memory
-RUN echo "maxmemory-policy allkeys-lru" >> /etc/redis/redis.conf
-
-# Disable saving the Redis database
-RUN echo "save \"\"" >> /etc/redis/redis.conf
-
-# Only provision 1 database
-RUN echo "databases 1" >> /etc/redis/redis.conf
-
-#####################
-# Build image start #
-#####################
-
-FROM debian:9.4-slim as BUILDER_IMAGE
-
-ARG APT_PROXY
-# Package caching
-RUN if [ "${APT_PROXY}" ]; then echo "Acquire::http { Proxy \"http://${APT_PROXY}\"; };" > /etc/apt/apt.conf.d/01proxy; fi
-
-WORKDIR /code
-
-COPY . /code
-
-RUN apt-get update && apt-get install make python-dev python-pip -y
-
-RUN pip install -r requirements/build.txt
-
-ENV LC_ALL C.UTF-8
-
-RUN touch docker/Makefile
-
-RUN make wheel
-
-RUN chmod 777 dist -R
-
-#####################
-# Final image start #
-#####################
+####
+# Final image - BASE_IMAGE + BUILDER_IMAGE artifact (Mayan install directory)
+####
 
 FROM BASE_IMAGE
 
-RUN mkdir -p /opt
+COPY --from=BUILDER_IMAGE --chown=mayan:mayan "${PROJECT_INSTALL_DIR}/" "${PROJECT_INSTALL_DIR}/"
 
-RUN python /usr/lib/python2.7/dist-packages/virtualenv.py $PROJECT_INSTALL_DIR
+USER root
 
-WORKDIR $PROJECT_INSTALL_DIR
+COPY docker/rootfs /
 
-COPY --from=BUILDER_IMAGE /code/dist/*.whl .
-
-COPY --from=BUILDER_IMAGE /code/contrib/scripts/docker/run-tests.sh .
-
-COPY --from=BUILDER_IMAGE /code/requirements/testing-base.txt requirements-testing.txt
-
-COPY --from=BUILDER_IMAGE /code/docker/version .
-
-# Fix ownership
-RUN chown -R mayan:mayan $PROJECT_INSTALL_DIR
-
-# Install build Mayan EDMS
-RUN sudo -u mayan $PYTHON_PIP install --no-cache-dir --no-use-pep517 *.whl && \
-rm *.whl
-
-# Install Python clients for librabbitmq, MySQL, PostgreSQL, REDIS
-RUN sudo -u mayan $PYTHON_PIP install --no-cache-dir --no-use-pep517 librabbitmq==1.6.1 mysql-python==1.2.5 psycopg2==2.7.3.2 redis==2.10.6
-
-# Setup supervisor
-COPY docker/etc/supervisor/mayan.conf /etc/supervisor/conf.d
-
-RUN mkdir /var/lib/mayan
 VOLUME ["/var/lib/mayan"]
 
-COPY docker/entrypoint.sh /usr/local/bin/
-RUN ln -s usr/local/bin/entrypoint.sh / # backwards compat
 ENTRYPOINT ["entrypoint.sh"]
 
 EXPOSE 8000
-CMD ["mayan"]
+CMD ["run_all"]
 
-RUN rm /root/.cache -R
-RUN rm -rf /tmp/*
-
-RUN apt-get -y autoremove --purge && apt-get -y autoclean && apt-get -y clean
-
-RUN rm -rf /usr/share/man/*
-RUN rm -rf /usr/share/doc/*
-
-RUN find /var/lib/apt -type f | xargs rm -f
-RUN find /var/cache -type f -exec rm -rf {} \;
-
-RUN find /var/log -type f | while read f; do echo -ne '' > $f; done;
+RUN ${PROJECT_INSTALL_DIR}/bin/mayan-edms.py platformtemplate supervisord_docker > /etc/supervisor/conf.d/mayan.conf \
+&& apt-get clean autoclean \
+&& apt-get autoremove --purge -y \
+&& rm -rf /var/lib/apt/lists/* \
+&& rm -f /var/cache/apt/archives/*.deb \
+# Remove temporary files owned by root from the platformtemplate step
+&& rm /tmp/* \
+# Keep displaying log messages to stdout
+&& find /var/log -type f | while read f; do echo -ne '' > $f; done;

docker/Dockerfile.ubuntu

@@ -1,150 +0,0 @@
-# vim:set ft=dockerfile:
-
-####################
-# Base image start #
-####################
-
-FROM ubuntu:16.04 as BASE_IMAGE
-
-MAINTAINER Roberto Rosario "roberto.rosario@mayan-edms.com"
-
-ENV DEBIAN_FRONTEND noninteractive
-ENV PYTHONUNBUFFERED 1
-ENV LANG en_US.UTF-8
-ENV PROJECT_INSTALL_DIR=/usr/local/lib/python2.7/dist-packages/mayan
-
-ARG APT_PROXY
-# Package caching
-RUN if [ "${APT_PROXY}" ]; then echo "Acquire::http { Proxy \"http://${APT_PROXY}\"; };" > /etc/apt/apt.conf.d/01proxy; fi
-
-# Install base Ubuntu libraries
-RUN apt-get update && \
-apt-get install -y --no-install-recommends \
-        curl \
-        g++ \
-        gcc \
-        gettext-base \
-        ghostscript \
-        gpgv \
-        graphviz \
-        libffi-dev \
-        libjpeg-dev \
-        libmagic1 \
-        libmysqlclient-dev \
-        libpng-dev \
-        libpq-dev \
-        libreoffice \
-        libtiff-dev \
-        locales \
-        netcat-openbsd \
-        poppler-utils \
-        python-dev \
-        python-pip \
-        python-setuptools \
-        python-wheel \
-        redis-server \
-        supervisor \
-        tesseract-ocr \
-        zlib1g-dev \
-&& \
-apt-get clean autoclean && \
-apt-get autoremove --purge -y && \
-rm -rf /var/lib/apt/lists/* && \
-rm -f /var/cache/apt/archives/*.deb
-
-# Switch to UTF locale
-RUN echo "LC_ALL=\"en_US.UTF-8\"" >> /etc/default/locale && \
-locale-gen en_US.UTF-8 && \
-update-locale LANG=en_US.UTF-8 && \
-export LC_ALL=en_US.UTF-8
-
-# Install apt-get-install
-ADD https://raw.githubusercontent.com/guilhem/apt-get-install/master/apt-get-install /usr/bin/
-RUN chmod +x /usr/bin/apt-get-install
-
-# Install Python clients for PostgreSQL, REDIS, librabbitmq
-RUN pip install psycopg2==2.7.3.2 redis==2.10.6 mysql-python==1.2.5 librabbitmq==1.6.1
-
-RUN adduser mayan --disabled-password --disabled-login --no-create-home --gecos ""
-
-# Pillow can't find zlib or libjpeg on aarch64
-RUN if [ "$(uname -m)" = "aarch64" ]; then \
-ln -s /usr/lib/aarch64-linux-gnu/libz.so /usr/lib/ && \
-ln -s /usr/lib/aarch64-linux-gnu/libjpeg.so /usr/lib/ \
-; fi
-
-#####################
-# Build image start #
-#####################
-
-FROM python:2-alpine3.7 as BUILDER_IMAGE
-
-WORKDIR /code
-
-COPY . /code
-
-RUN apk update && \
-    apk add make
-
-RUN pip install -r requirements/build.txt
-
-RUN make wheel
-
-RUN chmod 777 dist -R
-
-#####################
-# Final image start #
-#####################
-
-FROM BASE_IMAGE
-
-WORKDIR /root/
-
-COPY --from=BUILDER_IMAGE /code/dist/*.whl .
-
-# Install build Mayan EDMS
-RUN pip install *.whl && \
-rm *.whl
-
-# Setup supervisor
-#RUN mkdir /etc/supervisor.d/
-COPY docker/etc/supervisor/beat.conf /etc/supervisor/conf.d
-COPY docker/etc/supervisor/gunicorn.conf /etc/supervisor/conf.d
-COPY docker/etc/supervisor/redis.conf /etc/supervisor/conf.d
-COPY docker/etc/supervisor/workers.conf /etc/supervisor/conf.d
-
-# Create the directory for the logs
-RUN mkdir /var/log/mayan
-
-# Fix ownership
-RUN chown -R mayan:mayan $PROJECT_INSTALL_DIR
-
-# Allow flanker to autogenerate its PLY files
-RUN chown -R mayan:mayan /usr/local/lib/python2.7/dist-packages/flanker/
-
-RUN mkdir /var/lib/mayan
-VOLUME ["/var/lib/mayan"]
-
-COPY docker/entrypoint.sh /usr/local/bin/
-RUN ln -s usr/local/bin/entrypoint.sh / # backwards compat
-ENTRYPOINT ["entrypoint.sh"]
-
-# Healthcheck setup
-HEALTHCHECK --interval=15s --timeout=1s --retries=20 \
-  CMD curl -s -f http://localhost/authentication/login/ | grep 'form' > /dev/null || exit 1
-
-EXPOSE 8000
-CMD ["mayan"]
-
-RUN rm /root/.cache -R
-RUN rm -rf /tmp/*
-
-RUN apt-get -y autoremove --purge && apt-get -y autoclean && apt-get -y clean
-
-RUN rm -rf /usr/share/man/*
-RUN rm -rf /usr/share/doc/*
-
-RUN find /var/lib/apt -type f | xargs rm -f
-RUN find /var/cache -type f -exec rm -rf {} \;
-
-RUN find /var/log -type f | while read f; do echo -ne '' > $f; done;

docker/Makefile

@@ -1,33 +1,130 @@
-APT_PROXY ?= `/sbin/ip route|awk '/docker0/ { print $$9 }'`:3142
-IMAGE_VERSION ?= `cat docker/version`
+#!make
+include config.env
+
+HOST_IP = `/sbin/ip route|awk '/docker0/ { print $$9 }'`
+APT_PROXY ?= $(HOST_IP):3142
 CONSOLE_COLUMNS ?= `echo $$(tput cols)`
 CONSOLE_LINES ?= `echo $$(tput lines)`
+IMAGE_VERSION ?= `cat docker/rootfs/version`
+PIP_INDEX_URL ?= http://$(HOST_IP):3141/root/pypi/+simple/
+PIP_TRUSTED_HOST ?= $(HOST_IP)
 
-docker-help:
-	@echo
-	@echo "**** Docker makefile ****"
-	@echo "docker-build - Build a new image locally."
-	@echo "docker-build-with-proxy - Build a new image locally using an APT proxy."
-	@echo "docker-test-container - Build and run a test container."
-	@echo "docker-test-cleanup - Delete the test container and the test volume."
-	@echo "docker-test-all - Build and executed the test suite in a test container."
-	@echo "docker-shell - Launch a bash instance inside a running container. Pass the container name via DOCKER_CONTAINER."
-
-docker-build:
+docker-build: ## Build a new image locally.
 	docker build -t mayanedms/mayanedms:$(IMAGE_VERSION) -f docker/Dockerfile .
 
-docker-build-with-proxy:
-	docker build -t mayanedms/mayanedms:$(IMAGE_VERSION) -f docker/Dockerfile --build-arg APT_PROXY=$(APT_PROXY) .
+docker-build-with-proxy: ## Build a new image locally using an APT proxy as APT_PROXY.
+docker-build-with-proxy: devpi-start
+	docker build -t mayanedms/mayanedms:$(IMAGE_VERSION) -f docker/Dockerfile --build-arg APT_PROXY=$(APT_PROXY) --build-arg PIP_INDEX_URL=$(PIP_INDEX_URL) --build-arg PIP_TRUSTED_HOST=$(PIP_TRUSTED_HOST) --build-arg HTTP_PROXY=$(HTTP_PROXY) --build-arg HTTPS_PROXY=$(HTTPS_PROXY) .
+	$(MAKE) devpi-stop
 
-docker-shell:
+docker-shell: ## Launch a bash instance inside a running container. Pass the container name via DOCKER_CONTAINER.
 	docker exec -e TERM=$(TERM) -e "COLUMNS=$(CONSOLE_COLUMNS)" -e "LINES=$(CONSOLE_LINES)" -it $(DOCKER_CONTAINER) /bin/bash
 
-docker-test-container: docker-build-with-proxy docker-test-cleanup
-	docker run -d --name test-mayan-edms -p 80:8000 -v test-mayan_data:/var/lib/mayan mayanedms/mayanedms:$(DOCKER_VERSION)
+docker-runtest-container: ## Run a test container.
+docker-runtest-container: docker-test-cleanup
+	docker run \
+	-d \
+	--name test-mayan-edms \
+	-p 80:8000 \
+	-v test-mayan_data:/var/lib/mayan \
+	mayanedms/mayanedms:$(IMAGE_VERSION)
 
-docker-test-cleanup:
+docker-runtest-cleanup: ## Delete the test container and the test volume.
 	@docker rm -f test-mayan-edms || true
 	@docker volume rm test-mayan_data || true
 
-docker-test-all: docker-build-with-proxy
-	docker run --rm run-tests
+docker-runtest-all: ## Executed the test suite in a test container.
+	docker run --rm mayanedms/mayanedms:$(IMAGE_VERSION) run_tests
+
+docker-compose-build:
+	docker-compose -f docker/docker-compose.yml -p mayan-edms build
+
+docker-compose-build-with-proxy: devpi-start
+	docker-compose -f docker/docker-compose.yml -p mayan-edms build --build-arg APT_PROXY=$(APT_PROXY) --build-arg PIP_INDEX_URL=$(PIP_INDEX_URL) --build-arg PIP_TRUSTED_HOST=$(PIP_TRUSTED_HOST) --build-arg HTTP_PROXY=$(HTTP_PROXY) --build-arg HTTPS_PROXY=$(HTTPS_PROXY)
+	$(MAKE) devpi-stop
+
+docker-compose-up:
+	docker-compose -f docker/docker-compose.yml -p mayan-edms up
+
+docker-staging-network-create:
+	@docker network rm mayan-staging || true
+	docker network create mayan-staging
+
+docker-staging-container-postgresql-start:
+	docker run \
+	-d \
+	--name mayan-staging-postgres \
+	--network=mayan-staging \
+	-e POSTGRES_USER=mayan \
+	-e POSTGRES_DB=mayan \
+	-e POSTGRES_PASSWORD=mayanuserpass \
+	-v mayan-staging-postgres:/var/lib/postgresql/data \
+	$(DOCKER_POSTGRES_IMAGE_VERSION)
+
+docker-staging-container-redis-start:
+	docker run \
+	-d \
+	--name mayan-staging-redis \
+	--network=mayan-staging \
+	-v mayan-staging-redis:/data \
+	$(DOCKER_REDIS_IMAGE_VERSION) \
+	redis-server \
+	--databases \
+	"2" \
+	--maxmemory-policy \
+	allkeys-lru \
+	--save \
+	""
+
+docker-staging-container-rabbitmq-start:
+	docker run \
+	-d \
+	--name mayan-staging-rabbitmq \
+	--network=mayan-staging \
+	-v mayan-staging-rabbitmq:/var/lib/rabbitmq \
+	$(DOCKER_RABBITMQ_IMAGE_VERSION) \
+
+docker-staging-container-mayan-start:
+	sleep 5 && docker run \
+	-d \
+	--name mayan-staging-app \
+	--network=mayan-staging \
+	-p 80:8000 \
+	-e MAYAN_DATABASE_ENGINE=django.db.backends.postgresql \
+	-e MAYAN_DATABASE_HOST=mayan-staging-postgres \
+	-e MAYAN_DATABASE_NAME=mayan \
+	-e MAYAN_DATABASE_PASSWORD=mayanuserpass \
+	-e MAYAN_DATABASE_USER=mayan \
+	-e MAYAN_CELERY_BROKER_URL=$(MAYAN_CELERY_BROKER_URL) \
+	-e MAYAN_CELERY_RESULT_BACKEND="redis://mayan-staging-redis:6379/1" \
+	-v mayan-staging-app:/var/lib/mayan \
+	mayanedms/mayanedms:$(IMAGE_VERSION)
+
+docker-staging-start-with-rabbitmq: MAYAN_CELERY_BROKER_URL="amqp://guest:guest@mayan-staging-rabbitmq:5672/"
+docker-staging-start-with-rabbitmq: docker-staging-start
+
+docker-staging-start-with-redis: MAYAN_CELERY_BROKER_URL="redis://mayan-staging-redis:6379/0"
+docker-staging-start-with-redis: docker-staging-start
+
+docker-staging-start: docker-staging-cleanup docker-staging-network-create docker-staging-container-postgresql-start docker-staging-container-rabbitmq-start docker-staging-container-redis-start docker-staging-container-mayan-start
+	docker logs -f mayan-staging-app
+
+docker-staging-cleanup: ## Delete the test container and the test volume.
+	@docker rm -f mayan-staging-app || true
+	@docker rm -f mayan-staging-redis || true
+	@docker rm -f mayan-staging-rabbitmq || true
+	@docker rm -f mayan-staging-postgres || true
+	@docker volume rm mayan-staging-app || true
+	@docker volume rm mayan-staging-postgres || true
+	@docker volume rm mayan-staging-rabbitmq || true
+	@docker volume rm mayan-staging-redis || true
+	@docker network rm mayan-staging || true
+
+devpi-init:
+	devpi-server --init || true
+
+devpi-start: devpi-init
+	devpi-server --start --host=0.0.0.0 || true
+
+devpi-stop:
+	devpi-server --stop || true

docker/README-short.txt

@@ -1 +0,0 @@
-Mayan EDMS is a free open source electronic document management system.

docker/docker-compose-development.yml

@@ -1,72 +0,0 @@
-version: '2.1'
-
-volumes:
-    broker:
-        driver: local
-    app:
-        driver: local
-    db:
-        driver: local
-    results:
-        driver: local
-
-services:
-  broker:
-    container_name: mayan-edms-broker
-    image: healthcheck/rabbitmq
-    environment:
-      RABBITMQ_DEFAULT_USER: mayan
-      RABBITMQ_DEFAULT_PASS: mayan
-      RABBITMQ_DEFAULT_VHOST: mayan
-    volumes:
-      - broker:/var/lib/rabbitmq
-  results:
-    container_name: mayan-edms-results
-    image: healthcheck/redis
-    volumes:
-      - results:/data
-  #db:
-  #  container_name: mayan-edms-db
-  #  image: healthcheck/mysql
-  #  environment:
-  #    MYSQL_DATABASE: mayan
-  #    MYSQL_PASSWORD: mayan-password
-  #    MYSQL_ROOT_PASSWORD: root-password
-  #    MYSQL_USER: mayan
-  #  volumes:
-  #    - db:/var/lib/mysql
-  db:
-    container_name: mayan-edms-db
-    image: healthcheck/postgres
-    environment:
-      POSTGRES_DB: mayan
-      POSTGRES_PASSWORD: mayan-password
-      POSTGRES_USER: mayan
-    volumes:
-      - db:/var/lib/postgresql/data
-  mayan-edms:
-    container_name: mayan-edms-app
-    image: mayan-edms/next
-    build:
-      context: ./
-      args:
-        - APT_PROXY=172.18.0.1:3142
-    depends_on:
-      broker:
-        condition: service_healthy
-      db:
-        condition: service_healthy
-      results:
-        condition: service_healthy
-    environment:
-      MAYAN_BROKER_URL: amqp://mayan:mayan@broker:5672/mayan
-      MAYAN_CELERY_RESULT_BACKEND: redis://results:6379/0
-      MAYAN_DATABASE_ENGINE: django.db.backends.postgresql
-      MAYAN_DATABASE_HOST: db
-      MAYAN_DATABASE_NAME: mayan
-      MAYAN_DATABASE_PASSWORD: mayan-password
-      MAYAN_DATABASE_USER: mayan
-    ports:
-      - "80:80"
-    volumes:
-      - app:/var/lib/mayan

docker/docker-compose.yml

@@ -1,58 +1,130 @@
-version: '2.1'
+version: '3.7'
 
-volumes:
-    broker:
-        driver: local
-    app:
-        driver: local
-    db:
-        driver: local
-    results:
-        driver: local
+networks:
+  mayan-bridge:
+    driver: bridge
 
 services:
-  broker:
-    container_name: mayan-edms-broker
-    image: healthcheck/rabbitmq
-    environment:
-      RABBITMQ_DEFAULT_USER: mayan
-      RABBITMQ_DEFAULT_PASS: mayan
-      RABBITMQ_DEFAULT_VHOST: mayan
-    volumes:
-      - broker:/var/lib/rabbitmq
-  results:
-    container_name: mayan-edms-results
-    image: healthcheck/redis
-    volumes:
-      - results:/data
-  db:
-    container_name: mayan-edms-db
-    image: healthcheck/postgres
-    environment:
-      POSTGRES_DB: mayan
-      POSTGRES_PASSWORD: mayan-password
-      POSTGRES_USER: mayan
-    volumes:
-      - db:/var/lib/postgresql/data
-  mayan-edms:
-    container_name: mayan-edms-app
-    image: mayanedms/mayanedms:latest
+  app:
+    build:
+      context: ..
+      dockerfile: ./docker/Dockerfile
     depends_on:
-      broker:
-        condition: service_healthy
-      db:
-        condition: service_healthy
-      results:
-        condition: service_healthy
-    environment:
-      MAYAN_BROKER_URL: amqp://mayan:mayan@broker:5672/mayan
-      MAYAN_CELERY_RESULT_BACKEND: redis://results:6379/0
-      MAYAN_DATABASE_ENGINE: django.db.backends.postgresql
-      MAYAN_DATABASE_HOST: db
-      MAYAN_DATABASE_NAME: mayan
-      MAYAN_DATABASE_PASSWORD: mayan-password
-      MAYAN_DATABASE_USER: mayan
+      - postgresql
+      - redis
+      # Enable to use RabbitMQ
+      #- rabbitmq
+    environment: &mayan_env
+      # Enable to use RabbitMQ
+      # MAYAN_CELERY_BROKER_URL: amqp://mayan:mayanrabbitpass@broker:5672/mayan
+      # Disable Redis Broker to use RabbitMQ as Broker
+      MAYAN_CELERY_BROKER_URL: redis://redis:6379/0
+      MAYAN_CELERY_RESULT_BACKEND: redis://redis:6379/1
+      MAYAN_DATABASES: "{'default':{'ENGINE':'django.db.backends.postgresql','NAME':'mayan','PASSWORD':'mayandbpass','USER':'mayan','HOST':'postgresql'}}"
+    image: mayanedms/mayanedms:3.3
+    networks:
+      - mayan-bridge
     ports:
       - "80:8000"
+    restart: unless-stopped
     volumes:
-      - app:/var/lib/mayan
+      - /docker-volumes/mayan-edms/media:/var/lib/mayan
+
+  postgresql:
+    environment:
+      POSTGRES_DB: mayan
+      POSTGRES_PASSWORD: mayandbpass
+      POSTGRES_USER: mayan
+    image: postgres:9.6-alpine
+    networks:
+      - mayan-bridge
+    restart: unless-stopped
+    volumes:
+      - /docker-volumes/mayan-edms/postgres:/var/lib/postgresql/data
+
+  redis:
+    command:
+      - redis-server
+      - --databases
+      - "2"
+      - --maxmemory-policy
+      - allkeys-lru
+      - --save
+      - ""
+    image: redis:5.0-alpine
+    networks:
+      - mayan-bridge
+    restart: unless-stopped
+
+  # Optional services
+
+  # celery_flower:
+  #   command:
+  #     - run_celery
+  #     - flower
+  #   depends_on:
+  #     - postgresql
+  #     - redis
+  #     # Enable to use RabbitMQ
+  #     # - rabbitmq
+  #   environment:
+  #     <<: *mayan_env
+  #   image: mayanedms/mayanedms:3.3
+  #   networks:
+  #     - mayan-bridge
+  #   ports:
+  #     - "5555:5555"
+  #   restart: unless-stopped
+
+  # Enable to use RabbitMQ
+  # rabbitmq:
+  #   container_name: mayan-edms-rabbitmq
+  #   image: healthcheck/rabbitmq
+  #   environment:
+  #     RABBITMQ_DEFAULT_USER: mayan
+  #     RABBITMQ_DEFAULT_PASS: mayanrabbitpass
+  #     RABBITMQ_DEFAULT_VHOST: mayan
+  #   networks:
+  #     - mayan-bridge
+  #   restart: unless-stopped
+  #   volumes:
+  #     - /docker-volumes/mayan-edms/rabbitmq:/var/lib/rabbitmq
+
+  # Enable to run standalone workers
+  # worker_fast:
+  #   command:
+  #     - run_worker
+  #     - fast
+  #   depends_on:
+  #     - postgresql
+  #     - redis
+  #     # Enable to use RabbitMQ
+  #     # - rabbitmq
+  #   environment:
+  #     <<: *mayan_env
+  #   image: mayanedms/mayanedms:3.3
+  #   networks:
+  #     - mayan-bridge
+  #   restart: unless-stopped
+  #   volumes:
+  #     - /docker-volumes/mayan-edms/media:/var/lib/mayan
+
+  # Enable to run frontend gunicorn
+  # frontend:
+  #   command:
+  #     - run_frontend
+  #   depends_on:
+  #     - postgresql
+  #     - redis
+  #     # Enable to use RabbitMQ
+  #     # - rabbitmq
+  #   environment:
+  #     <<: *mayan_env
+  #   image: mayanedms/mayanedms:3.3
+  #   networks:
+  #     - mayan-bridge
+  #   ports:
+  #     - "81:8000"
+  #   restart: unless-stopped
+  #   volumes:
+  #     - /docker-volumes/mayan-edms/media:/var/lib/mayan

docker/entrypoint.sh

@@ -1,115 +0,0 @@
-#!/bin/sh
-
-set -e
-echo "mayan: starting entrypoint.sh"
-INSTALL_FLAG=/var/lib/mayan/system/SECRET_KEY
-CONCURRENCY_ARGUMENT=--concurrency=
-export DOCKER_ROOT=/opt/mayan-edms
-
-export MAYAN_DEFAULT_BROKER_URL=redis://127.0.0.1:6379/0
-export MAYAN_DEFAULT_CELERY_RESULT_BACKEND=redis://127.0.0.1:6379/0
-
-export MAYAN_ALLOWED_HOSTS='["*"]'
-export MAYAN_BIN=/opt/mayan-edms/bin/mayan-edms.py
-export MAYAN_BROKER_URL=${MAYAN_BROKER_URL:-${MAYAN_DEFAULT_BROKER_URL}}
-export MAYAN_CELERY_RESULT_BACKEND=${MAYAN_CELERY_RESULT_BACKEND:-${MAYAN_DEFAULT_CELERY_RESULT_BACKEND}}
-export MAYAN_INSTALL_DIR=/opt/mayan-edms
-export MAYAN_PYTHON_BIN_DIR=/opt/mayan-edms/bin/
-export MAYAN_MEDIA_ROOT=/var/lib/mayan
-export MAYAN_SETTINGS_MODULE=${MAYAN_SETTINGS_MODULE:-mayan.settings.production}
-
-export MAYAN_GUNICORN_BIN=${MAYAN_PYTHON_BIN_DIR}gunicorn
-export MAYAN_GUNICORN_WORKERS=${MAYAN_GUNICORN_WORKERS:-2}
-export MAYAN_PIP_BIN=${MAYAN_PYTHON_BIN_DIR}pip
-
-MAYAN_WORKER_FAST_CONCURRENCY=${MAYAN_WORKER_FAST_CONCURRENCY:-1}
-MAYAN_WORKER_MEDIUM_CONCURRENCY=${MAYAN_WORKER_MEDIUM_CONCURRENCY:-1}
-MAYAN_WORKER_SLOW_CONCURRENCY=${MAYAN_WORKER_SLOW_CONCURRENCY:-1}
-
-if [ "$MAYAN_WORKER_FAST_CONCURRENCY" -eq 0 ]; then
-    MAYAN_WORKER_FAST_CONCURRENCY=
-else
-    MAYAN_WORKER_FAST_CONCURRENCY="${CONCURRENCY_ARGUMENT}${MAYAN_WORKER_FAST_CONCURRENCY}"
-fi
-export MAYAN_WORKER_FAST_CONCURRENCY
-
-if [ "$MAYAN_WORKER_MEDIUM_CONCURRENCY" -eq 0 ]; then
-    MAYAN_WORKER_MEDIUM_CONCURRENCY=
-else
-    MAYAN_WORKER_MEDIUM_CONCURRENCY="${CONCURRENCY_ARGUMENT}${MAYAN_WORKER_MEDIUM_CONCURRENCY}"
-fi
-export MAYAN_WORKER_MEDIUM_CONCURRENCY
-
-if [ "$MAYAN_WORKER_SLOW_CONCURRENCY" -eq 0 ]; then
-    MAYAN_WORKER_SLOW_CONCURRENCY=
-else
-    MAYAN_WORKER_SLOW_CONCURRENCY="${CONCURRENCY_ARGUMENT}${MAYAN_WORKER_SLOW_CONCURRENCY}"
-fi
-export MAYAN_WORKER_SLOW_CONCURRENCY
-
-export CELERY_ALWAYS_EAGER=False
-export PYTHONPATH=$PYTHONPATH:$MAYAN_MEDIA_ROOT
-
-chown mayan:mayan /var/lib/mayan -R
-
-initialize() {
-    echo "mayan: initialize()"
-    su mayan -c "${MAYAN_BIN} initialsetup --force"
-    su mayan -c "${MAYAN_BIN} collectstatic --noinput --clear"
-}
-
-upgrade() {
-    echo "mayan: upgrade()"
-    su mayan -c "${MAYAN_BIN} performupgrade"
-    su mayan -c "${MAYAN_BIN} collectstatic --noinput --clear"
-}
-
-start() {
-    echo "mayan: start()"
-    rm -rf /var/run/supervisor.sock
-    exec /usr/bin/supervisord -nc /etc/supervisor/supervisord.conf
-}
-
-os_package_installs() {
-    echo "mayan: os_package_installs()"
-    if [ "${MAYAN_APT_INSTALLS}" ]; then
-        apt-get-install $MAYAN_APT_INSTALLS
-    fi
-}
-
-pip_installs() {
-    echo "mayan: pip_installs()"
-    if [ "${MAYAN_PIP_INSTALLS}" ]; then
-        $MAYAN_PIP_BIN install $MAYAN_PIP_INSTALLS
-    fi
-}
-
-os_package_installs || true
-pip_installs || true
-
-case "$1" in
-
-mayan) # Check if this is a new install, otherwise try to upgrade the existing
-       # installation on subsequent starts
-       if [ ! -f $INSTALL_FLAG ]; then
-           initialize
-       else
-           upgrade
-       fi
-       start
-       ;;
-
-run-tests) # Check if this is a new install, otherwise try to upgrade the existing
-           # installation on subsequent starts
-           if [ ! -f $INSTALL_FLAG ]; then
-               initialize
-           else
-               upgrade
-           fi
-           $DOCKER_ROOT/run-tests.sh
-           ;;
-
-*) su mayan -c "$@";
-   ;;
-
-esac

docker/etc/supervisor/mayan.conf

@@ -1,80 +0,0 @@
-[program:mayan-gunicorn]
-autorestart = false
-autostart = true
-command = /bin/bash -c "${MAYAN_GUNICORN_BIN} -w ${MAYAN_GUNICORN_WORKERS} mayan.wsgi --max-requests 500 --max-requests-jitter 50 --worker-class gevent --bind 0.0.0.0:8000  --env DJANGO_SETTINGS_MODULE=${MAYAN_SETTINGS_MODULE}" --timeout 120
-redirect_stderr = true
-stderr_logfile = /dev/fd/2
-stderr_logfile_maxbytes = 0
-stdout_logfile = /dev/fd/1
-stdout_logfile_maxbytes = 0
-user = mayan
-
-[program:redis]
-autorestart = false
-autostart = true
-command = /bin/bash -c "if [ ${MAYAN_BROKER_URL} == ${MAYAN_DEFAULT_BROKER_URL} ] && [ ${MAYAN_CELERY_RESULT_BACKEND} == ${MAYAN_DEFAULT_CELERY_RESULT_BACKEND} ];then /usr/bin/redis-server /etc/redis/;fi"
-stderr_logfile = /dev/fd/2
-stderr_logfile_maxbytes = 0
-stdout_logfile = /dev/fd/1
-stdout_logfile_maxbytes = 0
-user = root
-
-[program:mayan-worker-fast]
-autorestart = false
-autostart = true
-command = nice -n 1 /bin/bash -c "${MAYAN_BIN} celery --settings=${MAYAN_SETTINGS_MODULE} worker -Ofair -l ERROR -Q converter,sources_fast -n mayan-worker-fast.%%h ${MAYAN_WORKER_FAST_CONCURRENCY}"
-killasgroup = true
-numprocs = 1
-priority = 998
-startsecs = 10
-stderr_logfile = /dev/fd/2
-stderr_logfile_maxbytes = 0
-stdout_logfile = /dev/fd/1
-stdout_logfile_maxbytes = 0
-stopwaitsecs = 1
-user = mayan
-
-[program:mayan-worker-medium]
-autorestart = false
-autostart = true
-command = nice -n 18 /bin/bash -c "${MAYAN_BIN} celery --settings=${MAYAN_SETTINGS_MODULE} worker -Ofair -l ERROR -Q checkouts_periodic,documents_periodic,indexing,metadata,sources,sources_periodic,uploads,documents -n mayan-worker-medium.%%h ${MAYAN_WORKER_MEDIUM_CONCURRENCY}"
-killasgroup = true
-numprocs = 1
-priority = 998
-startsecs = 10
-stderr_logfile = /dev/fd/2
-stderr_logfile_maxbytes = 0
-stdout_logfile = /dev/fd/1
-stdout_logfile_maxbytes = 0
-stopwaitsecs = 1
-user = mayan
-
-[program:mayan-worker-slow]
-autorestart = false
-autostart = true
-command = nice -n 19 /bin/bash -c "${MAYAN_BIN} celery --settings=${MAYAN_SETTINGS_MODULE} worker -Ofair -l ERROR -Q mailing,tools,statistics,parsing,ocr -n mayan-worker-slow.%%h ${MAYAN_WORKER_SLOW_CONCURRENCY}"
-killasgroup = true
-numprocs = 1
-priority = 998
-startsecs = 10
-stderr_logfile = /dev/fd/2
-stderr_logfile_maxbytes = 0
-stdout_logfile = /dev/fd/1
-stdout_logfile_maxbytes = 0
-stopwaitsecs = 1
-user = mayan
-
-[program:mayan-celery-beat]
-autorestart = false
-autostart = true
-command = nice -n 1 /bin/bash -c "${MAYAN_BIN} celery --settings=${MAYAN_SETTINGS_MODULE} beat --pidfile= -l ERROR"
-killasgroup = true
-numprocs = 1
-priority = 998
-stderr_logfile = /dev/fd/2
-stderr_logfile_maxbytes = 0
-stdout_logfile = /dev/fd/1
-stdout_logfile_maxbytes = 0
-startsecs = 10
-stopwaitsecs = 1
-user = mayan

docker/rootfs/usr/local/bin/entrypoint.sh

@@ -0,0 +1,169 @@
+#!/bin/bash
+
+# Use bash and not sh to support argument slicing "${@:2}"
+# sh defaults to dash instead of bash.
+
+set -e
+echo "mayan: starting entrypoint.sh"
+INSTALL_FLAG=/var/lib/mayan/system/SECRET_KEY
+CONCURRENCY_ARGUMENT=--concurrency=
+
+DEFAULT_USER_UID=1000
+DEFAULT_USER_GID=1000
+
+MAYAN_USER_UID=${MAYAN_USER_UID:-${DEFAULT_USER_UID}}
+MAYAN_USER_GID=${MAYAN_USER_GID:-${DEFAULT_USER_GID}}
+
+export MAYAN_ALLOWED_HOSTS='["*"]'
+export MAYAN_BIN=/opt/mayan-edms/bin/mayan-edms.py
+export MAYAN_INSTALL_DIR=/opt/mayan-edms
+export MAYAN_PYTHON_BIN_DIR=/opt/mayan-edms/bin/
+export MAYAN_MEDIA_ROOT=/var/lib/mayan
+export MAYAN_SETTINGS_MODULE=${MAYAN_SETTINGS_MODULE:-mayan.settings.production}
+export DJANGO_SETTINGS_MODULE=${MAYAN_SETTINGS_MODULE}
+
+export MAYAN_GUNICORN_BIN=${MAYAN_PYTHON_BIN_DIR}gunicorn
+export MAYAN_GUNICORN_WORKERS=${MAYAN_GUNICORN_WORKERS:-2}
+export MAYAN_GUNICORN_TIMEOUT=${MAYAN_GUNICORN_TIMEOUT:-120}
+export MAYAN_PIP_BIN=${MAYAN_PYTHON_BIN_DIR}pip
+export MAYAN_STATIC_ROOT=${MAYAN_INSTALL_DIR}/static
+
+MAYAN_WORKER_FAST_CONCURRENCY=${MAYAN_WORKER_FAST_CONCURRENCY:-0}
+MAYAN_WORKER_MEDIUM_CONCURRENCY=${MAYAN_WORKER_MEDIUM_CONCURRENCY:-0}
+MAYAN_WORKER_SLOW_CONCURRENCY=${MAYAN_WORKER_SLOW_CONCURRENCY:-1}
+
+update_uid_gid() {
+    echo "mayan: update_uid_gid()"
+    groupmod mayan -g ${MAYAN_USER_GID} 2>/dev/null || true
+    usermod mayan -u ${MAYAN_USER_UID} -g ${MAYAN_USER_GID} 2>/dev/null
+
+    if [ ${MAYAN_USER_UID} -ne ${DEFAULT_USER_UID} ] || [ ${MAYAN_USER_GID} -ne ${DEFAULT_USER_GID} ]; then
+        echo "mayan: Updating file ownership. This might take a while if there are many documents."
+        chown -R mayan:mayan ${MAYAN_INSTALL_DIR} ${MAYAN_STATIC_ROOT} ${MAYAN_MEDIA_ROOT}
+    fi
+}
+
+update_uid_gid
+
+if [ "$MAYAN_WORKER_FAST_CONCURRENCY" -eq 0 ]; then
+    MAYAN_WORKER_FAST_CONCURRENCY=
+else
+    MAYAN_WORKER_FAST_CONCURRENCY="${CONCURRENCY_ARGUMENT}${MAYAN_WORKER_FAST_CONCURRENCY}"
+fi
+export MAYAN_WORKER_FAST_CONCURRENCY
+
+if [ "$MAYAN_WORKER_MEDIUM_CONCURRENCY" -eq 0 ]; then
+    MAYAN_WORKER_MEDIUM_CONCURRENCY=
+else
+    MAYAN_WORKER_MEDIUM_CONCURRENCY="${CONCURRENCY_ARGUMENT}${MAYAN_WORKER_MEDIUM_CONCURRENCY}"
+fi
+export MAYAN_WORKER_MEDIUM_CONCURRENCY
+
+if [ "$MAYAN_WORKER_SLOW_CONCURRENCY" -eq 0 ]; then
+    MAYAN_WORKER_SLOW_CONCURRENCY=
+else
+    MAYAN_WORKER_SLOW_CONCURRENCY="${CONCURRENCY_ARGUMENT}${MAYAN_WORKER_SLOW_CONCURRENCY}"
+fi
+export MAYAN_WORKER_SLOW_CONCURRENCY
+
+# Allow importing of user setting modules
+export PYTHONPATH=$PYTHONPATH:$MAYAN_MEDIA_ROOT
+
+apt_get_install() {
+    apt-get -q update
+    apt-get install -y --force-yes --no-install-recommends --auto-remove "$@"
+    apt-get -q clean
+    rm -rf /var/lib/apt/lists/*
+}
+
+initialsetup() {
+    echo "mayan: initialsetup()"
+    su mayan -c "${MAYAN_BIN} initialsetup --force --no-dependencies"
+}
+
+os_package_installs() {
+    echo "mayan: os_package_installs()"
+    if [ "${MAYAN_APT_INSTALLS}" ]; then
+        DEBIAN_FRONTEND=noninteractive apt_get_install $MAYAN_APT_INSTALLS
+    fi
+}
+
+pip_installs() {
+    echo "mayan: pip_installs()"
+    if [ "${MAYAN_PIP_INSTALLS}" ]; then
+        su mayan -c "${MAYAN_PIP_BIN} install $MAYAN_PIP_INSTALLS"
+    fi
+}
+
+run_all() {
+    echo "mayan: start()"
+    rm -rf /var/run/supervisor.sock
+    exec /usr/bin/supervisord -nc /etc/supervisor/supervisord.conf
+}
+
+performupgrade() {
+    echo "mayan: performupgrade()"
+    su mayan -c "${MAYAN_BIN} performupgrade --no-dependencies"
+}
+
+make_ready() {
+    # Check if this is a new install, otherwise try to upgrade the existing
+    # installation on subsequent starts
+    if [ ! -f $INSTALL_FLAG ]; then
+        initialsetup
+    else
+        performupgrade
+    fi
+}
+
+set_uid_guid() {
+    echo "mayan: changing uid/guid"
+    usermod mayan -u ${MAYAN_USER_UID:-${DEFAULT_USER_UID}}
+    groupmod mayan -g ${MAYAN_USER_GID:-${DEFAULT_USER_GID}}
+}
+
+os_package_installs || true
+pip_installs || true
+chown mayan:mayan /var/lib/mayan -R
+
+case "$1" in
+
+run_initialsetup)
+    initialsetup
+    ;;
+
+run_performupgrade)
+    performupgrade
+    ;;
+
+run_all)
+    make_ready
+    run_all
+    ;;
+
+run_celery)
+    run_celery.sh "${@:2}"
+    ;;
+
+run_command)
+    su mayan -c "${MAYAN_BIN} ${@:2}"
+    ;;
+
+run_frontend)
+    run_frontend.sh
+    ;;
+
+run_tests)
+    make_ready
+    run_tests.sh "${@:2}"
+    ;;
+
+run_worker)
+    run_worker.sh "${@:2}"
+    ;;
+
+*)
+    su mayan -c "$@"
+    ;;
+
+esac

docker/rootfs/usr/local/bin/run_celery.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+# Use -A and not --app. Both are the same but behave differently
+# -A can be located before the command while --app cannot.
+su mayan -c "${MAYAN_PYTHON_BIN_DIR}celery -A mayan $@"

docker/rootfs/usr/local/bin/run_frontend.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+MAYAN_GUNICORN_MAX_REQUESTS=${MAYAN_GUNICORN_MAX_REQUESTS:-500}
+MAYAN_GUNICORN_MAX_REQUESTS_JITTERS=${MAYAN_GUNICORN_MAX_REQUESTS_JITTERS:-50}
+MAYAN_GUNICORN_WORKER_CLASS=${MAYAN_GUNICORN_WORKER_CLASS:-sync}
+
+su mayan -c "${MAYAN_PYTHON_BIN_DIR}gunicorn -w ${MAYAN_GUNICORN_WORKERS} mayan.wsgi --max-requests ${MAYAN_GUNICORN_MAX_REQUESTS} --max-requests-jitter ${MAYAN_GUNICORN_MAX_REQUESTS_JITTERS} --worker-class ${MAYAN_GUNICORN_WORKER_CLASS} --bind 0.0.0.0:8000 --timeout ${MAYAN_GUNICORN_TIMEOUT}"

docker/rootfs/usr/local/bin/run_tests.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+export DEBIAN_FRONTEND=noninteractive
+TEST_ARGUMENT=${@:-"--mayan-apps"}
+
+apt-get update
+apt-get install -y --no-install-recommends gcc python3-dev tesseract-ocr-deu
+
+su mayan -c "${MAYAN_PIP_BIN} install -r ${MAYAN_INSTALL_DIR}/testing-base.txt"
+
+su mayan -c "${MAYAN_BIN} test ${TEST_ARGUMENT} --settings=mayan.settings.testing"

docker/rootfs/usr/local/bin/run_worker.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+export MAYAN_WORKER_NAME=${MAYAN_WORKER_NAME:-$1}
+
+QUEUE_LIST_DEFAULT=`su mayan -c "${MAYAN_PYTHON_BIN_DIR}mayan-edms.py platformtemplate worker_queues"`
+
+MAYAN_QUEUE_LIST=${MAYAN_QUEUE_LIST:-${QUEUE_LIST_DEFAULT}}
+
+# Use -A and not --app. Both are the same but behave differently
+# -A can be located before the command while --app cannot.
+# Pass ${@:2} to allow overriding the defaults arguments
+su mayan -c "${MAYAN_PYTHON_BIN_DIR}celery -A mayan worker -Ofair -l ERROR -Q ${MAYAN_QUEUE_LIST} ${@:2}"

docker/rootfs/version

@@ -0,0 +1 @@
+3.3.6

docker/stack.yml

@@ -9,24 +9,32 @@ volumes:
 services:
 
   db:
-    image: postgres
     environment:
       POSTGRES_DB: mayan
-      POSTGRES_PASSWORD: mayan-password
+      POSTGRES_PASSWORD: mayandbpass
       POSTGRES_USER: mayan
+    image: postgres
     volumes:
       - db:/var/lib/postgresql/data
 
   app:
+    environment:
+      MAYAN_CELERY_BROKER_URL: redis://redis:6379/1
+      MAYAN_CELERY_RESULT_BACKEND: redis://redis:6379/0
+      MAYAN_DATABASES: "{'default':{'ENGINE':'django.db.backends.postgresql','NAME':'mayan','PASSWORD':'mayandbpass','USER':'mayan','HOST':'db'}}"
     image: mayanedms/mayanedms:latest
     ports:
       - 80:8000
-    environment:
-      MAYAN_DATABASE_ENGINE: django.db.backends.postgresql
-      MAYAN_DATABASE_HOST: db
-      MAYAN_DATABASE_NAME: mayan
-      MAYAN_DATABASE_PASSWORD: mayan-password
-      MAYAN_DATABASE_USER: mayan
-      MAYAN_DATABASE_CONN_MAX_AGE: 60
     volumes:
       - app:/var/lib/mayan
+
+  redis:
+    command:
+      - redis-server
+      - --databases
+      - "2"
+      - --maxmemory-policy
+      - allkeys-lru
+      - --save
+      - ""
+    image: redis:5.0

docker/version

@@ -1 +0,0 @@
-3.1.10

docs/_ext/djangodocs.py

@@ -1,229 +0,0 @@
-"""
-Sphinx plugins for Django documentation.
-"""
-import os
-import re
-
-from docutils import nodes, transforms
-try:
-    import json
-except ImportError:
-    try:
-        import simplejson as json
-    except ImportError:
-        try:
-            from django.utils import simplejson as json
-        except ImportError:
-            json = None
-
-from sphinx import addnodes, roles, __version__ as sphinx_ver
-from sphinx.builders.html import StandaloneHTMLBuilder
-from sphinx.writers.html import SmartyPantsHTMLTranslator
-from sphinx.util.console import bold
-from sphinx.util.compat import Directive
-
-# RE for option descriptions without a '--' prefix
-simple_option_desc_re = re.compile(
-    r'([-_a-zA-Z0-9]+)(\s*.*?)(?=,\s+(?:/|-|--)|$)')
-
-def setup(app):
-    app.add_crossref_type(
-        directivename = "setting",
-        rolename      = "setting",
-        indextemplate = "pair: %s; setting",
-    )
-    app.add_crossref_type(
-        directivename = "templatetag",
-        rolename      = "ttag",
-        indextemplate = "pair: %s; template tag"
-    )
-    app.add_crossref_type(
-        directivename = "templatefilter",
-        rolename      = "tfilter",
-        indextemplate = "pair: %s; template filter"
-    )
-    app.add_crossref_type(
-        directivename = "fieldlookup",
-        rolename      = "lookup",
-        indextemplate = "pair: %s; field lookup type",
-    )
-    app.add_description_unit(
-        directivename = "django-admin",
-        rolename      = "djadmin",
-        indextemplate = "pair: %s; django-admin command",
-        parse_node    = parse_django_admin_node,
-    )
-    app.add_description_unit(
-        directivename = "django-admin-option",
-        rolename      = "djadminopt",
-        indextemplate = "pair: %s; django-admin command-line option",
-        parse_node    = parse_django_adminopt_node,
-    )
-    app.add_config_value('django_next_version', '0.0', True)
-    app.add_directive('versionadded', VersionDirective)
-    app.add_directive('versionchanged', VersionDirective)
-    app.add_builder(DjangoStandaloneHTMLBuilder)
-
-
-class VersionDirective(Directive):
-    has_content = True
-    required_arguments = 1
-    optional_arguments = 1
-    final_argument_whitespace = True
-    option_spec = {}
-
-    def run(self):
-        env = self.state.document.settings.env
-        arg0 = self.arguments[0]
-        is_nextversion = env.config.django_next_version == arg0
-        ret = []
-        node = addnodes.versionmodified()
-        ret.append(node)
-        if not is_nextversion:
-            if len(self.arguments) == 1:
-                linktext = 'Please see the release notes </releases/%s>' % (arg0)
-                xrefs = roles.XRefRole()('doc', linktext, linktext, self.lineno, self.state)
-                node.extend(xrefs[0])
-            node['version'] = arg0
-        else:
-            node['version'] = "Development version"
-        node['type'] = self.name
-        if len(self.arguments) == 2:
-            inodes, messages = self.state.inline_text(self.arguments[1], self.lineno+1)
-            node.extend(inodes)
-            if self.content:
-                self.state.nested_parse(self.content, self.content_offset, node)
-            ret = ret + messages
-        env.note_versionchange(node['type'], node['version'], node, self.lineno)
-        return ret
-
-
-class DjangoHTMLTranslator(SmartyPantsHTMLTranslator):
-    """
-    Django-specific reST to HTML tweaks.
-    """
-
-    # Don't use border=1, which docutils does by default.
-    def visit_table(self, node):
-        self._table_row_index = 0 # Needed by Sphinx
-        self.body.append(self.starttag(node, 'table', CLASS='docutils'))
-
-    # <big>? Really?
-    def visit_desc_parameterlist(self, node):
-        self.body.append('(')
-        self.first_param = 1
-        self.param_separator = node.child_text_separator
-
-    def depart_desc_parameterlist(self, node):
-        self.body.append(')')
-
-    if sphinx_ver < '1.0.8':
-        #
-        # Don't apply smartypants to literal blocks
-        #
-        def visit_literal_block(self, node):
-            self.no_smarty += 1
-            SmartyPantsHTMLTranslator.visit_literal_block(self, node)
-
-        def depart_literal_block(self, node):
-            SmartyPantsHTMLTranslator.depart_literal_block(self, node)
-            self.no_smarty -= 1
-
-    #
-    # Turn the "new in version" stuff (versionadded/versionchanged) into a
-    # better callout -- the Sphinx default is just a little span,
-    # which is a bit less obvious that I'd like.
-    #
-    # FIXME: these messages are all hardcoded in English. We need to change
-    # that to accomodate other language docs, but I can't work out how to make
-    # that work.
-    #
-    version_text = {
-        'deprecated':       'Deprecated in Django %s',
-        'versionchanged':   'Changed in Django %s',
-        'versionadded':     'New in Django %s',
-    }
-
-    def visit_versionmodified(self, node):
-        self.body.append(
-            self.starttag(node, 'div', CLASS=node['type'])
-        )
-        title = "%s%s" % (
-            self.version_text[node['type']] % node['version'],
-            len(node) and ":" or "."
-        )
-        self.body.append('<span class="title">%s</span> ' % title)
-
-    def depart_versionmodified(self, node):
-        self.body.append("</div>\n")
-
-    # Give each section a unique ID -- nice for custom CSS hooks
-    def visit_section(self, node):
-        old_ids = node.get('ids', [])
-        node['ids'] = ['s-' + i for i in old_ids]
-        node['ids'].extend(old_ids)
-        SmartyPantsHTMLTranslator.visit_section(self, node)
-        node['ids'] = old_ids
-
-def parse_django_admin_node(env, sig, signode):
-    command = sig.split(' ')[0]
-    env._django_curr_admin_command = command
-    title = "django-admin.py %s" % sig
-    signode += addnodes.desc_name(title, title)
-    return sig
-
-def parse_django_adminopt_node(env, sig, signode):
-    """A copy of sphinx.directives.CmdoptionDesc.parse_signature()"""
-    from sphinx.domains.std import option_desc_re
-    count = 0
-    firstname = ''
-    for m in option_desc_re.finditer(sig):
-        optname, args = m.groups()
-        if count:
-            signode += addnodes.desc_addname(', ', ', ')
-        signode += addnodes.desc_name(optname, optname)
-        signode += addnodes.desc_addname(args, args)
-        if not count:
-            firstname = optname
-        count += 1
-    if not count:
-        for m in simple_option_desc_re.finditer(sig):
-            optname, args = m.groups()
-            if count:
-                signode += addnodes.desc_addname(', ', ', ')
-            signode += addnodes.desc_name(optname, optname)
-            signode += addnodes.desc_addname(args, args)
-            if not count:
-                firstname = optname
-            count += 1
-    if not firstname:
-        raise ValueError
-    return firstname
-
-
-class DjangoStandaloneHTMLBuilder(StandaloneHTMLBuilder):
-    """
-    Subclass to add some extra things we need.
-    """
-
-    name = 'djangohtml'
-
-    def finish(self):
-        super(DjangoStandaloneHTMLBuilder, self).finish()
-        if json is None:
-            self.warn("cannot create templatebuiltins.js due to missing simplejson dependency")
-            return
-        self.info(bold("writing templatebuiltins.js..."))
-        xrefs = self.env.domaindata["std"]["objects"]
-        templatebuiltins = {
-            "ttags": [n for ((t, n), (l, a)) in xrefs.items()
-                        if t == "templatetag" and l == "ref/templates/builtins"],
-            "tfilters": [n for ((t, n), (l, a)) in xrefs.items()
-                        if t == "templatefilter" and l == "ref/templates/builtins"],
-        }
-        outfilename = os.path.join(self.outdir, "templatebuiltins.js")
-        f = open(outfilename, 'wb')
-        f.write('var django_template_builtins = ')
-        json.dump(templatebuiltins, f)
-        f.write(';\n')
-        f.close();

docs/_static/css/custom.css

@@ -0,0 +1,17 @@
+.menuselection {
+    color: white;
+    background: #2c3e50;
+    padding: 3px;
+}
+
+.document a.internal {
+    text-decoration: none;
+    border-bottom: 1px dotted #707070;
+}
+
+.document a.internal:visited {
+    color: #2980B9;
+    text-decoration: none;
+    border-bottom: 1px dotted #707070;
+}
+

docs/_templates/donate.html

@@ -1,16 +0,0 @@
-<h3>Support</h3>
-<p>
-    <a href='http://www.mayan-edms.com/providers/'>Consulting and support plans are available, click here</a>.
-</p>
-<hr />
-<p>
-    Or consider supporting Mayan EDMS by contributing to its development. (US tax payers, please note this contribution is not tax deductible).
-</p>
-<form action="https://www.paypal.com/cgi-bin/webscr" method="post" target="_top">
-    <input type="hidden" name="cmd" value="_s-xclick">
-    <input type="hidden" name="hosted_button_id" value="3PXN336XFXQNN">
-    <input type="image" src="https://www.paypalobjects.com/en_US/i/btn/btn_donateCC_LG.gif" border="0" name="submit" alt="PayPal - The safer, easier way to pay online!">
-    <img alt="" border="0" src="https://www.paypalobjects.com/en_US/i/scr/pixel.gif" width="1" height="1" style="display: none !important;">
-</form>
-
-<hr />

docs/_templates/message_area.html

@@ -0,0 +1,41 @@
+<style>
+.wy-body-for-nav #message-area {
+    color: #b3b3b3;
+    margin: 1em 2em 1em 1em;
+}
+#message-area {
+    border: 1px dotted #2980B9;
+    padding: .5em;
+    margin: 1em 0;
+    text-align: center;
+}
+</style>
+
+<div id="message-area">
+    <h3>Get the book!</h3>
+
+    <a href="https://www.mayan-edms.com/book/">
+        <img src="{{ pathto('_static/book_cover.jpg', 1) }}" />
+    </a>
+
+    <hr />
+
+    <p>
+        On-site consulting and support plans are available, <a href='https://www.mayan-edms.com/support/'>click here</a>.
+    </p>
+
+    <hr />
+
+    <p>
+        Or consider donating to support the continued development of the project.
+    </p>
+
+    <form action="https://www.paypal.com/cgi-bin/webscr" method="post" target="_top">
+        <input type="hidden" name="cmd" value="_s-xclick">
+        <input type="hidden" name="hosted_button_id" value="3PXN336XFXQNN">
+        <input type="image" src="https://www.paypalobjects.com/en_US/i/btn/btn_donateCC_LG.gif" border="0" name="submit" alt="PayPal - The safer, easier way to pay online!">
+        <img alt="" border="0" src="https://www.paypalobjects.com/en_US/i/scr/pixel.gif" width="1" height="1" style="display: none !important;">
+    </form>
+</div>
+
+

docs/callbacks.py

@@ -0,0 +1,8 @@
+from __future__ import unicode_literals
+
+
+def get_source_read_callback(substitutions):
+    def global_substitution_function(app, docname, source):
+        for old, new in substitutions:
+            source[0] = source[0].replace(old, new)
+    return global_substitution_function

docs/chapters/acls.rst

@@ -1,84 +0,0 @@
-********************
-Access control lists
-********************
-
-Besides the permissions system explained in :doc:`../chapters/permissions`,
-Mayan EDMS provides per object permission granting. This feature is used to
-grant a permission to a role, but this permission can only be executed for a
-limited number of objects (documents, folders, tags) instead of being
-effective system-wide.
-
-.. blockdiag::
-
-   blockdiag {
-      default_shape = roundedbox
-
-      document [ label = 'Document' ];
-      role [ label = 'Role' ];
-      permission [ label = 'Permission' ];
-
-       role -> permission -> document;
-   }
-
-Example:
-
-.. blockdiag::
-
-   blockdiag {
-      default_shape = roundedbox
-
-      document [ label = '2015 Payroll report.txt', width=200 ];
-      role [ label = 'Accountants' ];
-      permission [ label = 'View document' ];
-
-      role -> permission -> document;
-   }
-
-In this scenario only users in groups belonging to the ``Accountants`` role
-would be able to view the ``2015 Payroll report.txt`` document.
-
-Inherited access control
-========================
-
-It is also possible to grant a permission to a role for a specific document
-type (:doc:`../chapters/document_types`). Under this scheme all users in
-groups belonging to that role will inherit that permission for all documents
-of that type.
-
-.. blockdiag::
-
-   blockdiag {
-      default_shape = roundedbox
-      document_type [ label = 'Document type' ];
-      role [ label = 'Role' ];
-      permission [ label = 'Permission' ];
-      documents [shape = "note", stacked];
-
-       role ->  permission -> document_type ;
-       document_type -> documents [folded, label = "inherit" ];
-   }
-
-Example:
-
-.. blockdiag::
-
-   blockdiag {
-      default_shape = roundedbox
-      document_type [ label = 'Payroll reports', width=200 ];
-      role [ label = 'Accountants' ];
-      permission [ label = 'View document' ];
-      documents [shape = "note", stacked, label="payroll_report*.pdf" ];
-
-      role -> permission -> document_type ;
-       document_type -> documents [folded, label = "inherit" ];
-   }
-
-The role ``Accountants`` is given the permission ``document view`` for the
-document type ``Payroll reports``. Now all users in groups belonging to the
-``Accountants`` role can view all documents of the type ``Payroll reports``
-without needing to have that permissions granted for each particular
-``Payroll reports`` type document.
-
-If access control for the ``Payroll reports`` documents needs to be updated it
-only needs to be done for the document type and not for each document of the type
-``Payroll reports``.

docs/chapters/acls.txt

@@ -0,0 +1,84 @@
+====================
+Access control lists
+====================
+
+Besides the permissions system explained in :doc:`../chapters/permissions`,
+Mayan EDMS provides per object permission granting. This feature is used to
+grant a permission to a role, but this permission can only be executed for a
+limited number of objects (documents, folders, tags) instead of being
+effective system-wide.
+
+.. blockdiag::
+
+   blockdiag {
+      default_shape = roundedbox
+
+      document [ label = 'Document' ];
+      role [ label = 'Role' ];
+      permission [ label = 'Permission' ];
+
+       role -> permission -> document;
+   }
+
+Example:
+
+.. blockdiag::
+
+   blockdiag {
+      default_shape = roundedbox
+
+      document [ label = '2015 Payroll report.txt', width=200 ];
+      role [ label = 'Accountants' ];
+      permission [ label = 'View document' ];
+
+      role -> permission -> document;
+   }
+
+In this scenario only users in groups belonging to the ``Accountants`` role
+would be able to view the ``2015 Payroll report.txt`` document.
+
+Inherited access control
+========================
+
+It is also possible to grant a permission to a role for a specific document
+type (:doc:`../chapters/document_types`). Under this scheme all users in
+groups belonging to that role will inherit that permission for all documents
+of that type.
+
+.. blockdiag::
+
+   blockdiag {
+      default_shape = roundedbox
+      document_type [ label = 'Document type' ];
+      role [ label = 'Role' ];
+      permission [ label = 'Permission' ];
+      documents [shape = "note", stacked];
+
+       role ->  permission -> document_type ;
+       document_type -> documents [folded, label = "inherit" ];
+   }
+
+Example:
+
+.. blockdiag::
+
+   blockdiag {
+      default_shape = roundedbox
+      document_type [ label = 'Payroll reports', width=200 ];
+      role [ label = 'Accountants' ];
+      permission [ label = 'View document' ];
+      documents [shape = "note", stacked, label="payroll_report*.pdf" ];
+
+      role -> permission -> document_type ;
+       document_type -> documents [folded, label = "inherit" ];
+   }
+
+The role ``Accountants`` is given the permission ``document view`` for the
+document type ``Payroll reports``. Now all users in groups belonging to the
+``Accountants`` role can view all documents of the type ``Payroll reports``
+without needing to have that permissions granted for each particular
+``Payroll reports`` type document.
+
+If access control for the ``Payroll reports`` documents needs to be updated it
+only needs to be done for the document type and not for each document of the type
+``Payroll reports``.

docs/chapters/api.rst

@@ -1,102 +0,0 @@
-***
-API
-***
-
-Mayan EDMS provides an HTTP REST Application Program Interface (or API). This
-API allows integration with 3rd party software using simple HTTP requests.
-
-Several API authentication methods are provides: **Session**, **Token**,
-and **HTTP Basic**.
-
-The URL for the API can be found via the :menuselection:`Tools --> REST API
-menu. The API is also self-documenting. The live API documentation can be
-found in the :menuselection:`Tools --> API Documentation (Swagger)` menu for
-the Swagger version and in the
-:menuselection:`Tools --> API Documentation (ReDoc)` menu for the ReDoc version.
-
-The are a few ways to structure REST APIs. In the case of Mayan EDMS, API
-endpoints are structured by resource type. Examples:
-
-* /cabinets - To view or create new cabinets
-* /cabinets/<id> - To view the details, edit, or delete an existing cabinet.
-* /cabinets/<id>/documents - To view, add, or remove documents from an existing
-  cabinet.
-* /cabinets/<id>/documents/<id> - To view, add, or remove one document from an
-  existing cabinet.
-
-The API supports the HTTP verbs: **GET**, **POST**, **PUT**, **PATCH**,
-and **DELETE**.
-
-
-Example use
-===========
-
-Install Python Requests (http://docs.python-requests.org/en/master/)::
-
-    pip install requests
-
-Get a list of document types::
-
-    import requests
-
-    requests.get('http://127.0.0.1:8000/api/document_types/', auth=('username', 'password')).json()
-
-    {u'count': 1,
-     u'next': None,
-     u'previous': None,
-     u'results': [{u'delete_time_period': 30,
-       u'delete_time_unit': u'days',
-       u'documents_count': 12,
-       u'documents_url': u'http://127.0.0.1:8000/api/document_types/1/documents/',
-       u'filenames': [],
-       u'id': 1,
-       u'label': u'Default',
-       u'trash_time_period': None,
-       u'trash_time_unit': None,
-       u'url': u'http://127.0.0.1:8000/api/document_types/1/'}]}
-
-Upload a new document::
-
-    with open('test_document.pdf', mode='rb') as
-        requests.post('http://127.0.0.1:8000/api/documents/', auth=('username', 'password'), files={'file': file_object}, data={'document_type': 1}).json()
-
-    {u'description': u'',
-    u'document_type': 1,
-    u'id': 19,
-    u'label': u'test_document.pdf',
-    u'language': u'eng'}
-
-Use API tokens to avoid sending the username and password on every request. Obtain a token by making a POST request to ``/api/auth/token/obtain/?format=json``::
-
-    requests.post('http://127.0.0.1:8000/api/auth/token/obtain/?format=json', data={'username': 'username', 'password': 'password'}).json()
-
-    {u'token': u'4ccbc35b5eb327aa82dc3b7c9747b578900f02bb'}
-
-Add the API token to the request header::
-
-    headers = {'Authorization': 'Token 4ccbc35b5eb327aa82dc3b7c9747b578900f02bb'}
-
-    requests.get('http://127.0.0.1:8000/api/document_types/', headers=headers).json()
-
-    {u'description': u'',
-    u'document_type': 1,
-    u'id': 19,
-    u'label': u'test_document.pdf',
-    u'language': u'eng'}
-
-
-Use sessions to avoid having to add the headers on each request::
-
-    session = requests.Session()
-
-    headers = {'Authorization': 'Token 4ccbc35b5eb327aa82dc3b7c9747b578900f02bb'}
-
-    session.headers.update(headers)
-
-    session.get('http://127.0.0.1:8000/api/document_types/')
-
-    {u'description': u'',
-    u'document_type': 1,
-    u'id': 19,
-    u'label': u'test_document.pdf',
-    u'language': u'eng'}

docs/chapters/app_creation.rst

@@ -1,143 +0,0 @@
-************
-App creation
-************
-
-Mayan EDMS apps are essentially Django app with some extra code to register
-navigation, permissions and other relationships.
-
-
-App modules
-===========
-
-- __init__.py
-
-  Should be empty if possible. No initialization code should be here, use the
-  ready() method of the MayanAppConfig class in the apps.py module.
-
-- admin.py
-
-  Standard Django app module to define how models are to be presented in the
-  admin interface.
-
-- api_views.py
-
-  REST API views go here. Mayan EDMS uses Django REST Framework API view
-  classes.
-
-- apps.py
-
-  Contains the MayanAppConfig subclass as required by Django 1.7 and up. This
-  is a place to define the app name and translatable verbose name as well as
-  code to be execute when the modules of the app are ready.
-
-- classes.py
-
-  Hold python classes to be used internally or externally. Any class defined by
-  the app that is not a model.
-
-- events.py
-
-  Define event class instances that are later committed to a log by custom
-  code.
-
-- exceptions.py
-
-  Custom exceptions defined by the app.
-
-- fields.py
-
-  Place any custom form field classed you define here.
-
-- forms.py
-
-  Standard Django app module that hold custom form classes.
-
-- handlers.py
-
-  Contains the signal handlers, functions that will process a given signal
-  emitted from this or other apps. Connect the handler functions to the
-  corresponding signal in the ready() method of the MayanAppConfig subclass in
-  apps.py
-
-- links.py
-
-  Defines the links to be used by the app. Import only from the navigation app
-  and the local permissions.py file.
-
-- literals.py
-
-  Stores magic numbers, module choices (if static), settings defaults, and
-  constants. Should contain all capital case variables. Must not import from
-  any other module.
-
-- managers.py
-
-  Standard Django app module that hold custom model managers. These act as
-  model class method to performs actions in a series of model instances or
-  utilitarian actions on external models instances.
-
-- models.py
-
-  Standard Django app module that defines ORM persistent data schema.
-
-- permissions.py
-
-  Defines the permissions to be used to validate user access by links and views.
-  Imports only from the permissions app. Link or view conditions such as
-  testing for is_staff or is_super_user flag are defined in this same module.
-
-- runtime.py
-
-  Use this module when you need the same instance of a class for the entire app.
-  This module acts as a shared memory space for the other modules of the app or
-  other apps.
-
-- serializers.py
-
-  Hold Django REST Framework serializers used by the api_views.py module.
-
-- settings.py
-
-  Define the configuration settings instances that the app will use.
-
-- signals.py
-
-  Any custom defined signal goes here.
-
-- statistics.py
-
-  Provides functions that will compute any sort of statistical information on
-  the app’s data.
-
-- tasks.py
-
-  Code to be execute in the background or as an out-of-process action.
-
-- tests/ directory
-
-  Hold test modules. There should be one test_*.py module for each aspect being
-  tested, examples: test_api.py, test_views.py, test_parsers.py, test_permissions.py
-  Any shared constant data used by the tests should be added to tests/literals.py
-
-- utils.py
-
-  Holds utilitarian code that doesn't fit on any other app module or that is
-  used by several modules in the app. Anything used internally by the app that
-  is not a class or a literal (should be as little as possible)
-
-- widgets.py
-
-  HTML widgets go here. This should be the only place with presentation
-  directives in the app (aside the templates).
-
-
-Views
-=====
-
-The module common.generics provides custom generic class based views to be used.
-The basic views used to create, edit, view and delete objects in Mayan EDMS
-are: SingleObjectCreateView, SingleObjectDetailView, SingleObjectEditView,
-and SingleObjectListView
-
-These views handle aspects relating to view permissions, object permissions,
-post action redirection and template context generation.

docs/chapters/backups.rst

@@ -1,70 +0,0 @@
-*******
-Backups
-*******
-
-To backup your install of Mayan EDMS just copy the actual document files and
-the database content. If you are using the default storage backend, the
-document files should be found in the ``media`` folder of your installation.
-
-To dump the content of your database manager refer to the documentation chapter
-regarding database data "dumping".
-
-Here is an example of how to perform a backup and a restore of a PostgreSQL
-database.
-
-To dump the database into an SQL text file::
-
-    pg_dump -h <host> -U <database user> -c <database name> -W > `date +%Y-%m-%d"_"%H-%M-%S`.sql
-
-Example::
-
-    pg_dump -h 127.0.0.1 -U mayan -c mayan -W > `date +%Y-%m-%d"_"%H-%M-%S`.sql
-
-To restore the database from the SQL text file::
-
-    psql -h <host> -U <database user> -d <database name> -W -f <sql dump file>
-
-Example::
-
-    psql -h 127.0.0.1 -U mayan -d mayan -W -f 2018-06-07_18-10-56.sql
-
-Here is an example of how to perform a backup and a restore of a PostgreSQL
-Docker container using a compressed dump file. A dump file is not compatible or
-can be used interchangeable with an SQL text file.
-
-To backup a PostgreSQL Docker container::
-
-    docker exec <container name> pg_dump -U <database user> -Fc -c <database name> > `date +%Y-%m-%d"_"%H-%M-%S`.dump
-
-Example::
-
-    docker exec mayan-edms-db pg_dump -U mayan -Fc -c mayan > `date +%Y-%m-%d"_"%H-%M-%S`.dump
-
-This will produce a compressed dump file with the current date and time as the filename.
-
-To restore a PostgreSQL Docker container::
-
-    docker exec -i <container name> pg_restore -U <database user> -d <database name> < <dump file>
-
-Since it is not possible to drop a currently open PostgreSQL database, this
-command must be used on a new and empty PostsgreSQL container.
-
-Example::
-
-    docker run -d \
-    --name mayan-edms-pg-new \
-    --restart=always \
-    -p 5432:5432 \
-    -e POSTGRES_USER=mayan \
-    -e POSTGRES_DB=mayan \
-    -e POSTGRES_PASSWORD=mayanuserpass \
-    -v /docker-volumes/mayan-edms/postgres-new:/var/lib/postgresql/data \
-    -d postgres:9.5
-
-    docker exec -i mayan-edms-pg-new pg_restore -U mayan -d mayan < 2018-06-07_17-09-34.dump
-
-More information at:
-
- - PostgreSQL: https://www.postgresql.org/docs/current/static/backup.html
- - MySQL: https://dev.mysql.com/doc/refman/5.7/en/mysqldump.html
- - SQLite: Just copy the file ``mayan/media/db.sqlite3``

docs/chapters/backups.txt

@@ -0,0 +1,18 @@
+=======
+Backups
+=======
+
+To backup your install of Mayan EDMS just copy the ``media`` folder and
+the database content.
+
+If you are using the default storage backend, the ``media`` should be located
+at |MAYAN_MEDIA_ROOT|::
+
+    sudo tar -zcvf backup.tar.gz |MAYAN_MEDIA_ROOT|
+
+To dump the content of your database manager refer to the documentation chapter
+regarding database data "dumping".
+
+- PostgreSQL: https://www.postgresql.org/docs/current/backup-dump.html
+- MySQL: https://dev.mysql.com/doc/refman/5.7/en/mysqldump.html
+

docs/chapters/caa_entity.rst

@@ -1,233 +0,0 @@
-**************************************************
-Mayan EDMS Entity Contributor Assignment Agreement
-**************************************************
-
-Thank you for your interest in contributing to Mayan EDMS ("We" or "Us").
-
-This contributor agreement ("Agreement") documents the rights granted by
-contributors to Us. To make this document effective, please print it, sign it
-(by copyright holder or authorized party) and send it to Us by email to
-caa@mayan-edms.com. This is a legally binding document, so please read it
-carefully before agreeing to it. The Agreement may cover more than one
-software project managed by Us.
-
-
-1. Definitions
-==============
-"You" means any Legal Entity on behalf of whom a Contribution has been received
-by Us. "Legal Entity" means an entity which is not a natural person.
-"Affiliates" means other Legal Entities that control, are controlled by, or
-under common control with that Legal Entity. For the purposes of this
-definition, "control" means (i) the power, direct or indirect, to cause the
-direction or management of such Legal Entity, whether by contract or otherwise,
-(ii) ownership of fifty percent (50%) or more of the outstanding shares or
-securities which vote to elect the management or other persons who direct such
-Legal Entity or (iii) beneficial ownership of such entity.
-
-"Contribution" means any work of authorship that is Submitted by You to Us in
-which You own or assert ownership of the Copyright. We cannot accept
-contributions for which you do not own the Copyright or for which you don't
-have the necessary legal power to transfer.
-
-"Copyright" means all rights protecting works of authorship owned or controlled
-by You or Your Affiliates, including copyright, moral and neighboring rights,
-as appropriate, for the full term of their existence including any extensions
-by You.
-
-"Material" means the work of authorship which is made available by Us to third
-parties. When this Agreement covers more than one software project, the
-Material means the work of authorship to which the Contribution was Submitted.
-After You Submit the Contribution, it may be included in the Material.
-
-"Submit" means any form of electronic, verbal, or written communication sent
-to Us or our representatives, including but not limited to electronic mailing
-lists, source code control systems, and issue tracking systems that are managed
-by, or on behalf of, Us for the purpose of discussing and improving the
-Material, but excluding communication that is conspicuously marked or otherwise
-designated in writing by You as "Not a Contribution."
-
-"Submission Date" means the date on which You Submit a Contribution to Us.
-
-"Effective Date" means the date You execute this Agreement or the date You
-first Submit a Contribution to Us, whichever is earlier.
-
-
-2. Grant of Rights
-==================
-
-2.1 Copyright Assignment
-------------------------
-
-a. At the time the Contribution is Submitted, You assign to Us all right, title,
-   and interest worldwide in all Copyright covering the Contribution; provided
-   that this transfer is conditioned upon compliance with Section 2.3.
-
-b. To the extent that any of the rights in Section 2.1(a) cannot be assigned by
-   You to Us, You grant to Us a perpetual, worldwide, exclusive, royalty-free,
-   transferable, irrevocable license under such non-assigned rights, with rights
-   to sublicense through multiple tiers of sublicensees, to practice such
-   non-assigned rights, including, but not limited to, the right to reproduce,
-   modify, display, perform and distribute the Contribution; provided that this
-   license is conditioned upon compliance with Section 2.3.
-
-c. To the extent that any of the rights in Section 2.1(a) can neither be
-   assigned nor licensed by You to Us, You irrevocably waive and agree never to
-   assert such rights against Us, any of our successors in interest, or any of
-   our licensees, either direct or indirect; provided that this agreement not
-   to assert is conditioned upon compliance with Section 2.3.
-
-d. Upon such transfer of rights to Us, the Contribution will be licenses under
-   the terms of the Material.
-
-2.2 Patent License
-------------------
-
-For patent claims including, without limitation, method, process, and apparatus
-claims which You or Your Affiliates own, control or have the right to grant,
-now or in the future, You grant to Us a perpetual, worldwide, non-exclusive,
-transferable, royalty-free, irrevocable patent license, with the right to
-sublicense these rights to multiple tiers of sublicensees, to make, have made,
-use, sell, offer for sale, import and otherwise transfer the Contribution and
-the Contribution in combination with the Material (and portions of such
-combination). This license is granted only to the extent that the exercise of
-the licensed rights infringes such patent claims; and provided that this license
-is conditioned upon compliance with Section 2.3.
-
-2.3 Outbound License
---------------------
-
-As a condition on the grant of rights in Sections 2.1 and 2.2, We agree to
-license the Contribution only under the terms of the license or licenses which
-We are using on the Submission Date for the Material (including any rights to
-adopt any future version of a license if permitted).
-
-2.4 Moral Rights
-----------------
-
-If moral rights apply to the Contribution, to the maximum extent permitted by
-law, You waive and agree not to assert such moral rights against Us or our
-successors in interest, or any of our licensees, either direct or indirect.
-
-2.5 Our Rights
---------------
-
-You acknowledge that We are not obligated to use Your Contribution as part of
-the Material and may decide to include any Contribution We consider appropriate.
-
-2.6 Reservation of Rights
--------------------------
-
-Any rights not expressly assigned or licensed under this section are expressly
-reserved by You.
-
-
-3. Agreement
-============
-
-You confirm that:
-
-a. You have the legal authority to enter into this Agreement.
-
-b. You or Your Affiliates own the Copyright and patent claims covering the
-   Contribution which are required to grant the rights under Section 2.
-
-c. The grant of rights under Section 2 does not violate any grant of rights
-   which You or Your Affiliates have made to third parties.
-
-
-4. Disclaimer
-=============
-
-EXCEPT FOR THE EXPRESS WARRANTIES IN SECTION 3, THE CONTRIBUTION IS PROVIDED
-"AS IS". MORE PARTICULARLY, ALL EXPRESS OR IMPLIED WARRANTIES INCLUDING,
-WITHOUT LIMITATION, ANY IMPLIED WARRANTY OF MERCHANTABILITY, FITNESS FOR A
-PARTICULAR PURPOSE AND NON-INFRINGEMENT ARE EXPRESSLY DISCLAIMED BY YOU TO US
-AND BY US TO YOU. TO THE EXTENT THAT ANY SUCH WARRANTIES CANNOT BE DISCLAIMED,
-SUCH WARRANTY IS LIMITED IN DURATION TO THE MINIMUM PERIOD PERMITTED BY LAW.
-
-
-5. Consequential Damage Waiver
-==============================
-
-TO THE MAXIMUM EXTENT PERMITTED BY APPLICABLE LAW, IN NO EVENT WILL YOU OR US
-BE LIABLE FOR ANY LOSS OF PROFITS, LOSS OF ANTICIPATED SAVINGS, LOSS OF DATA,
-INDIRECT, SPECIAL, INCIDENTAL, CONSEQUENTIAL AND EXEMPLARY DAMAGES ARISING OUT
-OF THIS AGREEMENT REGARDLESS OF THE LEGAL OR EQUITABLE THEORY (CONTRACT, TORT
-OR OTHERWISE) UPON WHICH THE CLAIM IS BASED.
-
-
-6. Miscellaneous
-================
-
-6.1 Jurisdiction
-----------------
-
-This Agreement will be governed by and construed in accordance with the laws of
-Puerto Rico excluding its conflicts of law provisions. Under certain circumstances,
-the governing law in this section might be superseded by the United Nations
-Convention on Contracts for the International Sale of Goods ("UN Convention")
-and the parties intend to avoid the application of the UN Convention to this
-Agreement and, thus, exclude the application of the UN Convention in its
-entirety to this Agreement.
-
-6.2 Acceptance
---------------
-This Agreement sets out the entire agreement between You and Us for Your
-Contributions to Us and overrides all other agreements or understandings.
-
-6.3 Third parties
------------------
-If You or We assign the rights or obligations received through this Agreement
-to a third party, as a condition of the assignment, that third party must
-agree in writing to abide by all the rights and obligations in the Agreement.
-
-6.4 Unmet responsibilities
---------------------------
-The failure of either party to require performance by the other party of any
-provision of this Agreement in one situation shall not affect the right of a
-party to require such performance at any time in the future. A waiver of
-performance under a provision in one situation shall not be considered a
-waiver of the performance of the provision in the future or a waiver of the
-provision in its entirety.
-
-6.5 Continuation
-----------------
-If any provision of this Agreement is found void and unenforceable, such
-provision will be replaced to the extent possible with a provision that comes
-closest to the meaning of the original provision and which is enforceable.
-The terms and conditions set forth in this Agreement shall apply
-notwithstanding any failure of essential purpose of this Agreement or any
-limited remedy to the maximum extent possible under law.
-
-
-::
-
-
-    Name: _________________________________________________
-
-
-    Email: ________________________________________________
-
-
-    Address: ______________________________________________
-
-
-    Address (cont): _______________________________________
-
-
-    Country: ______________________________________________
-
-
-    Phone number: _________________________________________
-
-
-    Fax number: ___________________________________________
-
-
-
-    _______________________________________________________
-                             Signature
-
-
-
-Mayan EDMS Entity CAA Version 1.0 April 8, 2018

docs/chapters/caa_individual.rst

@@ -1,234 +0,0 @@
-******************************************************
-Mayan EDMS Individual Contributor Assignment Agreement
-******************************************************
-
-Thank you for your interest in contributing to Mayan EDMS ("We" or "Us").
-
-This contributor agreement ("Agreement") documents the rights granted by
-contributors to Us. To make this document effective, please print it, sign it
-(by copyright holder or authorized party) and send it to Us by email to
-caa@mayan-edms.com. This is a legally binding document, so please read it
-carefully before agreeing to it. The Agreement may cover more than one
-software project managed by Us.
-
-1. Definitions
-==============
-
-"You" means the individual who Submits a Contribution to Us.
-
-"Contribution" means any work of authorship that is Submitted by You to Us
-in which You own or assert ownership of the Copyright. We cannot accept
-contributions for which you do not own the Copyright or for which you don't
-have the necessary legal power to transfer.
-
-"Copyright" means all rights protecting works of authorship owned or
-controlled by You, including copyright, moral and neighboring rights,
-as appropriate, for the full term of their existence including any extensions
-by You.
-
-"Material" means the work of authorship which is made available by Us to third
-parties. When this Agreement covers more than one software project, the Material
-means the work of authorship to which the Contribution was Submitted. After
-You Submit the Contribution, it may be included in the Material.
-
-"Submit" means any form of electronic, verbal, or written communication
-sent to Us or our representatives, including but not limited to
-electronic mailing lists, source code control systems, and issue tracking systems
-that are managed by, or on behalf of, Us for the purpose of discussing and
-improving the Material, but excluding communication that is conspicuously marked
-or otherwise designated in writing by You as "Not a Contribution."
-
-"Submission Date" means the date on which You Submit a Contribution to Us.
-
-"Effective Date" means the date You execute this Agreement or the date You first
-Submit a Contribution to Us, whichever is earlier.
-
-2. Grant of Rights
-==================
-
-2.1 Copyright Assignment
-------------------------
-
-a. At the time the Contribution is Submitted, You assign to Us all right,
-   title, and interest worldwide in all Copyright covering the Contribution;
-   provided that this transfer is conditioned upon compliance with Section 2.3.
-
-b. To the extent that any of the rights in Section 2.1(a) cannot be assigned
-   by You to Us, You grant to Us a perpetual, worldwide, exclusive, royalty-free,
-   transferable, irrevocable license under such non-assigned rights, with rights
-   to sublicense through multiple tiers of sublicensees, to practice such
-   non-assigned rights, including, but not limited to, the right to reproduce,
-   modify, display, perform and distribute the Contribution; provided that
-   this license is conditioned upon compliance with Section 2.3.
-
-c. To the extent that any of the rights in Section 2.1(a) can neither be
-   assigned nor licensed by You to Us, You irrevocably waive and agree never to
-   assert such rights against Us, any of our successors in interest, or any of
-   our licensees, either direct or indirect; provided that this agreement not
-   to assert is conditioned upon compliance with Section 2.3.
-
-d. Upon such transfer of rights to Us, the Contribution will be licenses under
-   the terms of the Material.
-
-
-2.2 Patent License
-------------------
-
-For patent claims including, without limitation, method, process, and apparatus
-claims which You own, control or have the right to grant, now or in the future,
-You grant to Us a perpetual, worldwide, non-exclusive, transferable, royalty-free,
-irrevocable patent license, with the right to sublicense these rights to multiple
-tiers of sublicensees, to make, have made, use, sell, offer for sale, import and
-otherwise transfer the Contribution and the Contribution in combination with
-the Material (and portions of such combination). This license is granted only
-to the extent that the exercise of the licensed rights infringes such patent claims;
-and provided that this license is conditioned upon compliance with Section 2.3.
-
-
-2.3 Outbound License
---------------------
-As a condition on the grant of rights in Sections 2.1 and 2.2, We agree to
-license the Contribution only under the terms of the license or licenses which
-We are using on the Submission Date for the Material (including any rights to
-adopt any future version of a license if permitted).
-
-
-2.4 Moral Rights
-----------------
-
-If moral rights apply to the Contribution, to the maximum extent permitted by law,
-You waive and agree not to assert such moral rights against Us or our successors
-in interest, or any of our licensees, either direct or indirect.
-
-
-2.5 Our Rights
---------------
-
-You acknowledge that We are not obligated to use Your Contribution as part of
-the Material and may decide to include any Contribution We consider appropriate.
-
-
-2.6 Reservation of Rights
--------------------------
-
-Any rights not expressly assigned or licensed under this section are expressly
-reserved by You.
-
-
-3. Agreement
-============
-
-You confirm that:
-
-a. You have the legal authority to enter into this Agreement.
-
-b. You own the Copyright and patent claims covering the Contribution which are
-   required to grant the rights under Section 2.
-
-c. The grant of rights under Section 2 does not violate any grant of rights
-   which You have made to third parties, including Your employer. If You are an
-   employee, You have had Your employer approve this Agreement or sign the Entity
-   version of this document. If You are less than eighteen years old, please have
-   Your parents or guardian sign the Agreement.
-
-
-4. Disclaimer
-=============
-
-EXCEPT FOR THE EXPRESS WARRANTIES IN SECTION 3, THE CONTRIBUTION IS PROVIDED
-"AS IS". MORE PARTICULARLY, ALL EXPRESS OR IMPLIED WARRANTIES INCLUDING, WITHOUT
-LIMITATION, ANY IMPLIED WARRANTY OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
-PURPOSE AND NON-INFRINGEMENT ARE EXPRESSLY DISCLAIMED BY YOU TO US AND BY US
-TO YOU. TO THE EXTENT THAT ANY SUCH WARRANTIES CANNOT BE DISCLAIMED, SUCH
-WARRANTY IS LIMITED IN DURATION TO THE MINIMUM PERIOD PERMITTED BY LAW.
-
-
-5. Consequential Damage Waiver
-==============================
-
-TO THE MAXIMUM EXTENT PERMITTED BY APPLICABLE LAW, IN NO EVENT WILL YOU OR US
-BE LIABLE FOR ANY LOSS OF PROFITS, LOSS OF ANTICIPATED SAVINGS, LOSS OF DATA,
-INDIRECT, SPECIAL, INCIDENTAL, CONSEQUENTIAL AND EXEMPLARY DAMAGES ARISING OUT
-OF THIS AGREEMENT REGARDLESS OF THE LEGAL OR EQUITABLE THEORY (CONTRACT, TORT
-OR OTHERWISE) UPON WHICH THE CLAIM IS BASED.
-
-
-6. Miscellaneous
-================
-
-6.1 Jurisdiction
-----------------
-
-This Agreement will be governed by and construed in accordance with the laws of
-Puerto Rico excluding its conflicts of law provisions. Under certain circumstances,
-the governing law in this section might be superseded by the United Nations
-Convention on Contracts for the International Sale of Goods ("UN Convention")
-and the parties intend to avoid the application of the UN Convention to this
-Agreement and, thus, exclude the application of the UN Convention in its
-entirety to this Agreement.
-
-6.2 Acceptance
---------------
-
-This Agreement sets out the entire agreement between You and Us for Your
-Contributions to Us and overrides all other agreements or understandings.
-
-6.3 Third parties
------------------
-
-If You or We assign the rights or obligations received through this Agreement
-to a third party, as a condition of the assignment, that third party must
-agree in writing to abide by all the rights and obligations in the Agreement.
-
-6.4 Unmet responsibilities
---------------------------
-
-The failure of either party to require performance by the other party of any
-provision of this Agreement in one situation shall not affect the right of a
-party to require such performance at any time in the future. A waiver of
-performance under a provision in one situation shall not be considered a
-waiver of the performance of the provision in the future or a waiver of the
-provision in its entirety.
-
-6.5 Continuation
-----------------
-
-If any provision of this Agreement is found void and unenforceable,
-such provision will be replaced to the extent possible with a provision that
-comes closest to the meaning of the original provision and which is enforceable.
-The terms and conditions set forth in this Agreement shall apply notwithstanding
-any failure of essential purpose of this Agreement or any limited remedy to
-the maximum extent possible under law.
-
-
-::
-
-
-    Name: _________________________________________________
-
-
-    Email: ________________________________________________
-
-
-    Address: ______________________________________________
-
-
-    Address (cont): _______________________________________
-
-
-    Country: ______________________________________________
-
-
-    Phone number: _________________________________________
-
-
-    Fax number: ___________________________________________
-
-
-
-    _______________________________________________________
-                             Signature
-
-
-
-Mayan EDMS Individual CAA Version 1.0 April 8, 2018

docs/chapters/cabinets.rst

@@ -1,14 +0,0 @@
-Cabinets
-========
-
-Cabinets are a multi-level method to organize documents. Each cabinet can
-contain documents as well as other sub level cabinets. Cabinets can
-be added from the :menuselection:`Cabinets --> Add new` menu.
-
-With a cabinet structure created, the next step is to add documents to them.
-To add documents to a cabinet, select the cabinet tab of a document, and
-select :menuselection:`Actions --> Add to cabinets` link.
-
-Documents can also be added in bulk to one or more cabinets from any view
-that lists documents.
-

docs/chapters/cabinets.txt

@@ -0,0 +1,12 @@
+========
+Cabinets
+========
+
+Cabinets are a multi-level method to organize documents. Each cabinet can
+contain documents as well as other sub level cabinets.
+
+With a cabinet structure created, the next step is to add documents to them.
+
+Documents can also be added in bulk to one or more cabinets from any view
+that lists documents.
+

docs/chapters/checkouts.rst

@@ -1,14 +0,0 @@
-*********
-Checkouts
-*********
-
-Checkouts are a way to block certain accesses or actions of a document for a
-period of time.
-
-An user can choose to checkout a document to work on an update and block new
-versions of that document to be uploaded by other users. Document are checked
-out for a certain amount of time and if not manually checked in by the original
-user, will be checked in automatically by the system.
-
-To be able to check in documents that were checked out by other users, the
-permission 'Forcefully check in documents' is required.

docs/chapters/checkouts.txt

@@ -0,0 +1,14 @@
+=========
+Checkouts
+=========
+
+Checkouts are a way to block certain accesses or actions of a document for a
+period of time.
+
+An user can choose to checkout a document to work on an update and block new
+versions of that document to be uploaded by other users. Document are checked
+out for a certain amount of time and if not manually checked in by the original
+user, will be checked in automatically by the system.
+
+To be able to check in documents that were checked out by other users, the
+permission 'Forcefully check in documents' is required.

docs/chapters/database_conversion.rst

@@ -1,103 +0,0 @@
-*******************
-Database conversion
-*******************
-
-Version 3.1.x added a new management command to help convert data residing in
-an SQLite database to other database managers like PostgreSQL. Here is the
-conversion procedure.
-
-Direct install
-==============
-
-* Make a backup of your existing SQLite database and documents by copying the
-  ``/opt/mayan-edms/media`` folder.
-* :doc:`Upgrade to at least version 3.1.3. <../releases/3.1.3>`
-*  Migrate the existing SQLite database with the command ``performupgrade``::
-
-    sudo -u mayan MAYAN_MEDIA_ROOT=/opt/mayan-edms/media /opt/mayan-edms/bin/mayan-edms.py performupgrade
-
-* Install PostgreSQL::
-
-    sudo apt-get install postgresql libpq-dev
-
-* Provision a PostgreSQL database::
-
-    sudo -u postgres psql -c "CREATE USER mayan WITH password 'mayanuserpass';"
-    sudo -u postgres createdb -O mayan mayan
-
-* Install the Python client for PostgreSQL::
-
-    sudo -u mayan /opt/mayan-edms/bin/pip install --no-cache-dir --no-use-pep517 psycopg2==2.7.3.2
-
-* Copy the newly created fallback config file::
-
-    cp /opt/mayan-edms/media/config_backup.yml /opt/mayan-edms/media/config.yml
-
-* Edit the configuration file to add the entry for the PostgreSQL database and
-  rename the SQLite database to 'old'::
-
-    # Before
-    DATABASES:
-      default:
-        ATOMIC_REQUESTS: false
-        AUTOCOMMIT: true
-        CONN_MAX_AGE: 0
-        ENGINE: django.db.backends.sqlite3
-        HOST: ''
-        NAME: /opt/mayan-edms/media/db.sqlite3
-        OPTIONS: {}
-        PASSWORD: ''
-        PORT: ''
-        TEST: {CHARSET: null, COLLATION: null, MIRROR: null, NAME: null}
-        TIME_ZONE: null
-        USER: ''
-
-    # After
-    DATABASES:
-      old:
-        ATOMIC_REQUESTS: false
-        AUTOCOMMIT: true
-        CONN_MAX_AGE: 0
-        ENGINE: django.db.backends.sqlite3
-        HOST: ''
-        NAME: /opt/mayan-edms/media/db.sqlite3
-        OPTIONS: {}
-        PASSWORD: ''
-        PORT: ''
-        TEST: {CHARSET: null, COLLATION: null, MIRROR: null, NAME: null}
-        TIME_ZONE: null
-        USER: ''
-      default:
-        ATOMIC_REQUESTS: false
-        AUTOCOMMIT: true
-        CONN_MAX_AGE: 0
-        ENGINE: django.db.backends.postgresql
-        HOST: '127.0.0.1'
-        NAME: /opt/mayan-edms/media/db.sqlite3
-        OPTIONS: {}
-        PASSWORD: 'mayanuserpass'
-        PORT: ''
-        TEST: {CHARSET: null, COLLATION: null, MIRROR: null, NAME: null}
-        TIME_ZONE: null
-        USER: 'mayan'
-
-* Migrate the new database to create the empty tables::
-
-    sudo -u mayan MAYAN_DATABASE_ENGINE=django.db.backends.postgresql MAYAN_DATABASE_NAME=mayan MAYAN_DATABASE_PASSWORD=mayanuserpass MAYAN_DATABASE_USER=mayan MAYAN_DATABASE_HOST=127.0.0.1 MAYAN_MEDIA_ROOT=/opt/mayan-edms/media /opt/mayan-edms/bin/mayan-edms.py migrate
-
-* Convert the data in the SQLite and store it in the PostgreSQL database::
-
-    sudo -u mayan MAYAN_DATABASE_ENGINE=django.db.backends.postgresql MAYAN_DATABASE_NAME=mayan MAYAN_DATABASE_PASSWORD=mayanuserpass MAYAN_DATABASE_USER=mayan MAYAN_DATABASE_HOST=127.0.0.1 MAYAN_MEDIA_ROOT=/opt/mayan-edms/media /opt/mayan-edms/bin/mayan-edms.py convertdb --from=old --to=default --force
-
-* Update the supervisor config file to have Mayan EDMS run from the PostgreSQL database::
-
-    [supervisord]
-    environment=
-        <...>
-        MAYAN_DATABASE_ENGINE=django.db.backends.postgresql,
-        MAYAN_DATABASE_HOST=127.0.0.1,
-        MAYAN_DATABASE_NAME=mayan,
-        MAYAN_DATABASE_PASSWORD=mayanuserpass,
-        MAYAN_DATABASE_USER=mayan,
-        MAYAN_DATABASE_CONN_MAX_AGE=360,
-        <...>

docs/chapters/deploying.rst

@@ -1,356 +0,0 @@
-******************
-Direct deployments
-******************
-
-Mayan EDMS should be deployed like any other Django_ project and
-preferably using virtualenv_. Below are some ways to deploy and use Mayan EDMS.
-Do not use more than one method.
-
-Being a Django_ and a Python_ project, familiarity with these technologies is
-recommended to better understand why Mayan EDMS does some of the things it
-does.
-
-
-Basic deployment
-================
-This setup uses less memory and CPU resources at the expense of some speed.
-
-Binary dependencies
--------------------
-
-If using a Debian_ or Ubuntu_ based Linux distribution, get the executable
-requirements using::
-
-    sudo apt-get install g++ gcc ghostscript gnupg1 graphviz libfuse2 \
-    libjpeg-dev libmagic1 libpq-dev libpng-dev libreoffice libtiff-dev \
-    poppler-utils postgresql python-dev python-virtualenv redis-server \
-    sane-utils supervisor tesseract-ocr zlib1g-dev -y
-
-Create an user account for the installation:
---------------------------------------------
-::
-
-    sudo adduser mayan --disabled-password --disabled-login --no-create-home --gecos ""
-
-Create the parent directory where the project will be deployed:
----------------------------------------------------------------
-::
-
-    sudo mkdir /opt
-
-Create the Python virtual environment for the installation:
------------------------------------------------------------
-::
-
-    sudo virtualenv /opt/mayan-edms
-
-Make the mayan user the owner of the installation directory:
-------------------------------------------------------------
-::
-
-    sudo chown mayan:mayan /opt/mayan-edms -R
-
-Install Mayan EDMS from PyPI:
------------------------------
-::
-
-    sudo -u mayan /opt/mayan-edms/bin/pip install --no-cache-dir --no-use-pep517 mayan-edms
-
-Install the Python client for PostgreSQL and Redis:
----------------------------------------------------
-::
-
-    sudo -u mayan /opt/mayan-edms/bin/pip install --no-cache-dir --no-use-pep517 psycopg2==2.7.3.2 redis==2.10.6
-
-Create the database for the installation:
------------------------------------------
-::
-
-    sudo -u postgres psql -c "CREATE USER mayan WITH password 'mayanuserpass';"
-    sudo -u postgres createdb -O mayan mayan
-
-Initialize the project:
------------------------
-::
-
-    sudo -u mayan MAYAN_DATABASE_ENGINE=django.db.backends.postgresql MAYAN_DATABASE_NAME=mayan \
-    MAYAN_DATABASE_PASSWORD=mayanuserpass MAYAN_DATABASE_USER=mayan \
-    MAYAN_DATABASE_HOST=127.0.0.1 MAYAN_MEDIA_ROOT=/opt/mayan-edms/media \
-    /opt/mayan-edms/bin/mayan-edms.py initialsetup
-
-Collect the static files:
--------------------------
-::
-
-    sudo -u mayan MAYAN_MEDIA_ROOT=/opt/mayan-edms/media \
-    /opt/mayan-edms/bin/mayan-edms.py collectstatic --noinput
-
-Create the supervisor file at ``/etc/supervisor/conf.d/mayan.conf``:
---------------------------------------------------------------------
-::
-
-    [supervisord]
-    environment=
-        MAYAN_ALLOWED_HOSTS='["*"]',  # Allow access to other network hosts other than localhost
-        MAYAN_CELERY_RESULT_BACKEND="redis://127.0.0.1:6379/0",
-        MAYAN_BROKER_URL="redis://127.0.0.1:6379/0",
-        PYTHONPATH=/opt/mayan-edms/lib/python2.7/site-packages:/opt/mayan-edms/data,
-        MAYAN_MEDIA_ROOT=/opt/mayan-edms/media,
-        MAYAN_DATABASE_ENGINE=django.db.backends.postgresql,
-        MAYAN_DATABASE_HOST=127.0.0.1,
-        MAYAN_DATABASE_NAME=mayan,
-        MAYAN_DATABASE_PASSWORD=mayanuserpass,
-        MAYAN_DATABASE_USER=mayan,
-        MAYAN_DATABASE_CONN_MAX_AGE=60,
-        DJANGO_SETTINGS_MODULE=mayan.settings.production
-
-    [program:mayan-gunicorn]
-    autorestart = true
-    autostart = true
-    command = /opt/mayan-edms/bin/gunicorn -w 2 mayan.wsgi --max-requests 500 --max-requests-jitter 50 --worker-class gevent --bind 0.0.0.0:8000 --timeout 120
-    user = mayan
-
-    [program:mayan-worker-fast]
-    autorestart = true
-    autostart = true
-    command = nice -n 1 /opt/mayan-edms/bin/mayan-edms.py celery worker -Ofair -l ERROR -Q converter,sources_fast -n mayan-worker-fast.%%h --concurrency=1
-    killasgroup = true
-    numprocs = 1
-    priority = 998
-    startsecs = 10
-    stopwaitsecs = 1
-    user = mayan
-
-    [program:mayan-worker-medium]
-    autorestart = true
-    autostart = true
-    command = nice -n 18 /opt/mayan-edms/bin/mayan-edms.py celery worker -Ofair -l ERROR -Q checkouts_periodic,documents_periodic,indexing,metadata,sources,sources_periodic,uploads,documents -n mayan-worker-medium.%%h --concurrency=1
-    killasgroup = true
-    numprocs = 1
-    priority = 998
-    startsecs = 10
-    stopwaitsecs = 1
-    user = mayan
-
-    [program:mayan-worker-slow]
-    autorestart = true
-    autostart = true
-    command = nice -n 19 /opt/mayan-edms/bin/mayan-edms.py celery worker -Ofair -l ERROR -Q mailing,tools,statistics,parsing,ocr -n mayan-worker-slow.%%h --concurrency=1
-    killasgroup = true
-    numprocs = 1
-    priority = 998
-    startsecs = 10
-    stopwaitsecs = 1
-    user = mayan
-
-    [program:mayan-celery-beat]
-    autorestart = true
-    autostart = true
-    command = nice -n 1 /opt/mayan-edms/bin/mayan-edms.py celery beat --pidfile= -l ERROR
-    killasgroup = true
-    numprocs = 1
-    priority = 998
-    startsecs = 10
-    stopwaitsecs = 1
-    user = mayan
-
-Configure Redis to discard data when it runs out of memory, not save its database and only keep 1 database:
------------------------------------------------------------------------------------------------------------
-::
-
-    echo "maxmemory-policy allkeys-lru" >> /etc/redis/redis.conf
-    echo "save \"\"" >> /etc/redis/redis.conf
-    echo "databases 1" >> /etc/redis/redis.conf
-    systemctl restart redis
-
-Enable and restart the services [1_]:
--------------------------------------
-::
-
-    systemctl enable supervisor
-    systemctl restart supervisor
-
-
-.. _deployment_advanced:
-
-Advanced deployment
-===================
-
-This variation uses RabbitMQ as the message broker and removes the fast worker
-concurrency restriction. RabbitMQ consumes more memory but scales to thousands
-of messages. RabbitMQ messages are also persistent, this means that pending
-tasks are not lost in the case of a restart. The database connection lifetime
-is increased to 10 minutes. The Gunicorn workers are increased to 3.
-
-Binary dependencies
--------------------
-
-If using a Debian_ or Ubuntu_ based Linux distribution, get the executable
-requirements using::
-
-    sudo apt-get install g++ gcc ghostscript gnupg1 graphviz libfuse2 \
-    libjpeg-dev libmagic1 libpq-dev libpng-dev libreoffice libtiff-dev \
-    poppler-utils postgresql python-dev python-virtualenv rabbitmq-server \
-    redis-server sane-utils supervisor tesseract-ocr zlib1g-dev -y
-
-Create an user account for the installation:
---------------------------------------------
-::
-
-    sudo adduser mayan --disabled-password --disabled-login --no-create-home --gecos ""
-
-Create the parent directory where the project will be deployed:
----------------------------------------------------------------
-::
-
-    sudo mkdir /opt
-
-Create the Python virtual environment for the installation:
------------------------------------------------------------
-::
-
-    sudo virtualenv /opt/mayan-edms
-
-Make the mayan user the owner of the installation directory:
-------------------------------------------------------------
-::
-
-    sudo chown mayan:mayan /opt/mayan-edms -R
-
-Install Mayan EDMS from PyPI:
------------------------------
-::
-
-    sudo -u mayan /opt/mayan-edms/bin/pip install --no-cache-dir --no-use-pep517 mayan-edms
-
-Install the Python client for PostgreSQL and Redis:
----------------------------------------------------
-::
-
-    sudo -u mayan /opt/mayan-edms/bin/pip install --no-cache-dir --no-use-pep517 librabbitmq==2.0.0 psycopg2==2.7.3.2 redis==2.10.6
-
-Create the database for the installation:
------------------------------------------
-::
-
-    sudo -u postgres psql -c "CREATE USER mayan WITH password 'mayanuserpass';"
-    sudo -u postgres createdb -O mayan mayan
-
-Initialize the project:
------------------------
-::
-
-    sudo -u mayan MAYAN_DATABASE_ENGINE=django.db.backends.postgresql MAYAN_DATABASE_NAME=mayan \
-    MAYAN_DATABASE_PASSWORD=mayanuserpass MAYAN_DATABASE_USER=mayan \
-    MAYAN_DATABASE_HOST=127.0.0.1 MAYAN_MEDIA_ROOT=/opt/mayan-edms/media \
-    /opt/mayan-edms/bin/mayan-edms.py initialsetup
-
-Collect the static files:
--------------------------
-::
-
-    sudo -u mayan MAYAN_MEDIA_ROOT=/opt/mayan-edms/media \
-    /opt/mayan-edms/bin/mayan-edms.py collectstatic --noinput
-
-Create the RabbitMQ user and vhost:
------------------------------------
-::
-
-    sudo rabbitmqctl add_user mayan mayanrabbitmqpassword
-    sudo rabbitmqctl add_vhost mayan
-    sudo rabbitmqctl set_permissions -p mayan mayan ".*" ".*" ".*"
-
-Create the supervisor file at ``/etc/supervisor/conf.d/mayan.conf``:
---------------------------------------------------------------------
-::
-
-    [supervisord]
-    environment=
-        MAYAN_ALLOWED_HOSTS='["*"]',  # Allow access to other network hosts other than localhost
-        MAYAN_CELERY_RESULT_BACKEND="redis://127.0.0.1:6379/0",
-        MAYAN_BROKER_URL="amqp://mayan:mayanrabbitmqpassword@localhost:5672/mayan",
-        PYTHONPATH=/opt/mayan-edms/lib/python2.7/site-packages:/opt/mayan-edms/data,
-        MAYAN_MEDIA_ROOT=/opt/mayan-edms/media,
-        MAYAN_DATABASE_ENGINE=django.db.backends.postgresql,
-        MAYAN_DATABASE_HOST=127.0.0.1,
-        MAYAN_DATABASE_NAME=mayan,
-        MAYAN_DATABASE_PASSWORD=mayanuserpass,
-        MAYAN_DATABASE_USER=mayan,
-        MAYAN_DATABASE_CONN_MAX_AGE=360,
-        DJANGO_SETTINGS_MODULE=mayan.settings.production
-
-    [program:mayan-gunicorn]
-    autorestart = true
-    autostart = true
-    command = /opt/mayan-edms/bin/gunicorn -w 3 mayan.wsgi --max-requests 500 --max-requests-jitter 50 --worker-class gevent --bind 0.0.0.0:8000 --timeout 120
-    user = mayan
-
-    [program:mayan-worker-fast]
-    autorestart = true
-    autostart = true
-    command = nice -n 1 /opt/mayan-edms/bin/mayan-edms.py celery worker -Ofair -l ERROR -Q converter,sources_fast -n mayan-worker-fast.%%h
-    killasgroup = true
-    numprocs = 1
-    priority = 998
-    startsecs = 10
-    stopwaitsecs = 1
-    user = mayan
-
-    [program:mayan-worker-medium]
-    autorestart = true
-    autostart = true
-    command = nice -n 18 /opt/mayan-edms/bin/mayan-edms.py celery worker -Ofair -l ERROR -Q checkouts_periodic,documents_periodic,indexing,metadata,sources,sources_periodic,uploads,documents -n mayan-worker-medium.%%h --concurrency=1
-    killasgroup = true
-    numprocs = 1
-    priority = 998
-    startsecs = 10
-    stopwaitsecs = 1
-    user = mayan
-
-    [program:mayan-worker-slow]
-    autorestart = true
-    autostart = true
-    command = nice -n 19 /opt/mayan-edms/bin/mayan-edms.py celery worker -Ofair -l ERROR -Q mailing,tools,statistics,parsing,ocr -n mayan-worker-slow.%%h --concurrency=1
-    killasgroup = true
-    numprocs = 1
-    priority = 998
-    startsecs = 10
-    stopwaitsecs = 1
-    user = mayan
-
-    [program:mayan-celery-beat]
-    autorestart = true
-    autostart = true
-    command = nice -n 1 /opt/mayan-edms/bin/mayan-edms.py celery beat --pidfile= -l ERROR
-    killasgroup = true
-    numprocs = 1
-    priority = 998
-    startsecs = 10
-    stopwaitsecs = 1
-    user = mayan
-
-Configure Redis to discard data when it runs out of memory, not save its database and only keep 1 database:
------------------------------------------------------------------------------------------------------------
-::
-
-    echo "maxmemory-policy allkeys-lru" >> /etc/redis/redis.conf
-    echo "save \"\"" >> /etc/redis/redis.conf
-    echo "databases 1" >> /etc/redis/redis.conf
-    systemctl restart redis
-
-Enable and restart the services [1_]:
--------------------------------------
-::
-
-    systemctl enable supervisor
-    systemctl restart supervisor
-
-[1]: https://bugs.launchpad.net/ubuntu/+source/supervisor/+bug/1594740
-
-.. _Debian: https://www.debian.org/
-.. _Django: https://www.djangoproject.com/
-.. _Python: https://www.python.org/
-.. _SQLite: https://www.sqlite.org/
-.. _Ubuntu: http://www.ubuntu.com/
-.. _virtualenv: http://www.virtualenv.org/en/latest/index.html
-.. _1: https://bugs.launchpad.net/ubuntu/+source/supervisor/+bug/1594740

docs/chapters/deploying.txt

@@ -0,0 +1,273 @@
+.. _deployment_basic:
+
+=================
+Direct deployment
+=================
+
+Mayan EDMS should be deployed like any other Django_ project and
+preferably using virtualenv_. Below are some ways to deploy and use Mayan EDMS.
+
+Being a Django_ and a Python_ project, familiarity with these technologies is
+recommended to better understand why Mayan EDMS does some of the things it
+does.
+
+Compilers and development libraries will be installed to compile runtime
+libraries. LibreOffice and Poppler utils will also be installed as they are
+used to convert document files. Supervisor (https://supervisord.org/), a
+Process Control System, will be used to monitor and keep all Mayan processes
+running.
+
+
+Basic deployment
+================
+This setup uses less memory and CPU resources at the expense of some speed.
+For another setup that offers more performance and scalability refer to the
+`Advanced deployment`_ below.
+
+#. Install binary dependencies:
+
+   If using a Debian_ or Ubuntu_ based Linux distribution, get the executable
+   requirements using:
+
+   .. code-block:: bash
+
+       sudo apt-get install exiftool g++ gcc ghostscript gnupg1 graphviz \
+       libfuse2 libjpeg-dev libmagic1 libpq-dev libpng-dev libreoffice \
+       libtiff-dev poppler-utils postgresql python3-dev python3-virtualenv
+       redis-server sane-utils supervisor tesseract-ocr zlib1g-dev -y
+
+   .. note::
+
+       Platforms with the ARM CPU might also need additional requirements:
+
+       .. code-block:: bash
+
+           sudo apt-get install libffi-dev libssl-dev -y
+
+
+#. Create the user account for the installation:
+
+   This will create an unprivileged user account that is also unable to login:
+
+   .. code-block:: bash
+
+       sudo adduser mayan --disabled-password --disabled-login --gecos ""
+
+
+#. Create the parent directory where the project will be deployed:
+
+   ``/opt/`` is a good choice as it is meant is for "software and add-on packages
+   that are not part of the default installation". (https://www.tldp.org/LDP/Linux-Filesystem-Hierarchy/html/opt.html).
+   Create the ``/opt`` directory if it doesn't already exists:
+
+   .. code-block:: bash
+
+       sudo mkdir /opt
+
+
+#. Create the Python virtual environment:
+
+   This will keep all the Python packages installed here isolated from the rest
+   of the Python packages in the system:
+
+   .. code-block:: bash
+
+       sudo virtualenv |MAYAN_INSTALLATION_DIRECTORY| -p /usr/bin/python3
+
+
+#. Make the mayan user the owner of the installation directory:
+
+   .. code-block:: bash
+
+       sudo chown mayan:mayan |MAYAN_INSTALLATION_DIRECTORY| -R
+
+
+#. Install Mayan EDMS from PyPI:
+
+   .. code-block:: bash
+
+       sudo -u mayan |MAYAN_PIP_BIN| install --no-use-pep517 mayan-edms
+
+
+#. Install the Python client for PostgreSQL and Redis:
+
+   .. code-block:: bash
+
+       sudo -u mayan |MAYAN_PIP_BIN| install --no-use-pep517 psycopg2==|PYTHON_PSYCOPG2_VERSION| redis==|PYTHON_REDIS_VERSION|
+
+   .. note::
+
+       Platforms with the ARM CPU might also need additional requirements:
+
+       .. code-block:: bash
+
+           sudo -u mayan |MAYAN_PIP_BIN| install --no-use-pep517 psutil==|PYTHON_PSUTIL_VERSION|
+
+
+#. Create the database for the installation:
+
+   .. code-block:: bash
+
+       sudo -u postgres psql -c "CREATE USER mayan WITH password 'mayanuserpass';"
+       sudo -u postgres createdb -O mayan mayan
+
+
+#. Initialize the project:
+
+   This step will create all the database structures, download static media files
+   like JavaScript libraries and HTML frameworks, and create and initial admin
+   account with a random password.
+
+   .. note::
+
+       For simplicity, the ``MAYAN_MEDIA_ROOT`` folder is set to be a subfolder
+       of the installation. If you want to keep your files separated from
+       the installation files, change the value of the ``MAYAN_MEDIA_ROOT``
+       variable in this and all subsequent steps. Be sure to first create the
+       folder and give ownership of it to the ``mayan`` user with the ``chown``
+       command.
+
+   .. warning::
+
+       If this step is interrupted, even if it is later resumed, will
+       cause the automatic admin user to not be created in some cases. Make
+       sure all environment variables and values are correct. If this
+       happens, refer to the troubleshooting chapters:
+       :ref:`troubleshooting-autoadmin-account` and
+       :ref:`troubleshooting-admin-password`.
+
+   .. code-block:: bash
+
+       sudo -u mayan MAYAN_DATABASES="{'default':{'ENGINE':'django.db.backends.postgresql','NAME':'mayan','PASSWORD':'mayanuserpass','USER':'mayan','HOST':'127.0.0.1'}}" \
+       MAYAN_MEDIA_ROOT=|MAYAN_MEDIA_ROOT| \
+       |MAYAN_BIN| initialsetup
+
+
+#. Collect the static files:
+
+   This step merges and compressed static media files so they can be served more
+   effectively.
+
+   .. code-block:: bash
+
+       sudo -u mayan MAYAN_MEDIA_ROOT=|MAYAN_MEDIA_ROOT| \
+       |MAYAN_BIN| preparestatic --noinput
+
+
+#. Create the supervisor file at ``|MAYAN_SUPERVISOR_CONF|``:
+
+   .. code-block:: bash
+
+       sudo -u mayan MAYAN_DATABASES="{'default':{'ENGINE':'django.db.backends.postgresql','NAME':'mayan','PASSWORD':'mayanuserpass','USER':'mayan','HOST':'127.0.0.1'}}" \
+       MAYAN_MEDIA_ROOT=|MAYAN_MEDIA_ROOT| \
+       |MAYAN_BIN| platformtemplate supervisord | sudo sh -c "cat > |MAYAN_SUPERVISOR_CONF|"
+
+
+#. Configure Redis:
+
+   Configure Redis to discard data when it runs out of memory, not save its
+   database and only keep 2 database:
+
+   .. code-block:: bash
+
+       sudo echo "maxmemory-policy allkeys-lru" >> /etc/redis/redis.conf
+       sudo echo "save \"\"" >> /etc/redis/redis.conf
+       sudo echo "databases 2" >> /etc/redis/redis.conf
+       sudo systemctl restart redis
+
+#. Enable and restart the services [1_]:
+
+   .. code-block:: bash
+
+       sudo systemctl enable supervisor
+       sudo systemctl restart supervisor
+
+
+#. Cleaning up:
+
+   The following operating system dependencies are only needed during
+   installation and can be removed.
+
+   .. code-block:: bash
+
+       sudo apt-get remove --purge libjpeg-dev libpq-dev libpng-dev libtiff-dev zlib1g-dev
+
+
+.. _deployment_advanced:
+
+Advanced deployment
+===================
+
+This variation uses RabbitMQ as the message broker. RabbitMQ consumes more
+memory but scales to thousands of messages per second. RabbitMQ messages are also
+persistent by default, this means that pending tasks are not lost in the case
+of a restart or power failure. The Gunicorn workers are increased to 3.
+
+
+#. Install RabbitMQ:
+
+   If using a Debian_ or Ubuntu_ based Linux distribution, get the executable
+   requirements using:
+
+   .. code-block:: bash
+
+       sudo apt-get install rabbitmq-server -y
+
+
+#. Install the Python client for RabbitMQ:
+
+   .. code-block:: bash
+
+       sudo -u mayan |MAYAN_PIP_BIN| install --no-use-pep517 amqp==|PYTHON_AMQP_VERSION|
+
+
+#. Create the RabbitMQ user and vhost:
+
+   .. code-block:: bash
+
+       sudo rabbitmqctl add_user mayan mayanrabbitmqpassword
+       sudo rabbitmqctl add_vhost mayan
+       sudo rabbitmqctl set_permissions -p mayan mayan ".*" ".*" ".*"
+
+
+#. Edit the supervisor file at ``|MAYAN_SUPERVISOR_CONF|``:
+
+   Replace (paying attention to the comma at the end):
+
+   .. code-block:: ini
+
+       MAYAN_CELERY_BROKER_URL="redis://127.0.0.1:6379/0",
+
+   with:
+
+   .. code-block:: ini
+
+       MAYAN_CELERY_BROKER_URL="amqp://mayan:mayanrabbitmqpassword@localhost:5672/mayan",
+
+   increase the number of Gunicorn workers to 3 in the line (``-w 2`` section):
+
+   .. code-block:: ini
+
+       command = |MAYAN_GUNICORN_BIN| -w 2 mayan.wsgi --max-requests 1000 --max-requests-jitter 50 --worker-class gevent --bind 0.0.0.0:8000 --timeout 120
+
+   remove the concurrency limit (or increase it) of the fast worker (remove ``--concurrency=1``).
+
+
+#. Restart the services:
+
+   .. code-block:: bash
+
+       sudo supervisorctl reread
+       sudo supervisorctl restart all
+
+
+
+[1]: https://bugs.launchpad.net/ubuntu/+source/supervisor/+bug/1594740
+
+.. _Debian: https://www.debian.org/
+.. _Django: https://www.djangoproject.com/
+.. _Python: https://www.python.org/
+.. _SQLite: https://www.sqlite.org/
+.. _Ubuntu: http://www.ubuntu.com/
+.. _virtualenv: http://www.virtualenv.org/en/latest/index.html
+.. _1: https://bugs.launchpad.net/ubuntu/+source/supervisor/+bug/1594740

docs/chapters/development/caa_entity.txt

@@ -0,0 +1,247 @@
+==================================================
+Mayan EDMS Entity Contributor Assignment Agreement
+==================================================
+
+Thank you for your interest in contributing to Mayan EDMS ("We" or "Us").
+
+This contributor agreement ("Agreement") documents the rights granted by
+contributors to Us. To make this document effective, please print it, sign it
+(by copyright holder or authorized party) and send it to Us by email to
+caa@mayan-edms.com. This is a legally binding document, so please read it
+carefully before agreeing to it. The Agreement may cover more than one
+software project managed by Us.
+
+
+1. Definitions
+==============
+
+"You" means any Legal Entity on behalf of whom a Contribution has been received
+by Us. "Legal Entity" means an entity which is not a natural person.
+"Affiliates" means other Legal Entities that control, are controlled by, or
+under common control with that Legal Entity. For the purposes of this
+definition, "control" means (i) the power, direct or indirect, to cause the
+direction or management of such Legal Entity, whether by contract or otherwise,
+(ii) ownership of fifty percent (50%) or more of the outstanding shares or
+securities which vote to elect the management or other persons who direct such
+Legal Entity or (iii) beneficial ownership of such entity.
+
+"Contribution" means any work of authorship that is Submitted by You to Us in
+which You own or assert ownership of the Copyright. We cannot accept
+contributions for which you do not own the Copyright or for which you don't
+have the necessary legal power to transfer.
+
+"Copyright" means all rights protecting works of authorship owned or controlled
+by You or Your Affiliates, including copyright, moral and neighboring rights,
+as appropriate, for the full term of their existence including any extensions
+by You.
+
+"Material" means the work of authorship which is made available by Us to third
+parties. When this Agreement covers more than one software project, the
+Material means the work of authorship to which the Contribution was Submitted.
+After You Submit the Contribution, it may be included in the Material.
+
+"Submit" means any form of electronic, verbal, or written communication sent
+to Us or our representatives, including but not limited to electronic mailing
+lists, source code control systems, and issue tracking systems that are managed
+by, or on behalf of, Us for the purpose of discussing and improving the
+Material, but excluding communication that is conspicuously marked or otherwise
+designated in writing by You as "Not a Contribution."
+
+"Submission Date" means the date on which You Submit a Contribution to Us.
+
+"Effective Date" means the date You execute this Agreement or the date You
+first Submit a Contribution to Us, whichever is earlier.
+
+
+2. Grant of Rights
+==================
+
+
+2.1 Copyright Assignment
+------------------------
+
+a. At the time the Contribution is Submitted, You assign to Us all right, title,
+   and interest worldwide in all Copyright covering the Contribution; provided
+   that this transfer is conditioned upon compliance with Section 2.3.
+
+b. To the extent that any of the rights in Section 2.1(a) cannot be assigned by
+   You to Us, You grant to Us a perpetual, worldwide, exclusive, royalty-free,
+   transferable, irrevocable license under such non-assigned rights, with rights
+   to sublicense through multiple tiers of sublicensees, to practice such
+   non-assigned rights, including, but not limited to, the right to reproduce,
+   modify, display, perform and distribute the Contribution; provided that this
+   license is conditioned upon compliance with Section 2.3.
+
+c. To the extent that any of the rights in Section 2.1(a) can neither be
+   assigned nor licensed by You to Us, You irrevocably waive and agree never to
+   assert such rights against Us, any of our successors in interest, or any of
+   our licensees, either direct or indirect; provided that this agreement not
+   to assert is conditioned upon compliance with Section 2.3.
+
+d. Upon such transfer of rights to Us, the Contribution will be licenses under
+   the terms of the Material.
+
+2.2 Patent License
+------------------
+
+For patent claims including, without limitation, method, process, and apparatus
+claims which You or Your Affiliates own, control or have the right to grant,
+now or in the future, You grant to Us a perpetual, worldwide, non-exclusive,
+transferable, royalty-free, irrevocable patent license, with the right to
+sublicense these rights to multiple tiers of sublicensees, to make, have made,
+use, sell, offer for sale, import and otherwise transfer the Contribution and
+the Contribution in combination with the Material (and portions of such
+combination). This license is granted only to the extent that the exercise of
+the licensed rights infringes such patent claims; and provided that this license
+is conditioned upon compliance with Section 2.3.
+
+2.3 Outbound License
+--------------------
+
+As a condition on the grant of rights in Sections 2.1 and 2.2, We agree to
+license the Contribution only under the terms of the license or licenses which
+We are using on the Submission Date for the Material (including any rights to
+adopt any future version of a license if permitted).
+
+
+2.4 Moral Rights
+----------------
+
+If moral rights apply to the Contribution, to the maximum extent permitted by
+law, You waive and agree not to assert such moral rights against Us or our
+successors in interest, or any of our licensees, either direct or indirect.
+
+
+2.5 Our Rights
+--------------
+
+You acknowledge that We are not obligated to use Your Contribution as part of
+the Material and may decide to include any Contribution We consider appropriate.
+
+
+2.6 Reservation of Rights
+-------------------------
+
+Any rights not expressly assigned or licensed under this section are expressly
+reserved by You.
+
+
+3. Agreement
+============
+
+You confirm that:
+
+a. You have the legal authority to enter into this Agreement.
+
+b. You or Your Affiliates own the Copyright and patent claims covering the
+   Contribution which are required to grant the rights under Section 2.
+
+c. The grant of rights under Section 2 does not violate any grant of rights
+   which You or Your Affiliates have made to third parties.
+
+
+4. Disclaimer
+=============
+
+EXCEPT FOR THE EXPRESS WARRANTIES IN SECTION 3, THE CONTRIBUTION IS PROVIDED
+"AS IS". MORE PARTICULARLY, ALL EXPRESS OR IMPLIED WARRANTIES INCLUDING,
+WITHOUT LIMITATION, ANY IMPLIED WARRANTY OF MERCHANTABILITY, FITNESS FOR A
+PARTICULAR PURPOSE AND NON-INFRINGEMENT ARE EXPRESSLY DISCLAIMED BY YOU TO US
+AND BY US TO YOU. TO THE EXTENT THAT ANY SUCH WARRANTIES CANNOT BE DISCLAIMED,
+SUCH WARRANTY IS LIMITED IN DURATION TO THE MINIMUM PERIOD PERMITTED BY LAW.
+
+
+5. Consequential Damage Waiver
+==============================
+
+TO THE MAXIMUM EXTENT PERMITTED BY APPLICABLE LAW, IN NO EVENT WILL YOU OR US
+BE LIABLE FOR ANY LOSS OF PROFITS, LOSS OF ANTICIPATED SAVINGS, LOSS OF DATA,
+INDIRECT, SPECIAL, INCIDENTAL, CONSEQUENTIAL AND EXEMPLARY DAMAGES ARISING OUT
+OF THIS AGREEMENT REGARDLESS OF THE LEGAL OR EQUITABLE THEORY (CONTRACT, TORT
+OR OTHERWISE) UPON WHICH THE CLAIM IS BASED.
+
+
+6. Miscellaneous
+================
+
+
+6.1 Jurisdiction
+----------------
+
+This Agreement will be governed by and construed in accordance with the laws of
+Puerto Rico excluding its conflicts of law provisions. Under certain circumstances,
+the governing law in this section might be superseded by the United Nations
+Convention on Contracts for the International Sale of Goods ("UN Convention")
+and the parties intend to avoid the application of the UN Convention to this
+Agreement and, thus, exclude the application of the UN Convention in its
+entirety to this Agreement.
+
+
+6.2 Acceptance
+--------------
+
+This Agreement sets out the entire agreement between You and Us for Your
+Contributions to Us and overrides all other agreements or understandings.
+
+
+6.3 Third parties
+-----------------
+
+If You or We assign the rights or obligations received through this Agreement
+to a third party, as a condition of the assignment, that third party must
+agree in writing to abide by all the rights and obligations in the Agreement.
+
+
+6.4 Unmet responsibilities
+--------------------------
+
+The failure of either party to require performance by the other party of any
+provision of this Agreement in one situation shall not affect the right of a
+party to require such performance at any time in the future. A waiver of
+performance under a provision in one situation shall not be considered a
+waiver of the performance of the provision in the future or a waiver of the
+provision in its entirety.
+
+
+6.5 Continuation
+----------------
+
+If any provision of this Agreement is found void and unenforceable, such
+provision will be replaced to the extent possible with a provision that comes
+closest to the meaning of the original provision and which is enforceable.
+The terms and conditions set forth in this Agreement shall apply
+notwithstanding any failure of essential purpose of this Agreement or any
+limited remedy to the maximum extent possible under law.
+
+
+::
+
+
+    Name: _________________________________________________
+
+
+    Email: ________________________________________________
+
+
+    Address: ______________________________________________
+
+
+    Address (cont): _______________________________________
+
+
+    Country: ______________________________________________
+
+
+    Phone number: _________________________________________
+
+
+    Fax number: ___________________________________________
+
+
+
+    _______________________________________________________
+                             Signature
+
+
+
+Mayan EDMS Entity CAA Version 1.0 April 8, 2018

docs/chapters/development/caa_individual.txt

@@ -0,0 +1,243 @@
+======================================================
+Mayan EDMS Individual Contributor Assignment Agreement
+======================================================
+
+Thank you for your interest in contributing to Mayan EDMS ("We" or "Us").
+
+This contributor agreement ("Agreement") documents the rights granted by
+contributors to Us. To make this document effective, please print it, sign it
+(by copyright holder or authorized party) and send it to Us by email to
+caa@mayan-edms.com. This is a legally binding document, so please read it
+carefully before agreeing to it. The Agreement may cover more than one
+software project managed by Us.
+
+
+1. Definitions
+==============
+
+"You" means the individual who Submits a Contribution to Us.
+
+"Contribution" means any work of authorship that is Submitted by You to Us
+in which You own or assert ownership of the Copyright. We cannot accept
+contributions for which you do not own the Copyright or for which you don't
+have the necessary legal power to transfer.
+
+"Copyright" means all rights protecting works of authorship owned or
+controlled by You, including copyright, moral and neighboring rights,
+as appropriate, for the full term of their existence including any extensions
+by You.
+
+"Material" means the work of authorship which is made available by Us to third
+parties. When this Agreement covers more than one software project, the Material
+means the work of authorship to which the Contribution was Submitted. After
+You Submit the Contribution, it may be included in the Material.
+
+"Submit" means any form of electronic, verbal, or written communication
+sent to Us or our representatives, including but not limited to
+electronic mailing lists, source code control systems, and issue tracking systems
+that are managed by, or on behalf of, Us for the purpose of discussing and
+improving the Material, but excluding communication that is conspicuously marked
+or otherwise designated in writing by You as "Not a Contribution."
+
+"Submission Date" means the date on which You Submit a Contribution to Us.
+
+"Effective Date" means the date You execute this Agreement or the date You first
+Submit a Contribution to Us, whichever is earlier.
+
+
+2. Grant of Rights
+==================
+
+
+2.1 Copyright Assignment
+------------------------
+
+a. At the time the Contribution is Submitted, You assign to Us all right,
+   title, and interest worldwide in all Copyright covering the Contribution;
+   provided that this transfer is conditioned upon compliance with Section 2.3.
+
+b. To the extent that any of the rights in Section 2.1(a) cannot be assigned
+   by You to Us, You grant to Us a perpetual, worldwide, exclusive, royalty-free,
+   transferable, irrevocable license under such non-assigned rights, with rights
+   to sublicense through multiple tiers of sublicensees, to practice such
+   non-assigned rights, including, but not limited to, the right to reproduce,
+   modify, display, perform and distribute the Contribution; provided that
+   this license is conditioned upon compliance with Section 2.3.
+
+c. To the extent that any of the rights in Section 2.1(a) can neither be
+   assigned nor licensed by You to Us, You irrevocably waive and agree never to
+   assert such rights against Us, any of our successors in interest, or any of
+   our licensees, either direct or indirect; provided that this agreement not
+   to assert is conditioned upon compliance with Section 2.3.
+
+d. Upon such transfer of rights to Us, the Contribution will be licenses under
+   the terms of the Material.
+
+
+2.2 Patent License
+------------------
+
+For patent claims including, without limitation, method, process, and apparatus
+claims which You own, control or have the right to grant, now or in the future,
+You grant to Us a perpetual, worldwide, non-exclusive, transferable, royalty-free,
+irrevocable patent license, with the right to sublicense these rights to multiple
+tiers of sublicensees, to make, have made, use, sell, offer for sale, import and
+otherwise transfer the Contribution and the Contribution in combination with
+the Material (and portions of such combination). This license is granted only
+to the extent that the exercise of the licensed rights infringes such patent claims;
+and provided that this license is conditioned upon compliance with Section 2.3.
+
+
+2.3 Outbound License
+--------------------
+
+As a condition on the grant of rights in Sections 2.1 and 2.2, We agree to
+license the Contribution only under the terms of the license or licenses which
+We are using on the Submission Date for the Material (including any rights to
+adopt any future version of a license if permitted).
+
+
+2.4 Moral Rights
+----------------
+
+If moral rights apply to the Contribution, to the maximum extent permitted by law,
+You waive and agree not to assert such moral rights against Us or our successors
+in interest, or any of our licensees, either direct or indirect.
+
+
+2.5 Our Rights
+--------------
+
+You acknowledge that We are not obligated to use Your Contribution as part of
+the Material and may decide to include any Contribution We consider appropriate.
+
+
+2.6 Reservation of Rights
+-------------------------
+
+Any rights not expressly assigned or licensed under this section are expressly
+reserved by You.
+
+
+3. Agreement
+============
+
+You confirm that:
+
+a. You have the legal authority to enter into this Agreement.
+
+b. You own the Copyright and patent claims covering the Contribution which are
+   required to grant the rights under Section 2.
+
+c. The grant of rights under Section 2 does not violate any grant of rights
+   which You have made to third parties, including Your employer. If You are an
+   employee, You have had Your employer approve this Agreement or sign the Entity
+   version of this document. If You are less than eighteen years old, please have
+   Your parents or guardian sign the Agreement.
+
+
+4. Disclaimer
+=============
+
+EXCEPT FOR THE EXPRESS WARRANTIES IN SECTION 3, THE CONTRIBUTION IS PROVIDED
+"AS IS". MORE PARTICULARLY, ALL EXPRESS OR IMPLIED WARRANTIES INCLUDING, WITHOUT
+LIMITATION, ANY IMPLIED WARRANTY OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NON-INFRINGEMENT ARE EXPRESSLY DISCLAIMED BY YOU TO US AND BY US
+TO YOU. TO THE EXTENT THAT ANY SUCH WARRANTIES CANNOT BE DISCLAIMED, SUCH
+WARRANTY IS LIMITED IN DURATION TO THE MINIMUM PERIOD PERMITTED BY LAW.
+
+
+5. Consequential Damage Waiver
+==============================
+
+TO THE MAXIMUM EXTENT PERMITTED BY APPLICABLE LAW, IN NO EVENT WILL YOU OR US
+BE LIABLE FOR ANY LOSS OF PROFITS, LOSS OF ANTICIPATED SAVINGS, LOSS OF DATA,
+INDIRECT, SPECIAL, INCIDENTAL, CONSEQUENTIAL AND EXEMPLARY DAMAGES ARISING OUT
+OF THIS AGREEMENT REGARDLESS OF THE LEGAL OR EQUITABLE THEORY (CONTRACT, TORT
+OR OTHERWISE) UPON WHICH THE CLAIM IS BASED.
+
+
+6. Miscellaneous
+================
+
+
+6.1 Jurisdiction
+----------------
+
+This Agreement will be governed by and construed in accordance with the laws of
+Puerto Rico excluding its conflicts of law provisions. Under certain circumstances,
+the governing law in this section might be superseded by the United Nations
+Convention on Contracts for the International Sale of Goods ("UN Convention")
+and the parties intend to avoid the application of the UN Convention to this
+Agreement and, thus, exclude the application of the UN Convention in its
+entirety to this Agreement.
+
+
+6.2 Acceptance
+--------------
+
+This Agreement sets out the entire agreement between You and Us for Your
+Contributions to Us and overrides all other agreements or understandings.
+
+
+6.3 Third parties
+-----------------
+
+If You or We assign the rights or obligations received through this Agreement
+to a third party, as a condition of the assignment, that third party must
+agree in writing to abide by all the rights and obligations in the Agreement.
+
+
+6.4 Unmet responsibilities
+--------------------------
+
+The failure of either party to require performance by the other party of any
+provision of this Agreement in one situation shall not affect the right of a
+party to require such performance at any time in the future. A waiver of
+performance under a provision in one situation shall not be considered a
+waiver of the performance of the provision in the future or a waiver of the
+provision in its entirety.
+
+
+6.5 Continuation
+----------------
+
+If any provision of this Agreement is found void and unenforceable,
+such provision will be replaced to the extent possible with a provision that
+comes closest to the meaning of the original provision and which is enforceable.
+The terms and conditions set forth in this Agreement shall apply notwithstanding
+any failure of essential purpose of this Agreement or any limited remedy to
+the maximum extent possible under law.
+
+
+::
+
+
+    Name: _________________________________________________
+
+
+    Email: ________________________________________________
+
+
+    Address: ______________________________________________
+
+
+    Address (cont): _______________________________________
+
+
+    Country: ______________________________________________
+
+
+    Phone number: _________________________________________
+
+
+    Fax number: ___________________________________________
+
+
+
+    _______________________________________________________
+                             Signature
+
+
+
+Mayan EDMS Individual CAA Version 1.0 April 8, 2018

docs/chapters/development/code_structure.txt

@@ -0,0 +1,243 @@
+==============
+Code structure
+==============
+
+
+Follow PEP8
+-----------
+
+Whenever possible, but don't obsess over things like line length:
+
+.. code-block:: bash
+
+    $ flake8 --ignore=E501,E128,E122 |less
+
+To perform automatic PEP8 checks, install flake8's git hook using:
+
+.. code-block:: bash
+
+    $ flake8 --install-hook git
+
+
+Imports
+-------
+
+Import order should be:
+
+- Standard Python modules
+- Installed Python modules
+- Core Django modules
+- Installed Django modules
+- Mayan EDMS modules
+- Local imports
+
+Example:
+
+.. code-block:: bash
+
+    from __future__ import absolute_import
+
+    # Standard Python library
+    import base64
+
+    # 3rd party installed Python libraries
+    import requests
+
+    # Django core modules
+    from django.db.models import Q
+    from django.template.defaultfilters import slugify
+    from django.utils.translation import ugettext
+    from django.utils.translation import ugettext_lazy as _
+
+    # 3rd party installed Django libraries
+    from rest_framework import APIView
+
+    # Mayan apps
+    from metadata.classes import MetadataClass
+
+    # Local app imports (relative)
+    from .conf.settings import (
+        AVAILABLE_INDEXING_FUNCTIONS,
+        MAX_SUFFIX_COUNT, SLUGIFY_PATHS
+    )
+    from .exceptions import MaxSuffixCountReached
+    from .filesystem import (
+        fs_create_index_directory, fs_create_document_link,
+        fs_delete_document_link, fs_delete_index_directory,
+        assemble_suffixed_filename
+    )
+    from .models import Index, IndexInstanceNode, DocumentRenameCount
+
+All local app module imports are in relative form. Local app module name is to
+be referenced as little as possible, unless required by a specific feature,
+trick, restriction (e.g., Runtime modification of the module's attributes).
+
+Incorrect:
+
+.. code-block:: bash
+
+
+    # documents app views.py model
+    from documents.models import Document
+
+Correct:
+
+.. code-block:: bash
+
+    # documents app views.py model
+    from .models import Document
+
+
+Dependencies
+------------
+
+Mayan EDMS apps follow a hierarchical model of dependency. Apps import from
+their parents or siblings, never from their children. Think plugins. A parent
+app must never assume anything about a possible existing child app. The
+documents app and the Document model are the basic entities; they must never
+import anything else. The common and main apps are the base apps.
+
+
+Variables
+---------
+
+Naming of variables should follow a Major to Minor convention, usually
+including the purpose of the variable as the first piece of the name, using
+underscores as spaces. camelCase is not used in Mayan EDMS.
+
+Examples:
+
+Links:
+
+.. code-block:: bash
+
+    link_document_page_transformation_list = ...
+    link_document_page_transformation_create = ...
+    link_document_page_transformation_edit = ...
+    link_document_page_transformation_delete = ...
+
+Constants:
+
+.. code-block:: bash
+
+    PERMISSION_SMART_LINK_VIEW = ...
+    PERMISSION_SMART_LINK_CREATE = ...
+    PERMISSION_SMART_LINK_DELETE = ...
+    PERMISSION_SMART_LINK_EDIT = ...
+
+Classes:
+
+.. code-block:: bash
+
+    class Document(models.Model):
+    class DocumentPage(models.Model):
+    class DocumentPageTransformation(models.Model):
+    class DocumentType(models.Model):
+    class DocumentTypeFilename(models.Model):
+
+
+Strings
+-------
+
+Quotation character used in Mayan EDMS for strings is the single quote.
+Double quote is used for multiple line comments or HTML markup.
+
+
+Migrations
+----------
+
+Migrations should do only one thing (example: either create a table, move data
+to a new table or remove an old table) to aid retrying on failure.
+
+
+General
+-------
+
+Code should appear in their modules in alphabetic order or in their order of
+importance if it makes more sense for the specific application. This makes
+visual scanning easier on modules with a large number of imports, views or
+classes. Class methods that return a value should be pretended with a
+``get_`` to differentiate from an object’s properties. When a variable refers
+to a file it should be named as follows:
+
+- filename:  The file’s name and extension only.
+- filepath:  The entire path to the file including the filename.
+- path:  A path to a directory.
+
+Flash messages should end with a period as applicable for the language.
+Only exception is when the tail of the message contains an exceptions message
+as passed directly from the exception object.
+
+
+Steps to deploy a development version
+=====================================
+
+.. code-block:: bash
+
+    $ git clone |SOURCE_CODE_GIT|
+    $ cd mayan-edms
+    $ git checkout <Corresponding branch>
+    $ virtualenv venv
+    $ source venv/bin/activate
+    $ pip install -r requirements.txt
+    $ ./manage.py initialsetup
+    $ ./manage.py preparestatic
+    $ ./manage.py runserver
+
+
+
+Debugging
+=========
+
+Mayan EDMS makes extensive use of Django's new
+:django-docs:`logging capabilities <topics/logging>`.
+
+By default debug logging for all apps is turned on. If you wish to customize
+how logging is managed turn off automatic logging by setting
+`COMMON_AUTO_LOGGING` to ``False`` and add the following lines to your
+``settings/local.py`` file::
+
+    LOGGING = {
+        'version': 1,
+        'disable_existing_loggers': True,
+        'formatters': {
+            'verbose': {
+                'format': '%(levelname)s %(asctime)s %(name)s %(process)d %(thread)d %(message)s'
+            },
+            'intermediate': {
+                'format': '%(name)s <%(process)d> [%(levelname)s] "%(funcName)s() %(message)s"'
+            },
+            'simple': {
+                'format': '%(levelname)s %(message)s'
+            },
+        },
+        'handlers': {
+            'console':{
+                'level':'DEBUG',
+                'class':'logging.StreamHandler',
+                'formatter': 'intermediate'
+            }
+        },
+        'loggers': {
+            'documents': {
+                'handlers':['console'],
+                'propagate': True,
+                'level':'DEBUG',
+            },
+            'common': {
+                'handlers':['console'],
+                'propagate': True,
+                'level':'DEBUG',
+            },
+        }
+    }
+
+
+Likewise, to see the debug output of the ``tags`` app, just add the following inside the ``loggers`` block::
+
+
+    'tags': {
+        'handlers':['console'],
+        'propagate': True,
+        'level':'DEBUG',
+    },

docs/chapters/development/contributing.txt

@@ -0,0 +1,18 @@
+============
+Contributing
+============
+
+Mayan EDMS is under active development, and contributions are welcome.
+
+If you have a feature request, suggestion or bug report, please open a new
+issue on the `GitLab issue tracker`_. To submit patches, please send a merge
+request on GitLab_.
+
+.. _GitLab: |SOURCE_CODE_REPOSITORY|
+.. _`GitLab issue tracker`: |SOURCE_CODE_ISSUES|
+
+
+Contributing changes
+====================
+Follow the latest contributing guidelines outlined here:
+|SOURCE_CODE_REPOSITORY|blob/master/CONTRIBUTING.md

docs/chapters/development/documentation.txt

@@ -0,0 +1,24 @@
+=============
+Documentation
+=============
+
+The documentation is written in `reStructured Text`_ format, processed with
+Sphinx_, and resides in the ``docs`` directory. In order to build it, you will
+first need to install the documentation editing dependencies with::
+
+    $ pip install -r requirements/documentation.txt
+
+Then, to build an HTML version of the documentation, run the following command
+from the **docs** directory::
+
+    $ make docs-serve
+
+The generated documentation can be viewed by browsing to http://127.0.0.1:8000
+or by browsing to the ``docs/_build/html`` directory.
+
+You can also generate the documentation in formats other than HTML. Consult the
+Sphinx_ documentation for more details.
+
+.. _`reStructured Text`: http://docutils.sourceforge.net/rst.html
+.. _Sphinx: http://sphinx.pocoo.org
+

docs/chapters/development/index.txt

@@ -0,0 +1,29 @@
+==============
+For developers
+==============
+
+- :doc:`philosophies`
+- :doc:`code_structure`
+- :doc:`documentation`
+- :doc:`source_control`
+- :doc:`contributing`
+- :doc:`releases`
+- :doc:`translations`
+- :doc:`../../mercs/index`
+- :doc:`caa_entity`
+- :doc:`caa_individual`
+
+
+.. toctree::
+    :hidden:
+
+    philosophies
+    code_structure
+    documentation
+    source_control
+    contributing
+    releases
+    translations
+    ../../mercs/index
+    caa_entity
+    caa_individual

docs/chapters/development/philosophies.txt

@@ -0,0 +1,29 @@
+====================
+Project philosophies
+====================
+
+How to think about Mayan EDMS when doing changes or adding new features;
+why things are the way they are in Mayan EDMS:
+
+- Functionality must be as market/sector independent as possible, code for the
+  95% of use cases.
+- Each user must be able to configure and customize it to their needs after
+  install.
+- Abstract as much as possible, each app must be an expert in just one thing,
+  for other things they should use the API/classes/functions of other apps.
+- Assume as little as possible about anything outside the project
+  (hardware, OS, storage).
+- Provide Python based abstraction so that a default install runs with a single
+  step.
+- No hard dependencies on binaries unless there is no other choice.
+- Provide “drivers” or switchable backends to allow users to fine tune the
+  installation.
+- Call to binaries only when there is no other choice or the Python choices are
+  not viable/mature/efficient.
+- Each app is as independent and self contained as possible. Exceptions, the
+  basic requirements: navigation, permissions, common, main.
+- If an app is meant to be used by more than one other app, it should be as
+  generic as possible in regard to the project and another app will bridge the functionality.
+
+  - Example: since indexing (document_indexing) only applies to documents, the
+    app is specialized and depends on the documents app.

docs/chapters/development/releases.txt

@@ -0,0 +1,173 @@
+===================
+Installable package
+===================
+
+Source file package
+-------------------
+
+This is the sequence of step used to produce an installable package:
+
+1. Generate the packaged version (will produce dist/mayan-edms-x.y.z.tar.gz)::
+
+    $ make sdist
+
+2. Do a test install::
+
+    $ cd /tmp
+    $ virtualenv venv
+    $ source venv/bin/activate
+    $ pip install <path of the Git repository>/dist/mayan-edms-x.y.z.tar.gz
+    $ mayan-edms.py initialsetup
+    $ mayan-edms.py preparestatic
+    $ mayan-edms.py runserver
+
+
+Wheel package
+-------------
+
+1. Install the development requirements::
+
+    $ pip install -r requirements/development.txt
+
+2. Create wheel package using the makefile::
+
+    $ make wheel
+
+3. Do a test install::
+
+    $ cd /tmp
+    $ virtualenv venv
+    $ source venv/bin/activate
+    $ pip install <path of the Git repository>/dist/mayan_edms-x.y.z-py2-none-any.whl
+    $ mayan-edms.py initialsetup
+    $ mayan-edms.py preparestatic
+    $ mayan-edms.py runserver
+
+
+Version numbering
+=================
+
+Mayan EDMS uses the Semantic Versioning (http://semver.org/) method to choose
+version numbers along with Python's PEP-0440 (https://www.python.org/dev/peps/pep-0440/)
+to format them.
+
+X.YaN   # Alpha release
+X.YbN   # Beta release
+X.YrcN  # Release Candidate
+X.Y     # Final release
+
+
+Release checklist
+=================
+
+#. Check for missing migrations::
+
+    make check-missing-migrations
+
+#. Synchronize translations::
+
+    make translations-pull
+
+#. Compile translations::
+
+    make translations-compile
+
+#. Update changelog.
+#. Write release notes.
+#. Scan the code with flake8 for simple style warnings.
+#. Check README.rst format with::
+
+    python setup.py check -r -s
+
+   or with::
+
+       make check-readme
+
+#. Bump version in ``mayan/__init__.py`` and ``docker/rootfs/version``::
+
+    make increase-version PART=<major, minor or micro>
+
+#. Update requirements version in ``setup.py`` using:
+   ::
+
+       make generate-setup
+
+#. Commit as version bump.
+#. Build source package and test:
+   ::
+
+       make test-sdist-via-docker-ubuntu
+
+#. Build wheel package and test:
+   ::
+
+       make test-wheel-via-docker-ubuntu
+
+#. Tag version:
+   ::
+
+       git tag -a vX.Y.Z -m "Version X.Y.Z"
+
+#. Generate set ``setup.py`` again to update the build number::
+
+    make generate-setup
+
+#. Commit the new ``setup.py`` file.
+
+#. Release the version using one of the two following methods: GitLab CI or
+   manual
+
+
+Release using GitLab CI using the makefile
+------------------------------------------
+
+#. Use of the GitLab makefile targets: ``gitlab-release-all``,
+   ``gitlab-release-docker``, ``gitlab-release-documentation``,
+   ``gitlab-release-python``.
+
+
+Release using GitLab CI manually
+--------------------------------
+
+#. Push the current brach:
+   ::
+
+       git push
+
+
+#. Push the new tags:
+   ::
+
+       git push --tags
+
+
+#. Delete the corresponding ``releases/`` branch:
+   ::
+
+       git push origin :releases/<branch>
+
+#. Push the current branch to the corresponding origin ``releases/`` branch:
+   ::
+
+       git push origin <local branch>:releases/<branch>
+
+
+Manual release
+--------------
+
+#. Build and upload a test release:
+   ::
+
+       make release-test-via-docker-ubuntu
+
+#. Build and upload a final release:
+   ::
+
+       make release-via-docker-ubuntu
+
+Other steps
+-----------
+
+#. Update the contrib/scripts/install/docker.sh values
+
+#. Upload contrib/scripts/install/docker.sh to https://get.mayan-edms.com

docs/chapters/development/source_control.txt

@@ -0,0 +1,99 @@
+==============
+Source Control
+==============
+
+Mayan EDMS source is controlled with Git_.
+
+The project is publicly accessible, hosted and can be cloned from **GitLab** using::
+
+    $ git clone |SOURCE_CODE_GIT|
+
+
+Git branch structure
+--------------------
+
+Mayan EDMS follows a simplified model layout based on Vincent Driessen's
+`Successful Git Branching Model`_ blog post.
+
+``/versions/micro``
+    Working branch for the next bugfix release. Micro increment (third digit).
+    Only bug fixes, minor features, back-ported urgent features. This
+    branch is stable and safe for production.
+``/versions/minor``
+    Working branch for the next minor release (second digit). New features,
+    occasional breakage. Not for production but should run in test
+    environment most of the time. This is the branch you will want to
+    try out if you want to check out new features.
+``/versions/major``
+    Working branch for the next major release (first digit). New features,
+    incompatible changes to the user facing interfaces. Broken most of the
+    time, not for production and should only be cloned by developers
+    with experience with Mayan's development.
+``master``
+    Current production release (|version|). Points to the latest version of
+    the latest series. Production quality code.
+``features/``
+    Working branches for unfinished and unmerged feature. Likely unstable,
+    don't use in production. Once the feature is complete, it is merged
+    into one of the versions branches and deleted.
+
+Special branches:
+
+``releases/all``
+    Pushing code to this branch will trigger the build and release
+    a new Docker image, Documentation and Python package.
+``releases/docker``
+    Pushing code to this branch will trigger the build and release
+    of a new Docker image to Docker Hub.
+``releases/documentation``
+    Pushing code to this branch will trigger the build and release
+    of new documentation.
+``releases/python``
+    Pushing code to this branch will trigger the build and release
+    of a new Python package to PyPI.
+``nightly``
+    Pushing code to this branch will trigger the build and release
+    of a new Docker image based on development code to the GitLab image
+    repository only. The image will not be published to Docker Hub.
+
+Each release is tagged separately using annotated Git tags.
+
+When submitting patches, please place your code in its own ``feature/`` branch
+prior to opening a Merge Request on |SOURCE_CODE_ISSUES|.
+
+.. _Git: http://git-scm.org
+.. _`Successful Git Branching Model`: http://nvie.com/posts/a-successful-git-branching-model/
+
+
+Commit messages
+---------------
+
+#. Use English as the language for the commit messages.
+#. Provide a subject line composed of a tag and a short explanation::
+
+    Indexing: Add document base property reindex
+
+#. Keep the subject line to 50 or less characters.
+#. Capitalize the subject line.
+#. Don't end the subject line with a period, leave like a phrase in English.
+#. Use active voice in the. Say what the commit will do when applied not what
+   you did::
+
+       Add new properties to the model.
+
+   Vs.
+   ::
+
+       Added new properties to the model.
+
+#. Limit the body of the commit to 72 characters.
+#. When a commit fixes or improves an issue add the issue number in the commit
+   message. Either in the subject or in the body.
+#. Sign commit messages.
+#. Use explicit language even for minor commits. Don't do::
+
+       Fix typo
+
+   Use::
+
+       Document: Fix typo in label description

docs/chapters/development/translations.txt

@@ -0,0 +1,17 @@
+============
+Translations
+============
+
+Translations are handled online via the **Transifex** website:
+https://www.transifex.com/projects/p/mayan-edms/. To create a translation team
+for a new language or contribute to an already existing language translation,
+create a **Transifex** account and contact the team coordinator of the
+respective language in which you are interested.
+
+Feel free to open translation issues inside **Transifex** itself if you have a
+question about the usage or meaning of a source text string. If you open a
+translation issue, it will be your responsibility to close it after you get an
+answers that satisfies your question. Administrator will not close new issues
+as they have no way to determine if your question has been properly answered.
+However to avoid clutter, answered questions will be scanned periodically and
+closed if no activity is observed from the original poster in a period of time.

docs/chapters/docker.rst

@@ -1,417 +0,0 @@
-============
-Docker image
-============
-
-How to use this image
-=====================
-
-.. _docker_install:
-
-Start a Mayan EDMS image
-------------------------
-
-With Docker properly installed, proceed to download the Mayan EDMS image using the command::
-
-    docker pull mayanedms/mayanedms:<version>
-
-Then download version 9.5 of the Docker PostgreSQL image::
-
-    docker pull postgres:9.5
-
-Create and run a PostgreSQL container::
-
-    docker run -d \
-    --name mayan-edms-postgres \
-    --restart=always \
-    -p 5432:5432 \
-    -e POSTGRES_USER=mayan \
-    -e POSTGRES_DB=mayan \
-    -e POSTGRES_PASSWORD=mayanuserpass \
-    -v /docker-volumes/mayan-edms/postgres:/var/lib/postgresql/data \
-    -d postgres:9.5
-
-The PostgreSQL container will have one database named ``mayan``, with an user
-named ``mayan`` too, with a password of ``mayanuserpass``. The container will
-expose its internal 5432 port (PostgreSQL's default port) via the host's
-5432 port. The data of this container will reside on the host's
-``/docker-volumes/mayan-edms/postgres`` folder.
-
-Finally create and run a Mayan EDMS container. Change <version> with the
-latest version in numeric form (example: 2.7.3) or use the ``latest``
-identifier::
-
-    docker run -d \
-    --name mayan-edms \
-    --restart=always \
-    -p 80:8000 \
-    -e MAYAN_DATABASE_ENGINE=django.db.backends.postgresql \
-    -e MAYAN_DATABASE_HOST=172.17.0.1 \
-    -e MAYAN_DATABASE_NAME=mayan \
-    -e MAYAN_DATABASE_PASSWORD=mayanuserpass \
-    -e MAYAN_DATABASE_USER=mayan \
-    -e MAYAN_DATABASE_CONN_MAX_AGE=60 \
-    -v /docker-volumes/mayan-edms/media:/var/lib/mayan \
-    mayanedms/mayanedms:<version>
-
-The Mayan EDMS container will connect to the PostgreSQL container via the
-``172.17.0.1`` IP address (the Docker host's default IP address). It will
-connect using the ``django.db.backends.postgresql`` database driver and
-connect to the ``mayan`` database using the ``mayan`` user with the password
-``mayanuserpass``. The container will keep connections to the database
-for up to 60 seconds in an attempt to reuse them increasing response time
-and reducing memory usage. The files of the container will be store in the
-host's ``/docker-volumes/mayan-edms/media`` folder. The container will
-expose its web service running on port 8000 on the host's port 80.
-
-The container will be available by browsing to ``http://localhost`` or to
-the IP address of the computer running the container.
-
-If another web server is running on port 80 use a different port in the
-``-p`` option. For example: ``-p 81:8000``.
-
-
-Using a dedicated Docker network
---------------------------------
-Use this method to avoid having to expose PostreSQL port to the host's network
-or if you have other PostgreSQL instances but still want to use the default
-port of 5432 for this installation.
-
-Create the network::
-
-    docker network create mayan
-
-Launch the PostgreSQL container with the network option and remove the port
-binding (``-p 5432:5432``)::
-
-    docker run -d \
-    --name mayan-edms-postgres \
-    --network=mayan \
-    --restart=always \
-    -e POSTGRES_USER=mayan \
-    -e POSTGRES_DB=mayan \
-    -e POSTGRES_PASSWORD=mayanuserpass \
-    -v /docker-volumes/mayan-edms/postgres:/var/lib/postgresql/data \
-    -d postgres:9.5
-
-Launch the Mayan EDMS container with the network option and change the
-database hostname to the PostgreSQL container name (``mayan-edms-postgres``)
-instead of the IP address of the Docker host (``172.17.0.1``)::
-
-    docker run -d \
-    --name mayan-edms \
-    --network=mayan \
-    --restart=always \
-    -p 80:8000 \
-    -e MAYAN_DATABASE_ENGINE=django.db.backends.postgresql \
-    -e MAYAN_DATABASE_HOST=mayan-edms-postgres \
-    -e MAYAN_DATABASE_NAME=mayan \
-    -e MAYAN_DATABASE_PASSWORD=mayanuserpass \
-    -e MAYAN_DATABASE_USER=mayan \
-    -e MAYAN_DATABASE_CONN_MAX_AGE=60 \
-    -v /docker-volumes/mayan-edms/media:/var/lib/mayan \
-    mayanedms/mayanedms:<version>
-
-Stopping and starting the container
------------------------------------
-
-To stop the container use::
-
-    docker stop mayan-edms
-
-
-To start the container again::
-
-    docker start mayan-edms
-
-
-.. _docker_environment_variables:
-
-Environment Variables
----------------------
-
-The Mayan EDMS image can be configure via environment variables.
-
-``MAYAN_DATABASE_ENGINE``
-
-Defaults to ``None``. This environment variable configures the database
-backend to use. If left unset, SQLite will be used. The database backends
-supported by this Docker image are:
-
-- ``'django.db.backends.postgresql'``
-- ``'django.db.backends.mysql'``
-- ``'django.db.backends.sqlite3'``
-
-When using the SQLite backend, the database file will be saved in the Docker
-volume. The SQLite database as used by Mayan EDMS is meant only for development
-or testing, never use it in production.
-
-``MAYAN_DATABASE_NAME``
-
-Defaults to 'mayan'. This optional environment variable can be used to define
-the database name that Mayan EDMS will connect to. For more information read
-the pertinent Django documentation page:
-:django-docs:`Connecting to the database <ref/databases/#connecting-to-the-database>`
-
-``MAYAN_DATABASE_USER``
-
-Defaults to 'mayan'. This optional environment variable is used to set the
-username that will be used to connect to the database. For more information
-read the pertinent Django documentation page:
-:django-docs:`Settings, USER <ref/settings/#user>`
-
-``MAYAN_DATABASE_PASSWORD``
-
-Defaults to ''. This optional environment variable is used to set the
-password that will be used to connect to the database. For more information
-read the pertinent Django documentation page:
-:django-docs:`Settings, PASSWORD <ref/settings/#password>`
-
-``MAYAN_DATABASE_HOST``
-
-Defaults to `None`. This optional environment variable is used to set the
-hostname that will be used to connect to the database. This can be the
-hostname of another container or an IP address. For more information read
-the pertinent Django documentation page:
-:django-docs:`Settings, HOST <ref/settings/#host>`
-
-``MAYAN_DATABASE_PORT``
-
-Defaults to `None`. This optional environment variable is used to set the
-port number to use when connecting to the database. An empty string means
-the default port. Not used with SQLite. For more information read the
-pertinent Django documentation page:
-:django-docs:`Settings, PORT <ref/settings/#port>`
-
-``MAYAN_BROKER_URL``
-
-This optional environment variable determines the broker that Celery will use
-to relay task messages between the frontend code and the background workers.
-For more information read the pertinent Celery Kombu documentation page: `Broker URL`_
-
-.. _Broker URL: http://kombu.readthedocs.io/en/latest/userguide/connections.html#connection-urls
-
-This Docker image supports using Redis and RabbitMQ as brokers.
-
-Caveat: If the `MAYAN_BROKER_URL` and `MAYAN_CELERY_RESULT_BACKEND` environment
-variables are specified, the built-in Redis server inside the container will
-be disabled.
-
-``MAYAN_CELERY_RESULT_BACKEND``
-
-This optional environment variable determines the results backend that Celery
-will use to relay result messages from the background workers to the frontend
-code. For more information read the pertinent Celery Kombu documentation page:
-`Task result backend settings`_
-
-.. _Task result backend settings: http://docs.celeryproject.org/en/3.1/configuration.html#celery-result-backend
-
-This Docker image supports using Redis and RabbitMQ as result backends.
-
-Caveat: If the `MAYAN_BROKER_URL` and `MAYAN_CELERY_RESULT_BACKEND` environment
-variables are specified, the built-in Redis server inside the container will
-be disabled.
-
-``MAYAN_SETTINGS_MODULE``
-
-Optional. Allows loading an alternate settings file.
-
-``MAYAN_DATABASE_CONN_MAX_AGE``
-
-Amount in seconds to keep a database connection alive. Allow reuse of database
-connections. For more information read the pertinent Django documentation
-page: :django-docs:`Settings, CONN_MAX_AGE <ref/settings/#conn-max-age>`
-
-``MAYAN_GUNICORN_WORKERS``
-
-Optional. This environment variable controls the number of frontend workers
-that will be executed. If not specified the default is 2. For heavier loads,
-user a higher number. A formula recommended for this setting is the number
-of CPU cores + 1.
-
-``MAYAN_WORKER_FAST_CONCURRENCY``
-
-Optional. Changes the concurrency (number of child processes) of the Celery
-worker consuming the queues in the fast (low latency, short tasks) category.
-Default is 1. Use 0 to disable hardcoded concurrency and allow the Celery
-worker to launch its default number of child processes (equal to the number
-of CPUs detected).
-
-``MAYAN_WORKER_MEDIUM_CONCURRENCY``
-
-Optional. Changes the concurrency (number of child processes) of the Celery
-worker consuming the queues in the medium (medium latency, long running tasks)
-category. Default is 1. Use 0 to disable hardcoded concurrency and allow the
-Celery worker to launch its default number of child processes (equal to the
-number of CPUs detected).
-
-``MAYAN_WORKER_SLOW_CONCURRENCY``
-
-Optional. Changes the concurrency (number of child processes) of the Celery
-worker consuming the queues in the slow (high latency, very long running tasks)
-category. Default is 1. Use 0 to disable hardcoded concurrency and allow the
-Celery worker to launch its default number of child processes (equal to the
-number of CPUs detected).
-
-Accessing outside data
-======================
-
-To use Mayan EDMS's staging folders or watch folders from Docker, the data
-for these source must be made accessible to the container. This is done by
-mounting the folders in the host computer to folders inside the container.
-This is necessary because Docker containers do not have access to host data
-on purpose. For example, to make a folder in the host accessible as a watch
-folder, add the following to the Docker command line when starting the
-container::
-
-    -v /opt/scanned_files:/srv/watch_folder
-
-The command line would look like this::
-
-    docker run ... -v /opt/scanned_files:/srv/watch_folder mayanedms/mayanedms:latest
-
-Now create a watch folder in Mayan EDMS using the path ``/srv/watch_folder``
-and the documents from the host folder ``/opt/scanned_files`` will be
-automatically available. Use the same procedure to mount host folders to be
-used as staging folderes. In this example ``/srv/watch_folder`` was as the
-container directory, but any path can be used as long as it is not an
-already existing path or a path used by any other program.
-
-
-Performing backups
-==================
-
-To backup the existing data, stop the image and copy the content of the volume.
-For the example::
-
-    docker run -d --name mayan-edms --restart=always -p 80:8000 \
-    -v /docker-volumes/mayan:/var/lib/mayan \
-    -v /opt/scanned_files:/srv/watch_folder mayanedms/mayanedms:latest
-
-That would be the ``/docker-volumes/mayan folder``::
-
-    sudo tar -zcvf backup.tar.gz /docker-volumes/mayan
-    sudo chown `whoami` backup.tar.gz
-
-If using an external PostgreSQL or MySQL database or database containers, these
-too need to be backed up using their respective procedures. A simple solution
-is to copy the entire database container volume after the container has
-been stopped.
-
-Restoring from a backup
-=======================
-
-Uncompress the backup archive in the original docker volume using::
-
-    sudo tar -xvzf backup.tar.gz -C /
-
-Upgrading
-=========
-
-Upgrading a Mayan EDMS Docker container is actually a matter of stopping and
-deleting the container, downloading the most recent version of the image and
-starting a container again. The container will take care of updating the
-database structure to the newest version if necessary.
-
-**IMPORTANT!** Do not delete the volume storing the data, only the container.
-
-Stop the container to be upgraded::
-
-    docker stop mayan-edms
-
-
-Remove the container::
-
-    docker rm mayan-edms
-
-
-Pull the new image version::
-
-    docker pull mayanedms/mayanedms:latest
-
-
-Start the container again with the new image version::
-
-    docker run -d --name mayan-edms --restart=always -p 80:8000 -v /docker-volumes/mayan:/var/lib/mayan mayanedms/mayanedms:latest
-
-Building the image
-==================
-
-Clone the repository with::
-
-    git clone https://gitlab.com/mayan-edms/mayan-edms.git
-
-Change to the directory of the cloned repository::
-
-    cd mayan-edms
-
-Execute Docker's build command using the provided makefile::
-
-    make docker-build
-
-Or using an apt cacher to speed up the build::
-
-    make docker-build-with-proxy APT_PROXY=172.17.0.1:3142
-
-Replace the IP address `172.17.0.1` with the IP address of the computer
-running the APT proxy and caching service.
-
-Customizing the image
-=====================
-
-Simple method
--------------
-
-If you just need to add a few Ubuntu or Python packages to your installation,
-you can use the following environment variables:
-
-``MAYAN_APT_INSTALLS``
-
-Specifies a list of Ubuntu .deb packages to be installed via APT when the
-container is first created. The installed packages are not lost when the image
-is stopped. Example: To install the Tesseract OCR language packs for German
-and Spanish add the following in your ``docker start`` command line::
-
-    -e MAYAN_APT_INSTALLS="tesseract-ocr-deu tesseract-ocr-spa"
-
-``MAYAN_PIP_INSTALLS``
-
-Specifies a list of Python packages to be installed via ``pip``. Packages will
-be downloaded from the Python Package Index (https://pypi.python.org) by
-default.
-
-Using Docker compose
-====================
-
-To deploy a complete production stack using the included Docker compose file
-execute::
-
-    docker-compose -f docker-compose.yml up -d
-
-This Docker compose file will provision four containers:
-
-- Postgres as the database
-- Redis as the Celery result storage
-- RabbitMQ as the Celery broker
-- Mayan EDMS using the above service containers
-
-To stop the stack use::
-
-    docker-compose -f docker-compose.yml stop
-
-The stack will also create four volumes to store the data of each container.
-These are:
-
-- mayan_app - The Mayan EDMS data container, normally called `mayan_data` when not using Docker compose.
-- mayan_broker - The broker volume, in this case RabbitMQ.
-- mayan_db - The database volume, in this case Postgres.
-- mayan_results - The celery result backend volume, in this case Redis.
-
-
-Nightly images
-==============
-The continuous integration pipeline used for testing development builds also
-produces a resulting Docker image. These are build automatically and their
-stability is not guaranteed. They should never be used in production.
-If you want to try out the Docker images the development uses or want a sneak
-peek at the new features being worked on checkout the container registry at:
-https://gitlab.com/mayan-edms/mayan-edms/container_registry

docs/chapters/docker/accessing_data.txt

@@ -0,0 +1,30 @@
+.. _docker-accessing-outside-data:
+
+
+======================
+Accessing outside data
+======================
+
+To use Mayan EDMS's staging folders or watch folders from Docker, the data
+for these source must be made accessible to the container. This is done by
+mounting the folders in the host computer to folders inside the container.
+This is necessary because Docker containers do not have access to host data
+on purpose. For example, to make a folder in the host accessible as a watch
+folder, add the following to the Docker command line when starting the
+container::
+
+    -v /opt/scanned_files:/scanned_files
+
+The command line would look like this::
+
+    docker run ... -v /opt/scanned_files:/scanned_files mayanedms/mayanedms:latest
+
+Now create a watch folder in Mayan EDMS using the path ``/scanned_files``
+and the documents from the host folder ``/opt/scanned_files`` will be
+automatically available. Use the same procedure to mount host folders to be
+used as staging folders. In this example ``/scanned_files`` was used as the
+container directory, but any path can be used as long as:
+
+- the path not an already existing path
+- the path is not used by any other program
+- the path is a single level path

docs/chapters/docker/backups.txt

@@ -0,0 +1,47 @@
+.. _docker_backups:
+
+==================
+Performing backups
+==================
+
+#. Document files
+
+   To backup the existing data, stop the image and copy the content of the volume.
+   For the example::
+
+       docker run -d --name mayan-edms --restart=always -p 80:8000 \
+       -v /docker-volumes/mayan:/var/lib/mayan \
+       -v /opt/scanned_files:/scanned_files mayanedms/mayanedms:|DOCKER_MAYAN_IMAGE_VERSION|
+
+   That would be the ``/docker-volumes/mayan folder``::
+
+       sudo tar -zcvf backup.tar.gz /docker-volumes/mayan
+       sudo chown `whoami` backup.tar.gz
+
+#. Database
+
+   If using an external PostgreSQL or MySQL database or database containers, these
+   too need to be backed up using their respective procedures. A simple solution
+   is to copy the entire database container volume after the container has
+   been stopped.
+
+   - PostgreSQL: https://www.postgresql.org/docs/current/backup-dump.html
+   - MySQL: https://dev.mysql.com/doc/refman/5.7/en/mysqldump.html
+
+
+=======================
+Restoring from a backup
+=======================
+
+
+#. Document files
+
+   Uncompress the backup archive in the original docker volume using::
+
+       sudo tar -xvzf backup.tar.gz -C /
+
+
+#. Database
+
+   - PostgreSQL: https://www.postgresql.org/docs/current/backup-dump.html
+   - MySQL: https://dev.mysql.com/doc/refman/5.7/en/mysqldump.html

docs/chapters/docker/building.txt

@@ -0,0 +1,25 @@
+.. _docker_building:
+
+==================
+Building the image
+==================
+
+Clone the repository with::
+
+    git clone |SOURCE_CODE_GIT|
+
+Change to the directory of the cloned repository::
+
+    cd mayan-edms
+
+Execute Docker's build command using the provided makefile::
+
+    make docker-build
+
+Or using an APT cache to speed up the build::
+
+    make docker-build-with-proxy APT_PROXY=172.17.0.1:3142
+
+Replace the IP address `172.17.0.1` with the IP address of the computer
+running the APT proxy and caching service.
+

docs/chapters/docker/commands.txt

@@ -0,0 +1,10 @@
+.. _docker_commands:
+
+=================================
+Executing commands on a container
+=================================
+
+To perform Mayan EDMS commands in a running container that will take all the
+settings from the container, use the following command::
+
+    docker exec -ti <container name> /usr/local/bin/entrypoint.sh "/opt/mayan-edms/bin/mayan-edms.py <command>"

docs/chapters/docker/customizing.txt

@@ -0,0 +1,21 @@
+=====================
+Customizing the image
+=====================
+
+If you just need to add a few Ubuntu or Python packages to your installation,
+you can use the following environment variables:
+
+``MAYAN_APT_INSTALLS``
+
+Specifies a list of Ubuntu .deb packages to be installed via APT when the
+container is first created. The installed packages are not lost when the image
+is stopped. Example: To install the Tesseract OCR language packs for German
+and Spanish add the following in your ``docker start`` command line::
+
+    -e MAYAN_APT_INSTALLS="tesseract-ocr-deu tesseract-ocr-spa"
+
+``MAYAN_PIP_INSTALLS``
+
+Specifies a list of Python packages to be installed via ``pip``. Packages will
+be downloaded from the Python Package Index (https://pypi.python.org) by
+default.

docs/chapters/docker/docker_compose.txt

@@ -0,0 +1,30 @@
+.. _docker_compose:
+
+====================
+Using Docker compose
+====================
+
+To deploy a complete production stack using the included Docker compose file
+execute::
+
+    docker-compose -f docker-compose.yml up -d
+
+This Docker compose file will provision four containers:
+
+- Postgres as the database
+- Redis as the Celery result storage
+- RabbitMQ as the Celery broker
+- Mayan EDMS using the above service containers
+
+To stop the stack use::
+
+    docker-compose -f docker-compose.yml stop
+
+The stack will also create four volumes to store the data of each container.
+These are:
+
+- mayan_app - The Mayan EDMS data container, normally called `mayan_data` when not using Docker compose.
+- mayan_broker - The broker volume, in this case RabbitMQ.
+- mayan_db - The database volume, in this case Postgres.
+- mayan_results - The celery result backend volume, in this case Redis.
+

docs/chapters/docker/drivers.txt

@@ -0,0 +1,12 @@
+.. _docker_image_drivers:
+
+================
+Included drivers
+================
+
+The Docker image supports using Redis and RabbitMQ as result backends. For
+databases, the image includes support for PostgreSQL and MySQL/MariaDB.
+Support for additional brokers or databases may be added using the
+``MAYAN_APT_INSTALL`` environment variable.
+
+

docs/chapters/docker/environment_variables.txt

@@ -0,0 +1,63 @@
+.. index:: Docker environment variables
+.. _docker_environment_variables:
+
+
+=====================
+Environment Variables
+=====================
+
+The common set of settings can also be modified via environment variables when
+using the Docker image. In addition to the common set of settings, some Docker
+image specific environment variables are available.
+
+``MAYAN_SETTINGS_MODULE``
+
+Optional. Allows loading an alternate settings file.
+
+
+``MAYAN_GUNICORN_TIMEOUT``
+
+Optional. Changes the amount of time the frontend worker will wait for a
+request to finish before raising a timeout error. The default is 120
+seconds.
+
+``MAYAN_GUNICORN_WORKERS``
+
+Optional. This environment variable controls the number of frontend workers
+that will be executed. If not specified the default is 2. For heavier loads,
+user a higher number. A formula recommended for this setting is the number
+of CPU cores + 1.
+
+``MAYAN_WORKER_FAST_CONCURRENCY``
+
+Optional. Changes the concurrency (number of child processes) of the Celery
+worker consuming the queues in the fast (low latency, short tasks) category.
+Default is 1. Use 0 to disable hardcoded concurrency and allow the Celery
+worker to launch its default number of child processes (equal to the number
+of CPUs detected).
+
+``MAYAN_WORKER_MEDIUM_CONCURRENCY``
+
+Optional. Changes the concurrency (number of child processes) of the Celery
+worker consuming the queues in the medium (medium latency, long running tasks)
+category. Default is 1. Use 0 to disable hardcoded concurrency and allow the
+Celery worker to launch its default number of child processes (equal to the
+number of CPUs detected).
+
+``MAYAN_WORKER_SLOW_CONCURRENCY``
+
+Optional. Changes the concurrency (number of child processes) of the Celery
+worker consuming the queues in the slow (high latency, very long running tasks)
+category. Default is 1. Use 0 to disable hardcoded concurrency and allow the
+Celery worker to launch its default number of child processes (equal to the
+number of CPUs detected).
+
+``MAYAN_USER_UID``
+
+Optional. Changes the UID of the ``mayan`` user internal to the Docker
+container. Defaults to 1000.
+
+``MAYAN_USER_GID``
+
+Optional. Changes the GID of the ``mayan`` user internal to the Docker
+container. Defaults to 1000.

docs/chapters/docker/index.txt

@@ -0,0 +1,35 @@
+.. index:: Docker
+
+======
+Docker
+======
+
+Docker is a container technology. Containers are a standard unit of software
+that packages up code and all its dependencies.
+
+
+- Installation: :doc:`install_simple` | :doc:`install_docker_network` | :doc:`docker_compose` | :doc:`start_stop`
+- Setup: :doc:`environment_variables` | :doc:`drivers`
+- Integration: :doc:`accessing_data`
+- Development: :doc:`building` | :doc:`customizing` | :doc:`nightly`
+- Administration: :doc:`backups` | :doc:`upgrading` | :doc:`commands`
+
+
+.. toctree::
+    :hidden:
+
+    install_simple
+    install_docker_network
+    start_stop
+    environment_variables
+    drivers
+    accessing_data
+    backups
+    upgrading
+    building
+    customizing
+    docker_compose
+    nightly
+    commands
+
+

docs/chapters/docker/install_docker_network.txt

@@ -0,0 +1,67 @@
+.. _docker_install_networked:
+
+================================
+Using a dedicated Docker network
+================================
+
+Use this method to avoid having to expose PostreSQL port to the host's network
+or if you have other PostgreSQL instances but still want to use the default
+port of 5432 for this installation.
+
+#. Create the network:
+
+   .. code-block:: bash
+
+       docker network create mayan
+
+#. Launch the PostgreSQL container with the network option and remove the
+   port binding (``-p 5432:5432``):
+
+   .. code-block:: bash
+
+       docker run \
+       -d \
+       --name mayan-edms-postgres \
+       --network=mayan \
+       --restart=always \
+       -e POSTGRES_USER=mayan \
+       -e POSTGRES_DB=mayan \
+       -e POSTGRES_PASSWORD=mayanuserpass \
+       -v /docker-volumes/mayan-edms/postgres:/var/lib/postgresql/data \
+       |DOCKER_POSTGRES_IMAGE_VERSION|
+
+#. Launch the Redis container with the network option and remove the port
+   binding (``-p 6379:6379``):
+
+   .. code-block:: bash
+
+       docker run \
+       -d \
+       --name mayan-edms-redis \
+       --network=mayan \
+       |DOCKER_REDIS_IMAGE_VERSION| \
+       redis-server \
+       --databases \
+       "2" \
+       --maxmemory-policy \
+       allkeys-lru \
+       --save \
+       ""
+
+#. Launch the Mayan EDMS container with the network option and change the
+   database hostname to the PostgreSQL container name (``mayan-edms-postgres``)
+   instead of the IP address of the Docker host (``172.17.0.1``):
+
+   .. code-block:: bash
+
+       docker run \
+       -d \
+       --name mayan-edms \
+       --network=mayan \
+       --restart=always \
+       -p 80:8000 \
+       -e MAYAN_DATABASES="{'default':{'ENGINE':'django.db.backends.postgresql','NAME':'mayan','PASSWORD':'mayanuserpass','USER':'mayan','HOST':'mayan-edms-postgres'}}" \
+       -e MAYAN_CELERY_BROKER_URL="redis://mayan-edms-redis:6379/0" \
+       -e MAYAN_CELERY_RESULT_BACKEND="redis://mayan-edms-redis:6379/1" \
+       -v /docker-volumes/mayan-edms/media:/var/lib/mayan \
+       mayanedms/mayanedms:|DOCKER_MAYAN_IMAGE_VERSION|

docs/chapters/docker/install_simple.txt

@@ -0,0 +1,117 @@
+.. _docker_install:
+
+
+==========================
+Simple Docker installation
+==========================
+
+#. Install Docker:
+
+   .. code-block:: bash
+
+       wget -qO- https://get.docker.com/ | sh
+
+   If you don't want run an automated script, follow the instructions outlined
+   in their documentation: https://docs.docker.com/install/
+
+   Once the Docker installation is finished, proceed to the link below to install
+   the Docker image for Mayan EDMS.
+
+
+#. Download the Mayan EDMS Docker image:
+
+   With Docker properly installed, proceed to download the Mayan EDMS Docker
+   image using the command:
+
+   .. code-block:: bash
+
+       docker pull mayanedms/mayanedms:|DOCKER_MAYAN_IMAGE_VERSION|
+
+
+#. Download the PostgreSQL Docker image:
+
+   .. code-block:: bash
+
+       docker pull |DOCKER_POSTGRES_IMAGE_VERSION|
+
+
+#. Download the Redis Docker image:
+
+   .. code-block:: bash
+
+       docker pull |DOCKER_REDIS_IMAGE_VERSION|
+
+
+#. Create and run a PostgreSQL container:
+
+   .. code-block:: bash
+
+       docker run \
+       -d \
+       --name mayan-edms-postgres \
+       --restart=always \
+       -p 5432:5432 \
+       -e POSTGRES_USER=mayan \
+       -e POSTGRES_DB=mayan \
+       -e POSTGRES_PASSWORD=mayanuserpass \
+       -v /docker-volumes/mayan-edms/postgres:/var/lib/postgresql/data \
+       |DOCKER_POSTGRES_IMAGE_VERSION|
+
+   The PostgreSQL container will have one database named ``mayan``, with an user
+   named ``mayan`` too, with a password of ``mayanuserpass``. The container will
+   expose its internal 5432 port (PostgreSQL's default port) via the host's
+   5432 port. The data of this container will reside on the host's
+   ``/docker-volumes/mayan-edms/postgres`` folder.
+
+
+#. Create and run a Redis container:
+
+   .. code-block:: bash
+
+       docker run \
+       -d \
+       --name mayan-edms-redis \
+       --restart=always \
+       -p 6379:6379 \
+       |DOCKER_REDIS_IMAGE_VERSION| \
+       redis-server \
+       --databases \
+       "2" \
+       --maxmemory-policy \
+       allkeys-lru \
+       --save \
+       ""
+
+   The Redis container will have two databases, one for background task messages,
+   and the other to hold the results of those background tasks. Redis is
+   configure to not save its content to disk and to automatically clear up
+   memory.
+
+#. Create and run a Mayan EDMS container:
+
+   .. code-block:: bash
+
+       docker run \
+       -d \
+       --name mayan-edms \
+       --restart=always \
+       -p 80:8000 \
+       -e MAYAN_DATABASES="{'default':{'ENGINE':'django.db.backends.postgresql','NAME':'mayan','PASSWORD':'mayanuserpass','USER':'mayan','HOST':'172.17.0.1'}}" \
+       -e MAYAN_CELERY_BROKER_URL="redis://172.17.0.1:6379/0" \
+       -e MAYAN_CELERY_RESULT_BACKEND="redis://172.17.0.1:6379/1" \
+       -v /docker-volumes/mayan-edms/media:/var/lib/mayan \
+       mayanedms/mayanedms:|DOCKER_MAYAN_IMAGE_VERSION|
+
+   The Mayan EDMS container will connect to the PostgreSQL container via the
+   ``172.17.0.1`` IP address (the Docker host's default IP address). It will
+   connect using the ``django.db.backends.postgresql`` database driver and
+   connect to the ``mayan`` database using the ``mayan`` user with the password
+   ``mayanuserpass``. The files of the container will be store in the
+   host's ``/docker-volumes/mayan-edms/media`` folder. The container will
+   expose its web service running on port 8000 on the host's port 80.
+
+   The container will be available by browsing to ``http://localhost`` or to
+   the IP address of the computer running the container.
+
+   If another web server is running on port 80 use a different port in the
+   ``-p`` option. For example: ``-p 81:8000``.

docs/chapters/docker/nightly.txt

@@ -0,0 +1,12 @@
+==============
+Nightly images
+==============
+
+The continuous integration pipeline used for testing development builds also
+produces a resulting Docker image. These are build automatically and their
+stability is not guaranteed. They should never be used in production.
+If you want to try out the Docker images the development uses or want a sneak
+peek at the new features being worked on checkout the container registry at:
+|SOURCE_CODE_REPOSITORY|container_registry
+
+

docs/chapters/docker/start_stop.txt

@@ -0,0 +1,13 @@
+===================================
+Stopping and starting the container
+===================================
+
+To stop the container use::
+
+    docker stop mayan-edms
+
+
+To start the container again::
+
+    docker start mayan-edms
+

docs/chapters/docker/upgrading.txt

@@ -0,0 +1,30 @@
+=========
+Upgrading
+=========
+
+Upgrading a Mayan EDMS Docker container is actually a matter of stopping and
+deleting the container, downloading the most recent version of the image and
+starting a container again. The container will take care of updating the
+database structure to the newest version if necessary.
+
+**IMPORTANT!** Do not delete the volume storing the data, only the container.
+
+Stop the container to be upgraded::
+
+    docker stop mayan-edms
+
+
+Remove the container::
+
+    docker rm mayan-edms
+
+
+Pull the new image version::
+
+    docker pull mayanedms/mayanedms:latest
+
+
+Start the container again with the new image version::
+
+    docker run -d --name mayan-edms --restart=always -p 80:8000 -v /docker-volumes/mayan:/var/lib/mayan mayanedms/mayanedms:latest
+

docs/chapters/document_types.rst

@@ -1,72 +0,0 @@
-Document types
-==============
-
-The basic unit of data in Mayan EDMS is the **document type**. A document
-type can be interpreted also as a document category, a document class, or a
-document template. Every other aspect of the system will rely or be tied to
-one or more document type. Create one document type for each type or class of
-document you intend to upload into Mayan EDMS.
-
-Document types need to be created before documents can be
-uploaded. It is not possible to upload documents without assigning them a
-document type.
-
-Document types usually mirror the type of physical, paper document they
-represent.
-
-Example document types:
-
-* Letter
-* Invoice
-* Timesheet
-* Blueprint
-
-.. blockdiag::
-
-   blockdiag {
-      default_shape = roundedbox
-
-      document_type [ label = 'each document type' ];
-      documents [ label = 'many documents' ];
-
-      document_type -> documents;
-   }
-
-
-Examples:
-
-.. blockdiag::
-
-   blockdiag {
-      default_shape = roundedbox
-
-      document_type [ label = 'Invoice' ];
-      documents_1 [ label = 'invoice_001.pdf', width=200 ];
-      documents_2 [ label = 'invoice_032.pdf', width=200 ];
-
-      document_type -> documents_1, documents_2;
-   }
-
-
-.. blockdiag::
-
-   blockdiag {
-      default_shape = roundedbox
-
-      document_type [ label = 'Receipts' ];
-      documents_1 [ label = 'groceries_18-01-11.pdf', width=200  ];
-      documents_2 [ label = 'car_payment-17-01-02.png', width=200  ];
-
-      document_type -> documents_1, documents_2;
-   }
-
-
-Settings and attributes are applied to document types and documents will
-inherit those settings and attributes based on the document type they were
-assigned when uploaded into Mayan EDMS. A document can only be of one
-type at a given moment, but if needed, the type of a document can be changed.
-Upon changing its type, the document will lose its previous settings and
-attributes, and will inherit the settings and attributes of its new type.
-
-Document types are create in the
-:menuselection:`System --> Setup --> Document types` menu.

docs/chapters/document_types.txt

@@ -0,0 +1,70 @@
+==============
+Document types
+==============
+
+The basic unit of data in Mayan EDMS is the **document type**. A document
+type can be interpreted also as a document category, a document class, or a
+document template. Every other aspect of the system will rely or be tied to
+one or more document type. Create one document type for each type or class of
+document you intend to upload into Mayan EDMS.
+
+Document types need to be created before documents can be
+uploaded. It is not possible to upload documents without assigning them a
+document type.
+
+Document types usually mirror the type of physical, paper document they
+represent.
+
+Example document types:
+
+* Letter
+* Invoice
+* Timesheet
+* Blueprint
+
+.. blockdiag::
+
+   blockdiag {
+      default_shape = roundedbox
+
+      document_type [ label = 'each document type' ];
+      documents [ label = 'many documents' ];
+
+      document_type -> documents;
+   }
+
+
+Examples:
+
+.. blockdiag::
+
+   blockdiag {
+      default_shape = roundedbox
+
+      document_type [ label = 'Invoice' ];
+      documents_1 [ label = 'invoice_001.pdf', width=200 ];
+      documents_2 [ label = 'invoice_032.pdf', width=200 ];
+
+      document_type -> documents_1, documents_2;
+   }
+
+
+.. blockdiag::
+
+   blockdiag {
+      default_shape = roundedbox
+
+      document_type [ label = 'Receipts' ];
+      documents_1 [ label = 'groceries_18-01-11.pdf', width=200  ];
+      documents_2 [ label = 'car_payment-17-01-02.png', width=200  ];
+
+      document_type -> documents_1, documents_2;
+   }
+
+
+Settings and attributes are applied to document types and documents will
+inherit those settings and attributes based on the document type they were
+assigned when uploaded into Mayan EDMS. A document can only be of one
+type at a given moment, but if needed, the type of a document can be changed.
+Upon changing its type, the document will lose its previous settings and
+attributes, and will inherit the settings and attributes of its new type.

docs/chapters/file_storage.rst

@@ -1,36 +0,0 @@
-************
-File storage
-************
-
-The files are stored and placed under Mayan EDMS "control" to avoid
-filename clashes each file gets renamed to its ``UUID`` (Universally Unique ID),
-without extension, and stored in a simple flat arrangement in a directory.
-
-.. blockdiag::
-
-   blockdiag {
-      default_shape = roundedbox
-      file [ label = 'mayan_1-1.pdf', width=120];
-      document [ label = 'mayan/media/document_storage/ab6c1cfe-8a8f-4a30-96c9-f54f606b9248', width=450];
-      file -> document [label = "upload"];
-
-      file -> document;
-   }
-
-This doesn't stop access to the files but renaming, moving or updating
-directly them is not recommended because it would throw the database out
-of sync.
-
-Because Mayan EDMS components are as decoupled from each other as possible,
-storage in this case is decoupled and its behavior is controlled
-not by the project but by the ``Storage`` module class. All the other
-modules don't make any assumptions about how the actual document files are
-stored. This way files can be saved locally, over the network or even across
-the Internet and everything will still operate exactly the same.
-
-The default file storage backend: ``storage.backends.filebasedstorage.FileBasedStorage``
-is a simple backend that only supports paths and not IP addresses. In case you
-are interested in using remote volumes to store documents (NFS, SAMBA), first
-mount these volumes so that they appear as a directories to Mayan EDMS. For
-direct support for remote volumes a custom backend would be needed such as those
-provided by the Django Storages project (https://django-storages.readthedocs.org/en/latest/).

docs/chapters/file_storage.txt

@@ -0,0 +1,36 @@
+============
+File storage
+============
+
+The files are stored and placed under Mayan EDMS "control" to avoid
+filename clashes each file gets renamed to its ``UUID`` (Universally Unique ID),
+without extension, and stored in a simple flat arrangement in a directory.
+
+.. blockdiag::
+
+   blockdiag {
+      default_shape = roundedbox
+      file [ label = 'mayan_1-1.pdf', width=120];
+      document [ label = 'mayan/media/document_storage/ab6c1cfe-8a8f-4a30-96c9-f54f606b9248', width=450];
+      file -> document [label = "upload"];
+
+      file -> document;
+   }
+
+This doesn't stop access to the files but renaming, moving or updating
+directly them is not recommended because it would throw the database out
+of sync.
+
+Because Mayan EDMS components are as decoupled from each other as possible,
+storage in this case is decoupled and its behavior is controlled
+not by the project but by the ``Storage`` module class. All the other
+modules don't make any assumptions about how the actual document files are
+stored. This way files can be saved locally, over the network or even across
+the Internet and everything will still operate exactly the same.
+
+The default file storage backend: ``storage.backends.filebasedstorage.FileBasedStorage``
+is a simple backend that only supports paths and not IP addresses. In case you
+are interested in using remote volumes to store documents (NFS, SAMBA), first
+mount these volumes so that they appear as a directories to Mayan EDMS. For
+direct support for remote volumes a custom backend would be needed such as those
+provided by the Django Storages project (https://django-storages.readthedocs.org/en/latest/).

docs/chapters/index_examples.rst

@@ -1,115 +0,0 @@
-
-Index examples
---------------
-
-Index of document types
-^^^^^^^^^^^^^^^^^^^^^^^
-
-This index will create one level for each document type in the system and place
-links to the document of each respective type.
-
-#. Go to the :menuselection:`System --> Setup --> Indexes` menu.
-#. Create a new index using :menuselection:`Actions --> Create new`.
-#. Give it a label to describe it, and an internal name. The internal name is
-   used when referencing this index in other parts of the system.
-#. Press the :menuselection:`Template` link of the newly created index.
-#. Select :menuselection:`New child node` to create a new level in which the
-   following template code will be entered.
-   ::
-
-       {{ document.document_type }}
-
-#. Save the template.
-#. Go to :menuselection:`Document types` and associate this index with
-   existing document types in the system.
-#. Finally go to :menuselection:`Tools --> Rebuild indexes` to execute the
-   index template. The rebuild process is only necessary when changes are
-   made to the index templates. Otherwise they update automatically whenever
-   a new document is uploaded or existing documents properties are modified.
-#. A new index should appear under :menuselection:`Indexes` menu.
-
-You can also program different behavior based on the different document types,
-by use a comparison and a conditional statement. As the document_type itself
-is not a string you cannot directly use that for that comparison. You will have
-to use the label of the document type.
-
-For example::
-
-    {% if document.document_type.label == "Invoice" or document.document_type.label == "Letter" %}
-    Correspondence
-    {% else %}
-    {{ document.document_type }}
-    {% endif %}
-
-This will create and index level for each document type. Except for documents
-of types "Invoice" and "Letter", these will now go into the level "Correspondence".
-
-
-Index document by department, taken from the first character of the invoice number metadata
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-Requires one index node with the template::
-
-    {% if document.metadata_value_of.invoice_number.0 == "A" %}Accounting
-    {% if document.metadata_value_of.invoice_number.0 == "H" %}Human Resources
-    {% endif %}
-
-
-Nested date index from a date contained in a metadata
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-Assuming the metadata type is named **date_issued** with a date format
-of YYYY-MM-DD. The target is to have two levels: one for years and another
-sub level for months.
-
-**First level: Year**
-::
-
-    {{ document.metadata_value_of.date_issued|slice:"0:4" }}
-
-
-**Second level: Months**
-::
-
-    {{ document.metadata_value_of.date_issued|slice:"5:7" }}
-
-
-**Optional: Third level: Day**
-::
-
-    {{ document.metadata_value_of.date_issued|slice:"8:10" }}
-
-
-Index by OCR content
-^^^^^^^^^^^^^^^^^^^^
-
-This example indexes documents in a "quarterly report" level if they have the
-fragment “quarterly report” in the OCR text::
-
-    {% if "quarterly report" in document.latest_version.ocr_content|join:" "|lower %}Quarterly reports{% endif %}
-
-The same applies to text content extracted for the document::
-
-    {% if "quarterly report" in document.latest_version.content|join:" "|lower %}Quarterly reports{% endif %}
-
-
-
-Index documents not found in any cabinet
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-::
-
-    {% if document.cabinets.count == 0 %}No Cabinets{% endif %}
-
-
-Index untagged documents
-^^^^^^^^^^^^^^^^^^^^^^^^
-::
-
-    {% if document.tags.count == 0 %}No Tags{% endif %}
-
-
-Index documents specifically, by the year of a metadata field otherwise by their uploaded year
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-::
-
-    {% for tag in document.tags.all %}{% if tag.label == "Taxes" %}{% if document.metadata_value_of.tax_year|length_is:"4" %}{{ document.metadata_value_of.tax_year }}{% else %}{{ document.date_added|date:"Y" }}{% endif %}{% endif %}{% endfor %}

docs/chapters/indexes.rst

@@ -1,115 +0,0 @@
-Indexes
-=======
-
-Indexes are an automatic method to hierarchically organize documents in
-relation to their properties (:doc:`../chapters/metadata`, label, MIME type,
-etc). To use indexes you need to first create an index template. Once created,
-associate the index to one or more :doc:`../chapters/document_types`.
-
-Index are hierarchical models so a tree template needs to be specified for them.
-This tree template will contain references to document metadata or properties
-that will be replaced with the actual value for those metadata or properties.
-
-Example:
-
-- Document type: ``Product sheet``
-- Metadata type: ``Product year``, associated as a required metadata for the document type ``Product sheet``.
-
-- Index: ``Product sheets per year``, and associated to the document type ``Product sheet``.
-- Index slug: ``product-sheets-per-year``. Slugs are internal unique identifiers that can be used by other Mayan EDMS modules to reference each index.
-- Index tree template as follows:
-
-.. blockdiag::
-
-   blockdiag {
-      default_shape = roundedbox
-      span_width = 30;
-
-      index [ label = 'Product sheets per year', width=180 ];
-      root [ label = 'Root (Has document links? No)', width=450];
-      level_2 [ label = '{{ document.metadata_value_of.product_year }} (Has document links? Yes)', width=450];
-
-      group {
-        label = "Tree template";
-        color = "#dddddd";
-        style = dashed;
-        root; level_2;
-      }
-
-      index -> root
-      root -> level_2 [folded];
-   }
-
-Now every time a new ``Product sheet`` is uploaded a hierarchical unit with the value
-of the metadata type ``Product year`` is created and a link to the uploaded ``Product sheet`` added to it.
-
-Example:
-
-Suppose three ``Product sheets`` are uploaded with the following values as their
-``Product year`` metadata: 2001, 2002, 2001 respectively. The result index
-that will be generate based on the tree template would be as follows:
-
-.. blockdiag::
-
-   blockdiag {
-      default_shape = roundedbox
-
-      index [ label = 'Product sheets per year', width=180 ];
-      year_1 [ label = '2001', width = 60 ];
-      year_2 [ label = '2002', width = 60 ];
-      document_1 [ label = 'Product A data sheet (2001)', width = 200 ];
-      document_2 [ label = 'Product B data sheet (2002)', width = 200 ];
-      document_3 [ label = 'Product C data sheet (2001)', width = 200 ];
-
-      group {
-        label = "Index content";
-        color = "#dddddd";
-        style = dashed;
-        year_1, year_2, document_1, document_2, document_3;
-      }
-
-      index -> year_1;
-      index -> year_2;
-      year_1 -> document_1;
-      year_2 -> document_2;
-      year_1 -> document_3;
-
-   }
-
-.. include:: ../chapters/index_examples.rst
-
-
-Mirroring
----------
-
-Indexes can be exported as `FUSE <https://en.wikipedia.org/wiki/Filesystem_in_Userspace>`_
-filesystems. Using the management command ``mountindex`` we could export the
-previous example index as follows::
-
-    mkdir -p ~/indexes/products
-    mayan-edms.py mountindex product-sheets-per-year ~/indexes/products
-
-The ``~/indexes/products`` directory will now have a directory and files structure
-identical to that of the index. Once indexes are mounted with this command, they
-behave like any other filesystem directory and can even be further shared
-via the network with network file system software like
-`Samba <https://www.samba.org/>`_ or
-`NFS <https://en.wikipedia.org/wiki/Network_File_System>`_.
-
-.. blockdiag::
-
-    blockdiag {
-        orientation = portrait
-        span_width = 200;
-
-        index [ label = 'Product sheets per year', width=180 ];
-        block_device [ height = 100, label = "Block device\n(Hard drive)", shape = flowchart.database ];
-        network [ label = "Network", shape = cloud ];
-        user [ label = "Users", shape = actor ];
-
-        index -> block_device [ label = "mirroring", fontsize = 8 ];
-        block_device -> network -> user;
-   }
-
-Indexes and mirrored indexes are Read Only as they are generated as a result of
-prior activities like document uploads, metadata changes.

docs/chapters/indexes/examples.txt

@@ -0,0 +1,125 @@
+==============
+Index examples
+==============
+
+Index of document types
+=======================
+
+.. admonition:: Permissions required
+    :class: warning
+
+    - The "Create new document indexes" permission is required for this action.
+    - The "Edit document indexes" permission is required for this action, globally
+      of via an ACL for a document index.
+    - The "Edit document types" permission is required for this action, globally
+      of via an ACL for a document type.
+
+
+This index will create one level for each document type in the system and place
+links to the document of each respective type.
+
+#. Go to the :menuselection:`System --> Setup --> Indexes` menu.
+#. Create a new index using :guilabel:`Actions` > :guilabel:`Create new`.
+#. Give it a label to describe it, and an internal name. The internal name is
+   used when referencing this index in other parts of the system.
+#. Press the :guilabel:`Template` link of the newly created index.
+#. Select :guilabel:`New child node` to create a new level in which the
+   following template code will be entered.
+   ::
+
+       {{ document.document_type }}
+
+#. Save the template.
+#. Click on :guilabel:`Document types` and associate this index with
+   existing document types in the system.
+#. Finally go to :menuselection:`Tools --> Rebuild indexes` to execute the
+   index template. The rebuild process is only necessary when changes are
+   made to the index templates. Otherwise they update automatically whenever
+   a new document is uploaded or existing documents properties are modified.
+#. A new index should appear under :menuselection:`Indexes` menu.
+
+You can also program different behavior based on the different document types,
+by use a comparison and a conditional statement. As the document_type itself
+is not a string you cannot directly use that for that comparison. You will have
+to use the label of the document type.
+
+For example::
+
+    {% if document.document_type.label == "Invoice" or document.document_type.label == "Letter" %}
+    Correspondence
+    {% else %}
+    {{ document.document_type }}
+    {% endif %}
+
+This will create and index level for each document type. Except for documents
+of types "Invoice" and "Letter", these will now go into the level "Correspondence".
+
+
+Index document by department, taken from the first character of the invoice number metadata
+===========================================================================================
+
+Requires one index node with the template::
+
+    {% if document.metadata_value_of.invoice_number.0 == "A" %}Accounting
+    {% if document.metadata_value_of.invoice_number.0 == "H" %}Human Resources
+    {% endif %}
+
+
+Nested date index from a date contained in a metadata
+=====================================================
+
+Assuming the metadata type is named **date_issued** with a date format
+of YYYY-MM-DD. The target is to have two levels: one for years and another
+sub level for months.
+
+**First level: Year**
+::
+
+    {{ document.metadata_value_of.date_issued|slice:"0:4" }}
+
+
+**Second level: Months**
+::
+
+    {{ document.metadata_value_of.date_issued|slice:"5:7" }}
+
+
+**Optional: Third level: Day**
+::
+
+    {{ document.metadata_value_of.date_issued|slice:"8:10" }}
+
+
+Index by OCR content
+====================
+
+This example indexes documents in a "quarterly report" level if they have the
+fragment “quarterly report” in the OCR text::
+
+    {% if "quarterly report" in document.latest_version.ocr_content|join:" "|lower %}Quarterly reports{% endif %}
+
+The same applies to text content extracted for the document::
+
+    {% if "quarterly report" in document.latest_version.content|join:" "|lower %}Quarterly reports{% endif %}
+
+
+
+Index documents not found in any cabinet
+========================================
+::
+
+    {% if document.cabinets.count == 0 %}No Cabinets{% endif %}
+
+
+Index documents not tagged
+==========================
+::
+
+    {% if document.tags.count == 0 %}No Tags{% endif %}
+
+
+Index documents specifically, by the year of a metadata field otherwise by their uploaded year
+==============================================================================================
+::
+
+    {% for tag in document.tags.all %}{% if tag.label == "Taxes" %}{% if document.metadata_value_of.tax_year|length_is:"4" %}{{ document.metadata_value_of.tax_year }}{% else %}{{ document.date_added|date:"Y" }}{% endif %}{% endif %}{% endfor %}

docs/chapters/indexes/index.txt

@@ -0,0 +1,23 @@
+.. index:: Indexes
+
+================
+Document indexes
+================
+
+Document indexes are a multi level structure similar to folder in a filesystem,
+however the hierarchy of the levels is automatically created, updated, or deleted
+based on rules set by the administrator.
+
+- :doc:`indexes`
+- :doc:`examples`
+- :doc:`mirroring`
+
+.. toctree::
+  :hidden:
+
+  indexes
+  mirroring
+  examples
+
+
+

docs/chapters/indexes/indexes.txt

@@ -0,0 +1,78 @@
+===============
+Index structure
+===============
+
+Indexes are an automatic method to hierarchically organize documents in
+relation to their properties (:doc:`../metadata`, label, MIME type,
+etc). To use indexes you need to first create an index template. Once created,
+associate the index to one or more :doc:`../document_types`.
+
+Index are hierarchical models so a tree template needs to be specified for them.
+This tree template will contain references to document metadata or properties
+that will be replaced with the actual value for those metadata or properties.
+
+Example:
+
+- Document type: ``Product sheet``
+- Metadata type: ``Product year``, associated as a required metadata for the document type ``Product sheet``.
+
+- Index: ``Product sheets per year``, and associated to the document type ``Product sheet``.
+- Index slug: ``product-sheets-per-year``. Slugs are internal unique identifiers that can be used by other Mayan EDMS modules to reference each index.
+- Index tree template as follows:
+
+.. blockdiag::
+
+   blockdiag {
+      default_shape = roundedbox
+      span_width = 30;
+
+      index [ label = 'Product sheets per year', width=180 ];
+      root [ label = 'Root (Has document links? No)', width=450];
+      level_2 [ label = '{{ document.metadata_value_of.product_year }} (Has document links? Yes)', width=450];
+
+      group {
+        label = "Tree template";
+        color = "#dddddd";
+        style = dashed;
+        root; level_2;
+      }
+
+      index -> root
+      root -> level_2 [folded];
+   }
+
+Now every time a new ``Product sheet`` is uploaded a hierarchical unit with the value
+of the metadata type ``Product year`` is created and a link to the uploaded ``Product sheet`` added to it.
+
+Example:
+
+Suppose three ``Product sheets`` are uploaded with the following values as their
+``Product year`` metadata: 2001, 2002, 2001 respectively. The result index
+that will be generate based on the tree template would be as follows:
+
+.. blockdiag::
+
+   blockdiag {
+      default_shape = roundedbox
+
+      index [ label = 'Product sheets per year', width=180 ];
+      year_1 [ label = '2001', width = 60 ];
+      year_2 [ label = '2002', width = 60 ];
+      document_1 [ label = 'Product A data sheet (2001)', width = 200 ];
+      document_2 [ label = 'Product B data sheet (2002)', width = 200 ];
+      document_3 [ label = 'Product C data sheet (2001)', width = 200 ];
+
+      group {
+        label = "Index content";
+        color = "#dddddd";
+        style = dashed;
+        year_1, year_2, document_1, document_2, document_3;
+      }
+
+      index -> year_1;
+      index -> year_2;
+      year_1 -> document_1;
+      year_2 -> document_2;
+      year_1 -> document_3;
+
+   }