Fix document view test mixin

Signed-off-by: Roberto Rosario <roberto.rosario@mayan-edms.com>

.dockerignore

@@ -4,8 +4,7 @@ HISTORY.md
 mayan/media
 db.sqlite*
 docker
-!docker/etc
-!docker/entrypoint.sh
-!docker/version
+!docker/rootfs
 ./.*
 docs
+

.flake8

@@ -1,2 +1,3 @@
 [flake8]
-ignore = E501
+exclude = node_modules,static,templates
+ignore = E501, N801, N802, N804, N805, N806

.gitignore

@@ -7,6 +7,7 @@
 .coverage
 .coverage.tox*
 .idea/
+*.swp
 .tox/
 .vagrant
 _build/
@@ -29,4 +30,6 @@ static_collected/
 /venv/
 /venv3/
 /whoosh_index/
+google_fonts/
 node_modules/
+docs/build/

.gitlab-ci.yml

@@ -17,9 +17,9 @@ job_docker_build:
     - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
   script:
     - docker build --pull -t "$CI_REGISTRY_IMAGE" -f docker/Dockerfile .
-    - VERSION=`cat docker/version`
+    - VERSION=`cat docker/rootfs/version`
     - docker tag "$CI_REGISTRY_IMAGE" "$CI_REGISTRY_IMAGE:$VERSION"
-    - docker run --rm "$CI_REGISTRY_IMAGE:$VERSION" run-tests
+    - docker run --rm "$CI_REGISTRY_IMAGE:$VERSION" run_tests
     - docker push "$CI_REGISTRY_IMAGE:$VERSION"
     - docker push "$CI_REGISTRY_IMAGE:latest"
     - docker tag "$CI_REGISTRY_IMAGE:$VERSION" registry-1.docker.io/mayanedms/mayanedms:"$VERSION"
@@ -37,7 +37,7 @@ job_docker_push:
   before_script:
     - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
   script:
-    - VERSION=`cat docker/version`
+    - VERSION=`cat docker/rootfs/version`
     - docker pull "$CI_REGISTRY_IMAGE:$VERSION"
     - docker tag "$CI_REGISTRY_IMAGE:$VERSION" registry-1.docker.io/mayanedms/mayanedms:"$VERSION"
     - docker tag "$CI_REGISTRY_IMAGE:$VERSION" registry-1.docker.io/mayanedms/mayanedms:latest
@@ -58,7 +58,7 @@ job_docker_nightly:
     - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
   script:
     - docker build --pull -t "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG" -f docker/Dockerfile .
-    - docker run --rm "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG" run-tests
+    - docker run --rm "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG" run_tests
     - docker push "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG"
   only:
     - nightly
@@ -119,7 +119,7 @@ job_build_python:
     - update-locale LANG=en_US.UTF-8
     - export LC_ALL=en_US.UTF-8
     - pip install -r requirements/build.txt
-    - make wheel
+    - make python-wheel
   only:
     - releases/all
     - releases/python
@@ -140,7 +140,7 @@ job_push_python:
 
 .test_base: &test_base
   stage: test
-  image: ubuntu:16.04
+  image: ubuntu:18.04
   cache:
     paths:
       - ~/.cache/pip/
@@ -152,13 +152,14 @@ job_push_python:
     - locale-gen en_US.UTF-8
     - update-locale LANG=en_US.UTF-8
     - export LC_ALL=en_US.UTF-8
-    - apt-get install -qq curl gcc ghostscript gpgv gnupg graphviz libfuse2 libjpeg-dev libmagic1 libpng-dev libtiff-dev poppler-utils libreoffice poppler-utils python-dev python-pip tesseract-ocr tesseract-ocr-deu
-    - pip install -r requirements/testing.txt
+    - apt-get install -qq curl exiftool gcc ghostscript gnupg1 graphviz libfuse2 libjpeg-dev libmagic1 libpng-dev libtiff-dev poppler-utils libreoffice poppler-utils python-dev python-virtualenv python3-dev tesseract-ocr tesseract-ocr-deu
+    - virtualenv venv -p /usr/bin/python3
+    - . venv/bin/activate
+    - pip install -r requirements.txt -r requirements/testing-base.txt
   only:
     - releases/all
     - releases/docker
     - releases/python
-    - master
     - staging
     - nightly
 
@@ -171,7 +172,8 @@ test-mysql:
     - mysql:8.0.3
   script:
     - apt-get install -qq libmysqlclient-dev mysql-client
-    - pip install mysql-python
+    - . venv/bin/activate
+    - pip install mysqlclient
     - mysql -h"$MYSQL_PORT_3306_TCP_ADDR" -P"$MYSQL_PORT_3306_TCP_PORT" -uroot -p"$MYSQL_ENV_MYSQL_ROOT_PASSWORD" -e "set global character_set_server=utf8mb4;"
     - python manage.py test --mayan-apps --settings=mayan.settings.testing.gitlab-ci.db_mysql --nomigrations
   tags:
@@ -186,6 +188,7 @@ test-postgres:
     - postgres
   script:
     - apt-get install -qq libpq-dev
+    - . venv/bin/activate
     - pip install psycopg2
     - python manage.py test --mayan-apps --settings=mayan.settings.testing.gitlab-ci.db_postgres --nomigrations
   tags:
@@ -194,6 +197,7 @@ test-postgres:
 test-sqlite:
   <<: *test_base
   script:
+    - . venv/bin/activate
     - python manage.py test --mayan-apps --settings=mayan.settings.testing.gitlab-ci --nomigrations
 
 deploy_demo:

.tx/config

@@ -19,6 +19,12 @@ source_lang = en
 source_file = mayan/apps/authentication/locale/en/LC_MESSAGES/django.po
 type = PO
 
+[mayan-edms.autoadmin-2-0]
+file_filter = mayan/apps/autoadmin/locale/<lang>/LC_MESSAGES/django.po
+source_lang = en
+source_file = mayan/apps/autoadmin/locale/en/LC_MESSAGES/django.po
+type = PO
+
 [mayan-edms.cabinets-2-0]
 file_filter = mayan/apps/cabinets/locale/<lang>/LC_MESSAGES/django.po
 source_lang = en
@@ -43,18 +49,24 @@ source_lang = en
 source_file = mayan/apps/converter/locale/en/LC_MESSAGES/django.po
 type = PO
 
+[mayan-edms.dashboards-2-0]
+file_filter = mayan/apps/dashboards/locale/<lang>/LC_MESSAGES/django.po
+source_lang = en
+source_file = mayan/apps/dashboards/locale/en/LC_MESSAGES/django.po
+type = PO
+
+[mayan-edms.dependencies-3-0]
+file_filter = mayan/apps/dependencies/locale/<lang>/LC_MESSAGES/django.po
+source_lang = en
+source_file = mayan/apps/dependencies/locale/en/LC_MESSAGES/django.po
+type = PO
+
 [mayan-edms.django_gpg-2-0]
 file_filter = mayan/apps/django_gpg/locale/<lang>/LC_MESSAGES/django.po
 source_lang = en
 source_file = mayan/apps/django_gpg/locale/en/LC_MESSAGES/django.po
 type = PO
 
-[mayan-edms.documents-2-0]
-file_filter = mayan/apps/documents/locale/<lang>/LC_MESSAGES/django.po
-source_lang = en
-source_file = mayan/apps/documents/locale/en/LC_MESSAGES/django.po
-type = PO
-
 [mayan-edms.document_comments-2-0]
 file_filter = mayan/apps/document_comments/locale/<lang>/LC_MESSAGES/django.po
 source_lang = en
@@ -85,6 +97,12 @@ source_lang = en
 source_file = mayan/apps/document_states/locale/en/LC_MESSAGES/django.po
 type = PO
 
+[mayan-edms.documents-2-0]
+file_filter = mayan/apps/documents/locale/<lang>/LC_MESSAGES/django.po
+source_lang = en
+source_file = mayan/apps/documents/locale/en/LC_MESSAGES/django.po
+type = PO
+
 [mayan-edms.dynamic_search-2-0]
 file_filter = mayan/apps/dynamic_search/locale/<lang>/LC_MESSAGES/django.po
 source_lang = en
@@ -97,6 +115,18 @@ source_lang = en
 source_file = mayan/apps/events/locale/en/LC_MESSAGES/django.po
 type = PO
 
+[mayan-edms.file_caching-3-0]
+file_filter = mayan/apps/file_caching/locale/<lang>/LC_MESSAGES/django.po
+source_lang = en
+source_file = mayan/apps/file_caching/locale/en/LC_MESSAGES/django.po
+type = PO
+
+[mayan-edms.file_metadata-3-0]
+file_filter = mayan/apps/file_metadata/locale/<lang>/LC_MESSAGES/django.po
+source_lang = en
+source_file = mayan/apps/file_metadata/locale/en/LC_MESSAGES/django.po
+type = PO
+
 [mayan-edms.linking-2-0]
 file_filter = mayan/apps/linking/locale/<lang>/LC_MESSAGES/django.po
 source_lang = en
@@ -145,6 +175,12 @@ source_lang = en
 source_file = mayan/apps/permissions/locale/en/LC_MESSAGES/django.po
 type = PO
 
+[mayan-edms.platform-2-0]
+file_filter = mayan/apps/platform/locale/<lang>/LC_MESSAGES/django.po
+source_lang = en
+source_file = mayan/apps/platform/locale/en/LC_MESSAGES/django.po
+type = PO
+
 [mayan-edms.rest_api-2-0]
 file_filter = mayan/apps/rest_api/locale/<lang>/LC_MESSAGES/django.po
 source_lang = en
@@ -192,3 +228,10 @@ file_filter = mayan/apps/user_management/locale/<lang>/LC_MESSAGES/django.po
 source_lang = en
 source_file = mayan/apps/user_management/locale/en/LC_MESSAGES/django.po
 type = PO
+
+[mayan-edms.weblink-3-0]
+file_filter = mayan/apps/weblinks/locale/<lang>/LC_MESSAGES/django.po
+source_lang = en
+source_file = mayan/apps/weblinks/locale/en/LC_MESSAGES/django.po
+type = PO
+

CONTRIBUTING.md

@@ -23,7 +23,7 @@ with deployments, webservers, cloud providers, etc.
 Code
 ----
 
-1. Complete and mail, or scan and email the corresponding Contributor Assignment Agreement: [Mayan EDMS Individual Contributor Assignment Agreement](https://docs.mayan-edms.com/topics/caa_individual.rst) or [Mayan EDMS Entity Contributor Assignment Agreement](https://docs.mayan-edms.com/topics/topics/caa_entity.rst).
+1. Complete and mail, or scan and email the corresponding Contributor Assignment Agreement: [Mayan EDMS Individual Contributor Assignment Agreement](https://gitlab.com/mayan-edms/mayan-edms/blob/master/docs/chapters/caa_individual.rst) or [Mayan EDMS Entity Contributor Assignment Agreement](https://gitlab.com/mayan-edms/mayan-edms/blob/master/docs/chapters/caa_entity.rst).
 1. Fork [the repository](http://gitlab.com/mayan-edms/mayan-edms).
 1. Choose the version for which you want to develop. The code is divided in the
 following branches:

LICENSE

@@ -1,4 +1,4 @@
-   Copyright 2011-2018 Roberto Rosario
+   Copyright 2011 Roberto Rosario
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

MANIFEST.in

@@ -1,3 +1,6 @@
 include README.md LICENSE HISTORY.rst mayan/LICENSE
-recursive-include mayan/apps *.txt *.html *.css *.ico *.png *.jpg *.js *.po *.mo *.ttf *.woff *.woff2 *.gif *.eot *.svg *.doc *.pdf *.tiff *.sig *.asc *.gpg *.zip *.tar *.gz *.bz2 package.json
-global-exclude mayan/settings/local.py mayan/settings/travis/* mayan/media/*
+recursive-include mayan/apps *.txt *.html *.css *.ico *.png *.jpg *.js *.mo *.ttf *.woff *.woff2 *.gif *.eot *.svg *.doc *.pdf *.tiff *.sig *.asc *.gpg *.zip *.tar *.gz *.bz2 *.tmpl
+global-exclude mayan/settings/local.py *.po
+prune mayan/apps/*/static/*/node_modules/*
+prune mayan/settings/travis/*
+prune mayan/media/*

Makefile

@@ -1,90 +1,60 @@
 .PHONY: clean-pyc clean-build
 
-help: docker-help
-	@echo
-	@echo "**** Main makefile ****"
-	@echo "clean-build - Remove build artifacts."
-	@echo "clean-pyc - Remove Python artifacts."
-	@echo "clean - Remove Python and build artifacts."
-	@echo "generate-setup - Create and updated setup.py"
-	@echo "check-readme - Checks validity of the README.rst file for PyPI publication."
-	@echo "check-missing_migrations - Make sure all models have proper migrations."
+DOCKER_MYSQL_IMAGE = mysql:8.0
+DOCKER_ORACLE_IMAGE = wnameless/oracle-xe-11g
+DOCKER_POSTGRES_IMAGE = postgres:9.6-alpine
+DOCKER_REDIS_IMAGE = redis:5.0-alpine
 
-	@echo "test-all - Run all tests."
-	@echo "test MODULE=<python module name> - Run tests for a single app, module or test class."
-	@echo "test-with-postgres-all - Run all tests against a Postgres database container."
-	@echo "test-postgres MODULE=<python module name> - Run tests for a single app, module or test class against a Postgres database container."
-	@echo "test-with-mysql-all - Run all tests against a MySQL database container."
-	@echo "test-mysql MODULE=<python module name> - Run tests for a single app, module or test class against a MySQL database container."
-	@echo "test-with-oracle-all - Run all tests against a Oracle database container."
-	@echo "test-oracle MODULE=<python module name> - Run tests for a single app, module or test class against a Oracle database container."
+PYTHON_MYSQL_VERSION = 1.4.4
+PYTHON_PSYCOPG2_VERSION = 2.8.3
+PYTHON_RABBITMQ_VERSION = 2.0.0
+PYTHON_REDIS_VERSION = 3.2.1
 
-	@echo "docs-serve - Run the livehtml documentation generator."
-
-	@echo "translations-make - Refresh all translation files."
-	@echo "translations-compile - Compile all translation files."
-	@echo "translations-push - Upload all translation files to Transifex."
-	@echo "translations-pull - Download all translation files from Transifex."
-
-	@echo "sdist - Build the source distribution package."
-	@echo "wheel - Build the wheel distribution package."
-	@echo "release - Package (sdist and wheel) and upload a release."
-	@echo "test-release - Package (sdist and wheel) and upload to the PyPI test server."
-	@echo "release-test-via-docker-ubuntu - Package (sdist and wheel) and upload to the PyPI test server using an Ubuntu Docker builder."
-	@echo "release-via-docker-ubuntu - Package (sdist and wheel) and upload to PyPI using an Ubuntu Docker builder."
-	@echo "test-sdist-via-docker-ubuntu - Make an sdist packange and test it using an Ubuntu Docker container."
-	@echo "test-wheel-via-docker-ubuntu - Make a wheel package and test it using an Ubuntu Docker container."
-	@echo "runserver - Run the development server."
-	@echo "runserver_plus - Run the Django extension's development server."
-	@echo "shell_plus - Run the shell_plus command."
-
-	@echo "test-with-docker-services-on - Launch and initialize production-like services using Docker (Postgres and Redis)."
-	@echo "test-with-docker-services-off - Stop and delete the Docker production-like services."
-	@echo "test-with-docker-frontend - Launch a front end instance that uses the production-like services."
-	@echo "test-with-docker-worker - Launch a worker instance that uses the production-like services."
-	@echo "docker-mysql-on - Launch and initialize a MySQL Docker container."
-	@echo "docker-mysql-off - Stop and delete the MySQL Docker container."
-	@echo "docker-postgres-on - Launch and initialize a PostgreSQL Docker container."
-	@echo "docker-postgres-off - Stop and delete the PostgreSQL Docker container."
-
-	@echo "safety-check - Run a package safety check."
+help:
+	@echo "Usage: make <target>\n"
+	@awk 'BEGIN {FS = ":.*##"} /^[a-zA-Z_-]+:.*?## / { printf "  * %-40s -%s\n", $$1, $$2 }' $(MAKEFILE_LIST)|sort
 
 # Cleaning
 
+clean: ## Remove Python and build artifacts.
 clean: clean-build clean-pyc
 
-clean-build:
+clean-build: ## Remove build artifacts.
 	rm -fr build/
 	rm -fr dist/
 	rm -fr *.egg-info
 
-clean-pyc:
+clean-pyc: ## Remove Python artifacts.
 	find . -name '*.pyc' -exec rm -f {} +
 	find . -name '*.pyo' -exec rm -f {} +
 	find . -name '*~' -exec rm -f {} +
-
+	find . -name '__pycache__' -exec rm -R -f {} +
 
 # Testing
 
-test:
-	./manage.py test $(MODULE) --settings=mayan.settings.testing.development --nomigrations
+test: clean-pyc
+test: ## MODULE=<python module name> - Run tests for a single app, module or test class.
+	./manage.py test $(MODULE) --settings=mayan.settings.testing.development --nomigrations $(ARGUMENTS)
 
-test-all:
-	./manage.py test --mayan-apps --settings=mayan.settings.testing.development --nomigrations
+test-all: ## Run all tests.
+test-all: clean-pyc
+	./manage.py test --mayan-apps --settings=mayan.settings.testing.development --nomigrations $(ARGUMENTS)
 
 test-launch-postgres:
 	@docker rm -f test-postgres || true
 	@docker volume rm test-postgres || true
-	docker run -d --name test-postgres -p 5432:5432 -v test-postgres:/var/lib/postgresql/data healthcheck/postgres
-	sudo apt-get install -qq libpq-dev
-	pip install psycopg2
-	while ! docker inspect --format='{{json .State.Health}}' test-postgres|grep 'Status":"healthy"'; do sleep 1; done
+	docker run -d --name test-postgres -p 5432:5432 -v test-postgres:/var/lib/postgresql/data $(DOCKER_POSTGRES_IMAGE)
+	sudo apt-get install -q libpq-dev
+	pip install psycopg2==$(PYTHON_PSYCOPG2_VERSION)
+	while ! nc -z 127.0.0.1 5432; do sleep 1; done
 
+test-with-postgres: ## MODULE=<python module name> - Run tests for a single app, module or test class against a Postgres database container.
 test-with-postgres: test-launch-postgres
 	./manage.py test $(MODULE) --settings=mayan.settings.testing.docker.db_postgres --nomigrations
 	@docker rm -f test-postgres || true
 	@docker volume rm test-postgres || true
 
+test-with-postgres-all: ## Run all tests against a Postgres database container.
 test-with-postgres-all: test-launch-postgres
 	./manage.py test --mayan-apps --settings=mayan.settings.testing.docker.db_postgres --nomigrations
 	@docker rm -f test-postgres || true
@@ -93,17 +63,20 @@ test-with-postgres-all: test-launch-postgres
 test-launch-mysql:
 	@docker rm -f test-mysql || true
 	@docker volume rm test-mysql || true
-	docker run -d --name test-mysql -p 3306:3306 -e MYSQL_ALLOW_EMPTY_PASSWORD=True -e MYSQL_DATABASE=mayan -v test-mysql:/var/lib/mysql healthcheck/mysql
-	sudo apt-get install -qq libmysqlclient-dev mysql-client
-	pip install mysql-python
-	while ! docker inspect --format='{{json .State.Health}}' test-mysql|grep 'Status":"healthy"'; do sleep 1; done
+	docker run -d --name test-mysql -p 3306:3306 -e MYSQL_ALLOW_EMPTY_PASSWORD=True -e MYSQL_DATABASE=mayan -v test-mysql:/var/lib/mysql $(DOCKER_MYSQL_IMAGE)
+	sudo apt-get install -q libmysqlclient-dev mysql-client
+	pip install mysqlclient==$(PYTHON_MYSQL_VERSION)
+	while ! nc -z 127.0.0.1 3306; do sleep 1; done
 	mysql -h 127.0.0.1 -P 3306 -uroot  -e "set global character_set_server=utf8mb4;"
 
+test-with-mysql: ## MODULE=<python module name> - Run tests for a single app, module or test class against a MySQL database container.
 test-with-mysql: test-launch-mysql
 	./manage.py test $(MODULE) --settings=mayan.settings.testing.docker.db_mysql --nomigrations
 	@docker rm -f test-mysql || true
 	@docker volume rm test-mysql || true
 
+
+test-with-mysql-all: ## Run all tests against a MySQL database container.
 test-with-mysql-all: test-launch-mysql
 	./manage.py test --mayan-apps --settings=mayan.settings.testing.docker.db_mysql --nomigrations
 	@docker rm -f test-mysql || true
@@ -112,17 +85,19 @@ test-with-mysql-all: test-launch-mysql
 test-launch-oracle:
 	@docker rm -f test-oracle || true
 	@docker volume rm test-oracle || true
-	docker run -d --name test-oracle -p 49160:22 -p 49161:1521 -e ORACLE_ALLOW_REMOTE=true -v test-oracle:/u01/app/oracle wnameless/oracle-xe-11g
+	docker run -d --name test-oracle -p 49160:22 -p 49161:1521 -e ORACLE_ALLOW_REMOTE=true -v test-oracle:/u01/app/oracle $(DOCKER_ORACLE_IMAGE)
 	# https://gist.github.com/kimus/10012910
 	pip install cx_Oracle
 	while ! nc -z 127.0.0.1 49161; do sleep 1; done
 	sleep 10
 
+test-with-oracle: ## MODULE=<python module name> - Run tests for a single app, module or test class against a Oracle database container.
 test-with-oracle: test-launch-oracle
 	./manage.py test $(MODULE) --settings=mayan.settings.testing.docker.db_oracle --nomigrations
 	@docker rm -f test-oracle || true
 	@docker volume rm test-oracle || true
 
+test-with-oracle-all: ## Run all tests against a Oracle database container.
 test-with-oracle-all: test-launch-oracle
 	./manage.py test --mayan-apps --settings=mayan.settings.testing.docker.db_oracle --nomigrations
 	@docker rm -f test-oracle || true
@@ -130,58 +105,56 @@ test-with-oracle-all: test-launch-oracle
 
 # Documentation
 
-docs-serve:
+docs-serve: ## Run the livehtml documentation generator.
 	cd docs;make livehtml
 
-docs-spellcheck:
-
+docs-spellcheck: ## Spellcheck the documentation.
 	sphinx-build -b spelling -d docs/_build/ docs docs/_build/spelling
 
 # Translations
 
-translations-make:
+translations-make: ## Refresh all translation files.
 	contrib/scripts/process_messages.py -m
 
-translations-compile:
+translations-compile: ## Compile all translation files.
 	contrib/scripts/process_messages.py -c
 
-translations-push:
+translations-push: ## Upload all translation files to Transifex.
 	tx push -s
 
-translations-pull:
+translations-pull: ## Download all translation files from Transifex.
 	tx pull -f
 
-
-generate-setup:
-	@./generate_setup.py
-	@echo "Complete."
-
 # Releases
 
-increase-version:
+increase-version: ## Increase the version number of the entire project's files.
 	@VERSION=`grep "__version__ =" mayan/__init__.py| cut -d\' -f 2|./increase_version.py - $(PART)`; \
 	BUILD=`echo $$VERSION|awk '{split($$VERSION,a,"."); printf("0x%02d%02d%02d\n", a[1],a[2], a[3])}'`; \
 	sed -i -e "s/__build__ = 0x[0-9]*/__build__ = $${BUILD}/g" mayan/__init__.py; \
 	sed -i -e "s/__version__ = '[0-9\.]*'/__version__ = '$${VERSION}'/g" mayan/__init__.py; \
-	echo $$VERSION > docker/version
+	echo $$VERSION > docker/rootfs/version
 	make generate-setup
 
-test-release: clean wheel
+python-test-release: ## Package (sdist and wheel) and upload to the PyPI test server.
+python-test-release: clean wheel
 	twine upload dist/* -r testpypi
 	@echo "Test with: pip install -i https://testpypi.python.org/pypi mayan-edms"
 
-release: clean wheel
+python-release: ## Package (sdist and wheel) and upload a release.
+python-release: clean python-wheel
 	twine upload dist/* -r pypi
 
-sdist: clean
+python-sdist: ## Build the source distribution package.
+python-sdist: clean
 	python setup.py sdist
 	ls -l dist
 
-wheel: clean sdist
+python-wheel: ## Build the wheel distribution package.
+python-wheel: clean python-sdist
 	pip wheel --no-index --no-deps --wheel-dir dist dist/*.tar.gz
 	ls -l dist
 
-release-test-via-docker-ubuntu:
+python-release-test-via-docker-ubuntu: ## Package (sdist and wheel) and upload to the PyPI test server using an Ubuntu Docker builder.
 	docker run --rm --name mayan_release -v $(HOME):/host_home:ro -v `pwd`:/host_source -w /source ubuntu:16.04 /bin/bash -c "\
 	echo "LC_ALL=\"en_US.UTF-8\"" >> /etc/default/locale && \
 	locale-gen en_US.UTF-8 && \
@@ -194,7 +167,7 @@ release-test-via-docker-ubuntu:
 	cp -r /host_home/.pypirc ~/.pypirc && \
 	make test-release"
 
-release-via-docker-ubuntu:
+python-release-via-docker-ubuntu: ## Package (sdist and wheel) and upload to PyPI using an Ubuntu Docker builder.
 	docker run --rm --name mayan_release -v $(HOME):/host_home:ro -v `pwd`:/host_source -w /source ubuntu:16.04 /bin/bash -c "\
 	apt-get update && \
 	apt-get -y install locales && \
@@ -208,7 +181,7 @@ release-via-docker-ubuntu:
 	cp -r /host_home/.pypirc ~/.pypirc && \
 	make release"
 
-test-sdist-via-docker-ubuntu:
+test-sdist-via-docker-ubuntu: ## Make an sdist package and test it using an Ubuntu Docker container.
 	docker run --rm --name mayan_sdist_test -v $(HOME):/host_home:ro -v `pwd`:/host_source -w /source ubuntu:16.04 /bin/bash -c "\
 	cp -r /host_source/* . && \
 	echo "LC_ALL=\"en_US.UTF-8\"" >> /etc/default/locale && \
@@ -221,7 +194,7 @@ test-sdist-via-docker-ubuntu:
 	make sdist-test-suit \
 	"
 
-test-wheel-via-docker-ubuntu:
+test-wheel-via-docker-ubuntu: ## Make a wheel package and test it using an Ubuntu Docker container.
 	docker run --rm --name mayan_wheel_test -v $(HOME):/host_home:ro -v `pwd`:/host_source -w /source ubuntu:16.04 /bin/bash -c "\
 	cp -r /host_source/* . && \
 	echo "LC_ALL=\"en_US.UTF-8\"" >> /etc/default/locale && \
@@ -234,7 +207,7 @@ test-wheel-via-docker-ubuntu:
 	make wheel-test-suit \
 	"
 
-sdist-test-suit: sdist
+python-sdist-test-suit: sdist
 	rm -f -R _virtualenv
 	virtualenv _virtualenv
 	sh -c '\
@@ -245,7 +218,7 @@ sdist-test-suit: sdist
 	_virtualenv/bin/mayan-edms.py test --mayan-apps \
 	'
 
-wheel-test-suit: wheel
+python-wheel-test-suit: wheel
 	rm -f -R _virtualenv
 	virtualenv _virtualenv
 	sh -c '\
@@ -256,64 +229,77 @@ wheel-test-suit: wheel
 	_virtualenv/bin/mayan-edms.py test --mayan-apps \
 	'
 
+generate-setup: ## Create and update the setup.py file.
+generate-setup: generate-requirements
+	@./generate_setup.py
+	@echo "Complete."
+
+generate-requirements: ## Generate all requirements files from the project depedency declarations.
+	@./manage.py generaterequirements build > requirements/build.txt
+	@./manage.py generaterequirements development > requirements/development.txt
+	@./manage.py generaterequirements testing > requirements/testing-base.txt
+	@./manage.py generaterequirements production --exclude=django > requirements/base.txt
+	@./manage.py generaterequirements production --only=django > requirements/common.txt
+
 # Dev server
 
-runserver:
-	./manage.py runserver --settings=mayan.settings.development $(ADDRPORT)
+runserver: ## Run the development server.
+	./manage.py runserver --nothreading --settings=mayan.settings.development $(ADDRPORT)
 
-runserver_plus:
-	./manage.py runserver_plus --settings=mayan.settings.development $(ADDRPORT)
+runserver_plus: ## Run the Django extension's development server.
+	./manage.py runserver_plus --nothreading --settings=mayan.settings.development $(ADDRPORT)
 
-shell_plus:
+shell_plus: ## Run the shell_plus command.
 	./manage.py shell_plus --settings=mayan.settings.development
 
-test-with-docker-services-on:
-	docker run -d --name redis -p 6379:6379 redis
-	docker run -d --name postgres -p 5432:5432 postgres
+test-with-docker-services-on: ## Launch and initialize production-like services using Docker (Postgres and Redis).
+	docker run -d --name redis -p 6379:6379 $(DOCKER_REDIS_IMAGE)
+	docker run -d --name postgres -p 5432:5432 $(DOCKER_POSTGRES_IMAGE)
 	while ! nc -z 127.0.0.1 6379; do sleep 1; done
 	while ! nc -z 127.0.0.1 5432; do sleep 1; done
 	sleep 4
+	pip install psycopg2==$(PYTHON_PSYCOPG2_VERSION) redis==$(PYTHON_REDIS_VERSION)
 	./manage.py initialsetup --settings=mayan.settings.staging.docker
 
-test-with-docker-services-off:
+test-with-docker-services-off: ## Stop and delete the Docker production-like services.
 	docker stop postgres redis
 	docker rm postgres redis
 
-test-with-docker-frontend:
+test-with-docker-frontend: ## Launch a front end instance that uses the production-like services.
 	./manage.py runserver --settings=mayan.settings.staging.docker
 
-test-with-docker-worker:
-	./manage.py celery worker --settings=mayan.settings.staging.docker -B -l INFO -O fair
+test-with-docker-worker: ## Launch a worker instance that uses the production-like services.
+	DJANGO_SETTINGS_MODULE=mayan.settings.staging.docker ./manage.py celery worker -A mayan -B -l INFO -O fair
 
-docker-mysql-on:
-	docker run -d --name mysql -p 3306:3306 -e MYSQL_ALLOW_EMPTY_PASSWORD=True -e MYSQL_DATABASE=mayan_edms mysql
+docker-mysql-on: ## Launch and initialize a MySQL Docker container.
+	docker run -d --name mysql -p 3306:3306 -e MYSQL_ALLOW_EMPTY_PASSWORD=True -e MYSQL_DATABASE=mayan_edms $(DOCKER_MYSQL_IMAGE)
 	while ! nc -z 127.0.0.1 3306; do sleep 1; done
 
-docker-mysql-off:
+docker-mysql-off: ## Stop and delete the MySQL Docker container.
 	docker stop mysql
 	docker rm mysql
 
-docker-postgres-on:
-	docker run -d --name postgres -p 5432:5432 postgres
+docker-postgres-on: ## Launch and initialize a PostgreSQL Docker container.
+	docker run -d --name postgres -p 5432:5432 $(DOCKER_POSTGRES_IMAGE)
 	while ! nc -z 127.0.0.1 5432; do sleep 1; done
 
-docker-postgres-off:
+docker-postgres-off: ## Stop and delete the PostgreSQL Docker container.
 	docker stop postgres
 	docker rm postgres
 
 
 # Security
 
-safety-check:
+safety-check: ## Run a package safety check.
 	safety check
 
 
 # Other
-find-gitignores:
+find-gitignores: ## Find stray .gitignore files.
 	@export FIND_GITIGNORES=`find -name '.gitignore'| wc -l`; \
 	if [ $${FIND_GITIGNORES} -gt 1 ] ;then echo "More than one .gitignore found."; fi
 
-build:
+python-build:
 	docker rm -f mayan-edms-build || true && \
 	docker run --rm --name mayan-edms-build -v $(HOME):/host_home:ro -v `pwd`:/host_source -w /source python:2-slim sh -c "\
 	rm /host_source/dist -R || true && \
@@ -326,11 +312,13 @@ build:
 	make wheel && \
 	cp dist/* /host_source/dist/"
 
-check-readme:
+check-readme: ## Checks validity of the README.rst file for PyPI publication.
 	python setup.py check -r -s
 
-check-missing-migrations:
+check-missing-migrations: ## Make sure all models have proper migrations.
 	./manage.py makemigrations --dry-run --noinput --check
 
+setup-dev-environment: ## Bootstrap a virtualenv by install all dependencies to start developing.
+	pip install -r requirements.txt -r requirements/development.txt -r requirements/testing-base.txt -r requirements/documentation.txt -r requirements/build.txt
 
-include docker/Makefile
+-include docker/Makefile

README.md

@@ -1,11 +1,11 @@
+[![Donation](https://img.shields.io/badge/donation-PayPal-brightgreen)](https://paypal.me/MayanEDMS)
 [![pypi][pypi]][pypi-url]
 [![builds][builds]][builds-url]
-[![coverage][cover]][cover-url]
 ![python][python]
 ![license][license]
 [![Docker pulls](https://img.shields.io/docker/pulls/mayanedms/mayanedms.svg?maxAge=3600)](https://hub.docker.com/r/mayanedms/mayanedms/)
 [![Docker Stars](https://img.shields.io/docker/stars/mayanedms/mayanedms.svg?maxAge=3600)](https://hub.docker.com/r/mayanedms/mayanedms/)
-[![Docker layers](https://images.microbadger.com/badges/image/mayanedms/mayanedms.svg)](https://microbadger.com/images/mayanedms/mayanedms)
+
 
 [pypi]: http://img.shields.io/pypi/v/mayan-edms.svg
 [pypi-url]: http://badge.fury.io/py/mayan-edms
@@ -43,6 +43,20 @@
 
 </div>
 
+<h2 align="center">Book</h2>
+
+The pre-release version of the book "Exploring Mayan EDMS" available now at a discounted price!
+
+<p align="center">
+    <a href="https://sellfy.com/p/um2fkx/">
+        <img width="400" src="https://d12swbtw719y4s.cloudfront.net/images/v6RpxW40/aP0qKLjkPiAuXZhYuB45/wDAULAzFyx.jpeg?w=548">
+    </a>
+</p>
+
+Limited quantities. Buyers of the pre-release will get the full version for free when it is ready.
+
+The link is https://sellfy.com/p/um2fkx/
+
 <h2 align="center">Installation</h2>
 
 The easiest way to use Mayan EDMS is by using the official

README.rst

@@ -1,4 +1,4 @@
-|pypi| |builds| |coverage| |python| |license| |docker_pulls| |docker_stars| |docker_layers|
+|donation| |pypi| |builds| |coverage| |python| |license| |docker_pulls| |docker_stars| |docker_layers|
 
 
 .. image:: https://gitlab.com/mayan-edms/mayan-edms/raw/master/docs/_static/mayan_logo.png
@@ -25,6 +25,16 @@ at: https://docs.mayan-edms.com/topics/installation.html
 
 .. _Docker: https://www.docker.com/
 
+The pre-release version of the book "Exploring Mayan EDMS" available now at a discounted price!
+
+.. image:: https://camo.githubusercontent.com/89d2fe787507c9247aa7bb406e2682b53eb7a5f9/68747470733a2f2f64313273776274773731397934732e636c6f756466726f6e742e6e65742f696d616765732f76365270785734302f615030714b4c6a6b50694175585a6859754234352f774441554c417a4679782e6a7065673f773d353438
+    :align: center
+    :width: 300
+    :target: https://sellfy.com/p/um2fkx/
+
+Limited quantities. Buyers of the pre-release will get the full version for free when it is ready.
+Click the image or visit: https://sellfy.com/p/um2fkx/
+
 Hardware requirements
 
 - 2 Gigabytes of RAM (1 Gigabyte if OCR is turned off).
@@ -44,6 +54,8 @@ Important links
 - `Videos <https://www.youtube.com/channel/UCJOOXHP1MJ9lVA7d8ZTlHPw>`__
 
 
+.. |donation| image:: https://img.shields.io/badge/donation-PayPal-brightgreen
+   :target: https://paypal.me/MayanEDMS
 .. |pypi| image:: http://img.shields.io/pypi/v/mayan-edms.svg
    :target: http://badge.fury.io/py/mayan-edms
 .. |builds| image:: https://gitlab.com/mayan-edms/mayan-edms/badges/master/build.svg

__init__.py.tmpl

@@ -9,6 +9,6 @@ __author__ = 'Roberto Rosario'
 __author_email__ = 'roberto.rosario@mayan-edms.com'
 __description__ = 'Free Open Source Electronic Document Management System'
 __license__ = 'Apache 2.0'
-__copyright_short__ = '2011-2018 Roberto Rosario'
+__copyright_short__ = '2011 Roberto Rosario'
 __copyright__ = '{} {}'.format('Copyright', __copyright_short__)
 __website__ = 'https://www.mayan-edms.com'

contrib/scripts/docker/run-tests.sh

@@ -1,8 +1,10 @@
 #!/bin/sh
 
+export DEBIAN_FRONTEND=noninteractive
+
 apt-get update
 apt-get install -y --no-install-recommends tesseract-ocr-deu
 
-$MAYAN_PIP_BIN install -r $DOCKER_ROOT/requirements-testing.txt
+$MAYAN_PIP_BIN install -r ${MAYAN_INSTALL_DIR}/requirements-testing.txt
 
 $MAYAN_BIN test --mayan-apps --settings=mayan.settings.testing

contrib/scripts/install/development.sh

@@ -1,72 +0,0 @@
-#!/usr/bin/env bash
-
-INSTALLATION_DIRECTORY=/home/vagrant/mayan-edms/
-DB_NAME=mayan_edms
-DB_PASSWORD=test123
-
-cat << EOF | sudo tee -a /etc/motd.tail
-**********************************sudo apt
-
-Mayan EDMS Vagrant Development Box
-
-**********************************
-EOF
-
-# Update sources
-echo -e "\n -> Running apt-get update & upgrade \n"
-sudo apt-get -qq update
-sudo apt-get -y upgrade
-
-echo -e "\n -> Installing core binaries \n"
-sudo apt-get -y install git-core python-virtualenv gcc python-dev libjpeg-dev libpng-dev libtiff-dev tesseract-ocr poppler-utils libreoffice
-
-echo -e "\n -> Cloning development branch of repository \n"
-git clone /mayan-edms-repository/ $INSTALLATION_DIRECTORY
-cd $INSTALLATION_DIRECTORY
-git checkout development
-git reset HEAD --hard
-
-echo -e "\n -> Setting up virtual env \n"
-virtualenv venv
-source venv/bin/activate
-
-echo -e "\n -> Installing python dependencies \n"
-pip install -r requirements.txt
-
-echo -e "\n -> Running Mayan EDMS initial setup \n"
-./manage.py initialsetup
-
-echo -e "\n -> Installing Redis server \n"
-sudo apt-get install -y redis-server
-pip install redis
-
-echo -e "\n -> Installing testing software \n"
-pip install coverage
-
-echo -e "\n -> Installing MySQL \n"
-sudo debconf-set-selections <<< 'mysql-server mysql-server/root_password password '$DB_PASSWORD
-sudo debconf-set-selections <<< 'mysql-server mysql-server/root_password_again password '$DB_PASSWORD
-sudo apt-get install -y mysql-server libmysqlclient-dev
-# Create a passwordless root and travis users
-mysql -u root -p$DB_PASSWORD -e "SET PASSWORD = PASSWORD('');"
-mysql -u root -e "CREATE USER 'travis'@'localhost' IDENTIFIED BY '';GRANT ALL PRIVILEGES ON * . * TO 'travis'@'localhost';FLUSH PRIVILEGES;"
-mysql -u travis -e "CREATE DATABASE $DB_NAME;"
-pip install mysql-python
-
-echo -e "\n -> Installing PostgreSQL \n"
-sudo apt-get install -y postgresql postgresql-server-dev-all
-sudo -u postgres psql -c 'create database mayan_edms;' -U postgres
-sudo cat > /etc/postgresql/9.3/main/pg_hba.conf << EOF
-local   all             postgres                                trust
-
-# TYPE  DATABASE        USER            ADDRESS                 METHOD
-
-# "local" is for Unix domain socket connections only
-local   all             all                                     peer
-# IPv4 local connections:
-host    all             all             127.0.0.1/32            md5
-# IPv6 local connections:
-host    all             all             ::1/128                 md5
-EOF
-
-pip install -q psycopg2

contrib/scripts/install/docker.sh

@@ -5,21 +5,25 @@ set -e
 #   $ curl -fsSL get.mayan-edms.com -o get-mayan-edms.sh
 #   $ sh get-mayan-edms.sh
 #
-# NOTE: Make sure to verify the contents of the script
+# NOTE: Before executing, make sure to verify the contents of the script
 #       you downloaded matches the contents of docker.sh
 #       located at https://gitlab.com/mayan-edms/mayan-edms/blob/master/contrib/scripts/install/docker.sh
-#       before executing.
 
 : ${VERBOSE:=true}
 : ${INSTALL_DOCKER:=false}
 : ${DELETE_VOLUMES:=false}
+: ${USE_DOCKER_NETWORK:=true}
+: ${DOCKER_NETWORK_NAME:=mayan}
 : ${DATABASE_USER:=mayan}
 : ${DATABASE_NAME:=mayan}
 : ${DATABASE_PASSWORD:=mayanuserpass}
-: ${DOCKER_POSTGRES_IMAGE:=postgres:9.5}
+: ${DOCKER_POSTGRES_IMAGE:=postgres:9.6-alpine}
 : ${DOCKER_POSTGRES_CONTAINER:=mayan-edms-postgres}
 : ${DOCKER_POSTGRES_VOLUME:=/docker-volumes/mayan-edms/postgres}
 : ${DOCKER_POSTGRES_PORT:=5432}
+: ${DOCKER_REDIS_IMAGE:=redis:5.0-alpine}
+: ${DOCKER_REDIS_CONTAINER:=mayan-edms-redis}
+: ${DOCKER_REDIS_PORT:=6379}
 : ${DOCKER_MAYAN_IMAGE:=mayanedms/mayanedms:latest}
 : ${DOCKER_MAYAN_CONTAINER:=mayan-edms}
 : ${DOCKER_MAYAN_VOLUME:=/docker-volumes/mayan-edms/media}
@@ -44,6 +48,8 @@ echo "Variable values to be used:"
 echo "---------------------------"
 echo "INSTALL_DOCKER: $INSTALL_DOCKER"
 echo "DELETE_VOLUMES: $DELETE_VOLUMES"
+echo "USE_DOCKER_NETWORK: $USE_DOCKER_NETWORK"
+echo "DOCKER_NETWORK_NAME: $DOCKER_NETWORK_NAME"
 echo "DATABASE_USER: $DATABASE_USER"
 echo "DATABASE_NAME: $DATABASE_NAME"
 echo "DATABASE_PASSWORD: $DATABASE_PASSWORD"
@@ -51,10 +57,17 @@ echo "DOCKER_POSTGRES_IMAGE: $DOCKER_POSTGRES_IMAGE"
 echo "DOCKER_POSTGRES_CONTAINER: $DOCKER_POSTGRES_CONTAINER"
 echo "DOCKER_POSTGRES_VOLUME: $DOCKER_POSTGRES_VOLUME"
 echo "DOCKER_POSTGRES_PORT: $DOCKER_POSTGRES_PORT"
+echo "DOCKER_REDIS_IMAGE: $DOCKER_REDIS_IMAGE"
+echo "DOCKER_REDIS_CONTAINER: $DOCKER_REDIS_CONTAINER"
+echo "DOCKER_REDIS_PORT: $DOCKER_REDIS_PORT"
 echo "DOCKER_MAYAN_IMAGE: $DOCKER_MAYAN_IMAGE"
 echo "DOCKER_MAYAN_CONTAINER: $DOCKER_MAYAN_CONTAINER"
 echo "DOCKER_MAYAN_VOLUME: $DOCKER_MAYAN_VOLUME"
-echo "\nStarting in 10 seconds."
+echo
+echo "Override any of them by setting them before the script. "
+echo "Example: INSTALL_DOCKER=true sh get-mayan-edms.sh"
+
+echo "\nStarting in 10 seconds. Press CTRL+C to cancel."
 sleep 10
 fi
 
@@ -72,33 +85,62 @@ if [ -z `which docker`  ]; then
 fi
 
 echo -n "* Removing existing Mayan EDMS and PostgreSQL containers (no data will be lost)..."
-true || docker stop $DOCKER_MAYAN_CONTAINER >/dev/null 2>&1
-true || docker rm $DOCKER_MAYAN_CONTAINER >/dev/null 2>&1
-true || docker stop $DOCKER_POSTGRES_CONTAINER >/dev/null 2>&1
-true || docker rm $DOCKER_POSTGRES_CONTAINER >/dev/null 2>&1
+docker rm -f $DOCKER_REDIS_CONTAINER >/dev/null 2>&1  || true
+docker rm -f $DOCKER_POSTGRES_CONTAINER >/dev/null 2>&1  || true
+docker rm -f $DOCKER_MAYAN_CONTAINER >/dev/null 2>&1  || true
 echo "Done"
 
 if [ "$DELETE_VOLUMES" = true ]; then
-echo -n "* Deleting Docker volumes in 5 seconds (warning: this delete all document data)..."
+echo -n "* Deleting Docker volumes in 5 seconds (warning: this will delete all document data). Press CTRL+C to cancel..."
 sleep 5
-true || rm DOCKER_MAYAN_VOLUME -Rf
-true || rm DOCKER_POSTGRES_VOLUME -Rf
+rm DOCKER_MAYAN_VOLUME -Rf || true
+rm DOCKER_POSTGRES_VOLUME -Rf || true
 echo "Done"
 fi
 
-echo -n "* Pulling (downloading) the Mayan EDMS Docker image..."
-docker pull $DOCKER_MAYAN_IMAGE >/dev/null
+echo -n "* Pulling (downloading) the Redis Docker image..."
+docker pull $DOCKER_REDIS_IMAGE > /dev/null
 echo "Done"
 
 echo -n "* Pulling (downloading) the PostgreSQL Docker image..."
 docker pull $DOCKER_POSTGRES_IMAGE > /dev/null
 echo "Done"
 
+echo -n "* Pulling (downloading) the Mayan EDMS Docker image..."
+docker pull $DOCKER_MAYAN_IMAGE >/dev/null
+echo "Done"
+
+if [ "$USE_DOCKER_NETWORK" = true ]; then
+    echo -n "* Creating Docker network..."
+    docker network create $DOCKER_NETWORK_NAME 2> /dev/null || true
+    # Ignore error if the network already exists
+    echo "Done"
+fi
+
+if [ "$USE_DOCKER_NETWORK" = true ]; then
+    NETWORK_ARGUMENT="--network=$DOCKER_NETWORK_NAME"
+    POSTGRES_PORT_ARGUMENT=""
+    REDIS_PORT_ARGUMENT=""
+    MAYAN_DATABASE_PORT_ARGUMENT=""
+    MAYAN_DATABASE_HOST_ARGUMENT="-e MAYAN_DATABASE_HOST=$DOCKER_POSTGRES_CONTAINER"
+    MAYAN_BROKER_URL_ARGUMENT="-e MAYAN_BROKER_URL=redis://$DOCKER_REDIS_CONTAINER:6379/0"
+    MAYAN_CELERY_RESULT_BACKEND_ARGUMENT="-e MAYAN_CELERY_RESULT_BACKEND=redis://$DOCKER_REDIS_CONTAINER:6379/1"
+else
+    NETWORK_ARGUMENT=""
+    POSTGRES_PORT_ARGUMENT="-e $DOCKER_POSTGRES_PORT:5432"
+    REDIS_PORT_ARGUMENT="-e $DOCKER_REDIS_PORT:6379"
+    MAYAN_DATABASE_PORT_ARGUMENT="-e MAYAN_DATABASE_PORT=$DOCKER_POSTGRES_PORT"
+    MAYAN_DATABASE_HOST_ARGUMENT="-e MAYAN_DATABASE_HOST=172.17.0.1"
+    MAYAN_BROKER_URL_ARGUMENT="-e MAYAN_BROKER_URL=redis://172.17.0.1:6379/0"
+    MAYAN_CELERY_RESULT_BACKEND_ARGUMENT="-e MAYAN_CELERY_RESULT_BACKEND=redis://172.17.0.1:6379/1"
+fi
+
 echo -n "* Deploying the PostgreSQL container..."
 docker run -d \
 --name $DOCKER_POSTGRES_CONTAINER \
+$NETWORK_ARGUMENT \
 --restart=always \
--p $DOCKER_POSTGRES_PORT:5432 \
+$POSTGRES_PORT_ARGUMENT \
 -e POSTGRES_USER=$DATABASE_USER \
 -e POSTGRES_DB=$DATABASE_NAME \
 -e POSTGRES_PASSWORD=$DATABASE_PASSWORD \
@@ -106,6 +148,24 @@ docker run -d \
 $DOCKER_POSTGRES_IMAGE >/dev/null
 echo "Done"
 
+echo -n "* Deploying the Redis container..."
+docker run -d \
+--name $DOCKER_REDIS_CONTAINER \
+$NETWORK_ARGUMENT \
+--restart=always \
+$REDIS_PORT_ARGUMENT \
+$DOCKER_REDIS_IMAGE \
+redis-server \
+--appendonly no \
+--databases 2 \
+--maxmemory 100mb \
+--maxmemory-policy allkeys-lru \
+--maxclients 500 \
+--save "" \
+--tcp-backlog 256 \
+>/dev/null
+echo "Done"
+
 echo -n "* Waiting for the PostgreSQL container to be ready (10 seconds)..."
 sleep 10
 echo "Done"
@@ -113,15 +173,18 @@ echo "Done"
 echo -n "* Deploying Mayan EDMS container..."
 docker run -d \
 --name $DOCKER_MAYAN_CONTAINER \
+$NETWORK_ARGUMENT \
 --restart=always \
 -p 80:8000 \
 -e MAYAN_DATABASE_ENGINE=django.db.backends.postgresql \
--e MAYAN_DATABASE_HOST=172.17.0.1 \
+$MAYAN_DATABASE_HOST_ARGUMENT \
+$MAYAN_DATABASE_PORT_ARGUMENT \
 -e MAYAN_DATABASE_NAME=$DATABASE_NAME \
 -e MAYAN_DATABASE_PASSWORD=$DATABASE_PASSWORD \
 -e MAYAN_DATABASE_USER=$DATABASE_USER \
--e MAYAN_DATABASE_PORT=$DOCKER_POSTGRES_PORT \
--e MAYAN_DATABASE_CONN_MAX_AGE=60 \
+-e MAYAN_DATABASE_CONN_MAX_AGE=0 \
+$MAYAN_BROKER_URL_ARGUMENT \
+$MAYAN_CELERY_RESULT_BACKEND_ARGUMENT \
 -v $DOCKER_MAYAN_VOLUME:/var/lib/mayan \
 $DOCKER_MAYAN_IMAGE >/dev/null
 echo "Done"

contrib/scripts/install/production.sh

@@ -1,171 +0,0 @@
-#!/usr/bin/env bash
-
-# ====== CONFIG ======
-INSTALLATION_DIRECTORY=/usr/share/mayan-edms/
-DB_NAME=mayan_edms
-DB_USERNAME=mayan
-DB_PASSWORD=test123
-# ==== END CONFIG ====
-
-cat << EOF | tee -a /etc/motd.tail
-**********************************
-
-Mayan EDMS Vagrant Production Box
-
-**********************************
-EOF
-
-echo -e "\n -> Running apt-get update & upgrade \n"
-apt-get -qq update
-apt-get -y upgrade
-
-echo -e "\n -> Installing core binaries \n"
-apt-get install nginx supervisor redis-server postgresql libpq-dev libjpeg-dev libmagic1 libpng-dev libreoffice libtiff-dev gcc ghostscript gpgv python-dev python-virtualenv tesseract-ocr poppler-utils -y
-
-echo -e "\n -> Setting up virtualenv \n"
-rm -f ${INSTALLATION_DIRECTORY}
-virtualenv ${INSTALLATION_DIRECTORY}
-source ${INSTALLATION_DIRECTORY}bin/activate
-
-echo -e "\n -> Installing Mayan EDMS from PyPI \n"
-pip install mayan-edms
-
-echo -e "\n -> Installing Python client for PostgreSQL, Redis, and uWSGI \n"
-pip install psycopg2 redis uwsgi
-
-echo -e "\n -> Creating the database for the installation \n"
-echo "CREATE USER mayan WITH PASSWORD '$DB_PASSWORD';" | sudo -u postgres psql
-sudo -u postgres createdb -O $DB_USERNAME $DB_NAME
-
-echo -e "\n -> Creating the directories for the logs \n"
-mkdir /var/log/mayan
-
-echo -e "\n -> Making a convenience symlink \n"
-cd ${INSTALLATION_DIRECTORY}
-ln -s lib/python2.7/site-packages/mayan .
-
-echo -e "\n -> Creating an initial settings file \n"
-mayan-edms.py createsettings
-
-echo -e "\n -> Updating the mayan/settings/local.py file \n"
-cat >> mayan/settings/local.py << EOF
-DATABASES = {
-    'default': {
-        'ENGINE': 'django.db.backends.postgresql_psycopg2',
-        'NAME': '$DB_NAME',
-        'USER': '$DB_USERNAME',
-        'PASSWORD': '$DB_PASSWORD',
-        'HOST': 'localhost',
-        'PORT': '5432',
-    }
-}
-
-BROKER_URL = 'redis://127.0.0.1:6379/0'
-CELERY_RESULT_BACKEND = 'redis://127.0.0.1:6379/0'
-EOF
-
-echo -e "\n -> Migrating the database or initialize the project \n"
-mayan-edms.py initialsetup
-
-echo -e "\n -> Disabling the default NGINX site \n"
-rm -f /etc/nginx/sites-enabled/default
-
-echo -e "\n -> Creating a uwsgi.ini file \n"
-cat > uwsgi.ini << EOF
-[uwsgi]
-chdir = ${INSTALLATION_DIRECTORY}lib/python2.7/site-packages/mayan
-chmod-socket = 664
-chown-socket = www-data:www-data
-env = DJANGO_SETTINGS_MODULE=mayan.settings.production
-gid = www-data
-logto = /var/log/uwsgi/%n.log
-pythonpath = ${INSTALLATION_DIRECTORY}lib/python2.7/site-packages
-master = True
-max-requests = 5000
-socket = ${INSTALLATION_DIRECTORY}uwsgi.sock
-uid = www-data
-vacuum = True
-wsgi-file = ${INSTALLATION_DIRECTORY}lib/python2.7/site-packages/mayan/wsgi.py
-EOF
-
-echo -e "\n -> Creating the directory for the uWSGI log files \n"
-mkdir -p /var/log/uwsgi
-
-echo -e "\n -> Creating the NGINX site file for Mayan EDMS, /etc/nginx/sites-available/mayan \n"
-cat > /etc/nginx/sites-available/mayan << EOF
-server {
-    listen 80;
-    server_name localhost;
-
-    location / {
-        include uwsgi_params;
-        uwsgi_pass unix:${INSTALLATION_DIRECTORY}uwsgi.sock;
-
-        client_max_body_size 30M;  # Increse if your plan to upload bigger documents
-        proxy_read_timeout 30s;  # Increase if your document uploads take more than 30 seconds
-    }
-
-    location /static {
-        alias ${INSTALLATION_DIRECTORY}mayan/media/static;
-        expires 1h;
-    }
-
-    location /favicon.ico {
-        alias ${INSTALLATION_DIRECTORY}mayan/media/static/appearance/images/favicon.ico;
-        expires 1h;
-    }
-}
-EOF
-
-echo -e "\n -> Enabling the NGINX site for Mayan EDMS \n"
-ln -s /etc/nginx/sites-available/mayan /etc/nginx/sites-enabled/
-
-echo -e "\n -> Creating the supervisor file for the uWSGI process, /etc/supervisor/conf.d/mayan-uwsgi.conf \n"
-cat > /etc/supervisor/conf.d/mayan-uwsgi.conf << EOF
-[program:mayan-uwsgi]
-command = ${INSTALLATION_DIRECTORY}bin/uwsgi --ini ${INSTALLATION_DIRECTORY}uwsgi.ini
-user = root
-autostart = true
-autorestart = true
-redirect_stderr = true
-EOF
-
-echo -e "\n -> Creating the supervisor file for the Celery worker, /etc/supervisor/conf.d/mayan-celery.conf \n"
-cat > /etc/supervisor/conf.d/mayan-celery.conf << EOF
-[program:mayan-worker]
-command = ${INSTALLATION_DIRECTORY}bin/python ${INSTALLATION_DIRECTORY}bin/mayan-edms.py celery --settings=mayan.settings.production worker -Ofair -l ERROR
-directory = ${INSTALLATION_DIRECTORY}
-user = www-data
-stdout_logfile = /var/log/mayan/worker-stdout.log
-stderr_logfile = /var/log/mayan/worker-stderr.log
-autostart = true
-autorestart = true
-startsecs = 10
-stopwaitsecs = 10
-killasgroup = true
-priority = 998
-
-[program:mayan-beat]
-command = ${INSTALLATION_DIRECTORY}bin/python ${INSTALLATION_DIRECTORY}bin/mayan-edms.py celery --settings=mayan.settings.production beat -l ERROR
-directory = ${INSTALLATION_DIRECTORY}
-user = www-data
-numprocs = 1
-stdout_logfile = /var/log/mayan/beat-stdout.log
-stderr_logfile = /var/log/mayan/beat-stderr.log
-autostart = true
-autorestart = true
-startsecs = 10
-stopwaitsecs = 1
-killasgroup = true
-priority = 998
-EOF
-
-echo -e "\n -> Collecting the static files \n"
-mayan-edms.py collectstatic --noinput
-
-echo -e "\n -> Making the installation directory readable and writable by the webserver user \n"
-chown www-data:www-data ${INSTALLATION_DIRECTORY} -R
-
-echo -e "\n -> Restarting the services \n"
-/etc/init.d/nginx restart
-/etc/init.d/supervisor restart

contrib/scripts/process_messages.py

@@ -9,19 +9,22 @@ import sh
 
 
 APP_LIST = (
-    'acls', 'appearance', 'authentication', 'cabinets', 'checkouts', 'common',
-    'converter', 'django_gpg', 'document_comments', 'document_indexing',
-    'document_parsing', 'document_signatures', 'document_states', 'documents',
-    'dynamic_search', 'events', 'linking', 'lock_manager', 'mayan_statistics',
-    'mailer', 'metadata', 'mirroring', 'motd', 'navigation', 'ocr', 'permissions',
-    'rest_api', 'smart_settings', 'sources', 'storage', 'tags', 'task_manager',
-    'user_management'
+    'acls', 'appearance', 'authentication', 'autoadmin', 'cabinets',
+    'checkouts', 'common', 'converter', 'dashboards', 'dependencies',
+    'django_gpg', 'document_comments', 'document_indexing',
+    'document_parsing', 'document_signatures', 'document_states',
+    'documents', 'dynamic_search', 'events', 'file_caching',
+    'file_metadata', 'linking', 'lock_manager', 'mailer',
+    'mayan_statistics', 'metadata', 'mirroring', 'motd', 'navigation',
+    'ocr', 'permissions', 'platform', 'rest_api', 'smart_settings',
+    'sources', 'storage', 'tags', 'task_manager', 'user_management',
+    'weblinks'
 )
 
 LANGUAGE_LIST = (
-    'ar', 'bg', 'bs_BA', 'da', 'de_DE', 'en', 'es', 'fa', 'fr', 'hu', 'id',
-    'it', 'nl_NL', 'pl', 'pt', 'pt_BR', 'ro_RO', 'ru', 'sl_SI', 'tr_TR',
-    'vi_VN', 'zh_CN',
+    'ar', 'bg', 'bs_BA', 'cs', 'da_DK', 'de_DE', 'en', 'es', 'el', 'fa', 'fr',
+    'hu', 'id', 'it', 'lv', 'nl_NL', 'pl', 'pt', 'pt_BR', 'ro_RO', 'ru', 'sl_SI',
+    'tr_TR', 'vi_VN', 'zh',
 )
 
 makemessages = sh.Command('django-admin.py')

contrib/scripts/start_gunicorn.sh

@@ -1,35 +0,0 @@
-#!/bin/bash
-
-NAME="mayan-edms"
-DJANGODIR=/usr/share/mayan-edms
-SOCKFILE=/var/tmp/filesystem.sock
-USER=www-data
-GROUP=www-data
-NUM_WORKERS=3
-DJANGO_SETTINGS_MODULE=mayan.settings.production
-DJANGO_WSGI_MODULE=mayan.wsgi
-TIMEOUT=600
-
-echo "Starting $NAME as `whoami`"
-
-# Activate the virtual environment
-cd $DJANGODIR
-source bin/activate
-export DJANGO_SETTINGS_MODULE=$DJANGO_SETTINGS_MODULE
-export PYTHONPATH=$DJANGODIR:$PYTHONPATH
-
-# Create the run directory if it doesn't exist
-RUNDIR=$(dirname $SOCKFILE)
-test -d $RUNDIR || mkdir -p $RUNDIR
-
-# Start your Django Unicorn
-# Programs meant to be run under supervisor should not daemonize themselves (do not use --daemon)
-exec bin/gunicorn ${DJANGO_WSGI_MODULE}:application \
-  --name $NAME \
-  --workers $NUM_WORKERS \
-  --user=$USER --group=$GROUP \
-  --log-level=debug \
-  --bind=unix:$SOCKFILE \
-  --timeout=$TIMEOUT
-
-

docker/Dockerfile

@@ -1,164 +1,159 @@
 # vim:set ft=dockerfile:
 
-####################
-# Base image start #
-####################
+####
+# BASE_IMAGE - Bare bones image with the base packages needed to run Mayan EDMS
+####
 
-FROM debian:9.4-slim as BASE_IMAGE
+FROM debian:10.0-slim as BASE_IMAGE
 
-MAINTAINER Roberto Rosario "roberto.rosario@mayan-edms.com"
+LABEL maintainer="Roberto Rosario roberto.rosario@mayan-edms.com"
 
-ENV DEBIAN_FRONTEND noninteractive
-ENV PYTHONUNBUFFERED 1
-ENV LC_ALL C.UTF-8
-ENV PROJECT_INSTALL_DIR=/opt/mayan-edms
-ENV PYTHON_PIP=${PROJECT_INSTALL_DIR}/bin/pip
+ENV PYTHONUNBUFFERED=1 \
+    LC_ALL=C.UTF-8 \
+    PROJECT_INSTALL_DIR=/opt/mayan-edms
 
+# Debian package caching
 ARG APT_PROXY
-# Package caching
-RUN if [ "${APT_PROXY}" ]; then echo "Acquire::http { Proxy \"http://${APT_PROXY}\"; };" > /etc/apt/apt.conf.d/01proxy; fi
-
-# Install base Ubuntu libraries
-RUN apt-get update && \
-apt-get install -y --no-install-recommends \
-        g++ \
-        gcc \
-        ghostscript \
-        gpgv \
-        gnupg1 \
-        graphviz \
-        libffi-dev \
-        libfuse2 \
-        libjpeg-dev \
-        libmagic1 \
-        default-libmysqlclient-dev \
-        libpng-dev \
-        libpq-dev \
-        libreoffice \
-        libtiff-dev \
-        poppler-utils \
-        python-dev \
-        python-setuptools \
-        python-virtualenv \
-        python-wheel \
-        redis-server \
-        sane-utils \
-        sudo \
-        supervisor \
-        tesseract-ocr \
-        zlib1g-dev \
-&& \
-apt-get clean autoclean && \
-apt-get autoremove --purge -y && \
-rm -rf /var/lib/apt/lists/* && \
-rm -f /var/cache/apt/archives/*.deb
-
-# Install apt-get-install
-ADD https://raw.githubusercontent.com/guilhem/apt-get-install/master/apt-get-install /usr/bin/
-RUN chmod +x /usr/bin/apt-get-install
-
-RUN adduser mayan --disabled-password --disabled-login --no-create-home --gecos ""
-
+RUN set -x \
+&& if [ "${APT_PROXY}" ]; \
+    then echo "Acquire::http { Proxy \"http://${APT_PROXY}\"; };" > /etc/apt/apt.conf.d/01proxy \
+; fi \
+# Install base OS packages to run Mayan EDMS
+&& DEBIAN_FRONTEND=noninteractive \
+apt-get update \
+&& apt-get install -y --no-install-recommends \
+    ca-certificates \
+    exiftool \
+    fonts-arphic-uming \
+    fonts-arphic-ukai \
+    ghostscript \
+    gpgv \
+    gnupg1 \
+    graphviz \
+    libfuse2 \
+    libmagic1 \
+    libmariadb3 \
+    libreoffice \
+    libpq5 \
+    poppler-utils \
+    python3-distutils \
+    sane-utils \
+    sudo \
+    supervisor \
+    tesseract-ocr \
+# Remove make and build dependencies
+&& apt-get remove make libproxy-tools libreoffice-avmedia-backend-vlc libvlc-bin libvlc5 libvlccore9 adwaita-icon-theme gsettings-desktop-schemas libgstreamer-plugins-base1.0-0 -y \
+&& apt-get autoremove -y --purge \
+# Add mayan user
+&& adduser mayan --disabled-password --disabled-login --no-create-home --gecos "" \
 # Pillow can't find zlib or libjpeg on aarch64 (ODROID C2)
-RUN if [ "$(uname -m)" = "aarch64" ]; then \
-ln -s /usr/lib/aarch64-linux-gnu/libz.so /usr/lib/ && \
-ln -s /usr/lib/aarch64-linux-gnu/libjpeg.so /usr/lib/ \
-; fi
-
+&& if [ "$(uname -m)" = "aarch64" ]; then \
+    ln -s /usr/lib/aarch64-linux-gnu/libz.so /usr/lib/ \
+    && ln -s /usr/lib/aarch64-linux-gnu/libjpeg.so /usr/lib/ \
+; fi \
 # Pillow can't find zlib or libjpeg on armv7l (ODROID HC1)
-RUN if [ "$(uname -m)" = "armv7l" ]; then \
-apt-get install libssl-dev -y && \
-ln -s /usr/lib/arm-linux-gnueabihf/libz.so /usr/lib/ && \
-ln -s /usr/lib/arm-linux-gnueabihf/libjpeg.so /usr/lib/ \
+&& if [ "$(uname -m)" = "armv7l" ]; then \
+    ln -s /usr/lib/arm-linux-gnueabihf/libz.so /usr/lib/ \
+    && ln -s /usr/lib/arm-linux-gnueabihf/libjpeg.so /usr/lib/ \
 ; fi
 
-# Discard data when Redis runs out of memory
-RUN echo "maxmemory-policy allkeys-lru" >> /etc/redis/redis.conf
+####
+# BUILDER_IMAGE - This image builds the Python package and is discarded afterwards
+#                 only the build artifact is carried over to the next image.
+####
 
-# Disable saving the Redis database
-RUN echo "save \"\"" >> /etc/redis/redis.conf
+# Reuse image
+FROM BASE_IMAGE as BUILDER_IMAGE
 
-# Only provision 1 database
-RUN echo "databases 1" >> /etc/redis/redis.conf
+# Python libraries caching
+ARG PIP_INDEX_URL
+ARG PIP_TRUSTED_HOST
 
-#####################
-# Build image start #
-#####################
+WORKDIR /src
 
-FROM debian:9.4-slim as BUILDER_IMAGE
+# Copy the source files needed to build the Python package
+COPY --chown=mayan:mayan requirements /src/requirements
+COPY --chown=mayan:mayan \
+    HISTORY.rst \
+    LICENSE \
+    MANIFEST.in \
+    README.md \
+    README.rst \
+    setup.py \
+    /src/
 
-ARG APT_PROXY
-# Package caching
-RUN if [ "${APT_PROXY}" ]; then echo "Acquire::http { Proxy \"http://${APT_PROXY}\"; };" > /etc/apt/apt.conf.d/01proxy; fi
+COPY --chown=mayan:mayan mayan /src/mayan
 
-WORKDIR /code
+# Install development packages needed to build the Python packages
+RUN DEBIAN_FRONTEND=noninteractive \
+apt-get install -y --no-install-recommends \
+    default-libmysqlclient-dev \
+    libffi-dev \
+    libjpeg-dev \
+    libpng-dev \
+    libpq-dev \
+    libtiff-dev \
+    zlib1g-dev \
+    libssl-dev \
+    g++ \
+    gcc \
+    python3-dev \
+    python3-venv \
+&& mkdir -p "${PROJECT_INSTALL_DIR}" \
+&& chown -R mayan:mayan "${PROJECT_INSTALL_DIR}" \
+&& chown -R mayan:mayan /src
 
-COPY . /code
+USER mayan
+RUN python3 -m venv "${PROJECT_INSTALL_DIR}" \
+&& . "${PROJECT_INSTALL_DIR}/bin/activate" \
+&& pip install --no-cache-dir \
+    librabbitmq==2.0.0 \
+    mysqlclient==1.4.2.post1 \
+    psycopg2==2.8.3 \
+    redis==3.2.1 \
+    flower==0.9.3 \
+# psutil is needed by ARM builds otherwise gevent and gunicorn fail to start
+&& UNAME=`uname -m` && if [ "${UNAME#*arm}" != $UNAME ]; then \
+    pip install --no-cache-dir \
+    psutil==5.6.2 \
+; fi \
+# Install the Python packages needed to build Mayan EDMS
+&& pip install --no-cache-dir -r /src/requirements/build.txt \
+# Build Mayan EDMS
+&& python3 setup.py sdist \
+# Install the built Mayan EDMS package
+&& pip install --no-cache-dir dist/mayan* \
+# Install the static content
+&& mayan-edms.py installdependencies \
+&& MAYAN_STATIC_ROOT=${PROJECT_INSTALL_DIR}/static mayan-edms.py preparestatic --link --noinput
 
-RUN apt-get update && apt-get install make python-dev python-pip -y
+COPY --chown=mayan:mayan requirements/testing-base.txt "${PROJECT_INSTALL_DIR}"
 
-RUN pip install -r requirements/build.txt
-
-ENV LC_ALL C.UTF-8
-
-RUN touch docker/Makefile
-
-RUN make wheel
-
-RUN chmod 777 dist -R
-
-#####################
-# Final image start #
-#####################
+####
+# Final image - BASE_IMAGE + BUILDER_IMAGE artifact (Mayan install directory)
+####
 
 FROM BASE_IMAGE
 
-RUN mkdir -p /opt
+COPY --from=BUILDER_IMAGE --chown=mayan:mayan "${PROJECT_INSTALL_DIR}/" "${PROJECT_INSTALL_DIR}/"
 
-RUN python /usr/lib/python2.7/dist-packages/virtualenv.py $PROJECT_INSTALL_DIR
+USER root
 
-WORKDIR $PROJECT_INSTALL_DIR
+COPY docker/rootfs /
 
-COPY --from=BUILDER_IMAGE /code/dist/*.whl .
-
-COPY --from=BUILDER_IMAGE /code/contrib/scripts/docker/run-tests.sh .
-
-COPY --from=BUILDER_IMAGE /code/requirements/testing-base.txt requirements-testing.txt
-
-COPY --from=BUILDER_IMAGE /code/docker/version .
-
-# Fix ownership
-RUN chown -R mayan:mayan $PROJECT_INSTALL_DIR
-
-# Install build Mayan EDMS
-RUN sudo -u mayan $PYTHON_PIP install --no-cache-dir *.whl && \
-rm *.whl
-
-# Install Python clients for librabbitmq, MySQL, PostgreSQL, REDIS
-RUN sudo -u mayan $PYTHON_PIP install --no-cache-dir librabbitmq==1.6.1 mysql-python==1.2.5 psycopg2==2.7.3.2 redis==2.10.6
-
-# Setup supervisor
-COPY docker/etc/supervisor/mayan.conf /etc/supervisor/conf.d
-
-RUN mkdir /var/lib/mayan
 VOLUME ["/var/lib/mayan"]
 
-COPY docker/entrypoint.sh /usr/local/bin/
-RUN ln -s usr/local/bin/entrypoint.sh / # backwards compat
 ENTRYPOINT ["entrypoint.sh"]
 
 EXPOSE 8000
-CMD ["mayan"]
+CMD ["run_all"]
 
-RUN rm /root/.cache -R
-RUN rm -rf /tmp/*
-
-RUN apt-get -y autoremove --purge && apt-get -y autoclean && apt-get -y clean
-
-RUN rm -rf /usr/share/man/*
-RUN rm -rf /usr/share/doc/*
-
-RUN find /var/lib/apt -type f | xargs rm -f
-RUN find /var/cache -type f -exec rm -rf {} \;
-
-RUN find /var/log -type f | while read f; do echo -ne '' > $f; done;
+RUN ${PROJECT_INSTALL_DIR}/bin/mayan-edms.py platformtemplate supervisord_docker > /etc/supervisor/conf.d/mayan.conf \
+&& apt-get clean autoclean \
+&& apt-get autoremove --purge -y \
+&& rm -rf /var/lib/apt/lists/* \
+&& rm -f /var/cache/apt/archives/*.deb \
+# Remove temporary files owned by root from the platformtemplate step
+&& rm /tmp/* \
+# Keep displaying log messages to stdout
+&& find /var/log -type f | while read f; do echo -ne '' > $f; done;

docker/Dockerfile.ubuntu

@@ -1,150 +0,0 @@
-# vim:set ft=dockerfile:
-
-####################
-# Base image start #
-####################
-
-FROM ubuntu:16.04 as BASE_IMAGE
-
-MAINTAINER Roberto Rosario "roberto.rosario@mayan-edms.com"
-
-ENV DEBIAN_FRONTEND noninteractive
-ENV PYTHONUNBUFFERED 1
-ENV LANG en_US.UTF-8
-ENV PROJECT_INSTALL_DIR=/usr/local/lib/python2.7/dist-packages/mayan
-
-ARG APT_PROXY
-# Package caching
-RUN if [ "${APT_PROXY}" ]; then echo "Acquire::http { Proxy \"http://${APT_PROXY}\"; };" > /etc/apt/apt.conf.d/01proxy; fi
-
-# Install base Ubuntu libraries
-RUN apt-get update && \
-apt-get install -y --no-install-recommends \
-        curl \
-        g++ \
-        gcc \
-        gettext-base \
-        ghostscript \
-        gpgv \
-        graphviz \
-        libffi-dev \
-        libjpeg-dev \
-        libmagic1 \
-        libmysqlclient-dev \
-        libpng-dev \
-        libpq-dev \
-        libreoffice \
-        libtiff-dev \
-        locales \
-        netcat-openbsd \
-        poppler-utils \
-        python-dev \
-        python-pip \
-        python-setuptools \
-        python-wheel \
-        redis-server \
-        supervisor \
-        tesseract-ocr \
-        zlib1g-dev \
-&& \
-apt-get clean autoclean && \
-apt-get autoremove --purge -y && \
-rm -rf /var/lib/apt/lists/* && \
-rm -f /var/cache/apt/archives/*.deb
-
-# Switch to UTF locale
-RUN echo "LC_ALL=\"en_US.UTF-8\"" >> /etc/default/locale && \
-locale-gen en_US.UTF-8 && \
-update-locale LANG=en_US.UTF-8 && \
-export LC_ALL=en_US.UTF-8
-
-# Install apt-get-install
-ADD https://raw.githubusercontent.com/guilhem/apt-get-install/master/apt-get-install /usr/bin/
-RUN chmod +x /usr/bin/apt-get-install
-
-# Install Python clients for PostgreSQL, REDIS, librabbitmq
-RUN pip install psycopg2==2.7.3.2 redis==2.10.6 mysql-python==1.2.5 librabbitmq==1.6.1
-
-RUN adduser mayan --disabled-password --disabled-login --no-create-home --gecos ""
-
-# Pillow can't find zlib or libjpeg on aarch64
-RUN if [ "$(uname -m)" = "aarch64" ]; then \
-ln -s /usr/lib/aarch64-linux-gnu/libz.so /usr/lib/ && \
-ln -s /usr/lib/aarch64-linux-gnu/libjpeg.so /usr/lib/ \
-; fi
-
-#####################
-# Build image start #
-#####################
-
-FROM python:2-alpine3.7 as BUILDER_IMAGE
-
-WORKDIR /code
-
-COPY . /code
-
-RUN apk update && \
-    apk add make
-
-RUN pip install -r requirements/build.txt
-
-RUN make wheel
-
-RUN chmod 777 dist -R
-
-#####################
-# Final image start #
-#####################
-
-FROM BASE_IMAGE
-
-WORKDIR /root/
-
-COPY --from=BUILDER_IMAGE /code/dist/*.whl .
-
-# Install build Mayan EDMS
-RUN pip install *.whl && \
-rm *.whl
-
-# Setup supervisor
-#RUN mkdir /etc/supervisor.d/
-COPY docker/etc/supervisor/beat.conf /etc/supervisor/conf.d
-COPY docker/etc/supervisor/gunicorn.conf /etc/supervisor/conf.d
-COPY docker/etc/supervisor/redis.conf /etc/supervisor/conf.d
-COPY docker/etc/supervisor/workers.conf /etc/supervisor/conf.d
-
-# Create the directory for the logs
-RUN mkdir /var/log/mayan
-
-# Fix ownership
-RUN chown -R mayan:mayan $PROJECT_INSTALL_DIR
-
-# Allow flanker to autogenerate its PLY files
-RUN chown -R mayan:mayan /usr/local/lib/python2.7/dist-packages/flanker/
-
-RUN mkdir /var/lib/mayan
-VOLUME ["/var/lib/mayan"]
-
-COPY docker/entrypoint.sh /usr/local/bin/
-RUN ln -s usr/local/bin/entrypoint.sh / # backwards compat
-ENTRYPOINT ["entrypoint.sh"]
-
-# Healthcheck setup
-HEALTHCHECK --interval=15s --timeout=1s --retries=20 \
-  CMD curl -s -f http://localhost/authentication/login/ | grep 'form' > /dev/null || exit 1
-
-EXPOSE 8000
-CMD ["mayan"]
-
-RUN rm /root/.cache -R
-RUN rm -rf /tmp/*
-
-RUN apt-get -y autoremove --purge && apt-get -y autoclean && apt-get -y clean
-
-RUN rm -rf /usr/share/man/*
-RUN rm -rf /usr/share/doc/*
-
-RUN find /var/lib/apt -type f | xargs rm -f
-RUN find /var/cache -type f -exec rm -rf {} \;
-
-RUN find /var/log -type f | while read f; do echo -ne '' > $f; done;

docker/Makefile

@@ -1,33 +1,40 @@
-APT_PROXY ?= `/sbin/ip route|awk '/docker0/ { print $$9 }'`:3142
-IMAGE_VERSION ?= `cat docker/version`
+HOST_IP = `/sbin/ip route|awk '/docker0/ { print $$9 }'`
+
+APT_PROXY ?= $(HOST_IP):3142
+PIP_INDEX_URL ?= http://$(HOST_IP):3141/root/pypi/+simple/
+PIP_TRUSTED_HOST ?= $(HOST_IP)
+
+IMAGE_VERSION ?= `cat docker/rootfs/version`
 CONSOLE_COLUMNS ?= `echo $$(tput cols)`
 CONSOLE_LINES ?= `echo $$(tput lines)`
 
-docker-help:
-	@echo
-	@echo "**** Docker makefile ****"
-	@echo "docker-build - Build a new image locally."
-	@echo "docker-build-with-proxy - Build a new image locally using an APT proxy."
-	@echo "docker-test-container - Build and run a test container."
-	@echo "docker-test-cleanup - Delete the test container and the test volume."
-	@echo "docker-test-all - Build and executed the test suite in a test container."
-	@echo "docker-shell - Launch a bash instance inside a running container. Pass the container name via DOCKER_CONTAINER."
-
-docker-build:
+docker-build: ## Build a new image locally.
 	docker build -t mayanedms/mayanedms:$(IMAGE_VERSION) -f docker/Dockerfile .
 
-docker-build-with-proxy:
-	docker build -t mayanedms/mayanedms:$(IMAGE_VERSION) -f docker/Dockerfile --build-arg APT_PROXY=$(APT_PROXY) .
+docker-build-with-proxy: ## Build a new image locally using an APT proxy as APT_PROXY.
+	docker build -t mayanedms/mayanedms:$(IMAGE_VERSION) -f docker/Dockerfile --build-arg APT_PROXY=$(APT_PROXY) --build-arg PIP_INDEX_URL=$(PIP_INDEX_URL) --build-arg PIP_TRUSTED_HOST=$(PIP_TRUSTED_HOST) --build-arg HTTP_PROXY=$(HTTP_PROXY) --build-arg HTTPS_PROXY=$(HTTPS_PROXY) .
 
-docker-shell:
+docker-shell: ## Launch a bash instance inside a running container. Pass the container name via DOCKER_CONTAINER.
 	docker exec -e TERM=$(TERM) -e "COLUMNS=$(CONSOLE_COLUMNS)" -e "LINES=$(CONSOLE_LINES)" -it $(DOCKER_CONTAINER) /bin/bash
 
+docker-test-container: ## Build and run a test container.
 docker-test-container: docker-build-with-proxy docker-test-cleanup
 	docker run -d --name test-mayan-edms -p 80:8000 -v test-mayan_data:/var/lib/mayan mayanedms/mayanedms:$(DOCKER_VERSION)
 
-docker-test-cleanup:
+docker-test-cleanup: ## Delete the test container and the test volume.
 	@docker rm -f test-mayan-edms || true
 	@docker volume rm test-mayan_data || true
 
+docker-test-all: ## Build and executed the test suite in a test container.
 docker-test-all: docker-build-with-proxy
 	docker run --rm run-tests
+
+docker-compose-build:
+	docker-compose -f docker/docker-compose.yml -p mayan-edms build
+
+docker-compose-build-with-proxy:
+	docker-compose -f docker/docker-compose.yml -p mayan-edms build --build-arg APT_PROXY=$(APT_PROXY) --build-arg PIP_INDEX_URL=$(PIP_INDEX_URL) --build-arg PIP_TRUSTED_HOST=$(PIP_TRUSTED_HOST) --build-arg HTTP_PROXY=$(HTTP_PROXY) --build-arg HTTPS_PROXY=$(HTTPS_PROXY)
+
+docker-compose-up:
+	docker-compose -f docker/docker-compose.yml -p mayan-edms up
+

docker/README-short.txt

@@ -1 +0,0 @@
-Mayan EDMS is a free open source electronic document management system.

docker/docker-compose-development.yml

@@ -1,72 +0,0 @@
-version: '2.1'
-
-volumes:
-    broker:
-        driver: local
-    app:
-        driver: local
-    db:
-        driver: local
-    results:
-        driver: local
-
-services:
-  broker:
-    container_name: mayan-edms-broker
-    image: healthcheck/rabbitmq
-    environment:
-      RABBITMQ_DEFAULT_USER: mayan
-      RABBITMQ_DEFAULT_PASS: mayan
-      RABBITMQ_DEFAULT_VHOST: mayan
-    volumes:
-      - broker:/var/lib/rabbitmq
-  results:
-    container_name: mayan-edms-results
-    image: healthcheck/redis
-    volumes:
-      - results:/data
-  #db:
-  #  container_name: mayan-edms-db
-  #  image: healthcheck/mysql
-  #  environment:
-  #    MYSQL_DATABASE: mayan
-  #    MYSQL_PASSWORD: mayan-password
-  #    MYSQL_ROOT_PASSWORD: root-password
-  #    MYSQL_USER: mayan
-  #  volumes:
-  #    - db:/var/lib/mysql
-  db:
-    container_name: mayan-edms-db
-    image: healthcheck/postgres
-    environment:
-      POSTGRES_DB: mayan
-      POSTGRES_PASSWORD: mayan-password
-      POSTGRES_USER: mayan
-    volumes:
-      - db:/var/lib/postgresql/data
-  mayan-edms:
-    container_name: mayan-edms-app
-    image: mayan-edms/next
-    build:
-      context: ./
-      args:
-        - APT_PROXY=172.18.0.1:3142
-    depends_on:
-      broker:
-        condition: service_healthy
-      db:
-        condition: service_healthy
-      results:
-        condition: service_healthy
-    environment:
-      MAYAN_BROKER_URL: amqp://mayan:mayan@broker:5672/mayan
-      MAYAN_CELERY_RESULT_BACKEND: redis://results:6379/0
-      MAYAN_DATABASE_ENGINE: django.db.backends.postgresql
-      MAYAN_DATABASE_HOST: db
-      MAYAN_DATABASE_NAME: mayan
-      MAYAN_DATABASE_PASSWORD: mayan-password
-      MAYAN_DATABASE_USER: mayan
-    ports:
-      - "80:80"
-    volumes:
-      - app:/var/lib/mayan

docker/docker-compose.yml

@@ -1,58 +1,130 @@
-version: '2.1'
+version: '3.7'
 
-volumes:
-    broker:
-        driver: local
-    app:
-        driver: local
-    db:
-        driver: local
-    results:
-        driver: local
+networks:
+  mayan-bridge:
+    driver: bridge
 
 services:
-  broker:
-    container_name: mayan-edms-broker
-    image: healthcheck/rabbitmq
-    environment:
-      RABBITMQ_DEFAULT_USER: mayan
-      RABBITMQ_DEFAULT_PASS: mayan
-      RABBITMQ_DEFAULT_VHOST: mayan
-    volumes:
-      - broker:/var/lib/rabbitmq
-  results:
-    container_name: mayan-edms-results
-    image: healthcheck/redis
-    volumes:
-      - results:/data
-  db:
-    container_name: mayan-edms-db
-    image: healthcheck/postgres
-    environment:
-      POSTGRES_DB: mayan
-      POSTGRES_PASSWORD: mayan-password
-      POSTGRES_USER: mayan
-    volumes:
-      - db:/var/lib/postgresql/data
-  mayan-edms:
-    container_name: mayan-edms-app
-    image: mayanedms/mayanedms:latest
+  app:
+    build:
+      context: ..
+      dockerfile: ./docker/Dockerfile
     depends_on:
-      broker:
-        condition: service_healthy
-      db:
-        condition: service_healthy
-      results:
-        condition: service_healthy
-    environment:
-      MAYAN_BROKER_URL: amqp://mayan:mayan@broker:5672/mayan
-      MAYAN_CELERY_RESULT_BACKEND: redis://results:6379/0
-      MAYAN_DATABASE_ENGINE: django.db.backends.postgresql
-      MAYAN_DATABASE_HOST: db
-      MAYAN_DATABASE_NAME: mayan
-      MAYAN_DATABASE_PASSWORD: mayan-password
-      MAYAN_DATABASE_USER: mayan
+      - postgresql
+      - redis
+      # Enable to use RabbitMQ
+      #- rabbitmq
+    environment: &mayan_env
+      # Enable to use RabbitMQ
+      # MAYAN_CELERY_BROKER_URL: amqp://mayan:mayanrabbitpass@broker:5672/mayan
+      # Disable Redis Broker to use RabbitMQ as Broker
+      MAYAN_CELERY_BROKER_URL: redis://redis:6379/1
+      MAYAN_CELERY_RESULT_BACKEND: redis://redis:6379/0
+      MAYAN_DATABASES: "{'default':{'ENGINE':'django.db.backends.postgresql','NAME':'mayan','PASSWORD':'mayandbpass','USER':'mayan','HOST':'postgresql'}}"
+    image: mayanedms/mayanedms:3.2.6
+    networks:
+      - mayan-bridge
     ports:
       - "80:8000"
+    restart: unless-stopped
     volumes:
-      - app:/var/lib/mayan
+      - /docker-volumes/mayan-edms/media:/var/lib/mayan
+
+  postgresql:
+    environment:
+      POSTGRES_DB: mayan
+      POSTGRES_PASSWORD: mayandbpass
+      POSTGRES_USER: mayan
+    image: postgres:9.6
+    networks:
+      - mayan-bridge
+    restart: unless-stopped
+    volumes:
+      - /docker-volumes/mayan-edms/postgres:/var/lib/postgresql/data
+
+  redis:
+    command:
+      - redis-server
+      - --databases
+      - "2"
+      - --maxmemory-policy
+      - allkeys-lru
+      - --save
+      - ""
+    image: redis:5.0
+    networks:
+      - mayan-bridge
+    restart: unless-stopped
+
+  # Optional services
+
+  # celery_flower:
+  #   command:
+  #     - run_celery
+  #     - flower
+  #   depends_on:
+  #     - postgresql
+  #     - redis
+  #     # Enable to use RabbitMQ
+  #     # - rabbitmq
+  #   environment:
+  #     <<: *mayan_env
+  #   image: mayanedms/mayanedms:3.2.6
+  #   networks:
+  #     - mayan-bridge
+  #   ports:
+  #     - "5555:5555"
+  #   restart: unless-stopped
+
+  # Enable to use RabbitMQ
+  # rabbitmq:
+  #   container_name: mayan-edms-rabbitmq
+  #   image: healthcheck/rabbitmq
+  #   environment:
+  #     RABBITMQ_DEFAULT_USER: mayan
+  #     RABBITMQ_DEFAULT_PASS: mayanrabbitpass
+  #     RABBITMQ_DEFAULT_VHOST: mayan
+  #   networks:
+  #     - mayan-bridge
+  #   restart: unless-stopped
+  #   volumes:
+  #     - /docker-volumes/mayan-edms/rabbitmq:/var/lib/rabbitmq
+
+  # Enable to run stand alone workers
+  # worker_fast:
+  #   command:
+  #     - run_worker
+  #     - fast
+  #   depends_on:
+  #     - postgresql
+  #     - redis
+  #     # Enable to use RabbitMQ
+  #     # - rabbitmq
+  #   environment:
+  #     <<: *mayan_env
+  #   image: mayanedms/mayanedms:3.2.6
+  #   networks:
+  #     - mayan-bridge
+  #   restart: unless-stopped
+  #   volumes:
+  #     - /docker-volumes/mayan-edms/media:/var/lib/mayan
+
+  # Enable to run stand frontend gunicorn
+  # frontend:
+  #   command:
+  #     - run_frontend
+  #   depends_on:
+  #     - postgresql
+  #     - redis
+  #     # Enable to use RabbitMQ
+  #     # - rabbitmq
+  #   environment:
+  #     <<: *mayan_env
+  #   image: mayanedms/mayanedms:3.2.6
+  #   networks:
+  #     - mayan-bridge
+  #   ports:
+  #     - "81:8000"
+  #   restart: unless-stopped
+  #   volumes:
+  #     - /docker-volumes/mayan-edms/media:/var/lib/mayan

docker/entrypoint.sh

@@ -1,115 +0,0 @@
-#!/bin/sh
-
-set -e
-echo "mayan: starting entrypoint.sh"
-INSTALL_FLAG=/var/lib/mayan/system/SECRET_KEY
-CONCURRENCY_ARGUMENT=--concurrency=
-export DOCKER_ROOT=/opt/mayan-edms
-
-export MAYAN_DEFAULT_BROKER_URL=redis://127.0.0.1:6379/0
-export MAYAN_DEFAULT_CELERY_RESULT_BACKEND=redis://127.0.0.1:6379/0
-
-export MAYAN_ALLOWED_HOSTS='["*"]'
-export MAYAN_BIN=/opt/mayan-edms/bin/mayan-edms.py
-export MAYAN_BROKER_URL=${MAYAN_BROKER_URL:-${MAYAN_DEFAULT_BROKER_URL}}
-export MAYAN_CELERY_RESULT_BACKEND=${MAYAN_CELERY_RESULT_BACKEND:-${MAYAN_DEFAULT_CELERY_RESULT_BACKEND}}
-export MAYAN_INSTALL_DIR=/opt/mayan-edms
-export MAYAN_PYTHON_BIN_DIR=/opt/mayan-edms/bin/
-export MAYAN_MEDIA_ROOT=/var/lib/mayan
-export MAYAN_SETTINGS_MODULE=${MAYAN_SETTINGS_MODULE:-mayan.settings.production}
-
-export MAYAN_GUNICORN_BIN=${MAYAN_PYTHON_BIN_DIR}gunicorn
-export MAYAN_GUNICORN_WORKERS=${MAYAN_GUNICORN_WORKERS:-2}
-export MAYAN_PIP_BIN=${MAYAN_PYTHON_BIN_DIR}pip
-
-MAYAN_WORKER_FAST_CONCURRENCY=${MAYAN_WORKER_FAST_CONCURRENCY:-1}
-MAYAN_WORKER_MEDIUM_CONCURRENCY=${MAYAN_WORKER_MEDIUM_CONCURRENCY:-1}
-MAYAN_WORKER_SLOW_CONCURRENCY=${MAYAN_WORKER_SLOW_CONCURRENCY:-1}
-
-if [ "$MAYAN_WORKER_FAST_CONCURRENCY" -eq 0 ]; then
-    MAYAN_WORKER_FAST_CONCURRENCY=
-else
-    MAYAN_WORKER_FAST_CONCURRENCY="${CONCURRENCY_ARGUMENT}${MAYAN_WORKER_FAST_CONCURRENCY}"
-fi
-export MAYAN_WORKER_FAST_CONCURRENCY
-
-if [ "$MAYAN_WORKER_MEDIUM_CONCURRENCY" -eq 0 ]; then
-    MAYAN_WORKER_MEDIUM_CONCURRENCY=
-else
-    MAYAN_WORKER_MEDIUM_CONCURRENCY="${CONCURRENCY_ARGUMENT}${MAYAN_WORKER_MEDIUM_CONCURRENCY}"
-fi
-export MAYAN_WORKER_MEDIUM_CONCURRENCY
-
-if [ "$MAYAN_WORKER_SLOW_CONCURRENCY" -eq 0 ]; then
-    MAYAN_WORKER_SLOW_CONCURRENCY=
-else
-    MAYAN_WORKER_SLOW_CONCURRENCY="${CONCURRENCY_ARGUMENT}${MAYAN_WORKER_SLOW_CONCURRENCY}"
-fi
-export MAYAN_WORKER_SLOW_CONCURRENCY
-
-export CELERY_ALWAYS_EAGER=False
-export PYTHONPATH=$PYTHONPATH:$MAYAN_MEDIA_ROOT
-
-chown mayan:mayan /var/lib/mayan -R
-
-initialize() {
-    echo "mayan: initialize()"
-    su mayan -c "${MAYAN_BIN} initialsetup --force"
-    su mayan -c "${MAYAN_BIN} collectstatic --noinput --clear"
-}
-
-upgrade() {
-    echo "mayan: upgrade()"
-    su mayan -c "${MAYAN_BIN} performupgrade"
-    su mayan -c "${MAYAN_BIN} collectstatic --noinput --clear"
-}
-
-start() {
-    echo "mayan: start()"
-    rm -rf /var/run/supervisor.sock
-    exec /usr/bin/supervisord -nc /etc/supervisor/supervisord.conf
-}
-
-os_package_installs() {
-    echo "mayan: os_package_installs()"
-    if [ "${MAYAN_APT_INSTALLS}" ]; then
-        apt-get-install $MAYAN_APT_INSTALLS
-    fi
-}
-
-pip_installs() {
-    echo "mayan: pip_installs()"
-    if [ "${MAYAN_PIP_INSTALLS}" ]; then
-        $MAYAN_PIP_BIN install $MAYAN_PIP_INSTALLS
-    fi
-}
-
-os_package_installs || true
-pip_installs || true
-
-case "$1" in
-
-mayan) # Check if this is a new install, otherwise try to upgrade the existing
-       # installation on subsequent starts
-       if [ ! -f $INSTALL_FLAG ]; then
-           initialize
-       else
-           upgrade
-       fi
-       start
-       ;;
-
-run-tests) # Check if this is a new install, otherwise try to upgrade the existing
-           # installation on subsequent starts
-           if [ ! -f $INSTALL_FLAG ]; then
-               initialize
-           else
-               upgrade
-           fi
-           $DOCKER_ROOT/run-tests.sh
-           ;;
-
-*) su mayan -c "$@";
-   ;;
-
-esac

docker/etc/supervisor/mayan.conf

@@ -1,80 +0,0 @@
-[program:mayan-gunicorn]
-autorestart = false
-autostart = true
-command = /bin/bash -c "${MAYAN_GUNICORN_BIN} -w ${MAYAN_GUNICORN_WORKERS} mayan.wsgi --max-requests 500 --max-requests-jitter 50 --worker-class gevent --bind 0.0.0.0:8000  --env DJANGO_SETTINGS_MODULE=${MAYAN_SETTINGS_MODULE}" --timeout 120
-redirect_stderr = true
-stderr_logfile = /dev/fd/2
-stderr_logfile_maxbytes = 0
-stdout_logfile = /dev/fd/1
-stdout_logfile_maxbytes = 0
-user = mayan
-
-[program:redis]
-autorestart = false
-autostart = true
-command = /bin/bash -c "if [ ${MAYAN_BROKER_URL} == ${MAYAN_DEFAULT_BROKER_URL} ] && [ ${MAYAN_CELERY_RESULT_BACKEND} == ${MAYAN_DEFAULT_CELERY_RESULT_BACKEND} ];then /usr/bin/redis-server /etc/redis/;fi"
-stderr_logfile = /dev/fd/2
-stderr_logfile_maxbytes = 0
-stdout_logfile = /dev/fd/1
-stdout_logfile_maxbytes = 0
-user = root
-
-[program:mayan-worker-fast]
-autorestart = false
-autostart = true
-command = nice -n 1 /bin/bash -c "${MAYAN_BIN} celery --settings=${MAYAN_SETTINGS_MODULE} worker -Ofair -l ERROR -Q converter,sources_fast -n mayan-worker-fast.%%h ${MAYAN_WORKER_FAST_CONCURRENCY}"
-killasgroup = true
-numprocs = 1
-priority = 998
-startsecs = 10
-stderr_logfile = /dev/fd/2
-stderr_logfile_maxbytes = 0
-stdout_logfile = /dev/fd/1
-stdout_logfile_maxbytes = 0
-stopwaitsecs = 1
-user = mayan
-
-[program:mayan-worker-medium]
-autorestart = false
-autostart = true
-command = nice -n 18 /bin/bash -c "${MAYAN_BIN} celery --settings=${MAYAN_SETTINGS_MODULE} worker -Ofair -l ERROR -Q checkouts_periodic,documents_periodic,indexing,metadata,sources,sources_periodic,uploads,documents -n mayan-worker-medium.%%h ${MAYAN_WORKER_MEDIUM_CONCURRENCY}"
-killasgroup = true
-numprocs = 1
-priority = 998
-startsecs = 10
-stderr_logfile = /dev/fd/2
-stderr_logfile_maxbytes = 0
-stdout_logfile = /dev/fd/1
-stdout_logfile_maxbytes = 0
-stopwaitsecs = 1
-user = mayan
-
-[program:mayan-worker-slow]
-autorestart = false
-autostart = true
-command = nice -n 19 /bin/bash -c "${MAYAN_BIN} celery --settings=${MAYAN_SETTINGS_MODULE} worker -Ofair -l ERROR -Q mailing,tools,statistics,parsing,ocr -n mayan-worker-slow.%%h ${MAYAN_WORKER_SLOW_CONCURRENCY}"
-killasgroup = true
-numprocs = 1
-priority = 998
-startsecs = 10
-stderr_logfile = /dev/fd/2
-stderr_logfile_maxbytes = 0
-stdout_logfile = /dev/fd/1
-stdout_logfile_maxbytes = 0
-stopwaitsecs = 1
-user = mayan
-
-[program:mayan-celery-beat]
-autorestart = false
-autostart = true
-command = nice -n 1 /bin/bash -c "${MAYAN_BIN} celery --settings=${MAYAN_SETTINGS_MODULE} beat --pidfile= -l ERROR"
-killasgroup = true
-numprocs = 1
-priority = 998
-stderr_logfile = /dev/fd/2
-stderr_logfile_maxbytes = 0
-stdout_logfile = /dev/fd/1
-stdout_logfile_maxbytes = 0
-startsecs = 10
-stopwaitsecs = 1
-user = mayan

docker/rootfs/usr/local/bin/entrypoint.sh

@@ -0,0 +1,165 @@
+#!/bin/bash
+
+# Use bash and not sh to support argument slicing "${@:2}"
+# sh defaults to dash instead of bash.
+
+set -e
+echo "mayan: starting entrypoint.sh"
+INSTALL_FLAG=/var/lib/mayan/system/SECRET_KEY
+CONCURRENCY_ARGUMENT=--concurrency=
+
+DEFAULT_USER_UID=1000
+DEFAULT_USER_GID=1000
+
+MAYAN_USER_UID=${MAYAN_USER_UID:-${DEFAULT_USER_UID}}
+MAYAN_USER_GID=${MAYAN_USER_GID:-${DEFAULT_USER_GID}}
+
+export MAYAN_ALLOWED_HOSTS='["*"]'
+export MAYAN_BIN=/opt/mayan-edms/bin/mayan-edms.py
+export MAYAN_INSTALL_DIR=/opt/mayan-edms
+export MAYAN_PYTHON_BIN_DIR=/opt/mayan-edms/bin/
+export MAYAN_MEDIA_ROOT=/var/lib/mayan
+export MAYAN_SETTINGS_MODULE=${MAYAN_SETTINGS_MODULE:-mayan.settings.production}
+export DJANGO_SETTINGS_MODULE=${MAYAN_SETTINGS_MODULE}
+
+export MAYAN_GUNICORN_BIN=${MAYAN_PYTHON_BIN_DIR}gunicorn
+export MAYAN_GUNICORN_WORKERS=${MAYAN_GUNICORN_WORKERS:-2}
+export MAYAN_GUNICORN_TIMEOUT=${MAYAN_GUNICORN_TIMEOUT:-120}
+export MAYAN_PIP_BIN=${MAYAN_PYTHON_BIN_DIR}pip
+export MAYAN_STATIC_ROOT=${MAYAN_INSTALL_DIR}/static
+
+MAYAN_WORKER_FAST_CONCURRENCY=${MAYAN_WORKER_FAST_CONCURRENCY:-0}
+MAYAN_WORKER_MEDIUM_CONCURRENCY=${MAYAN_WORKER_MEDIUM_CONCURRENCY:-0}
+MAYAN_WORKER_SLOW_CONCURRENCY=${MAYAN_WORKER_SLOW_CONCURRENCY:-1}
+
+update_uid_gid() {
+    echo "mayan: update_uid_gid()"
+    groupmod mayan -g ${MAYAN_USER_GID} 2>/dev/null || true
+    usermod mayan -u ${MAYAN_USER_UID} -g ${MAYAN_USER_GID} 2>/dev/null
+
+    if [ ${MAYAN_USER_UID} -ne ${DEFAULT_USER_UID} ] || [ ${MAYAN_USER_GID} -ne ${DEFAULT_USER_GID} ]; then
+        echo "mayan: Updating file ownership. This might take a while if there are many documents."
+        chown -R mayan:mayan ${MAYAN_INSTALL_DIR} ${MAYAN_STATIC_ROOT} ${MAYAN_MEDIA_ROOT}
+    fi
+}
+
+update_uid_gid
+
+if [ "$MAYAN_WORKER_FAST_CONCURRENCY" -eq 0 ]; then
+    MAYAN_WORKER_FAST_CONCURRENCY=
+else
+    MAYAN_WORKER_FAST_CONCURRENCY="${CONCURRENCY_ARGUMENT}${MAYAN_WORKER_FAST_CONCURRENCY}"
+fi
+export MAYAN_WORKER_FAST_CONCURRENCY
+
+if [ "$MAYAN_WORKER_MEDIUM_CONCURRENCY" -eq 0 ]; then
+    MAYAN_WORKER_MEDIUM_CONCURRENCY=
+else
+    MAYAN_WORKER_MEDIUM_CONCURRENCY="${CONCURRENCY_ARGUMENT}${MAYAN_WORKER_MEDIUM_CONCURRENCY}"
+fi
+export MAYAN_WORKER_MEDIUM_CONCURRENCY
+
+if [ "$MAYAN_WORKER_SLOW_CONCURRENCY" -eq 0 ]; then
+    MAYAN_WORKER_SLOW_CONCURRENCY=
+else
+    MAYAN_WORKER_SLOW_CONCURRENCY="${CONCURRENCY_ARGUMENT}${MAYAN_WORKER_SLOW_CONCURRENCY}"
+fi
+export MAYAN_WORKER_SLOW_CONCURRENCY
+
+# Allow importing of user setting modules
+export PYTHONPATH=$PYTHONPATH:$MAYAN_MEDIA_ROOT
+
+apt_get_install() {
+    apt-get -q update
+    apt-get install -y --force-yes --no-install-recommends --auto-remove "$@"
+    apt-get -q clean
+    rm -rf /var/lib/apt/lists/*
+}
+
+initialsetup() {
+    echo "mayan: initialsetup()"
+    su mayan -c "${MAYAN_BIN} initialsetup --force --no-dependencies"
+}
+
+os_package_installs() {
+    echo "mayan: os_package_installs()"
+    if [ "${MAYAN_APT_INSTALLS}" ]; then
+        DEBIAN_FRONTEND=noninteractive apt_get_install $MAYAN_APT_INSTALLS
+    fi
+}
+
+pip_installs() {
+    echo "mayan: pip_installs()"
+    if [ "${MAYAN_PIP_INSTALLS}" ]; then
+        su mayan -c "${MAYAN_PIP_BIN} install $MAYAN_PIP_INSTALLS"
+    fi
+}
+
+run_all() {
+    echo "mayan: start()"
+    rm -rf /var/run/supervisor.sock
+    exec /usr/bin/supervisord -nc /etc/supervisor/supervisord.conf
+}
+
+performupgrade() {
+    echo "mayan: performupgrade()"
+    su mayan -c "${MAYAN_BIN} performupgrade --no-dependencies"
+}
+
+make_ready() {
+    # Check if this is a new install, otherwise try to upgrade the existing
+    # installation on subsequent starts
+    if [ ! -f $INSTALL_FLAG ]; then
+        initialsetup
+    else
+        performupgrade
+    fi
+}
+
+set_uid_guid() {
+    echo "mayan: changing uid/guid"
+    usermod mayan -u ${MAYAN_USER_UID:-${DEFAULT_USER_UID}}
+    groupmod mayan -g ${MAYAN_USER_GID:-${DEFAULT_USER_GID}}
+}
+
+os_package_installs || true
+pip_installs || true
+chown mayan:mayan /var/lib/mayan -R
+
+case "$1" in
+
+run_initialsetup)
+    initialsetup
+    ;;
+
+run_performupgrade)
+    performupgrade
+    ;;
+
+run_all)
+    make_ready
+    run_all
+    ;;
+
+run_celery)
+    run_celery.sh "${@:2}"
+    ;;
+
+run_frontend)
+    run_frontend.sh
+    ;;
+
+run_tests)
+    make_ready
+    run_tests.sh
+    ;;
+
+run_worker)
+    run_worker.sh "${@:2}"
+    ;;
+
+*)
+    su mayan -c "$@"
+    ;;
+
+esac

docker/rootfs/usr/local/bin/run_celery.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+# Use -A and not --app. Both are the same but behave differently
+# -A can be located before the command while --app cannot.
+su mayan -c "${MAYAN_PYTHON_BIN_DIR}celery -A mayan $@"

docker/rootfs/usr/local/bin/run_frontend.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+MAYAN_GUNICORN_MAX_REQUESTS=${MAYAN_GUNICORN_MAX_REQUESTS:-500}
+MAYAN_GUNICORN_MAX_REQUESTS_JITTERS=${MAYAN_GUNICORN_MAX_REQUESTS_JITTERS:-50}
+MAYAN_GUNICORN_WORKER_CLASS=${MAYAN_GUNICORN_WORKER_CLASS:-sync}
+
+su mayan -c "${MAYAN_PYTHON_BIN_DIR}gunicorn -w ${MAYAN_GUNICORN_WORKERS} mayan.wsgi --max-requests ${MAYAN_GUNICORN_MAX_REQUESTS} --max-requests-jitter ${MAYAN_GUNICORN_MAX_REQUESTS_JITTERS} --worker-class ${MAYAN_GUNICORN_WORKER_CLASS} --bind 0.0.0.0:8000 --timeout ${MAYAN_GUNICORN_TIMEOUT}"

docker/rootfs/usr/local/bin/run_tests.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+export DEBIAN_FRONTEND=noninteractive
+
+apt-get update
+apt-get install -y --no-install-recommends gcc python-dev tesseract-ocr-deu
+
+su mayan -c "$MAYAN_PIP_BIN install -r ${MAYAN_INSTALL_DIR}/testing-base.txt"
+
+su mayan -c "$MAYAN_BIN test --mayan-apps --settings=mayan.settings.testing"

docker/rootfs/usr/local/bin/run_worker.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+QUEUE_LIST=`MAYAN_WORKER_NAME=$1 su mayan -c "${MAYAN_PYTHON_BIN_DIR}mayan-edms.py platformtemplate worker_queues"`
+
+# Use -A and not --app. Both are the same but behave differently
+# -A can be located before the command while --app cannot.
+# Pass ${@:2} to allow overriding the defaults arguments
+su mayan -c "${MAYAN_PYTHON_BIN_DIR}celery -A mayan worker -Ofair -l ERROR -Q $QUEUE_LIST ${@:2}"

docker/rootfs/version

@@ -0,0 +1 @@
+3.3beta1

docker/stack.yml

@@ -9,24 +9,32 @@ volumes:
 services:
 
   db:
-    image: postgres
     environment:
       POSTGRES_DB: mayan
-      POSTGRES_PASSWORD: mayan-password
+      POSTGRES_PASSWORD: mayandbpass
       POSTGRES_USER: mayan
+    image: postgres
     volumes:
       - db:/var/lib/postgresql/data
 
   app:
+    environment:
+      MAYAN_CELERY_BROKER_URL: redis://redis:6379/1
+      MAYAN_CELERY_RESULT_BACKEND: redis://redis:6379/0
+      MAYAN_DATABASES: "{'default':{'ENGINE':'django.db.backends.postgresql','NAME':'mayan','PASSWORD':'mayandbpass','USER':'mayan','HOST':'db'}}"
     image: mayanedms/mayanedms:latest
     ports:
       - 80:8000
-    environment:
-      MAYAN_DATABASE_ENGINE: django.db.backends.postgresql
-      MAYAN_DATABASE_HOST: db
-      MAYAN_DATABASE_NAME: mayan
-      MAYAN_DATABASE_PASSWORD: mayan-password
-      MAYAN_DATABASE_USER: mayan
-      MAYAN_DATABASE_CONN_MAX_AGE: 60
     volumes:
       - app:/var/lib/mayan
+
+  redis:
+    command:
+      - redis-server
+      - --databases
+      - "2"
+      - --maxmemory-policy
+      - allkeys-lru
+      - --save
+      - ""
+    image: redis:5.0

docker/version

@@ -1 +0,0 @@
-3.1.9

docs/_static/css/custom.css

@@ -0,0 +1,5 @@
+.menuselection {
+    color: white;
+    background: #2c3e50;
+    padding: 3px;
+}

docs/chapters/api.rst

@@ -26,3 +26,77 @@ endpoints are structured by resource type. Examples:
 
 The API supports the HTTP verbs: **GET**, **POST**, **PUT**, **PATCH**,
 and **DELETE**.
+
+
+Example use
+===========
+
+Install Python Requests (http://docs.python-requests.org/en/master/)::
+
+    pip install requests
+
+Get a list of document types::
+
+    import requests
+
+    requests.get('http://127.0.0.1:8000/api/document_types/', auth=('username', 'password')).json()
+
+    {u'count': 1,
+     u'next': None,
+     u'previous': None,
+     u'results': [{u'delete_time_period': 30,
+       u'delete_time_unit': u'days',
+       u'documents_count': 12,
+       u'documents_url': u'http://127.0.0.1:8000/api/document_types/1/documents/',
+       u'filenames': [],
+       u'id': 1,
+       u'label': u'Default',
+       u'trash_time_period': None,
+       u'trash_time_unit': None,
+       u'url': u'http://127.0.0.1:8000/api/document_types/1/'}]}
+
+Upload a new document::
+
+    with open('test_document.pdf', mode='rb') as
+        requests.post('http://127.0.0.1:8000/api/documents/', auth=('username', 'password'), files={'file': file_object}, data={'document_type': 1}).json()
+
+    {u'description': u'',
+    u'document_type': 1,
+    u'id': 19,
+    u'label': u'test_document.pdf',
+    u'language': u'eng'}
+
+Use API tokens to avoid sending the username and password on every request. Obtain a token by making a POST request to ``/api/auth/token/obtain/?format=json``::
+
+    requests.post('http://127.0.0.1:8000/api/auth/token/obtain/?format=json', data={'username': 'username', 'password': 'password'}).json()
+
+    {u'token': u'4ccbc35b5eb327aa82dc3b7c9747b578900f02bb'}
+
+Add the API token to the request header::
+
+    headers = {'Authorization': 'Token 4ccbc35b5eb327aa82dc3b7c9747b578900f02bb'}
+
+    requests.get('http://127.0.0.1:8000/api/document_types/', headers=headers).json()
+
+    {u'description': u'',
+    u'document_type': 1,
+    u'id': 19,
+    u'label': u'test_document.pdf',
+    u'language': u'eng'}
+
+
+Use sessions to avoid having to add the headers on each request::
+
+    session = requests.Session()
+
+    headers = {'Authorization': 'Token 4ccbc35b5eb327aa82dc3b7c9747b578900f02bb'}
+
+    session.headers.update(headers)
+
+    session.get('http://127.0.0.1:8000/api/document_types/')
+
+    {u'description': u'',
+    u'document_type': 1,
+    u'id': 19,
+    u'label': u'test_document.pdf',
+    u'language': u'eng'}

docs/chapters/app_creation.rst

@@ -59,6 +59,17 @@ App modules
   corresponding signal in the ready() method of the MayanAppConfig subclass in
   apps.py
 
+- html_widgets.py
+
+  Classes to render an HTML widget. HTML widget are not the same as Django's
+  native form widgets. Form widgets only work as part of a form field.
+  HTML widgets are for use outside of forms, such as in a table cell.
+
+- licenses.py
+
+  This module outlines the license text of the third party content used in 
+  the app. It could be other Python libraries, JavaScript libraries, etc.  
+
 - links.py
 
   Defines the links to be used by the app. Import only from the navigation app
@@ -92,6 +103,11 @@ App modules
   This module acts as a shared memory space for the other modules of the app or
   other apps.
 
+- search.py
+
+  Search model definitions. Define which field of the app's models are
+  searchable.
+
 - serializers.py
 
   Hold Django REST Framework serializers used by the api_views.py module.
@@ -127,7 +143,7 @@ App modules
 
 - widgets.py
 
-  HTML widgets go here. This should be the only place with presentation
+  Custom form widgets go here. This should be the only place with presentation
   directives in the app (aside the templates).
 
 

docs/chapters/backups.rst

@@ -59,7 +59,7 @@ Example::
     -e POSTGRES_DB=mayan \
     -e POSTGRES_PASSWORD=mayanuserpass \
     -v /docker-volumes/mayan-edms/postgres-new:/var/lib/postgresql/data \
-    -d postgres:9.5
+    -d postgres:9.6
 
     docker exec -i mayan-edms-pg-new pg_restore -U mayan -d mayan < 2018-06-07_17-09-34.dump
 

docs/chapters/database_conversion.rst

@@ -27,7 +27,7 @@ Direct install
 
 * Install the Python client for PostgreSQL::
 
-    sudo -u mayan /opt/mayan-edms/bin/pip install --no-cache-dir psycopg2==2.7.3.2
+    sudo -u mayan /opt/mayan-edms/bin/pip install --no-cache-dir --no-use-pep517 psycopg2==2.7.3.2
 
 * Copy the newly created fallback config file::
 
@@ -99,5 +99,5 @@ Direct install
         MAYAN_DATABASE_NAME=mayan,
         MAYAN_DATABASE_PASSWORD=mayanuserpass,
         MAYAN_DATABASE_USER=mayan,
-        MAYAN_DATABASE_CONN_MAX_AGE=360,
+        MAYAN_DATABASE_CONN_MAX_AGE=0,
         <...>

docs/chapters/development.rst

@@ -226,35 +226,46 @@ Git branch structure
 Mayan EDMS follows a simplified model layout based on Vincent Driessen's
 `Successful Git Branching Model`_ blog post.
 
-``development``
-    The "next release" branch, likely unstable, don't use in production.
+``/versions/micro``
+    Working branch for the next bugfix release. Micro increment (third digit).
+    Only bug fixes, minor features, back-ported urgent features. This
+    branch is stable and safe for production.
+``/versions/minor``
+    Working branch for the next minor release (second digit). New features,
+    occasional breakage. Not for production but should run in test
+    environment most of the time. This is the branch you will want to
+    try out if you want to check out new features.
+``/versions/major``
+    Working branch for the next major release (first digit). New features,
+    incompatible changes to the user facing interfaces. Broken most of the
+    time, not for production and should only be cloned by developers
+    with experience with Mayan's development.
 ``master``
     Current production release (|version|). Points to the latest version of
     the latest series. Production quality code.
-``feature/``
-    Unfinished/unmerged feature. Likely unstable, don't use in production.
-
+``features/``
+    Working branches for unfinished and unmerged feature. Likely unstable,
+    don't use in production. Once the feature is complete, it is merged
+    into one of the versions branches and deleted.
+    
 Special branches:
 
 ``releases/all``
-    Used by the continuous delivery system to trigger the build and release
+    Pushing code to this branch will trigger the build and release
     a new Docker image, Documentation and Python package.
 ``releases/docker``
-    Used by the continuous delivery system to trigger the build and release
+    Pushing code to this branch will trigger the build and release
     of a new Docker image to Docker Hub.
 ``releases/documentation``
-    Used by the continuous delivery system to trigger the build and release
+    Pushing code to this branch will trigger the build and release
     of new documentation.
 ``releases/python``
-    Used by the continuous delivery system to trigger the build and release
-    of a new Python package to PyPI.
-``releases/python``
-    Used by the continuous delivery system to trigger the build and release
+    Pushing code to this branch will trigger the build and release
     of a new Python package to PyPI.
 ``nightly``
-    Used by the continuous delivery system to trigger the build and release
+    Pushing code to this branch will trigger the build and release
     of a new Docker image based on development code to the GitLab image
-    repository.
+    repository only. The image will not be published to Docker Hub.
 
 Each release is tagged separately using annotated Git tags.
 
@@ -482,7 +493,7 @@ Release checklist
 
        make check-readme
 
-#. Bump version in ``mayan/__init__.py`` and ``docker/version``::
+#. Bump version in ``mayan/__init__.py`` and ``docker/rootfs/version``::
 
     make increase-version PART=<major, minor or micro>
 
@@ -522,7 +533,7 @@ Release using GitLab CI
    ::
 
        git checkout releases/all
-       git merge versions/next
+       git merge <corresponding branch>
 
 #. Push code to trigger builds:
    ::

docs/chapters/docker.rst

@@ -7,16 +7,23 @@ How to use this image
 
 .. _docker_install:
 
-Start a Mayan EDMS image
-------------------------
+Start a Mayan EDMS Docker image
+-------------------------------
 
-With Docker properly installed, proceed to download the Mayan EDMS image using the command::
+With Docker properly installed, proceed to download the Mayan EDMS image using
+the command::
 
     docker pull mayanedms/mayanedms:<version>
 
-Then download version 9.5 of the Docker PostgreSQL image::
+Instead of a specific version tag you may use then generic ``:latest`` tag
+to the get latest version available automatically. If you use the ``:latest``
+tag here, remember to do so in the next steps also.::
 
-    docker pull postgres:9.5
+    docker pull mayanedms/mayanedms:latest
+
+Then download version 9.6 of the Docker PostgreSQL image::
+
+    docker pull postgres:9.6
 
 Create and run a PostgreSQL container::
 
@@ -28,7 +35,7 @@ Create and run a PostgreSQL container::
     -e POSTGRES_DB=mayan \
     -e POSTGRES_PASSWORD=mayanuserpass \
     -v /docker-volumes/mayan-edms/postgres:/var/lib/postgresql/data \
-    -d postgres:9.5
+    -d postgres:9.6
 
 The PostgreSQL container will have one database named ``mayan``, with an user
 named ``mayan`` too, with a password of ``mayanuserpass``. The container will
@@ -36,20 +43,13 @@ expose its internal 5432 port (PostgreSQL's default port) via the host's
 5432 port. The data of this container will reside on the host's
 ``/docker-volumes/mayan-edms/postgres`` folder.
 
-Finally create and run a Mayan EDMS container. Change <version> with the
-latest version in numeric form (example: 2.7.3) or use the ``latest``
-identifier::
+Finally create and run a Mayan EDMS container::
 
     docker run -d \
     --name mayan-edms \
     --restart=always \
     -p 80:8000 \
-    -e MAYAN_DATABASE_ENGINE=django.db.backends.postgresql \
-    -e MAYAN_DATABASE_HOST=172.17.0.1 \
-    -e MAYAN_DATABASE_NAME=mayan \
-    -e MAYAN_DATABASE_PASSWORD=mayanuserpass \
-    -e MAYAN_DATABASE_USER=mayan \
-    -e MAYAN_DATABASE_CONN_MAX_AGE=60 \
+    -e MAYAN_DATABASES="{'default':{'ENGINE':'django.db.backends.postgresql','NAME':'mayan','PASSWORD':'mayanuserpass','USER':'mayan','HOST':'172.17.0.1'}}" \
     -v /docker-volumes/mayan-edms/media:/var/lib/mayan \
     mayanedms/mayanedms:<version>
 
@@ -72,6 +72,7 @@ If another web server is running on port 80 use a different port in the
 
 Using a dedicated Docker network
 --------------------------------
+
 Use this method to avoid having to expose PostreSQL port to the host's network
 or if you have other PostgreSQL instances but still want to use the default
 port of 5432 for this installation.
@@ -91,7 +92,7 @@ binding (``-p 5432:5432``)::
     -e POSTGRES_DB=mayan \
     -e POSTGRES_PASSWORD=mayanuserpass \
     -v /docker-volumes/mayan-edms/postgres:/var/lib/postgresql/data \
-    -d postgres:9.5
+    -d postgres:9.6
 
 Launch the Mayan EDMS container with the network option and change the
 database hostname to the PostgreSQL container name (``mayan-edms-postgres``)
@@ -102,15 +103,11 @@ instead of the IP address of the Docker host (``172.17.0.1``)::
     --network=mayan \
     --restart=always \
     -p 80:8000 \
-    -e MAYAN_DATABASE_ENGINE=django.db.backends.postgresql \
-    -e MAYAN_DATABASE_HOST=mayan-edms-postgres \
-    -e MAYAN_DATABASE_NAME=mayan \
-    -e MAYAN_DATABASE_PASSWORD=mayanuserpass \
-    -e MAYAN_DATABASE_USER=mayan \
-    -e MAYAN_DATABASE_CONN_MAX_AGE=60 \
+    -e MAYAN_DATABASES="{'default':{'ENGINE':'django.db.backends.postgresql','NAME':'mayan','PASSWORD':'mayanuserpass','USER':'mayan','HOST':'mayan-edms-postgres'}}" \
     -v /docker-volumes/mayan-edms/media:/var/lib/mayan \
     mayanedms/mayanedms:<version>
 
+
 Stopping and starting the container
 -----------------------------------
 
@@ -126,100 +123,24 @@ To start the container again::
 
 .. _docker_environment_variables:
 
+
 Environment Variables
 ---------------------
 
-The Mayan EDMS image can be configure via environment variables.
-
-``MAYAN_DATABASE_ENGINE``
-
-Defaults to ``None``. This environment variable configures the database
-backend to use. If left unset, SQLite will be used. The database backends
-supported by this Docker image are:
-
-- ``'django.db.backends.postgresql'``
-- ``'django.db.backends.mysql'``
-- ``'django.db.backends.sqlite3'``
-
-When using the SQLite backend, the database file will be saved in the Docker
-volume. The SQLite database as used by Mayan EDMS is meant only for development
-or testing, never use it in production.
-
-``MAYAN_DATABASE_NAME``
-
-Defaults to 'mayan'. This optional environment variable can be used to define
-the database name that Mayan EDMS will connect to. For more information read
-the pertinent Django documentation page:
-:django-docs:`Connecting to the database <ref/databases/#connecting-to-the-database>`
-
-``MAYAN_DATABASE_USER``
-
-Defaults to 'mayan'. This optional environment variable is used to set the
-username that will be used to connect to the database. For more information
-read the pertinent Django documentation page:
-:django-docs:`Settings, USER <ref/settings/#user>`
-
-``MAYAN_DATABASE_PASSWORD``
-
-Defaults to ''. This optional environment variable is used to set the
-password that will be used to connect to the database. For more information
-read the pertinent Django documentation page:
-:django-docs:`Settings, PASSWORD <ref/settings/#password>`
-
-``MAYAN_DATABASE_HOST``
-
-Defaults to `None`. This optional environment variable is used to set the
-hostname that will be used to connect to the database. This can be the
-hostname of another container or an IP address. For more information read
-the pertinent Django documentation page:
-:django-docs:`Settings, HOST <ref/settings/#host>`
-
-``MAYAN_DATABASE_PORT``
-
-Defaults to `None`. This optional environment variable is used to set the
-port number to use when connecting to the database. An empty string means
-the default port. Not used with SQLite. For more information read the
-pertinent Django documentation page:
-:django-docs:`Settings, PORT <ref/settings/#port>`
-
-``MAYAN_BROKER_URL``
-
-This optional environment variable determines the broker that Celery will use
-to relay task messages between the frontend code and the background workers.
-For more information read the pertinent Celery Kombu documentation page: `Broker URL`_
-
-.. _Broker URL: http://kombu.readthedocs.io/en/latest/userguide/connections.html#connection-urls
-
-This Docker image supports using Redis and RabbitMQ as brokers.
-
-Caveat: If the `MAYAN_BROKER_URL` and `MAYAN_CELERY_RESULT_BACKEND` environment
-variables are specified, the built-in Redis server inside the container will
-be disabled.
-
-``MAYAN_CELERY_RESULT_BACKEND``
-
-This optional environment variable determines the results backend that Celery
-will use to relay result messages from the background workers to the frontend
-code. For more information read the pertinent Celery Kombu documentation page:
-`Task result backend settings`_
-
-.. _Task result backend settings: http://docs.celeryproject.org/en/3.1/configuration.html#celery-result-backend
-
-This Docker image supports using Redis and RabbitMQ as result backends.
-
-Caveat: If the `MAYAN_BROKER_URL` and `MAYAN_CELERY_RESULT_BACKEND` environment
-variables are specified, the built-in Redis server inside the container will
-be disabled.
+The common set of settings can also be modified via environment variables when
+using the Docker image. In addition to the common set of settings, some Docker
+image specific environment variables are available.
 
 ``MAYAN_SETTINGS_MODULE``
 
 Optional. Allows loading an alternate settings file.
 
-``MAYAN_DATABASE_CONN_MAX_AGE``
 
-Amount in seconds to keep a database connection alive. Allow reuse of database
-connections. For more information read the pertinent Django documentation
-page: :django-docs:`Settings, CONN_MAX_AGE <ref/settings/#conn-max-age>`
+``MAYAN_GUNICORN_TIMEOUT``
+
+Optional. Changes the amount of time the frontend worker will wait for a
+request to finish before raising a timeout error. The default is 120
+seconds.
 
 ``MAYAN_GUNICORN_WORKERS``
 
@@ -252,6 +173,28 @@ category. Default is 1. Use 0 to disable hardcoded concurrency and allow the
 Celery worker to launch its default number of child processes (equal to the
 number of CPUs detected).
 
+``MAYAN_USER_UID``
+
+Optional. Changes the UID of the ``mayan`` user internal to the Docker
+container. Defaults to 1000.
+
+``MAYAN_USER_GID``
+
+Optional. Changes the GID of the ``mayan`` user internal to the Docker
+container. Defaults to 1000.
+
+
+Included drivers
+----------------
+
+The Docker image supports using Redis and RabbitMQ as result backends. For
+databases, the image includes support for PostgreSQL and MySQL/MariaDB.
+Support for additional brokers or databases may be added using the
+``MAYAN_APT_INSTALL`` environment variable.
+
+
+.. _docker-accessing-outside-data:
+
 Accessing outside data
 ======================
 
@@ -263,18 +206,21 @@ on purpose. For example, to make a folder in the host accessible as a watch
 folder, add the following to the Docker command line when starting the
 container::
 
-    -v /opt/scanned_files:/srv/watch_folder
+    -v /opt/scanned_files:/scanned_files
 
 The command line would look like this::
 
-    docker run ... -v /opt/scanned_files:/srv/watch_folder mayanedms/mayanedms:latest
+    docker run ... -v /opt/scanned_files:/scanned_files mayanedms/mayanedms:latest
 
-Now create a watch folder in Mayan EDMS using the path ``/srv/watch_folder``
+Now create a watch folder in Mayan EDMS using the path ``/scanned_files``
 and the documents from the host folder ``/opt/scanned_files`` will be
 automatically available. Use the same procedure to mount host folders to be
-used as staging folderes. In this example ``/srv/watch_folder`` was as the
-container directory, but any path can be used as long as it is not an
-already existing path or a path used by any other program.
+used as staging folders. In this example ``/scanned_files`` was used as the
+container directory, but any path can be used as long as:
+
+- the path not an already existing path
+- the path is not used by any other program
+- the path is a single level path
 
 
 Performing backups
@@ -285,7 +231,7 @@ For the example::
 
     docker run -d --name mayan-edms --restart=always -p 80:8000 \
     -v /docker-volumes/mayan:/var/lib/mayan \
-    -v /opt/scanned_files:/srv/watch_folder mayanedms/mayanedms:latest
+    -v /opt/scanned_files:/scanned_files mayanedms/mayanedms:latest
 
 That would be the ``/docker-volumes/mayan folder``::
 
@@ -297,6 +243,7 @@ too need to be backed up using their respective procedures. A simple solution
 is to copy the entire database container volume after the container has
 been stopped.
 
+
 Restoring from a backup
 =======================
 
@@ -304,6 +251,7 @@ Uncompress the backup archive in the original docker volume using::
 
     sudo tar -xvzf backup.tar.gz -C /
 
+
 Upgrading
 =========
 
@@ -333,6 +281,7 @@ Start the container again with the new image version::
 
     docker run -d --name mayan-edms --restart=always -p 80:8000 -v /docker-volumes/mayan:/var/lib/mayan mayanedms/mayanedms:latest
 
+
 Building the image
 ==================
 
@@ -348,13 +297,14 @@ Execute Docker's build command using the provided makefile::
 
     make docker-build
 
-Or using an apt cacher to speed up the build::
+Or using an APT cache to speed up the build::
 
     make docker-build-with-proxy APT_PROXY=172.17.0.1:3142
 
 Replace the IP address `172.17.0.1` with the IP address of the computer
 running the APT proxy and caching service.
 
+
 Customizing the image
 =====================
 
@@ -379,6 +329,7 @@ Specifies a list of Python packages to be installed via ``pip``. Packages will
 be downloaded from the Python Package Index (https://pypi.python.org) by
 default.
 
+
 Using Docker compose
 ====================
 
@@ -409,6 +360,7 @@ These are:
 
 Nightly images
 ==============
+
 The continuous integration pipeline used for testing development builds also
 produces a resulting Docker image. These are build automatically and their
 stability is not guaranteed. They should never be used in production.

docs/chapters/document_types.rst

@@ -68,5 +68,21 @@ type at a given moment, but if needed, the type of a document can be changed.
 Upon changing its type, the document will lose its previous settings and
 attributes, and will inherit the settings and attributes of its new type.
 
-Document types are create in the
-:menuselection:`System --> Setup --> Document types` menu.
+
+Creating document types
+=======================
+
+.. admonition:: Permissions required
+    :class: warning
+
+    The "Create document types" permission is required for this action.
+
+
+#. Go to the :menuselection:`System --> Setup --> Document types` menu.
+#. From the :guilabel:`Actions` dropdown select :guilabel:`Create document type`.
+#. Enter a label to be shown to users when using this document type.
+#. Optional: Enter a period of time after which documents of this type will be moved to the trash.
+#. Optional: Select the unit of time for the period after which the document will be moved to the trash.
+#. Optional: Enter a period of time after which documents moved to the trash will be permanently deleted.
+#. Optional: Select the unit of time for the period of time after which the documents moved to the trash will be permanently deleted
+#. Press :guilabel:`Submit`.

docs/chapters/index_examples.rst

@@ -5,22 +5,32 @@ Index examples
 Index of document types
 ^^^^^^^^^^^^^^^^^^^^^^^
 
+.. admonition:: Permissions required
+    :class: warning
+
+    - The "Create new document indexes" permission is required for this action.
+    - The "Edit document indexes" permission is required for this action, globally
+      of via an ACL for a document index.
+    - The "Edit document types" permission is required for this action, globally
+      of via an ACL for a document type.
+
+
 This index will create one level for each document type in the system and place
 links to the document of each respective type.
 
 #. Go to the :menuselection:`System --> Setup --> Indexes` menu.
-#. Create a new index using :menuselection:`Actions --> Create new`.
+#. Create a new index using :guilabel:`Actions` > :guilabel:`Create new`.
 #. Give it a label to describe it, and an internal name. The internal name is
    used when referencing this index in other parts of the system.
-#. Press the :menuselection:`Template` link of the newly created index.
-#. Select :menuselection:`New child node` to create a new level in which the
+#. Press the :guilabel:`Template` link of the newly created index.
+#. Select :guilabel:`New child node` to create a new level in which the
    following template code will be entered.
    ::
 
        {{ document.document_type }}
 
 #. Save the template.
-#. Go to :menuselection:`Document types` and associate this index with
+#. Click on :guilabel:`Document types` and associate this index with
    existing document types in the system.
 #. Finally go to :menuselection:`Tools --> Rebuild indexes` to execute the
    index template. The rebuild process is only necessary when changes are
@@ -101,8 +111,8 @@ Index documents not found in any cabinet
     {% if document.cabinets.count == 0 %}No Cabinets{% endif %}
 
 
-Index untagged documents
-^^^^^^^^^^^^^^^^^^^^^^^^
+Index documents not tagged
+^^^^^^^^^^^^^^^^^^^^^^^^^^
 ::
 
     {% if document.tags.count == 0 %}No Tags{% endif %}

docs/chapters/languages.rst

@@ -6,7 +6,7 @@ The list of languages choices in the language dropdown used for documents is
 based on the current ISO 639 list. This list can be quite extensive. To reduce
 the number of languages available use the setting ``DOCUMENTS_LANGUAGE_CODES``,
 and set it to a nested list of abbreviations. This setting can be found in the
-:menuselection:`System --> Setup -> Settings -> Common` menu.
+:menuselection:`System --> Setup --> Settings --> Common` menu.
 
 For example, to reduce the list to just English and Spanish use
 ::

docs/chapters/metadata.rst

@@ -26,3 +26,74 @@ interpret and modify the value provided by the user to a conform to a specific
 format. An example of a provided parser is the date parser which will interpret
 and correct dates provided by users regardless of the format in which they are
 entered.
+
+
+Creating metadata types
+=======================
+
+.. admonition:: Permissions required
+    :class: warning
+
+    The "Create new metadata types" permission is required for this action.
+
+
+#. Go to the :menuselection:`System --> Setup --> Metadata types` menu.
+#. From the :guilabel:`Actions` dropdown select :guilabel:`Create new`.
+#. Provide a name to reference this metadata type in other parts of the system.
+#. Enter a label to be shown to users when using this metadata type.
+#. Optional: Enter a default value for the metadata type.
+#. Optional: Provide a comma separated list of options to restrict the data entry
+   when using this metadata type.
+#. Optional: Select a validator and a parser to validate and cleanup the data
+   entry when not using a predetermined list of values.
+#. Press :guilabel:`Submit`.
+
+
+Assigning a metadata type to a document type
+============================================
+
+.. admonition:: Permissions required
+    :class: warning
+
+    - The "Edit metadata types" permission is required for this action, globally or
+      via an ACL for a metadata type.
+    - Also the "Edit document type" permission
+      is required, globally or via an ACL for a document type.
+
+
+This action can be performed in two ways.
+
+Option 1: Via the metadata type view
+------------------------------------
+
+#. Go to the :menuselection:`System --> Setup --> Metadata types` menu.
+#. Click on the button :guilabel:`Document types` of the metadata type you which
+   to associate.
+#. From the list of existing document types press either:
+
+   - :guilabel:`None` if this metadata type will not be available for documents
+     of the type.
+   - :guilabel:`Optional` if this metadata type will be available and is
+     optional to provide a value for documents of the type.
+   - :guilabel:`Required` if this metadata type will be available and is
+     required to provide a value for documents of the type.
+
+#. Press :guilabel:`Save`.
+
+
+Option 2: Via the document type view
+------------------------------------
+
+#. Go to the :menuselection:`System --> Setup --> Document types` menu.
+#. Click on the button :guilabel:`Metadata types` of the metadata type you which
+   to associate.
+#. From the list of existing metadata types press either:
+
+   - :guilabel:`None` if this metadata type will not be available for documents
+     of the type.
+   - :guilabel:`Optional` if this metadata type will be available and is
+     optional to provide a value for documents of the type.
+   - :guilabel:`Required` if this metadata type will be available and is
+     required to provide a value for documents of the type.
+
+#. Press :guilabel:`Save`.

docs/chapters/object_storage.rst

@@ -29,32 +29,3 @@ to use a S3 style storage for documents.
     '{access_key: <your S3 access key>, secret_key: <your S3 secret key>, bucket_name: <S3 bucket name>}'
 
 5. Save and restart your Mayan EDMS installation for the setting to take effect.
-
-
-Storage
-=======
-Mayan EDMS stores documents in their original file format only changing the
-filename to avoid collision. For best input and output speed use a block
-based local filesystem for the ``/media`` sub folder of the path specified by
-the MEDIA_ROOT setting. For increased storage capacity use an object storage
-filesystem like S3.
-
-To use a S3 compatible object storage do the following:
-
-* Install the Python packages ``django-storages`` and ``boto3``:
-
-  * Using Python::
-
-      pip install django-storages boto3
-
-  * Using Docker::
-
-    -e MAYAN_PIP_INSTALLS='django-storages boto3'
-
-On the Mayan EDMS user interface, go to ``System``, ``Setup``, ``Settings``,
-``Documents`` and change the following setting:
-
-* ``DOCUMENTS_STORAGE_BACKEND`` to ``storages.backends.s3boto3.S3Boto3Storage``
-* ``DOCUMENTS_STORAGE_BACKEND_ARGUMENTS`` to ``'{access_key: <your access key>, secret_key: <your secret key>, bucket_name: <bucket name>}'``.
-
-Restart Mayan EDMS for the changes to take effect.

docs/chapters/quick_labels.rst

@@ -0,0 +1,70 @@
+************
+Quick labels
+************
+
+Quick labels are predetermined filenames that allow the quick renaming of
+documents as they are uploaded or after they have been uploaded.
+
+Quick labels are added and associated to a document type.
+
+Example of quick label: Invoice, Receipt from X store, Purchase order.
+
+It is possible to preserve the file extension when using quick labels.
+Extensions are required for some operating system to be able to detect the
+correct file type to access the content.
+
+For example if a document file is named "file0001.pdf" and the quick label
+"Receipt from X store" is applied, the resulting document label will be
+"Receipt from X store.pdf".
+
+
+Creating quick labels
+=====================
+
+.. admonition:: Permissions required
+    :class: warning
+
+    The "Edit document types" permission is required for this action, either
+    globally or via an ACL for a document type.
+
+
+Since quick labels are associated with document types, creating quick labels
+must be done from the document type view.
+
+#. Go to the :menuselection:`System --> Setup --> Document types` menu.
+#. In the document type list, click on the :guilabel:`Quick labels` button of
+   the document type for which you wish to create a quick label.
+#. In the view titled "Quick labels for document type: <your document type>",
+   from the :guilabel:`Actions` dropdown select :guilabel:`Add quick label to document type`.
+#. At the quick label creation form enter the desired label and press :guilabel:`Save`.
+
+
+Using quick labels during upload
+================================
+
+#. Use the new document upload wizard from :menuselection:`Documents --> New document`.
+#. Select a document type and navigate to the penultimate step, where you have
+   the option to drag and drop files to upload.
+#. Select a an option from the :guilabel:`Quick document rename` dropdown.
+#. Optionally select the :guilabel:`Preserve extension` checkbox to keep the file
+   extension.
+#. Upload your documents.
+
+
+Using quick labels for existing documents
+=========================================
+
+.. admonition:: Permissions required
+    :class: warning
+
+    The "Edit document properties" permission is required for this action, either
+    globally or via an ACL for a document or document type.
+
+
+#. Navigate to the document preview view of the document to rename. Make sure
+   quick labels for the document type of the document select have been created.
+#. From the :guilabel:`Actions` dropdown select :guilabel:`Edit Properties`.
+#. Select a an option from the :guilabel:`Quick document rename` dropdown.
+#. Optionally select the :guilabel:`Preserve extension` checkbox to keep the file
+   extension.
+#. Press :guilabel:`Save` to rename the document.

docs/chapters/scaling_up.rst

@@ -6,15 +6,19 @@ The default installation method fits most use cases. If you use case requires
 more speed or capacity here are some suggestion that can help you improve the
 performance of your installation.
 
+
 Change the database manager
 ===========================
+
 Use PostgreSQL or MySQL as the database manager.
 Tweak the memory setting of the database manager to increase memory allocation.
 More PostgreSQL specific examples are available in their wiki page:
 https://wiki.postgresql.org/wiki/Performance_Optimization
 
+
 Increase the number of Gunicorn workers
 =======================================
+
 The Gunicorn workers process HTTP requests and affect the speed at which the
 website responds.
 
@@ -33,6 +37,7 @@ And increase the value of the ``-w 2`` argument. This line is found in the
 
 Background task processing
 ==========================
+
 The Celery workers are system processes that take care of the background
 tasks requested by the frontend interactions like document image rendering
 and periodic tasks like OCR. There are several dozen tasks defined in the code.
@@ -45,37 +50,9 @@ low volume, medium duration tasks. It is not advisable to have the same
 worker processing OCR to process image rendering too. If the worker is
 processing several OCR tasks it will not be able to provide fast images
 when an user is browsing the user interface. This is why by default the
-queues are split into 3 workers: fast, medium, and slow.
+queues are split into 3 workers: fast, medium, and slow. Each worker will handle
+queues based on the latency required by each queue group.
 
-The fast worker handles the queues:
-
-* converter: Handles document page rendering
-* sources_fast: Does staging file image rendering
-
-The medium worker handles the queues:
-
-* checkouts_periodic: Scheduled tasks that check if a document's checkout
-  period has expired
-* documents_periodic:
-* indexing: Does reindexing of documents in the background when their
-  properties change
-* metadata:
-* sources:
-* sources_periodic: Checking email accounts and watch folders for new
-  documents.
-* uploads: Processes files to turn the into Mayan documents. Processing
-  encompasses MIME type detection, page count detection.
-* documents:
-
-The slow worker handles the queues:
-
-* mailing: Does the actual sending of documents via email as requested by
-  users via the mailing profiles
-* tools: Executes in the background maintenance requests from the options
-  in the tools menu
-* statistics: Recalculates statistics and charts
-* parsing: Parses documents to extract actual text content
-* ocr: Performs OCR to transcribe page images to text
 
 Optimizations
 -------------
@@ -88,15 +65,16 @@ Optimizations
 * By default each worker process uses 1 thread. You can increase the thread
   count of each worker process with the Docker environment options:
 
-  * MAYAN_WORKER_FAST_CONCURRENCY
-  * MAYAN_WORKER_MEDIUM_CONCURRENCY
-  * MAYAN_WORKER_SLOW_CONCURRENCY
+  * ``MAYAN_WORKER_FAST_CONCURRENCY``
+  * ``MAYAN_WORKER_MEDIUM_CONCURRENCY``
+  * ``MAYAN_WORKER_SLOW_CONCURRENCY``
 
-* If using direct deployment, increase the value of the --concurrency=1
+* If using direct deployment, increase the value of the ``--concurrency=1``
   argument of each worker in the supervisor file. You can also remove this
   argument and let the Celery algorithm choose the number of threads to
   launch. Usually this defaults to the number of CPU cores + 1.
 
+
 Change the message broker
 =========================
 Messages are the method of communication between front end interactive code
@@ -116,22 +94,24 @@ For the Docker image, launch a separate RabbitMQ container
 
     docker run -d --name mayan-edms-rabbitmq -e RABBITMQ_DEFAULT_USER=mayan -e RABBITMQ_DEFAULT_PASS=mayanrabbitmqpassword -e RABBITMQ_DEFAULT_VHOST=mayan rabbitmq:3
 
-Pass the MAYAN_BROKER_URL environment variable (https://kombu.readthedocs.io/en/latest/userguide/connections.html#connection-urls)
+Pass the MAYAN_CELERY_BROKER_URL environment variable (https://kombu.readthedocs.io/en/latest/userguide/connections.html#connection-urls)
 to the Mayan EDMS container so that it uses the RabbitMQ container the
 message broker::
 
-    -e MAYAN_BROKER_URL="amqp://mayan:mayanrabbitmqpassword@localhost:5672/mayan",
+    -e MAYAN_CELERY_BROKER_URL="amqp://mayan:mayanrabbitmqpassword@localhost:5672/mayan",
 
 When tasks finish, they leave behind a return status or the result of a
 calculation, these are stored for a while so that whoever requested the
 background task, is able retrieve the result. These results are stored in the
 result storage. By default a Redis server is launched inside the Mayan EDMS
 container. You can launch a separate Docker Redis container and tell the Mayan
-EDMS container to use this via the MAYAN_CELERY_RESULT_BACKEND environment
+EDMS container to use this via the ``MAYAN_CELERY_RESULT_BACKEND`` environment
 variable. The format of this variable is explained here: http://docs.celeryproject.org/en/3.1/configuration.html#celery-result-backend
 
+
 Deployment type
 ===============
+
 Docker provides a faster deployment and the overhead is not high on modern
 systems. It is however memory and CPU limited by default and you need to
 increase this limits. The settings to change the container resource limits
@@ -140,8 +120,10 @@ are here: https://docs.docker.com/config/containers/resource_constraints/#limit-
 For the best performance possible use the advanced deployment method on a
 host dedicated to serving only Mayan EDMS.
 
+
 Storage
 =======
+
 For best input and output speed use a block based local filesystem on an
 SSD drive for the ``/media`` sub folder. The location of the ``/media`` folder
 will be specified by the ``MEDIA_ROOT`` setting.
@@ -149,8 +131,10 @@ will be specified by the ``MEDIA_ROOT`` setting.
 If capacity is your bottom line, switch to an
 :doc:`object storage <../chapters/object_storage>` system.
 
+
 Use additional hosts
 ====================
+
 When one host is not enough you can use multiple hosts and share the load.
 Make sure that all hosts share the ``/media`` folder as specified by the
 ``MEDIA_ROOT`` setting, also the database, the broker, and the result storage.

docs/chapters/signatures.rst

@@ -11,6 +11,7 @@ signatures sections of a document.
 .. blockdiag::
 
    blockdiag {
+
       orientation = portrait
       span_width = 240;
 

docs/chapters/sources.rst

@@ -15,7 +15,8 @@ The current document sources supported are:
 - IMAP email - Same as the ``POP3`` email source but for email accounts using
   the ``IMAP`` protocol.
 - Watch folder - A filesystem folder that is scanned periodically for files.
-  Any file in the watch folder is automatically uploaded.
+  Any file in the watch folder is automatically uploaded. When the upload for a
+  file is completed, the file is removed from source folder.
 - Staging folder - Folder where networked attached scanned can save image
   files. The files in these staging folders are scanned and a preview is
   generated to help the process of upload. Staging folders and Watch folders
@@ -51,7 +52,29 @@ The current document sources supported are:
         web -> manual;
     }
 
-Document source can be configure to allow document bundles to uploaded as
+Document sources can be configure to allow document bundles to uploaded as
 compressed files which are decompressed and their content uploaded as separate
 documents. This feature is useful when migrating from another document
 manager system.
+
+
+.. note::
+
+    If you deployed a Mayan EDMS Docker container and want to use watched folders
+    or staging folder, refer to the Docker chapter :ref:`docker-accessing-outside-data`.
+
+
+Creating new sources
+====================
+
+.. admonition:: Permissions required
+    :class: warning
+
+    The "Create new document sources" permission is required for this action.
+
+
+#. Go to the :menuselection:`System --> Setup --> Sources` menu.
+#. From the :guilabel:`Actions` dropdown select the new type of source to create.
+#. Each source type will have different fields to customize its behavior. Enter
+   the required information in each field based on the help text provided.
+#. Press :guilabel:`Save`.

docs/chapters/tags.rst

@@ -5,10 +5,6 @@ Tags are color coded properties that can be attached or removed from documents.
 Tags allow giving documents a binary property. Documents can also be tagged
 with more than one tag.
 
-Tags are created from the :menuselection:`Tags --> Create` menu.
-
-To view all created tags use the :menuselection:`Tags --> All` menu.
-
 Once tagged, documents can be searched by their tags. It is also possible to
 show all the documents tagged with a particular
 tag via the **Documents** link of each tag.
@@ -21,3 +17,30 @@ more tags to the document and another to remove tags from the document.
 It is also possible to tag documents in bulk by selecting several documents
 from any view that displays a list documents and selecting the **Attach tag**
 or **Remove tags** from the dropdown menu.
+
+
+Creating tags
+=============
+
+.. admonition:: Permissions required
+    :class: warning
+
+    The "Create new tags" permission is required for this action.
+
+
+#. Go to the :menuselection:`Tags --> Create` menu.
+#. Enter a label to identify the tag.
+#. Select a color for the tag.
+#. Press :guilabel:`Submit`.
+
+
+View all tags
+=============
+
+.. admonition:: Permissions required
+    :class: warning
+
+    The "View tags" permission is required for this action, globally or via an
+    ACL for a tag.
+
+#. Go to the :menuselection:`Tags --> All` menu.

docs/chapters/versioning.rst

@@ -46,3 +46,35 @@ There is no limit to the number of versions a document can have.
 
 By default, the last version will be showed when working with the document
 but any of the versions can be inspected and viewed.
+
+
+View a document version list
+============================
+
+.. admonition:: Permissions required
+    :class: warning
+
+    The "View documents' versions list" permission is required for this action,
+    either globally or via an ACL for a document or document type.
+
+
+#. Navigate to the select document's preview view.
+#. Click on the sibebar's :guilabel:`Versions` button.
+
+
+Uploading a new document version
+================================
+
+.. admonition:: Permissions required
+    :class: warning
+
+    The "Create new document versions" permission is required for this action,
+    either globally or via an ACL for a document or document type.
+
+
+#. Navigate to the select document's versions list view.
+#. From the :guilabel:`Actions` dropdown select :guilabel:`Upload new version`.
+#. Optionally type a comment explaining the changes in the new version.
+#. Press the :guilabel:`Browse` button and select a new file.
+#. Press :guilabel:`Save` upload the new version.
+

docs/chapters/workflows.rst

@@ -5,7 +5,7 @@ Workflows
 Introduction
 ============
 
-Workflows provide a structure method for storing a sequence of states over
+Workflows provide a structured method for storing a sequence of states over
 which the a document will progress. Workflows keep track how a document has
 been processed so far.
 
@@ -20,6 +20,36 @@ provide and order for the sequence of possible states changes.
 Transitions can be executed manually by users if they have the required access
 level as configure by the system administrator.
 
+.. blockdiag::
+   :caption: Example workflow. Circles represent states, rectangles represent transitions.
+
+      span_width = 30;
+
+      submitted [shape = circle, width = 60, height = 60];
+      approved [shape = circle, width = 60, height = 60];
+      rejected [shape = circle, width = 60, height = 60];
+      archived [shape = circle, width = 60, height = 60];
+
+      approve [label = approve];
+      reject [label = reject];
+      reopen [label = "re-open"];
+
+
+      submitted -> approve -> approved;
+      approved -> archived;
+      submitted -> reject -> rejected;
+      rejected -> archived;
+      archived -> reopen -> submitted;
+
+
+
+Workflows in Mayan EDMS are implemented as finite state machines
+(https://en.wikipedia.org/wiki/Finite-state_machine).
+
+Workflows are mainly used to represent business processes. But they can also be
+used an automation system to customized Mayan EDMS and have it perform some
+tasks automatically.
+
 
 Automation
 ==========
@@ -66,26 +96,18 @@ Implementation
 ==============
 
 Internally, workflows are implemented as a finite state machines
-(https://en.wikipedia.org/wiki/Finite-state_machine). And have the limitation
-that only one state can be the current active state for a workflow being
-executed. The other limitation of the current implementation is that every
-workflow needs at least one state marked as the initial state. These limitations
-are the result of a compromised in the design between flexibility and ease of
-use.
+(https://en.wikipedia.org/wiki/Finite-state_machine). To make them simpler to
+use, workflow have been designed so that only one state can be the current
+active state for a workflow being executed. Another design decision is that
+every workflow needs at least one state marked as the initial state.
 
 
 Visualizations
 ==============
 
-The graphical representation of a workflow (or a finite state machine style
-in Mayan EDMS's case) is similar to a flowchart. The states are represented
-with circles. The transitions are represented with arrows. Circle with a
-double border represent the initial state of the workflow.
+The graphical representation of a workflow is similar to a flowchart. The
+states are represented with circles. The transitions are represented with
+arrows. Circle with a double border represent the initial state of the workflow.
 
 To view the graphical representations of workflow use **Preview** button of
 the workflow in the setup view.
-
-
-
-We are working now on workflow transition trigger filters to have
-the trigger move the state of the workflow on certain conditions. For example: Attach a tag if there is a specific word found in the OCR text.

docs/conf.py

@@ -22,7 +22,9 @@ import mayan
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
-sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "_ext")))
+sys.path.append(
+    os.path.abspath(os.path.join(os.path.dirname(__file__), '_ext'))
+)
 
 # -- General configuration -----------------------------------------------------
 
@@ -34,12 +36,13 @@ sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "_ext"))
 #extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode']
 #extensions = ["djangodocs", "sphinx.ext.intersphinx"]
 extensions = [
-    'sphinx.ext.extlinks', 'sphinxcontrib.blockdiag', 'sphinxcontrib.spelling'
+    'sphinx.ext.extlinks', 'sphinxcontrib.blockdiag',
+    'sphinxcontrib.spelling'
 ]
 
 blockdiag_antialias = True
-blockdiag_html_image_format = "SVG"
-blockdiag_latex_image_format = "PDF"
+blockdiag_html_image_format = 'SVG'
+blockdiag_latex_image_format = 'PDF'
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
@@ -106,12 +109,14 @@ pygments_style = 'sphinx'
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
-html_theme = 'classic'
+html_theme = 'sphinx_rtd_theme'
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
 # documentation.
-#html_theme_options = {}
+html_theme_options = {
+    'analytics_id': 'UA-52965619-6',
+}
 
 # Add any paths that contain custom themes here, relative to this directory.
 #html_theme_path = []
@@ -139,7 +144,7 @@ html_static_path = ['_static']
 
 # If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
 # using the given strftime format.
-#html_last_updated_fmt = '%b %d, %Y'
+html_last_updated_fmt = '%b %d, %Y'
 
 # If true, SmartyPants will be used to convert quotes and dashes to
 # typographically correct entities.
@@ -162,7 +167,7 @@ html_static_path = ['_static']
 #html_split_index = False
 
 # If true, links to the reST sources are added to the pages.
-#html_show_sourcelink = True
+html_show_sourcelink = False
 
 # If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
 #html_show_sphinx = True
@@ -181,8 +186,6 @@ html_static_path = ['_static']
 # Output file base name for HTML help builder.
 htmlhelp_basename = 'MayanEDMSdoc'
 
-html_show_sourcelink = False
-
 html_show_sphinx = False
 
 # -- Options for LaTeX output --------------------------------------------------
@@ -233,8 +236,6 @@ man_pages = [
      [mayan.__author__], 1)
 ]
 
-html_theme = 'sphinx_rtd_theme'
-
 # -- External links --
 extlinks = {
     'django-docs': (
@@ -242,6 +243,9 @@ extlinks = {
             mayan.__django_version__
         ), 'Django documentation section: '
     ),
+    'forum-topic': (
+        'https://forum.mayan-edms.com/viewtopic.php?t=%s', 'Forum topic #'
+    ),
     'github-issue': (
         'https://github.com/mayan-edms/mayan-edms/issues/%s', 'GitHub issue #'
     ),
@@ -249,3 +253,7 @@ extlinks = {
         'https://gitlab.com/mayan-edms/mayan-edms/issues/%s', 'GitLab issue #'
     )
 }
+
+
+def setup(app):
+    app.add_stylesheet('css/custom.css')

docs/index.rst

@@ -21,14 +21,17 @@ repository for electronic documents.
     topics/installation
     releases/index
     topics/getting_started
+    topics/adding_documents
     topics/access_control
     topics/categorization
     topics/collaboration
     topics/settings
     topics/storage
     topics/integration
+    topics/docker
     topics/advanced
     topics/administration
+    topics/troubleshooting
     topics/development
     topics/faq
     topics/license

docs/mercs/0001-merc-process.rst

@@ -31,9 +31,9 @@ for Mayan EDMS. Most MERCs will be Feature MERCs.
 2. An **Informational** MERC describes a Mayan EDMS design issue, or
 provides general guidelines or information to the Mayan EDMS community,
 but does not propose a new feature. Informational MERCs do not
-necessarily represent a community consensus or
-recommendation, so users and implementers are free to ignore
-Informational MERCs or follow their advice.
+necessarily represent a community consensus or recommendation, so users
+and implementers are free to ignore Informational MERCs or follow their
+advice.
 
 3. A **Process** MERC describes a process surrounding Mayan EDMS, or
 proposes a change to (or an event in) a process.  Process MERCs are

docs/mercs/0002-test-writing.rst

@@ -1,6 +1,6 @@
-=====================
+====================
 MERC 2: Test writing
-=====================
+====================
 
 :MERC: 2
 :Author: Michael Price

docs/mercs/0005-explicit-arguments.rst

@@ -0,0 +1,149 @@
+==========================
+MERC 5: Explicit arguments
+==========================
+
+:MERC: 5
+:Author: Roberto Rosario
+:Status: Accepted
+:Type: Feature
+:Created: 2018-12-30
+:Last-Modified: 2018-12-31
+
+.. contents:: Table of Contents
+   :depth: 3
+   :local:
+
+
+Abstract
+========
+
+This MERC proposes the adoption of a new methodology when performing calls.
+It seeks to reduce the use of positional arguments in favor of keyword
+arguments in as many places as possible.
+
+
+Motivation
+==========
+
+As the project grows, legibility of code becomes more important. Keyword
+argument help document the use of services, classes and functions. Refactors
+that affect the interface of services are also easier to find and update and
+fix. Positional argument can cause a call to continue working as long as the
+datatype of the argument remains the same. Usage of keyword arguments will
+automatically raise and error that will prevent such situations. Keyword
+argument further eliminate the relevance of position or the arguments, and
+the arguments can be sorted alphabetically for easier visual scanning or by
+semantic significance improving code readability.
+
+
+Specification
+=============
+
+Adoption of this MERC will require an audit of existing calls and the use
+of the method proposed for new calls. Every call regardless of the type or
+origin of the source callable will name each argument used. By type it is
+meant: classes, functions, methods. Origin means: local from the project,
+from the framework, third party libraries or the standard library.
+
+
+Backwards Compatibility
+=======================
+
+No backwards compatibility issues are expected. New errors arising from the use
+if keyword arguments could be interpreted as existing latent issues that
+have not been uncovered.
+
+
+Reference Implementation
+========================
+
+Example:
+
+Before:
+
+.. code-block:: python
+
+    from mayan.apps.common.classes import Template
+
+    Template(
+        'menu_main', 'appearance/menu_main.html'
+    )
+
+
+After:
+
+.. code-block:: python
+
+    from mayan.apps.common.classes import Template
+
+    Template(
+        name='menu_main', template_name='appearance/menu_main.html'
+    )
+
+
+When calls use a mixture or positional and keyword arguments, the keywords
+arguments can only be found after the positional arguments. Complete use
+of keyword arguments allow the reposition of arguments for semantic
+purposes.
+
+Example:
+
+Before:
+
+.. code-block:: python
+
+    from django.conf.urls import url
+
+    from .views import AboutView, HomeView, RootView
+
+    urlpatterns = [
+        url(r'^$', RootView.as_view(), name='root'),
+        url(r'^home/$', HomeView.as_view(), name='home'),
+        url(r'^about/$', AboutView.as_view(), name='about_view'),
+    ]
+
+
+After:
+
+.. code-block:: python
+
+    from django.conf.urls import url
+
+    from .views import AboutView, HomeView, RootView
+
+    urlpatterns = [
+        url(regex=r'^$', name='root', view=RootView.as_view()),
+        url(regex=r'^home/$', name='home', view=HomeView.as_view()),
+        url(regex=r'^about/$', name='about_view', view=AboutView.as_view()),
+    ]
+
+
+Keyword arguments should also be used for callables that pass those to others
+down the line like Django's ``reverse`` function. Any change to the name of
+the ``pk`` URL parameter will raise an exception in this code alerting to
+any possible incompatible use.
+
+
+Example:
+
+.. code-block:: python
+
+    def get_absolute_url(self):
+        return reverse(
+            viewname='documents:document_preview', kwargs={'pk': self.pk}
+        )
+
+
+This becomes even more important when multiple URL parameters are used. Since
+the API documentation is auto generated from the code itself, it would make
+sense to rename the first URL parameter from ``pk`` to ``document_pk``. Such
+change will cause all address to view resolutions to break forcing their
+update and allowing all consumers' interface usage to remain synchronized to the
+callable's interface.
+
+.. code-block:: python
+
+    url(
+        regex=r'^documents/(?P<pk>[0-9]+)/versions/(?P<document_version_pk>[0-9]+)/pages/(?P<document_page_pk>[0-9]+)/image/$',
+        name='documentpage-image', view=APIDocumentPageImageView.as_view()
+    ),

docs/mercs/0006-lower-information-disclose.rst

@@ -0,0 +1,81 @@
+==================================
+MERC 6: Lower information disclose
+==================================
+
+:MERC: 6
+:Author: Michael Price
+:Status: Accepted
+:Type: Feature
+:Created: 2018-12-30
+:Last-Modified: 2018-12-31
+
+.. contents:: Table of Contents
+   :depth: 3
+   :local:
+
+Abstract
+========
+
+This MERC proposes the use of errors that don't disclose the existence of a
+resource in the event that the requester doesn't have the required credentials.
+
+Motivation
+==========
+
+When an user tries to perform an action like opening a view to a document for
+which the required permission is missing, a permission required or access
+denied error is presented. This is semantically correct, but from the stand
+point of security it is still failing because it is letting the user know
+that such document exists in the first place. This MERC proposes changing the
+error message for existing resource to one that doesn't divulge any information
+to unauthorized parties, like "Not Found".
+
+Specification
+=============
+
+Out of the 4 basic CRUD operations, Read, Update and Delete should return an
+HTTP 404 error instead of an HTTP 403 error. Only the Create operation will
+continue returning the current HTTP 403 error, unless it is creating a
+new resource that is related to an existing resource.
+
+Since most view use the internal custom CRUD classes making a change to the
+``ObjectPermissionCheckMixin`` class to raise an HTTP 404 on object access
+failure will fulfill the proposal of this MERC.
+
+Adding the ``object_permission_raise_404`` class attribute and setting it
+to default to False will allow fulfilling the goal of this MERC while
+keeping the existing functionality intact.
+
+
+Example:
+
+.. code-block:: python
+
+    class ObjectPermissionCheckMixin(object):
+        """
+        If object_permission_raise_404 is True an HTTP 404 error will be raised
+        instead of the normal 403.
+        """
+        object_permission = None
+        object_permission_raise_404 = False
+
+        def get_permission_object(self):
+            return self.get_object()
+
+        def dispatch(self, request, *args, **kwargs):
+            if self.object_permission:
+                try:
+                    AccessControlList.objects.check_access(
+                        permissions=self.object_permission, user=request.user,
+                        obj=self.get_permission_object(),
+                        related=getattr(self, 'object_permission_related', None)
+                    )
+                except PermissionDenied:
+                    if self.object_permission_raise_404:
+                        raise Http404
+                    else:
+                        raise
+
+            return super(
+                ObjectPermissionCheckMixin, self
+            ).dispatch(request, *args, **kwargs)

docs/mercs/index.rst

@@ -20,6 +20,8 @@ Accepted
    ../mercs/0001-merc-process
    ../mercs/0002-test-writing
    ../mercs/0003-using-javascript-libraries
+   ../mercs/0005-explicit-arguments
+   ../mercs/0006-lower-information-disclose
 
 Draft
 -----
@@ -49,3 +51,5 @@ Feature
 
    ../mercs/0002-test-writing
    ../mercs/0003-using-javascript-libraries
+   ../mercs/0005-explicit-arguments
+   ../mercs/0006-lower-information-disclose

docs/mercs/merging-roles-and-groups.rst

@@ -63,5 +63,5 @@ Changes needed:
    the Role model's permissions many to many field.
 4. Update the ``AccessControlList`` models roles field to point to the group
    models.
-5. Update the role checks in the ``check_access`` and ``filter_by_access``
+5. Update the role checks in the ``check_access`` and ``restrict_queryset``
    ``AccessControlList`` model manager methods.

docs/releases/0.11.rst

@@ -17,11 +17,11 @@ Version 0.11
 * Added a view to delete the document image cache, useful when switching
   converter backends or doing diagnostics.
 * Added South to the requirements.
-* Merged documents' filename and extension database fiels into a single
+* Merged documents' filename and extension database fields into a single
   filename field, filename are store as uploaded not manipulation is done
-  Users with existing data must install South and run the appropiate
+  Users with existing data must install South and run the appropriate
   migrate commands::
-  
+
     $ pip install -r requirements/production.txt
     $ ./manager syncdb
     $ ./manage.py migrate documents 0001 --fake
@@ -30,5 +30,5 @@ Version 0.11
 * Added new office document mimetype
   * application/vnd.ms-office
 * Fixed documents not saving the file encoding
-* Removed extra slash in ajax-loader.gif URL fixes #15, thanks to 
+* Removed extra slash in ajax-loader.gif URL fixes #15, thanks to
   IHLeanne for finding this one

docs/releases/3.1.10.rst

@@ -0,0 +1,188 @@
+Version 3.1.10
+==============
+
+Released: April 04, 2019
+
+
+Changes
+-------
+
+Test improvements
+^^^^^^^^^^^^^^^^^
+
+A number of test improvements were backported from the development branch. These
+include a mixin to assign random primary keys to the test objects. Normally
+primary keys are sequential and their predictability can hide edge cases that
+would normally only manifest in production. The tests case code was also split
+into smaller mixin units separating them by purpose. All code to perform HTTP
+requests now resides in the ``ClientMethodsTestCaseMixin``. Changes in the
+``ContentTypeCheckTestCaseMixin`` were added to enable this mixin for the API views
+in a future minor version. The test view code was moved into its own mixin
+called ``TestViewTestCaseMixin``. This mixin allows creating dynamic views to test
+object and link resolution among other things. The API base test case class is
+now part of the same class hierarchy, this means that the API test code will
+now behave the same and will benefit from all improvements in the base test case
+classes. Some indexing tests were failing randomly. This was caused by the way
+the test were built and not because of faulty code. The tests were updated and
+now operate as expected. While there is no official support for installing
+Mayan EDMS as a sub URL updates to the ``TestViewTestCaseMixin`` will allow it to
+execute with custom ``ROOT_URLCONF`` configurations. Further work and feedback is
+needed to officially support this installation method.
+
+
+Fixes
+^^^^^
+
+The advanced search API endpoint was fixed and is now usable. Tests for all
+the issues fixed were added to avoid future regressions. A fix was applied
+to the compressed class that caused compressed files uploads to fail from the
+web form.
+
+Docker
+^^^^^^
+
+Updates were added to the Docker image that allow it to build on armv7l
+platforms like the RasperryPi, Odroid XU4, and Odroid HC2. This doesn't imply
+official support for these platforms. Further experimentation and optimization
+is needed and this changed will now allow users to build the image and provide
+feedback in an easier manner.
+
+
+Installation
+^^^^^^^^^^^^
+
+The installation process has been updated to work around the Python pip
+issue #6197 (https://github.com/pypa/pip/issues/6197).
+
+
+User interface
+^^^^^^^^^^^^^^
+
+Improvements in the AJAX menu rendering were conflicting with an old method
+of refreshing the unread notification counter badge. This conflict would cause
+the unread notification counter badge to blink and/or disappear. The older
+badge refresh method was removed and improvements to the menu rendering from
+the development branch were backported to support displaying link badges from
+the templates completing fixing the issue. An API entrypoint was also backported
+showing a list of all the server side AJAX templates. This change doesn't affect
+the API layout and it remains compatible with the previous version. Newlines
+are now explicitly removed from the rendered AJAX templates avoiding rendering
+issues on some browsers.
+
+
+Emails
+^^^^^^
+
+An update was added to will now cause Mayan EDMS to reject all email attachments
+with a size 0 even if the attachment is valid. Documents of size 0 are not
+consideref valid in Mayan EDMS since they can't be introspected for MIME type,
+don't contain at least one version, and don't contain a single page. All these
+are requirements for a valid document in Mayan EDMS.
+
+
+Memory usage
+^^^^^^^^^^^^
+
+The ``MIMETYPE_FILE_READ_SIZE`` setting was added to limit the number of bytes that
+will be read into memory to determine the MIME type of a new document. For
+compatibility with the current bevahor this setting defaults to 0 which means
+that it is disabled. Disabling the setting will cause the entire document's
+file to be loaded into memory. If documents are not processing due to out of
+memory errors (large documents or devices with limited memory), set
+``MIMETYPE_FILE_READ_SIZE`` to a value other than 0. Limited tests suggest 1024
+to be a good alternative as most "magic numbers" used for MIME type detection
+are located at the start of the file and just reading the first 1024 bytes will
+result in a positive identification with little memory usage.
+
+
+Other changes
+^^^^^^^^^^^^^
+
+* Add missing document index API view create permission.
+* Fix index list API view. Add index create, delete, detail API tests.
+* Add support for skipping a default set of tests. Tests to be excluded
+  by default should be tagged as 'exclude'.
+
+Removals
+--------
+
+* None
+
+
+Upgrading from a previous version
+---------------------------------
+
+If installed via Python's PIP
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Remove deprecated requirements::
+
+    $ curl https://gitlab.com/mayan-edms/mayan-edms/raw/master/removals.txt | pip uninstall -r /dev/stdin
+
+Type in the console::
+
+    $ pip install mayan-edms==3.1.10
+
+the requirements will also be updated automatically.
+
+
+Using Git
+^^^^^^^^^
+
+If you installed Mayan EDMS by cloning the Git repository issue the commands::
+
+    $ git reset --hard HEAD
+    $ git pull
+
+otherwise download the compressed archived and uncompress it overriding the
+existing installation.
+
+Remove deprecated requirements::
+
+    $ pip uninstall -y -r removals.txt
+
+Next upgrade/add the new requirements::
+
+    $ pip install --upgrade -r requirements.txt
+
+
+Common steps
+^^^^^^^^^^^^
+
+Perform these steps after updating the code from either step above.
+
+Migrate existing database schema with::
+
+    $ mayan-edms.py performupgrade
+
+Add new static media::
+
+    $ mayan-edms.py collectstatic --noinput
+
+The upgrade procedure is now complete.
+
+
+Backward incompatible changes
+-----------------------------
+
+* None
+
+
+Bugs fixed or issues closed
+---------------------------
+
+* :gitlab-issue:`557` Break workflows with invalid input
+* :gitlab-issue:`559` IndexTestCase.test_dual_level_dual_document_index failure
+* :gitlab-issue:`562` events.links.link_user_notifications_list should use
+  reverse
+* :gitlab-issue:`564` API REST /api/indexes/ generates an "Internal Server Error"
+* :gitlab-issue:`566` apps.common.tests.base.GenericViewTestCase doesn't work
+  with a custom ROOT_URLCONF
+* :gitlab-issue:`568` Tornado 6.0 doesn't support Python 2.7
+* :gitlab-issue:`572` Error when sending compressed files: ziparchive object
+  has no attribute children
+* :gitlab-issue:`574` import of E-Mails with empty attachment fails
+* :gitlab-issue:`576` Exception with access check for cabinets containing
+  NON-ASCII chars
+
+.. _PyPI: https://pypi.python.org/pypi/mayan-edms/

docs/releases/3.1.11.rst

@@ -0,0 +1,148 @@
+Version 3.1.11
+==============
+
+Released: April XX, 2019
+
+
+Changes
+-------
+
+Memory usage
+^^^^^^^^^^^^
+
+The ``DOCUMENTS_HASH_BLOCK_SIZE`` setting was added to limit the number of
+bytes that will be read into memory when calculating the checksum of a new
+document. For compatibility with the current bevahor this setting defaults to
+0 which means that it is disabled. Disabling the setting will cause the
+entire document's file to be loaded into memory. If documents are not
+processing due to out of memory errors (large documents or devices with
+limited memory), set ``DOCUMENTS_HASH_BLOCK_SIZE`` to a value other than 0.
+Limited tests suggest 65535 to be a good alternative.
+
+
+Tag wizard step
+^^^^^^^^^^^^^^^
+
+The tag wizard step was fixed and will now allow attaching multple tags to a
+new document.
+
+
+Permissions
+^^^^^^^^^^^
+
+Previously the document checkout information link required one of the following
+permissions: document check in, document check in override, or document
+checkout. Meanwhile the document checkout information view would require the
+document checkout detail view permission. This difference in permissions
+has been eliminated and the link will now required the document checkout
+detail view permission, same as the view. Update your user role permissions
+accordingly.
+
+
+Other changes
+^^^^^^^^^^^^^
+
+* Lower the log severity when links don't resolve.
+
+
+Removals
+--------
+
+* None
+
+
+Upgrading from a previous version
+---------------------------------
+
+If installed via Python's PIP
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Remove deprecated requirements::
+
+    $ curl https://gitlab.com/mayan-edms/mayan-edms/raw/master/removals.txt | pip uninstall -r /dev/stdin
+
+Type in the console::
+
+    $ pip install mayan-edms==3.1.11
+
+the requirements will also be updated automatically.
+
+Migrate existing database schema with::
+
+    $ mayan-edms.py performupgrade
+
+Add new static media::
+
+    $ mayan-edms.py collectstatic --noinput
+
+The upgrade procedure is now complete.
+
+
+If installed using a direct deployment
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Remove deprecated requirements::
+
+    $ curl https://gitlab.com/mayan-edms/mayan-edms/raw/master/removals.txt | sudo -u mayan /opt/mayan-edms/bin/pip uninstall -r /dev/stdin
+
+Download and install the new version::
+
+    $ sudo -u mayan /opt/mayan-edms/bin/pip install --no-cache-dir --no-use-pep517 mayan-edms==3.1.11
+
+the requirements will also be updated automatically.
+
+Run the upgrade command::
+
+    $ sudo -u mayan MAYAN_DATABASE_ENGINE=django.db.backends.postgresql \
+    MAYAN_DATABASE_NAME=mayan MAYAN_DATABASE_PASSWORD=mayanuserpass \
+    MAYAN_DATABASE_USER=mayan MAYAN_DATABASE_HOST=127.0.0.1 \
+    MAYAN_MEDIA_ROOT=/opt/mayan-edms/media /opt/mayan-edms/bin/mayan-edms.py performupgrade
+
+Add any new static files::
+
+    $ sudo -u mayan MAYAN_MEDIA_ROOT=/opt/mayan-edms/media /opt/mayan-edms/bin/mayan-edms.py collectstatic --noinput
+
+The upgrade procedure is now complete.
+
+Using Git
+^^^^^^^^^
+
+If you installed Mayan EDMS by cloning the Git repository issue the commands::
+
+    $ git reset --hard HEAD
+    $ git pull
+
+otherwise download the compressed archived and uncompress it overriding the
+existing installation.
+
+Remove deprecated requirements::
+
+    $ pip uninstall -y -r removals.txt
+
+Next upgrade/add the new requirements::
+
+    $ pip install --upgrade -r requirements.txt
+
+Migrate existing database schema with::
+
+    $ mayan-edms.py performupgrade
+
+Add new static media::
+
+    $ mayan-edms.py collectstatic --noinput
+
+The upgrade procedure is now complete.
+
+
+Backward incompatible changes
+-----------------------------
+
+* None
+
+
+Bugs fixed or issues closed
+---------------------------
+
+* None
+
+.. _PyPI: https://pypi.python.org/pypi/mayan-edms/

docs/releases/3.2.1.rst

@@ -0,0 +1,119 @@
+Version 3.2.1
+=============
+
+Released: June 14, 2019
+
+
+Changes
+-------
+
+- Fix sub cabinet creation view. Thanks to Frédéric Sheedy
+  (@fsheedy) for the report.
+- Add PostgreSQL troubleshooting entry. Closes GitLab
+  issues #523 and #602
+- Use YAML SafeDumper to avoid adding YAML datatype tags.
+  Closes GitLab issue #599. Thanks to Frédéric Sheedy
+  (@fsheedy) for the report and debug information.
+- Add check for app references and point users to release notes for details.
+  GitLab issue #603. Thanks to Vikas Kedia (@vikaskedia) for the report.
+- Remove sidebar floar right.
+  Fixed GitLab issue #600. Thanks to Frédéric Sheedy
+  (@fsheedy) for the report and debug information.
+- Collapse sidebar on small screen
+  Display sidebar at the bottom of the screen on small displays.
+
+Removals
+--------
+
+- None
+
+
+Upgrading from a previous version
+---------------------------------
+
+If installed via Python's PIP
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Remove deprecated requirements::
+
+    $ curl https://gitlab.com/mayan-edms/mayan-edms/raw/master/removals.txt | pip uninstall -r /dev/stdin
+
+Type in the console::
+
+    $ pip install mayan-edms==3.2.1
+
+the requirements will also be updated automatically.
+
+
+Using Git
+^^^^^^^^^
+
+If you installed Mayan EDMS by cloning the Git repository issue the commands::
+
+    $ git reset --hard HEAD
+    $ git pull
+
+otherwise download the compressed archived and uncompress it overriding the
+existing installation.
+
+Remove deprecated requirements::
+
+    $ pip uninstall -y -r removals.txt
+
+Next upgrade/add the new requirements::
+
+    $ pip install --upgrade -r requirements.txt
+
+
+Common steps
+^^^^^^^^^^^^
+
+Perform these steps after updating the code from either step above.
+
+Make a backup of your supervisord file::
+
+    sudo cp /etc/supervisor/conf.d/mayan.conf /etc/supervisor/conf.d/mayan.conf.bck
+
+Update the supervisord configuration file. Replace the environment
+variables values show here with your respective settings. This step will refresh
+the supervisord configuration file with the new queues and the latest
+recommended layout::
+
+    MAYAN_DATABASE_ENGINE=django.db.backends.postgresql MAYAN_DATABASE_NAME=mayan \
+    MAYAN_DATABASE_PASSWORD=mayanuserpass MAYAN_DATABASE_USER=mayan \
+    MAYAN_DATABASE_HOST=127.0.0.1 MAYAN_MEDIA_ROOT=/opt/mayan-edms/media \
+    /opt/mayan-edms/bin/mayan-edms.py platformtemplate supervisord > /etc/supervisor/conf.d/mayan.conf
+
+Edit the supervisord configuration file and update any setting the template
+generator missed::
+
+    vi /etc/supervisor/conf.d/mayan.conf
+
+Migrate existing database schema with::
+
+    $ mayan-edms.py performupgrade
+
+Add new static media::
+
+    $ mayan-edms.py preparestatic --noinput
+
+The upgrade procedure is now complete.
+
+
+Backward incompatible changes
+-----------------------------
+
+- None
+
+
+Bugs fixed or issues closed
+---------------------------
+
+- :gitlab-issue:`523` PostgreSQL error about insufficient connections
+- :gitlab-issue:`599` Settings display !!python/unicode with values since 3.2
+- :gitlab-issue:`600` Layout broken if we change locale, since 3.2
+- :gitlab-issue:`601` Error when creating new cabinet level
+- :gitlab-issue:`602` System stops responding for a minute every 10 minutes or so
+- :gitlab-issue:`603` ImportError: No module named appearance
+
+.. _PyPI: https://pypi.python.org/pypi/mayan-edms/

docs/releases/3.2.2.rst

@@ -0,0 +1,109 @@
+Version 3.2.2
+=============
+
+Released: June 19, 2019
+
+
+Changes
+-------
+
+- Fix document type change view. Closes GitLab issue #614.
+  Thanks to Christoph Roeder (@brightdroid) for the report.
+- Fix document parsing tool view typo. Closes GitLab issue #615.
+  Thanks to Tyler Page (@iamtpage) for the report.
+- Update the task_check_interval_source reference
+  GitLab issue #617. Thanks to Lukas Gill (@lukkigi) for
+  the report and debug information.
+
+Removals
+--------
+
+- None
+
+
+Upgrading from a previous version
+---------------------------------
+
+If installed via Python's PIP
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Remove deprecated requirements::
+
+    $ curl https://gitlab.com/mayan-edms/mayan-edms/raw/master/removals.txt | pip uninstall -r /dev/stdin
+
+Type in the console::
+
+    $ pip install mayan-edms==3.2.2
+
+the requirements will also be updated automatically.
+
+
+Using Git
+^^^^^^^^^
+
+If you installed Mayan EDMS by cloning the Git repository issue the commands::
+
+    $ git reset --hard HEAD
+    $ git pull
+
+otherwise download the compressed archived and uncompress it overriding the
+existing installation.
+
+Remove deprecated requirements::
+
+    $ pip uninstall -y -r removals.txt
+
+Next upgrade/add the new requirements::
+
+    $ pip install --upgrade -r requirements.txt
+
+
+Common steps
+^^^^^^^^^^^^
+
+Perform these steps after updating the code from either step above.
+
+Make a backup of your supervisord file::
+
+    sudo cp /etc/supervisor/conf.d/mayan.conf /etc/supervisor/conf.d/mayan.conf.bck
+
+Update the supervisord configuration file. Replace the environment
+variables values show here with your respective settings. This step will refresh
+the supervisord configuration file with the new queues and the latest
+recommended layout::
+
+    MAYAN_DATABASE_ENGINE=django.db.backends.postgresql MAYAN_DATABASE_NAME=mayan \
+    MAYAN_DATABASE_PASSWORD=mayanuserpass MAYAN_DATABASE_USER=mayan \
+    MAYAN_DATABASE_HOST=127.0.0.1 MAYAN_MEDIA_ROOT=/opt/mayan-edms/media \
+    /opt/mayan-edms/bin/mayan-edms.py platformtemplate supervisord > /etc/supervisor/conf.d/mayan.conf
+
+Edit the supervisord configuration file and update any setting the template
+generator missed::
+
+    vi /etc/supervisor/conf.d/mayan.conf
+
+Migrate existing database schema with::
+
+    $ mayan-edms.py performupgrade
+
+Add new static media::
+
+    $ mayan-edms.py preparestatic --noinput
+
+The upgrade procedure is now complete.
+
+
+Backward incompatible changes
+-----------------------------
+
+- None
+
+
+Bugs fixed or issues closed
+---------------------------
+
+- :gitlab-issue:`614` change type exception
+- :gitlab-issue:`615` TypeError: success() got an unexpected keyword argument 'requrest'
+- :gitlab-issue:`617` Watcher Task not running
+
+.. _PyPI: https://pypi.python.org/pypi/mayan-edms/

docs/releases/3.2.3.rst

@@ -0,0 +1,114 @@
+Version 3.2.3
+=============
+
+Released: June 21, 2019
+
+
+Changes
+-------
+
+- Add support for disabling the random primary key
+  test mixin.
+- Fix mailing profile log columns mappings.
+  GitLab issue #626. Thanks to Jesaja Everling (@jeverling)
+  for the report.
+- Fix the Django SMTP backend username field name.
+  GitLab issue #625. Thanks to Jesaja Everling (@jeverling)
+  for the report and the research.
+- Increase the Django STMP username.
+  GitLab issue #625. Thanks to Jesaja Everling (@jeverling)
+  for the report and the research.
+
+
+Removals
+--------
+
+- None
+
+
+Upgrading from a previous version
+---------------------------------
+
+If installed via Python's PIP
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Remove deprecated requirements::
+
+    $ curl https://gitlab.com/mayan-edms/mayan-edms/raw/master/removals.txt | pip uninstall -r /dev/stdin
+
+Type in the console::
+
+    $ pip install mayan-edms==3.2.1
+
+the requirements will also be updated automatically.
+
+
+Using Git
+^^^^^^^^^
+
+If you installed Mayan EDMS by cloning the Git repository issue the commands::
+
+    $ git reset --hard HEAD
+    $ git pull
+
+otherwise download the compressed archived and uncompress it overriding the
+existing installation.
+
+Remove deprecated requirements::
+
+    $ pip uninstall -y -r removals.txt
+
+Next upgrade/add the new requirements::
+
+    $ pip install --upgrade -r requirements.txt
+
+
+Common steps
+^^^^^^^^^^^^
+
+Perform these steps after updating the code from either step above.
+
+Make a backup of your supervisord file::
+
+    sudo cp /etc/supervisor/conf.d/mayan.conf /etc/supervisor/conf.d/mayan.conf.bck
+
+Update the supervisord configuration file. Replace the environment
+variables values show here with your respective settings. This step will refresh
+the supervisord configuration file with the new queues and the latest
+recommended layout::
+
+    MAYAN_DATABASE_ENGINE=django.db.backends.postgresql MAYAN_DATABASE_NAME=mayan \
+    MAYAN_DATABASE_PASSWORD=mayanuserpass MAYAN_DATABASE_USER=mayan \
+    MAYAN_DATABASE_HOST=127.0.0.1 MAYAN_MEDIA_ROOT=/opt/mayan-edms/media \
+    /opt/mayan-edms/bin/mayan-edms.py platformtemplate supervisord > /etc/supervisor/conf.d/mayan.conf
+
+Edit the supervisord configuration file and update any setting the template
+generator missed::
+
+    vi /etc/supervisor/conf.d/mayan.conf
+
+Migrate existing database schema with::
+
+    $ mayan-edms.py performupgrade
+
+Add new static media::
+
+    $ mayan-edms.py preparestatic --noinput
+
+The upgrade procedure is now complete.
+
+
+Backward incompatible changes
+-----------------------------
+
+- None
+
+
+Bugs fixed or issues closed
+---------------------------
+
+- :gitlab-issue:`619` poplib.POP3_SSL and poplib.POP3 initialized with wrong kwarg
+- :gitlab-issue:`625` mayan.apps.mailer.mailers.DjangoSMTP uses "user", but django.core.mail.backends.smtp.EmailBackend expects "username"
+- :gitlab-issue:`626` Mailing profile error log is empty, despite errors
+
+.. _PyPI: https://pypi.python.org/pypi/mayan-edms/

docs/releases/3.2.4.rst

@@ -0,0 +1,131 @@
+Version 3.2.4
+=============
+
+Released: June 29, 2019
+
+
+Changes
+-------
+
+- Support configurable GUnicorn timeouts. Defaults to
+  current value of 120 seconds.
+- Fix help text of the platformtemplate command.
+- Fix IMAP4 mailbox.store flags argument. Python's documentation
+  incorrectly state it is named flag_list. Closes GitLab issue
+  #606. Thanks to Samuel Aebi (@samuelaebi) for the report and
+  debug information.
+- Support configurable GUnicorn timeouts. Defaults to
+  current value of 120 seconds.
+- Fix help text of the platformtemplate command.
+- Fix IMAP4 mailbox.store flags argument. Python's documentation
+  incorrectly state it is named flag_list. Closes GitLab issue
+  #606.
+- Improve the workflow preview generation. Use polylines
+  instead of splines. Add state actions to the preview.
+  Highlight the initial state.
+- Add help text to the workflow transition form comment field.
+- Fix direct deployment instructions.
+- Add user, group, and role dashboard widgets.
+- Add test mixin detect database connection leaks.
+- Remove tag create event registration from the tag
+  instances. The tag create event is not applicable to
+  existing tags.
+- Add proper redirection after moving a document to the
+  trash.
+- Remove the INSTALLED_APPS setting. Replace it with
+  the new COMMON_EXTRA_APPS and COMMON_DISABLED_APPS.
+- Improve email metadata support. Can now work on
+  email with nested parts. Also the metadata.yaml
+  attachment no longer needs to be the first attachment.
+
+Removals
+--------
+
+- None
+
+
+Upgrading from a previous version
+---------------------------------
+
+If installed via Python's PIP
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Remove deprecated requirements::
+
+    $ curl https://gitlab.com/mayan-edms/mayan-edms/raw/master/removals.txt | pip uninstall -r /dev/stdin
+
+Type in the console::
+
+    $ pip install mayan-edms==3.2.4
+
+the requirements will also be updated automatically.
+
+
+Using Git
+^^^^^^^^^
+
+If you installed Mayan EDMS by cloning the Git repository issue the commands::
+
+    $ git reset --hard HEAD
+    $ git pull
+
+otherwise download the compressed archived and uncompress it overriding the
+existing installation.
+
+Remove deprecated requirements::
+
+    $ pip uninstall -y -r removals.txt
+
+Next upgrade/add the new requirements::
+
+    $ pip install --upgrade -r requirements.txt
+
+
+Common steps
+^^^^^^^^^^^^
+
+Perform these steps after updating the code from either step above.
+
+Make a backup of your supervisord file::
+
+    sudo cp /etc/supervisor/conf.d/mayan.conf /etc/supervisor/conf.d/mayan.conf.bck
+
+Update the supervisord configuration file. Replace the environment
+variables values show here with your respective settings. This step will refresh
+the supervisord configuration file with the new queues and the latest
+recommended layout::
+
+    sudo MAYAN_DATABASE_ENGINE=django.db.backends.postgresql MAYAN_DATABASE_NAME=mayan \
+    MAYAN_DATABASE_PASSWORD=mayanuserpass MAYAN_DATABASE_USER=mayan \
+    MAYAN_DATABASE_HOST=127.0.0.1 MAYAN_MEDIA_ROOT=/opt/mayan-edms/media \
+    /opt/mayan-edms/bin/mayan-edms.py platformtemplate supervisord > /etc/supervisor/conf.d/mayan.conf
+
+Edit the supervisord configuration file and update any setting the template
+generator missed::
+
+    sudo vi /etc/supervisor/conf.d/mayan.conf
+
+Migrate existing database schema with::
+
+    $ mayan-edms.py performupgrade
+
+Add new static media::
+
+    $ mayan-edms.py preparestatic --noinput
+
+The upgrade procedure is now complete.
+
+
+Backward incompatible changes
+-----------------------------
+
+- None
+
+
+Bugs fixed or issues closed
+---------------------------
+
+- :gitlab-issue:`606` Delete after IMAP Processing
+- :gitlab-issue:`628` mailbox.user in POP3Email gets passed keyword argument, but only accepts "user" or positional argument
+
+.. _PyPI: https://pypi.python.org/pypi/mayan-edms/

docs/releases/3.2.5.rst

@@ -0,0 +1,117 @@
+Version 3.2.5
+=============
+
+Released: July 05, 2019
+
+
+Changes
+-------
+
+- Don't error out if the EXTRA_APPS or the DISABLED_APPS settings
+  are set to blank.
+- Update troubleshooting documentation topic.
+- Add data migration to the file metadata app. Synchronizes the
+  document type settings model of existing document types.
+- Fix cabinet and tags upload wizard steps missing some entries.
+  GitLab issue #632. Thanks to Matthias Urhahn (@d4rken) for the
+  report.
+- Add alert when settings are changed and util the installation
+  is restarted. GitLab issue #605. Thanks to
+  Vikas Kedia (@vikaskedia) to the report.
+- Update Django to version 1.11.22, PyYAML to version 5.1.1,
+  django-widget-tweaks to version 1.4.5, pathlib2 to version 2.3.4,
+  Werkzeug to version 0.15.4, django-extensions to version 2.1.9,
+  django-rosetta to version 0.9.3, psutil to version 5.6.3.
+
+Removals
+--------
+
+- None
+
+
+Upgrading from a previous version
+---------------------------------
+
+If installed via Python's PIP
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Remove deprecated requirements::
+
+    $ curl https://gitlab.com/mayan-edms/mayan-edms/raw/master/removals.txt | pip uninstall -r /dev/stdin
+
+Type in the console::
+
+    $ pip install mayan-edms==3.2.5
+
+the requirements will also be updated automatically.
+
+
+Using Git
+^^^^^^^^^
+
+If you installed Mayan EDMS by cloning the Git repository issue the commands::
+
+    $ git reset --hard HEAD
+    $ git pull
+
+otherwise download the compressed archived and uncompress it overriding the
+existing installation.
+
+Remove deprecated requirements::
+
+    $ pip uninstall -y -r removals.txt
+
+Next upgrade/add the new requirements::
+
+    $ pip install --upgrade -r requirements.txt
+
+
+Common steps
+^^^^^^^^^^^^
+
+Perform these steps after updating the code from either step above.
+
+Make a backup of your supervisord file::
+
+    sudo cp /etc/supervisor/conf.d/mayan.conf /etc/supervisor/conf.d/mayan.conf.bck
+
+Update the supervisord configuration file. Replace the environment
+variables values show here with your respective settings. This step will refresh
+the supervisord configuration file with the new queues and the latest
+recommended layout::
+
+    sudo MAYAN_DATABASE_ENGINE=django.db.backends.postgresql MAYAN_DATABASE_NAME=mayan \
+    MAYAN_DATABASE_PASSWORD=mayanuserpass MAYAN_DATABASE_USER=mayan \
+    MAYAN_DATABASE_HOST=127.0.0.1 MAYAN_MEDIA_ROOT=/opt/mayan-edms/media \
+    /opt/mayan-edms/bin/mayan-edms.py platformtemplate supervisord > /etc/supervisor/conf.d/mayan.conf
+
+Edit the supervisord configuration file and update any setting the template
+generator missed::
+
+    sudo vi /etc/supervisor/conf.d/mayan.conf
+
+Migrate existing database schema with::
+
+    $ mayan-edms.py performupgrade
+
+Add new static media::
+
+    $ mayan-edms.py preparestatic --noinput
+
+The upgrade procedure is now complete.
+
+
+Backward incompatible changes
+-----------------------------
+
+- None
+
+
+Bugs fixed or issues closed
+---------------------------
+
+- :gitlab-issue:`605` Project title fluctuates between default value and new value [Video]
+- :gitlab-issue:`629` Cannot Upgrade to 3.2.X Docker Image
+- :gitlab-issue:`632` Tags get lost when uploading through the webui
+
+.. _PyPI: https://pypi.python.org/pypi/mayan-edms/

docs/releases/3.2.6.rst

@@ -0,0 +1,111 @@
+Version 3.2.6
+=============
+
+Released: July 10, 2019
+
+
+Changes
+-------
+
+- Remove the smart settings app * import. Following MERC 0005.
+- Encode settings YAML before hashing. Avoids unicode issues with Python 3.
+- Fix document icon used in the workflow runtime links.
+- Add trashed date time label.
+- Fix thumbnail generation issue. GitLab issue #637.
+  Thanks to Giacomo Cariello (@giacomocariello) for the report
+  and the merge request fixing the issue.
+
+Removals
+--------
+
+- None
+
+
+Upgrading from a previous version
+---------------------------------
+
+If installed via Python's PIP
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Remove deprecated requirements::
+
+    sudo -u mayan curl https://gitlab.com/mayan-edms/mayan-edms/raw/master/removals.txt -o /tmp/removals.txt && sudo -u mayan /opt/mayan-edms/bin/pip uninstall -y -r /tmp/removals.txt
+
+Type in the console::
+
+    sudo -u mayan /opt/mayan-edms/bin/pip install mayan-edms==3.2.6
+
+the requirements will also be updated automatically.
+
+
+Using Git
+^^^^^^^^^
+
+If you installed Mayan EDMS by cloning the Git repository issue the commands::
+
+    $ git reset --hard HEAD
+    $ git pull
+
+otherwise download the compressed archived and uncompress it overriding the
+existing installation.
+
+Remove deprecated requirements::
+
+    $ pip uninstall -y -r removals.txt
+
+Next upgrade/add the new requirements::
+
+    $ pip install --upgrade -r requirements.txt
+
+
+Common steps
+^^^^^^^^^^^^
+
+Perform these steps after updating the code from either step above.
+
+Make a backup of your supervisord file::
+
+    sudo cp /etc/supervisor/conf.d/mayan.conf /etc/supervisor/conf.d/mayan.conf.bck
+
+Update the supervisord configuration file. Replace the environment
+variables values show here with your respective settings. This step will refresh
+the supervisord configuration file with the new queues and the latest
+recommended layout::
+
+    sudo MAYAN_DATABASE_ENGINE=django.db.backends.postgresql MAYAN_DATABASE_NAME=mayan \
+    MAYAN_DATABASE_PASSWORD=mayanuserpass MAYAN_DATABASE_USER=mayan \
+    MAYAN_DATABASE_HOST=127.0.0.1 MAYAN_MEDIA_ROOT=/opt/mayan-edms/media \
+    /opt/mayan-edms/bin/mayan-edms.py platformtemplate supervisord > /etc/supervisor/conf.d/mayan.conf
+
+Edit the supervisord configuration file and update any setting the template
+generator missed::
+
+    sudo vi /etc/supervisor/conf.d/mayan.conf
+
+Migrate existing database schema with::
+
+    sudo -u mayan MAYAN_DATABASE_ENGINE=django.db.backends.postgresql MAYAN_DATABASE_NAME=mayan \
+    MAYAN_DATABASE_PASSWORD=mayanuserpass MAYAN_DATABASE_USER=mayan \
+    MAYAN_DATABASE_HOST=127.0.0.1 MAYAN_MEDIA_ROOT=/opt/mayan-edms/media \
+    /opt/mayan-edms/bin/mayan-edms.py performupgrade
+
+Add new static media::
+
+    sudo -u mayan MAYAN_MEDIA_ROOT=/opt/mayan-edms/media \
+    /opt/mayan-edms/bin/mayan-edms.py preparestatic --noinput
+
+The upgrade procedure is now complete.
+
+
+Backward incompatible changes
+-----------------------------
+
+- None
+
+
+Bugs fixed or issues closed
+---------------------------
+
+- :gitlab-issue:`637` Thumbnail generation bug
+
+.. _PyPI: https://pypi.python.org/pypi/mayan-edms/

docs/releases/3.2.7.rst

@@ -0,0 +1,145 @@
+Version 3.2.7
+=============
+
+Released: August 28, 2019
+
+
+Changes
+-------
+
+- Fix checkouts form bug. Fixes GitLab issue #654
+  Thanks to Lucius Schaerer (@lschaer1) for the report.
+- Disable pagination current page button
+  Current page button was clickable and would cause the
+  single page navigation to jump to the home view.
+- Remove redundant Celery queue declarations from the
+  file_metadata app.
+- Add internal_name field to workflow serializer.
+  Fixes workflow API creation view.
+- Fix document cabinet list API view. Thanks for forum user
+  "jere" for the report. Forum topic 1039.
+- Fix document template column field. GitLab issue #655.
+  Thanks to Christian Wiegand (@christianwgd) for the
+  report.
+- Increase mailing profile password field max length
+  from 48 to 128 characters. GitLab issue #657.
+  Thanks to sigsec (@sigsec) for the report.
+- Update the Docker entrypoint to update the ownership
+  of files when the UID of GUID are changed.
+  GitLab issue #650. Thanks to Fabian (@ruffy91)
+  for the report.
+- Update the Docker entrypoint to allow changing
+  the GID of the mayan user to existing values.
+  GitLab issue #652. Thanks to Fabian (@ruffy91)
+  for the report.
+- Rename the MAYAN_USER_GUID environment variable
+  to MAYAN_USER_GID.
+- Backport individual index rebuild support.
+- Add automatic adjustment of HTML body on navigation
+  bar changes. Closes GitLab issue #643. Thanks to
+  Light Templar (@LightTemplar) for the report.
+- Unify all line endings to be Linux style.
+- Make sure system alerts don't appear under
+  floating elements.
+
+Removals
+--------
+
+- None
+
+
+Upgrading from a previous version
+---------------------------------
+
+If installed via Python's PIP
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Remove deprecated requirements::
+
+    sudo -u mayan curl https://gitlab.com/mayan-edms/mayan-edms/raw/master/removals.txt -o /tmp/removals.txt && sudo -u mayan /opt/mayan-edms/bin/pip uninstall -y -r /tmp/removals.txt
+
+Type in the console::
+
+    sudo -u mayan /opt/mayan-edms/bin/pip install mayan-edms==3.2.7
+
+the requirements will also be updated automatically.
+
+
+Using Git
+^^^^^^^^^
+
+If you installed Mayan EDMS by cloning the Git repository issue the commands::
+
+    $ git reset --hard HEAD
+    $ git pull
+
+otherwise download the compressed archived and uncompress it overriding the
+existing installation.
+
+Remove deprecated requirements::
+
+    $ pip uninstall -y -r removals.txt
+
+Next upgrade/add the new requirements::
+
+    $ pip install --upgrade -r requirements.txt
+
+
+Common steps
+^^^^^^^^^^^^
+
+Perform these steps after updating the code from either step above.
+
+Make a backup of your supervisord file::
+
+    sudo cp /etc/supervisor/conf.d/mayan.conf /etc/supervisor/conf.d/mayan.conf.bck
+
+Update the supervisord configuration file. Replace the environment
+variables values show here with your respective settings. This step will refresh
+the supervisord configuration file with the new queues and the latest
+recommended layout::
+
+    sudo MAYAN_DATABASE_ENGINE=django.db.backends.postgresql MAYAN_DATABASE_NAME=mayan \
+    MAYAN_DATABASE_PASSWORD=mayanuserpass MAYAN_DATABASE_USER=mayan \
+    MAYAN_DATABASE_HOST=127.0.0.1 MAYAN_MEDIA_ROOT=/opt/mayan-edms/media \
+    /opt/mayan-edms/bin/mayan-edms.py platformtemplate supervisord > /etc/supervisor/conf.d/mayan.conf
+
+Edit the supervisord configuration file and update any setting the template
+generator missed::
+
+    sudo vi /etc/supervisor/conf.d/mayan.conf
+
+Migrate existing database schema with::
+
+    sudo -u mayan MAYAN_DATABASE_ENGINE=django.db.backends.postgresql MAYAN_DATABASE_NAME=mayan \
+    MAYAN_DATABASE_PASSWORD=mayanuserpass MAYAN_DATABASE_USER=mayan \
+    MAYAN_DATABASE_HOST=127.0.0.1 MAYAN_MEDIA_ROOT=/opt/mayan-edms/media \
+    /opt/mayan-edms/bin/mayan-edms.py performupgrade
+
+Add new static media::
+
+    sudo -u mayan MAYAN_MEDIA_ROOT=/opt/mayan-edms/media \
+    /opt/mayan-edms/bin/mayan-edms.py preparestatic --noinput
+
+The upgrade procedure is now complete.
+
+
+Backward incompatible changes
+-----------------------------
+
+- None
+
+
+Bugs fixed or issues closed
+---------------------------
+
+- :gitlab-issue:`643` The "Actions" button won't show up, if window size between 992 and 1248 px
+- :gitlab-issue:`650` Permission denied when MAYAN_USER_UID and MAYAN_USER_GUID are set
+- :gitlab-issue:`652` MAYAN_USER_GUID cannot be set to specific values
+- :gitlab-issue:`654` Internal Server Error, Document Checkout
+- :gitlab-issue:`655` Index setup tree view shows two times the "enabled" field instead of "Link documents"
+- :gitlab-issue:`657` Mailer password length restriction is too short
+- :forum-topic:`1039` Re: /api/documents/{id}/cabinets returns 500
+- :forum-topic:`1050` edit/delete sub-cabinet
+
+.. _PyPI: https://pypi.python.org/pypi/mayan-edms/

docs/releases/3.2.8.rst

@@ -0,0 +1,161 @@
+Version 3.2.8
+=============
+
+Released: October 1, 2019
+
+
+Changes
+-------
+
+
+API
+^^^
+
+Fix an error when accessing some API entry points without
+being authenticated. Accessing API endpoints without being authenticated
+will now always return empty results.
+
+
+Cabinets
+^^^^^^^^
+
+Tweaked the jstree component's appearance to cope with long labels.
+Added a scrollbar, reduced the font size, switched to a sans serif font,
+and reduced padding. Thanks for forum user @briboe for the report.
+
+Workflow actions to add and remove documents from cabinets was added.
+
+
+Dependencies
+^^^^^^^^^^^^
+
+The Django version used was updated to version 1.11.24. The jQuery version
+used was updated to version 3.4.1. Both as fully backwards compatible with
+their previous versions.
+
+
+OCR
+^^^
+
+Support was added to delete the content of document's OCR or parsed content.
+Events for both situations was added allowing content deletion to be used
+as workflow transition triggers.
+
+
+Docker
+^^^^^^
+
+A missing recursive option was added to the Docker entrypoint
+command "chown" to change the ownership of files when specifying a custom
+UID or GID. Closes GitLab issue #668. Thanks to John Wice (@brilthor)
+for the report.
+
+Two fonts were added to the Docker image to support rendering Chinese office
+documents. Closes GitLab issue #666. Thanks to javawcy (@javawcy) and forum
+user @leoliu for the report and help closing this issue.
+
+
+Usability
+^^^^^^^^^
+
+Descriptions for screenreaders was added via image alt tag. The user interface
+will also now allow scaling.
+
+
+Removals
+--------
+
+- None
+
+
+Upgrading from a previous version
+---------------------------------
+
+If installed via Python's PIP
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Remove deprecated requirements::
+
+    sudo -u mayan curl https://gitlab.com/mayan-edms/mayan-edms/raw/master/removals.txt -o /tmp/removals.txt && sudo -u mayan /opt/mayan-edms/bin/pip uninstall -y -r /tmp/removals.txt
+
+Type in the console::
+
+    sudo -u mayan /opt/mayan-edms/bin/pip install mayan-edms==3.2.8
+
+the requirements will also be updated automatically.
+
+
+Using Git
+^^^^^^^^^
+
+If you installed Mayan EDMS by cloning the Git repository issue the commands::
+
+    $ git reset --hard HEAD
+    $ git pull
+
+otherwise download the compressed archived and uncompress it overriding the
+existing installation.
+
+Remove deprecated requirements::
+
+    $ pip uninstall -y -r removals.txt
+
+Next upgrade/add the new requirements::
+
+    $ pip install --upgrade -r requirements.txt
+
+
+Common steps
+^^^^^^^^^^^^
+
+Perform these steps after updating the code from either step above.
+
+Make a backup of your supervisord file::
+
+    sudo cp /etc/supervisor/conf.d/mayan.conf /etc/supervisor/conf.d/mayan.conf.bck
+
+Update the supervisord configuration file. Replace the environment
+variables values show here with your respective settings. This step will refresh
+the supervisord configuration file with the new queues and the latest
+recommended layout::
+
+    sudo MAYAN_DATABASE_ENGINE=django.db.backends.postgresql MAYAN_DATABASE_NAME=mayan \
+    MAYAN_DATABASE_PASSWORD=mayanuserpass MAYAN_DATABASE_USER=mayan \
+    MAYAN_DATABASE_HOST=127.0.0.1 MAYAN_MEDIA_ROOT=/opt/mayan-edms/media \
+    /opt/mayan-edms/bin/mayan-edms.py platformtemplate supervisord > /etc/supervisor/conf.d/mayan.conf
+
+Edit the supervisord configuration file and update any setting the template
+generator missed::
+
+    sudo vi /etc/supervisor/conf.d/mayan.conf
+
+Migrate existing database schema with::
+
+    sudo -u mayan MAYAN_DATABASE_ENGINE=django.db.backends.postgresql MAYAN_DATABASE_NAME=mayan \
+    MAYAN_DATABASE_PASSWORD=mayanuserpass MAYAN_DATABASE_USER=mayan \
+    MAYAN_DATABASE_HOST=127.0.0.1 MAYAN_MEDIA_ROOT=/opt/mayan-edms/media \
+    /opt/mayan-edms/bin/mayan-edms.py performupgrade
+
+Add new static media::
+
+    sudo -u mayan MAYAN_MEDIA_ROOT=/opt/mayan-edms/media \
+    /opt/mayan-edms/bin/mayan-edms.py preparestatic --noinput
+
+The upgrade procedure is now complete.
+
+
+Backward incompatible changes
+-----------------------------
+
+- None
+
+
+Bugs fixed or issues closed
+---------------------------
+
+- :gitlab-issue:`666` Chinese document such as .doc can't display well.
+- :gitlab-issue:`668` Permission denied errors with custom uid persist (650 needs re-open)
+- :forum-topic:`1120` Cabinet Presentation
+- :forum-topic:`2202` Cannot display Chinese character and cannot identify Excel files
+
+.. _PyPI: https://pypi.python.org/pypi/mayan-edms/

docs/releases/3.3.rst

@@ -0,0 +1,213 @@
+Version 3.3
+===========
+
+Released: XX XX, 2019
+
+
+Changes
+-------
+
+- Add support for icon shadows.
+- Add icons and no-result template to the object error log view and
+  links.
+- Use Select2 widget for the document type selection form.
+- Backport the vertical main menu update. This update splits the previous
+  main menu into a new menu in the same location as the previous one
+  now called the top bar, and a new vertical main menu on the left side.
+  The vertical menu remain open even when clicking on items and upon
+  a browser refresh will also restore its state to match the selected
+  view.
+- Backport workflow preview refactor. GitLab issue #532.
+- Add support for source column inheritance.
+- Add support for source column exclusion.
+- Backport workflow context support.
+- Backport workflow transitions field support.
+- Backport workflow email action.
+- Backport individual index rebuild support.
+- Rename the installjavascript command to installdependencies.
+- Remove database conversion command.
+- Remove support for quoted configuration entries. Support unquoted,
+  nested dictionaries in the configuration. Requires manual
+  update of existing config.yml files.
+- Support user specified locations for the configuration file with the
+  CONFIGURATION_FILEPATH (MAYAN_CONFIGURATION_FILEPATH environment variable), and
+  CONFIGURATION_LAST_GOOD_FILEPATH
+  (MAYAN_CONFIGURATION_LAST_GOOD_FILEPATH environment variable) settings.
+- Move bootstrapped settings code to their own module in the smart_settings apps.
+- Remove individual database configuration options. All database configuration
+  is now done using MAYAN_DATABASES to mirror Django way of doing database setup.
+- Added support for YAML encoded environment variables to the platform
+  templates apps.
+- Move YAML code to its own module. Code now resides in common.serialization
+  in the form of two new functions: yaml_load and yaml_dump.
+- Move Django and Celery settings. Django settings now reside in the smart
+  settings app. Celery settings now reside in the task manager app.
+- Backport FakeStorageSubclass from versions/next. Placeholder class to allow
+  serializing the real storage subclass to support migrations.
+  Used by all configurable storages.
+- Support checking in and out multiple documents.
+- Remove encapsulate helper.
+- Add support for menu inheritance.
+- Emphasize source column labels.
+- Backport file cache manager app.
+- Convert document image cache to use file cache manager app.
+  Add setting DOCUMENTS_CACHE_MAXIMUM_SIZE defaults to 500 MB.
+- Update Celery to version 4.3.0. Settings changed:
+  MAYAN_BROKER_URL to MAYAN_CELERY_BROKER_URL,
+  MAYAN_CELERY_ALWAYS_EAGER to MAYAN_CELERY_TASK_ALWAYS_EAGER.
+- Replace djcelery and replace it with django-celery-beat.
+- Update Celery to version 4.3.0 with 55e9b2263cbdb9b449361412fd18d8ee0a442dd3
+  from versions/next, code from GitLab issue #594 and GitLab merge request !55.
+  Thanks to Jakob Haufe (@sur5r) and Jesaja Everling (@jeverling)
+  for much of the research and code updates.
+- Support wildcard MIME type associations for the file metadata drivers.
+- Rename MAYAN_GUID to MAYAN_GID
+- Update Gunicorn to use sync workers.
+- Include devpi-server as a development dependency.
+- Update default Docker stack file.
+- Remove Redis from the Docker image.
+- Add Celery flower to the Docker image.
+- Allow PIP proxying to the Docker image during build.
+- Default Celery worker concurrency to 0 (auto).
+- Set DJANGO_SETTINGS_MODULE environment variable to make it
+  available to sub processes.
+- Add entrypoint commands to run single workers, single gunicorn
+  or single celery commands like "flower".
+- Add platform template to return queues for a worker.
+- Remove task inspection from task manager app.
+- Move pagination navigation inside the toolbar.
+- Remove document image clear link and view.
+  This is now handled by the file caching app.
+- Add web links app.
+- Add support to display column help text
+  as a tooltip.
+- Update numeric dashboard widget to display
+  thousand commas.
+- Add support for disabling document pages.
+- Add support for converter layers.
+- Add redactions app.
+- Unify all line endings to be Linux style.
+- Add support for changing the system messages position.
+  GitLab issue #640. Thanks to Matthias Urhahn (@d4rken).
+
+Removals
+--------
+
+- Database conversion. Reason for removal: The database conversions support
+  provided by this feature (SQLite to PostgreSQL) was being confused with
+  database migrations and upgrades.
+
+  Database upgrades are the responsibility of the app and the framework.
+  Database conversions however are not the responsibility of the app (Mayan),
+  they are the responsibility of the framework.
+
+  Database conversion is outside the scope of what Mayan does but we added
+  the code, management command, instructions and testing setup to provide
+  this to our users until the framework (Django) decided to add this
+  themselves (like they did with migrations).
+
+  Continued confusion about the purpose of the feature and confusion about
+  how errors with this feature were a reflexion of the code quality of
+  Mayan necessitated the removal of the database conversion feature.
+
+- Django environ
+
+
+Upgrading from a previous version
+---------------------------------
+
+If installed via Python's PIP
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Remove deprecated requirements::
+
+    sudo -u mayan curl https://gitlab.com/mayan-edms/mayan-edms/raw/master/removals.txt -o /tmp/removals.txt && sudo -u mayan /opt/mayan-edms/bin/pip uninstall -y -r /tmp/removals.txt
+
+Type in the console::
+
+    /opt/mayan-edms/bin/pip install mayan-edms==3.3
+
+the requirements will also be updated automatically.
+
+
+Using Git
+^^^^^^^^^
+
+If you installed Mayan EDMS by cloning the Git repository issue the commands::
+
+    git reset --hard HEAD
+    git pull
+
+otherwise download the compressed archived and uncompress it overriding the
+existing installation.
+
+Remove deprecated requirements::
+
+    pip uninstall -y -r removals.txt
+
+Next upgrade/add the new requirements::
+
+    pip install --upgrade -r requirements.txt
+
+
+Common steps
+^^^^^^^^^^^^
+
+Perform these steps after updating the code from either step above.
+
+Make a backup of your supervisord file::
+
+    sudo cp /etc/supervisor/conf.d/mayan.conf /etc/supervisor/conf.d/mayan.conf.bck
+
+Update the supervisord configuration file. Replace the environment
+variables values show here with your respective settings. This step will refresh
+the supervisord configuration file with the new queues and the latest
+recommended layout::
+
+    sudo MAYAN_DATABASES="{'default':{'ENGINE':'django.db.backends.postgresql','NAME':'mayan','PASSWORD':'mayanuserpass','USER':'mayan','HOST':'127.0.0.1'}}" \
+    MAYAN_MEDIA_ROOT=/opt/mayan-edms/media \
+    /opt/mayan-edms/bin/mayan-edms.py platformtemplate supervisord > /etc/supervisor/conf.d/mayan.conf
+
+Edit the supervisord configuration file and update any setting the template
+generator missed::
+
+    sudo vi /etc/supervisor/conf.d/mayan.conf
+
+Migrate existing database schema with::
+
+    sudo -u mayan MAYAN_MEDIA_ROOT=/opt/mayan-edms/media /opt/mayan-edms/bin/mayan-edms.py performupgrade
+
+Add new static media::
+
+    sudo -u mayan MAYAN_MEDIA_ROOT=/opt/mayan-edms/media /opt/mayan-edms/bin/mayan-edms.py preparestatic --noinput
+
+The upgrade procedure is now complete.
+
+
+Backward incompatible changes
+-----------------------------
+
+- Update quoted settings to be unquoted:
+
+  - COMMON_SHARED_STORAGE_ARGUMENTS
+  - CONVERTER_GRAPHICS_BACKEND_ARGUMENTS
+  - DOCUMENTS_CACHE_STORAGE_BACKEND_ARGUMENTS
+  - DOCUMENTS_STORAGE_BACKEND_ARGUMENTS
+  - FILE_METADATA_DRIVERS_ARGUMENTS
+  - SIGNATURES_STORAGE_BACKEND_ARGUMENTS
+
+
+Bugs fixed or issues closed
+---------------------------
+
+- :gitlab-issue:`526` RuntimeWarning: Never call result.get() within a task!
+- :gitlab-issue:`532` Workflow preview isn't updated right after transitions are modified
+- :gitlab-issue:`540` hint-outdated/update documentation
+- :gitlab-issue:`594` 3.2b1: Unable to install/run under Python 3.5/3.6/3.7
+- :gitlab-issue:`634` Failing docker entrypoint when using secret config
+- :gitlab-issue:`635` Build a docker image for Python3
+- :gitlab-issue:`640` UX: "Toast" Popup position prevents access to actions
+- :gitlab-issue:`644` Update sane-utils package in docker image.
+
+
+.. _PyPI: https://pypi.python.org/pypi/mayan-edms/

docs/releases/index.rst

@@ -20,6 +20,18 @@ versions of the documentation contain the release notes for any later releases.
 .. toctree::
    :maxdepth: 1
 
+   3.3
+   3.2.8
+   3.2.7
+   3.2.6
+   3.2.5
+   3.2.4
+   3.2.3
+   3.2.2
+   3.2.1
+   3.2
+   3.1.11
+   3.1.10
    3.1.9
    3.1.8
    3.1.7

docs/spelling_wordlist.txt

@@ -1,3 +1,5 @@
+Archlinux
+archlinux
 adf
 ajax
 ajaxForm
@@ -141,6 +143,7 @@ storages
 Storages
 subclasses
 subclassing
+subfolder
 sublicense
 sublicensees
 swedish
@@ -173,3 +176,15 @@ YaN
 yasg
 YbN
 YrcN
+metacopy
+overlayfs
+sys
+supervisord
+Monkeypatch
+Bootswatch
+sortable
+Odroid
+callables
+ModelPermission
+mixins
+kwargs

docs/topics/adding_documents.rst

@@ -0,0 +1,5 @@
+################
+Adding documents
+################
+
+.. include:: ../chapters/sources.rst

docs/topics/advanced.rst

@@ -8,3 +8,4 @@ Advanced topics
 .. include:: ../chapters/metadata.rst
 .. include:: ../chapters/transformations.rst
 .. include:: ../chapters/versioning.rst
+.. include:: ../chapters/quick_labels.rst

docs/topics/categorization.rst

@@ -7,10 +7,12 @@ allows showing users different views of document groups. These are divided
 into ones requiring manual maintenance and others that are updated
 automatically.
 
+
 **************
 Manual methods
 **************
 
+.. include:: ../chapters/document_types.rst
 .. include:: ../chapters/cabinets.rst
 .. include:: ../chapters/tags.rst
 

docs/topics/docker.rst

@@ -0,0 +1,8 @@
+######
+Docker
+######
+
+Docker is a container technology. Containers are a standard unit of software
+that packages up code and all its dependencies.
+
+.. include:: ../chapters/docker.rst

docs/topics/faq.rst

@@ -6,6 +6,88 @@ FAQ
 Non technical
 *************
 
+Contact
+=======
+
+Can I contact members of the development team?
+----------------------------------------------
+
+Direct access to members of the development team is reserved for clients
+of a support or consultation contract.
+
+
+I can't pay for support/consultation
+------------------------------------
+
+A community forum is available at https://forum.mayan-edms.com. Members of
+the development team visit the forum on occasions and might answer some
+questions on a voluntary basis.
+
+
+I found a bug
+-------------
+
+For bugs, open an issue at https://gitlab.com/mayan-edms/mayan-edms/issues.
+
+
+I found a security issue
+------------------------
+
+Open an issue at https://gitlab.com/mayan-edms/mayan-edms/issues and mark it as
+Confidential. Allow us at least 48 hours to find and release a fix
+for the issue before submitting it to the CVE database.
+
+
+I want to join the development team
+-----------------------------------
+
+Perfect! We need all the help we can get. Tell us where we can see what you've
+done using Mayan EDMS. It could be a fork, a new UI, an API client, a custom app,
+or anything else that you think will be able to evidence your dominion over the
+codebase.
+
+
+I submitted an issue and it has not been fixed
+----------------------------------------------
+
+There are many reason an issue might not get resolved and remain open for an
+extended period of time. We document on the issue itself the progress and when
+resolved will reference what commit fixed it and the version or approximate time
+for the code to make it into a production release. Some reason why issues remain
+open with no resolution are:
+
+- It was not an issue with the code. It happens that sometimes the problem lies
+  with the operating system, the filesystem being used, the platform or packaging
+  method.
+- It was not an issue but a question or a comment. The issue system is for reporting
+  problems with the code. That said, we sometimes answer questions submitted as
+  issues. For questions or discussions use the community forum.
+- We are not able to reproduce the issue. We need to be able to recreate the
+  conditions that trigger the issue so that we can pinpoint the cause. After that
+  we create a test to make sure the issue is really fixed. Some issues can
+  be triggered manually easily but are hard to trigger programatically.
+- The issue is not clearly explained. Issue descriptions like: "It doesn't work",
+  or "It showed an error but I forgot to write it down" are almost impossible to
+  resolve.
+- We understand the cause of the problem but don't yet a solution to implement.
+  We have studied the issue and have been able to reproduce it, but have not
+  reached a consensus on how it should be fixed. Could be that the solution is
+  beyond the collective expertise of the development team or that a design
+  decision of big impact is needed before code changes can be implemented.
+- The issue is no real. The issue might be for an obsolete version. It could be
+  for a fork or a program that is not developed by us. Sometimes they are just
+  fake issues.
+
+
+I submitted a merge request and has not been merged
+---------------------------------------------------
+
+Merge requests must follow the development standards of the code as close a
+possible. They must also be atomic and as small as possible. The code must also
+not change the behavior of the project. We can't accept merge request that
+customize it in some way based on your own preferences or needs.
+
+
 Distribution
 ============
 
@@ -44,6 +126,7 @@ Restricting distribution or sale would conflict with the license terms. This
 would possible for a commercial version of Mayan EDMS with separate licensing
 terms.
 
+
 Is there a commercial partnership program?
 ------------------------------------------
 
@@ -83,54 +166,6 @@ moment this could cause problems is when running South migrations during
 upgrades, if a migration fails the database structure is left in a transitory
 state and has to be reverted manually before trying again.
 
-_mysql_exceptions. OperationalError: (1267, "Illegal mix of collations (latin1_swedish_ci, IMPLICIT) and (utf8_general_ci, COERCIBLE) for operation ‘='”)
----------------------------------------------------------------------------------------------------------------------------------------------------------
-
-::
-
-    $ mayan-edms.py shell
-    >>> from django.db import connection
-    >>> cursor = connection.cursor()
-    >>> cursor.execute('SHOW TABLES')
-    >>> results=[]
-    >>> for row in cursor.fetchall(): results.append(row)
-    >>> for row in results: cursor.execute('ALTER TABLE %s CONVERT TO CHARACTER SET utf8 COLLATE utf8_general_ci;' % (row[0]))
-
-References:
-
-* http://stackoverflow.com/questions/1073295/django-character-set-with-mysql-weirdness
-
-
-Incorrect string value: ``'xE2x80x95rs6…'`` for column ``'content'`` at row 1
------------------------------------------------------------------------------
-
-When using MySQL and doing OCR on languages other than English
-
-Use utf-8 collation on MySQL server, or at least in table
-‘documents_documentpage', ‘content' field
-
-References:
-
-* http://groups.google.com/group/django-users/browse_thread/thread/429447086fca6412
-* http://markmail.org/message/bqajx2utvmtriixi
-
-Error "django.db.utils.IntegrityError IntegrityError: (1452, ‘Cannot add or update a child row: a foreign key constraint fails (`…`.`…`, CONSTRAINT `…_refs_id_b0252274` FOREIGN KEY (`…`) REFERENCES `…` (`…`))')
-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
-
-Solution:
-Convert all MySQL tables to the same type, either all MyISAM or InnoDB
-
-
-Document versions
-=================
-
-How do you upload a new version of an existing file?
-----------------------------------------------------
-
-Choose a document, and go to the versions tab, on the right menu at the bottom
-under Other available action there is Upload new version. Clicking it will
-take you to a very similar view as the Upload new document but you will be
-able to specify version number and comments for the new version being uploaded.
 
 LDAP
 ====
@@ -142,6 +177,7 @@ A sample settings file called ldap_connection_settings.py is included in the
 contrib/settings/ folder of the repository showing how to setup LDAP
 authentication.
 
+
 Operating systems
 =================
 
@@ -185,20 +221,24 @@ Django's development server doesn't serve static files unless the DEBUG option
 is set to True, this mode of operation should only be used for development or
 testing. For production deployments the management command::
 
-    $ mayan-edms.py collectstatic
+    $ mayan-edms.py preparestatic
 
 should be used and the resulting static folder served from a webserver.
 For more information check the
 :django-docs:`howto/static-files/`
 
-Watchfolders
-============
+
+Watch folders
+=============
 
 The watched folder feature is not working
 -----------------------------------------
 
 Make sure that the Celery BEAT scheduler is running correctly as it is the
-element that triggers the periodics tasks.
+element that triggers the periodic tasks. Check that the user running the Mayan
+EDMS services has read and write permissions for the watch folder.
+
+
 
 Other
 =====
@@ -225,8 +265,3 @@ Example::
 
 Reference:
 * http://www.samba.org/samba/docs/man/manpages-3/smb.conf.5.html
-
-Can you change the display order of documents…i.e can they be in alphabetical order?
-------------------------------------------------------------------------------------
-
-A the moment no, but it is something being worked on.

docs/topics/features.rst

@@ -92,7 +92,7 @@ Features
   * It is very easy to use 3rd party plugins such as the ones available for
     Amazon EC2.
 
-* Color coded <../chapters/tags>`.
+* :doc:`Color coded tags <../chapters/tags>`.
 
   * Labeled and color coded tags can be assigned for intuitive recognition.
 

docs/topics/getting_started.rst

@@ -4,6 +4,19 @@ Getting started
 
 These are the basic concepts you need to understand to start using Mayan EDMS.
 
+The absolute minimum setup to start using Mayan EDMS is:
+
+#. A document type
+#. A document source to add new documents
+
+that's it!
+
+To make things even easier, a default document type (named "Default") and a
+default document source (also named "Default") were created for you during the
+installation. So you don't need to do anything more to start adding documents
+right now, however take your time reading the information below to get a deeper
+understanding about how these concepts work.
+
 .. include:: ../chapters/document_types.rst
 .. include:: ../chapters/sources.rst
 

docs/topics/installation.rst

@@ -12,7 +12,7 @@ Minimum hardware requirements
 
 - 2 Gigabytes of RAM (1 Gigabyte if OCR is turned off).
 - Multiple core CPU (64 bit, faster than 1 GHz recommended).
-- Unix-like operating system like Linux and OpenBSD. For other operating systems
+- Unix-like operating system like GNU/Linux. For other operating systems
   user container technologies like Docker or virtual machines.
 
 ****************

docs/topics/troubleshooting.rst

@@ -0,0 +1,183 @@
+###############
+Troubleshooting
+###############
+
+
+********
+Database
+********
+
+MySQL error: ``OperationalError: (1267, "Illegal mix of collations (latin1_swedish_ci, IMPLICIT) and (utf8_general_ci, COERCIBLE) for operation '='”)``
+=======================================================================================================================================================
+
+::
+
+    $ mayan-edms.py shell
+    >>> from django.db import connection
+    >>> cursor = connection.cursor()
+    >>> cursor.execute('SHOW TABLES')
+    >>> results=[]
+    >>> for row in cursor.fetchall(): results.append(row)
+    >>> for row in results: cursor.execute('ALTER TABLE %s CONVERT TO CHARACTER SET utf8 COLLATE utf8_general_ci;' % (row[0]))
+
+References:
+
+* http://stackoverflow.com/questions/1073295/django-character-set-with-mysql-weirdness
+
+
+MySQL error: ``Incorrect string value: `'xE2x80x95rs6…'` for column `'content'` at row 1``
+==========================================================================================
+
+When using MySQL and doing OCR on languages other than English
+
+Use utf-8 collation on MySQL server, or at least in table
+‘documents_documentpage', ‘content' field
+
+References:
+
+* http://groups.google.com/group/django-users/browse_thread/thread/429447086fca6412
+* http://markmail.org/message/bqajx2utvmtriixi
+
+
+MySQL error: ``Error "django.db.utils.IntegrityError IntegrityError: (1452, 'Cannot add or update a child row: a foreign key constraint fails (`…`.`…`, CONSTRAINT `…_refs_id_b0252274` FOREIGN KEY (`…`) REFERENCES `…` (`…`))')``
+===================================================================================================================================================================================================================================
+
+Solution:
+Convert all MySQL tables to the same type, either all MyISAM or InnoDB
+
+
+PostgreSQL error: ``OperationalError: FATAL:  sorry, too many clients already``
+===============================================================================
+
+Set ``MAYAN_DATABASE_CONN_MAX_AGE`` to 0
+
+This setting keeps a database connection alive. It allows reuse of database
+connections. When Mayan EDMS is deployed with Gunicorn a microthreads backend,
+the database connections are not shared and this setting has the reverse effect
+of exhausting the available PostgreSQL connections available. To avoid this,
+Setting ``MAYAN_DATABASE_CONN_MAX_AGE`` to 0 will cause all microthreads to
+release their connections, by closing them when finished.
+
+References:
+
+- https://serverfault.com/questions/635100/django-conn-max-age-persists-connections-but-doesnt-reuse-them-with-postgresq
+- https://github.com/benoitc/gunicorn/issues/996
+
+
+******
+Docker
+******
+
+MAYAN_APT_INSTALLS does not work for Archlinux with kernels > 4.14
+==================================================================
+
+This is caused by a change from kernel 4.18 - 4.19. Metacopy on these kernels
+is set to yes in archlinux kernels (/sys/module/overlay/parameters/metacopy)
+and overlayfs should override this which it does not at the moment.
+
+The workaround is to disable metacopy::
+
+    echo N | sudo tee /sys/module/overlay/parameters/metacopy
+
+References:
+
+* https://bbs.archlinux.org/viewtopic.php?id=241866
+* https://www.spinics.net/lists/linux-unionfs/msg06316.html
+
+
+
+*********
+Passwords
+*********
+
+.. _troubleshooting-admin-password:
+
+Admin password reset
+====================
+
+To reset the password of the admin account use the following command::
+
+    MAYAN_MEDIA_ROOT=<your Mayan media root setting> <installation directory>/bin/mayan-edms.py changepassword admin
+
+If you followed the deploying instructions from the documentation your
+``MAYAN_MEDIA_ROOT`` will be ``/opt/mayan-edms/media``.
+
+If using a Docker image, execute the command inside the container. First you
+need to know the name of the Docker container running Mayan EDMS on your setup
+with::
+
+    docker ps
+
+Then execute the password reset command inside the Docker container::
+
+    docker exec -ti <your docker container name> /opt/mayan-edms/bin/mayan-edms.py changepassword admin
+
+Another way to do this is to execute a shell inside the container to get a
+command prompt::
+
+    docker exec -ti <your docker container name> /bin/bash
+
+And then execute the command::
+
+    /opt/mayan-edms/bin/mayan-edms.py changepassword admin
+
+
+.. _troubleshooting-autoadmin-account:
+
+Missing automatic admin account after installation
+==================================================
+
+This is caused when the ``initialsetup`` command is interrupted as the admin
+user is created outside of the database migrations.
+
+To create an admin super user account manually use the command::
+
+    MAYAN_MEDIA_ROOT=<your Mayan media root setting> <installation directory>/bin/mayan-edms.py createsuperuser
+
+If you followed the deploying instructions from the documentation your
+``MAYAN_MEDIA_ROOT`` will be ``/opt/mayan-edms/media``.
+
+If using a Docker image, execute the command inside the container. First
+find you container name with::
+
+    docker ps
+
+Then execute the command inside the container::
+
+    docker exec -ti <your docker container name> /opt/mayan-edms/bin/mayan-edms.py createsuperuser
+
+Another way to do this is to execute a shell inside the container to get a
+command prompt::
+
+    docker exec -ti <your docker container name> /bin/bash
+
+And then execute the command::
+
+    /opt/mayan-edms/bin/mayan-edms.py createsuperuser
+
+
+************
+Watchfolders
+************
+
+Incomplete files uploaded
+=========================
+
+To avoid uploading files are they are being copied to the watchfolder, copy the
+files to a temporary directory on the same partition as the watchfolder first.
+Then move the files to the watchfolder. The move will be executed as an atomic
+operation and will prevent the files to be uploaded in the middle of the
+copying process.
+
+************
+Dependencies
+************
+
+Error: ``unable to execute 'x86_64-linux-gnu-gcc': No such file or directory``
+==============================================================================
+
+This happens when using the ``MAYAN_APT_INSTALLS`` feature. It means that the
+``GCC`` package is required to compile the packages specified with
+``MAYAN_APT_INSTALLS``.
+
+Solution: Include ``gcc`` in the list of packages specified with ``MAYAN_APT_INSTALLS``.

mayan/LICENSE

@@ -1,4 +1,4 @@
-   Copyright 2011-2018 Roberto Rosario
+   Copyright 2011 Roberto Rosario
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

mayan/__init__.py

@@ -1,14 +1,14 @@
 from __future__ import unicode_literals
 
 __title__ = 'Mayan EDMS'
-__version__ = '3.1.9'
-__build__ = 0x030109
-__build_string__ = 'v3.1.9_Thu Nov 1 03:55:06 2018 -0400'
+__version__ = '3.3beta1'
+__build__ = 0x030300
+__build_string__ = 'v3.3beta1-9-g1b327b99f0_Tue Oct 8 15:15:08 2019 -0400'
 __django_version__ = '1.11'
 __author__ = 'Roberto Rosario'
 __author_email__ = 'roberto.rosario@mayan-edms.com'
 __description__ = 'Free Open Source Electronic Document Management System'
 __license__ = 'Apache 2.0'
-__copyright_short__ = '2011-2018 Roberto Rosario'
+__copyright_short__ = '2011 Roberto Rosario'
 __copyright__ = '{} {}'.format('Copyright', __copyright_short__)
 __website__ = 'https://www.mayan-edms.com'

mayan/apps/acls/__init__.py

@@ -1,5 +1,3 @@
 from __future__ import unicode_literals
 
-from .classes import ModelPermission  # NOQA
-
-default_app_config = 'acls.apps.ACLsApp'
+default_app_config = 'mayan.apps.acls.apps.ACLsApp'

mayan/apps/acls/api_views.py

@@ -21,12 +21,12 @@ class APIObjectACLListView(generics.ListCreateAPIView):
     """
     def get_content_object(self):
         content_type = get_object_or_404(
-            ContentType, app_label=self.kwargs['app_label'],
+            klass=ContentType, app_label=self.kwargs['app_label'],
             model=self.kwargs['model']
         )
 
         content_object = get_object_or_404(
-            content_type.model_class(), pk=self.kwargs['object_pk']
+            klass=content_type.model_class(), pk=self.kwargs['object_id']
         )
 
         if self.request.method == 'GET':
@@ -35,8 +35,8 @@ class APIObjectACLListView(generics.ListCreateAPIView):
             permission_required = permission_acl_edit
 
         AccessControlList.objects.check_access(
-            permissions=permission_required, user=self.request.user,
-            obj=content_object
+            obj=content_object, permissions=(permission_required,),
+            user=self.request.user
         )
 
         return content_object
@@ -62,7 +62,9 @@ class APIObjectACLListView(generics.ListCreateAPIView):
         if not self.request:
             return None
 
-        return super(APIObjectACLListView, self).get_serializer(*args, **kwargs)
+        return super(APIObjectACLListView, self).get_serializer(
+            *args, **kwargs
+        )
 
     def get_serializer_class(self):
         if self.request.method == 'GET':
@@ -85,17 +87,17 @@ class APIObjectACLView(generics.RetrieveDestroyAPIView):
             permission_required = permission_acl_edit
 
         content_type = get_object_or_404(
-            ContentType, app_label=self.kwargs['app_label'],
+            klass=ContentType, app_label=self.kwargs['app_label'],
             model=self.kwargs['model']
         )
 
         content_object = get_object_or_404(
-            content_type.model_class(), pk=self.kwargs['object_pk']
+            klass=content_type.model_class(), pk=self.kwargs['object_id']
         )
 
         AccessControlList.objects.check_access(
-            permissions=permission_required, user=self.request.user,
-            obj=content_object
+            obj=content_object, permissions=(permission_required,),
+            user=self.request.user
         )
 
         return content_object
@@ -111,22 +113,27 @@ class APIObjectACLPermissionListView(generics.ListCreateAPIView):
     """
     def get_acl(self):
         return get_object_or_404(
-            self.get_content_object().acls, pk=self.kwargs['pk']
+            klass=self.get_content_object().acls, pk=self.kwargs['pk']
         )
 
     def get_content_object(self):
         content_type = get_object_or_404(
-            ContentType, app_label=self.kwargs['app_label'],
+            klass=ContentType, app_label=self.kwargs['app_label'],
             model=self.kwargs['model']
         )
 
         content_object = get_object_or_404(
-            content_type.model_class(), pk=self.kwargs['object_pk']
+            klass=content_type.model_class(), pk=self.kwargs['object_id']
         )
 
+        if self.request.method == 'GET':
+            permission = permission_acl_view
+        else:
+            permission = permission_acl_edit
+
         AccessControlList.objects.check_access(
-            permissions=permission_acl_view, user=self.request.user,
-            obj=content_object
+            obj=content_object, permissions=(permission,),
+            user=self.request.user
         )
 
         return content_object
@@ -147,7 +154,9 @@ class APIObjectACLPermissionListView(generics.ListCreateAPIView):
             return WritableAccessControlListPermissionSerializer
 
     def get_serializer_context(self):
-        context = super(APIObjectACLPermissionListView, self).get_serializer_context()
+        context = super(
+            APIObjectACLPermissionListView, self
+        ).get_serializer_context()
         if self.kwargs:
             context.update(
                 {
@@ -168,22 +177,27 @@ class APIObjectACLPermissionView(generics.RetrieveDestroyAPIView):
 
     def get_acl(self):
         return get_object_or_404(
-            self.get_content_object().acls, pk=self.kwargs['pk']
+            klass=self.get_content_object().acls, pk=self.kwargs['pk']
         )
 
     def get_content_object(self):
         content_type = get_object_or_404(
-            ContentType, app_label=self.kwargs['app_label'],
+            klass=ContentType, app_label=self.kwargs['app_label'],
             model=self.kwargs['model']
         )
 
         content_object = get_object_or_404(
-            content_type.model_class(), pk=self.kwargs['object_pk']
+            klass=content_type.model_class(), pk=self.kwargs['object_id']
         )
 
+        if self.request.method == 'GET':
+            permission = permission_acl_view
+        else:
+            permission = permission_acl_edit
+
         AccessControlList.objects.check_access(
-            permissions=permission_acl_view, user=self.request.user,
-            obj=content_object
+            obj=content_object, permissions=(permission,),
+            user=self.request.user
         )
 
         return content_object
@@ -192,7 +206,9 @@ class APIObjectACLPermissionView(generics.RetrieveDestroyAPIView):
         return self.get_acl().permissions.all()
 
     def get_serializer_context(self):
-        context = super(APIObjectACLPermissionView, self).get_serializer_context()
+        context = super(
+            APIObjectACLPermissionView, self
+        ).get_serializer_context()
         if self.kwargs:
             context.update(
                 {

mayan/apps/acls/apps.py

@@ -2,35 +2,56 @@ from __future__ import unicode_literals
 
 from django.utils.translation import ugettext_lazy as _
 
-from common import MayanAppConfig, menu_object, menu_sidebar
-from navigation import SourceColumn
+from mayan.apps.common.apps import MayanAppConfig
+from mayan.apps.common.menus import menu_object, menu_secondary
+from mayan.apps.events.classes import ModelEventType
+from mayan.apps.events.links import (
+    link_events_for_object, link_object_event_types_user_subcriptions_list
+)
+from mayan.apps.navigation.classes import SourceColumn
 
+from .classes import ModelPermission
+from .events import event_acl_created, event_acl_edited
 from .links import link_acl_create, link_acl_delete, link_acl_permissions
 
 
 class ACLsApp(MayanAppConfig):
+    app_namespace = 'acls'
+    app_url = 'acls'
     has_rest_api = True
     has_tests = True
-    name = 'acls'
+    name = 'mayan.apps.acls'
     verbose_name = _('ACLs')
 
     def ready(self):
         super(ACLsApp, self).ready()
+        from actstream import registry
 
-        AccessControlList = self.get_model('AccessControlList')
+        AccessControlList = self.get_model(model_name='AccessControlList')
 
-        SourceColumn(
-            source=AccessControlList, label=_('Role'), attribute='role'
+        ModelEventType.register(
+            event_types=(event_acl_created, event_acl_edited),
+            model=AccessControlList
         )
+
+        ModelPermission.register_inheritance(
+            model=AccessControlList, related='content_object',
+        )
+
         SourceColumn(
-            source=AccessControlList, label=_('Permissions'),
-            attribute='get_permission_titles'
+            attribute='role', is_sortable=True, source=AccessControlList,
         )
 
         menu_object.bind_links(
-            links=(link_acl_permissions, link_acl_delete),
+            links=(
+                link_acl_permissions, link_acl_delete,
+                link_events_for_object,
+                link_object_event_types_user_subcriptions_list
+            ),
             sources=(AccessControlList,)
         )
-        menu_sidebar.bind_links(
+        menu_secondary.bind_links(
             links=(link_acl_create,), sources=('acls:acl_list',)
         )
+
+        registry.register(AccessControlList)

mayan/apps/acls/classes.py

@@ -1,30 +1,25 @@
 from __future__ import unicode_literals, absolute_import
 
+import itertools
 import logging
 
 from django.apps import apps
+from django.utils.encoding import force_text
 
 logger = logging.getLogger(__name__)
 
 
 class ModelPermission(object):
-    _registry = {}
-    _proxies = {}
+    _functions = {}
     _inheritances = {}
+    _manager_names = {}
+    _registry = {}
 
     @classmethod
-    def register(cls, model, permissions):
-        from django.contrib.contenttypes.fields import GenericRelation
-
-        cls._registry.setdefault(model, [])
-        for permission in permissions:
-            cls._registry[model].append(permission)
-
-        AccessControlList = apps.get_model(
-            app_label='acls', model_name='AccessControlList'
-        )
-
-        model.add_to_class('acls', GenericRelation(AccessControlList))
+    def deregister(cls, model):
+        cls._registry.pop(model, None)
+        # TODO: Find method to revert the add_to_class('acls'...)
+        # delattr doesn't work.
 
     @classmethod
     def get_classes(cls, as_content_type=False):
@@ -45,8 +40,21 @@ class ModelPermission(object):
             return cls._registry.keys()
 
     @classmethod
-    def get_for_class(cls, klass):
-        return cls._registry.get(klass, ())
+    def get_for_class(cls, klass, as_choices=False):
+        if as_choices:
+            results = []
+
+            for namespace, permissions in itertools.groupby(cls.get_for_class(klass=klass, as_choices=False), lambda entry: entry.namespace):
+                permission_options = [
+                    (force_text(permission.pk), permission) for permission in permissions
+                ]
+                results.append(
+                    (namespace, permission_options)
+                )
+
+            return results
+        else:
+            return cls._registry.get(klass, ())
 
     @classmethod
     def get_for_instance(cls, instance):
@@ -61,24 +69,61 @@ class ModelPermission(object):
         if class_permissions:
             permissions.extend(class_permissions)
 
-        proxy = cls._proxies.get(type(instance))
-
-        if proxy:
-            permissions.extend(cls._registry.get(proxy))
-
         pks = [
             permission.stored_permission.pk for permission in set(permissions)
         ]
         return StoredPermission.objects.filter(pk__in=pks)
 
     @classmethod
-    def register_proxy(cls, source, model):
-        cls._proxies[model] = source
+    def get_function(cls, model):
+        return cls._functions[model]
+
+    @classmethod
+    def get_inheritance(cls, model):
+        return cls._inheritances[model]
+
+    @classmethod
+    def get_manager(cls, model):
+        try:
+            manager_name = cls.get_manager_name(model=model)
+        except KeyError:
+            manager_name = None
+
+        if manager_name:
+            manager = getattr(model, manager_name)
+        else:
+            manager = model._meta.default_manager
+
+        return manager
+
+    @classmethod
+    def get_manager_name(cls, model):
+        return cls._manager_names[model]
+
+    @classmethod
+    def register(cls, model, permissions):
+        from django.contrib.contenttypes.fields import GenericRelation
+
+        cls._registry.setdefault(model, [])
+        for permission in permissions:
+            cls._registry[model].append(permission)
+
+        AccessControlList = apps.get_model(
+            app_label='acls', model_name='AccessControlList'
+        )
+
+        model.add_to_class(
+            name='acls', value=GenericRelation(AccessControlList)
+        )
+
+    @classmethod
+    def register_function(cls, model, function):
+        cls._functions[model] = function
 
     @classmethod
     def register_inheritance(cls, model, related):
         cls._inheritances[model] = related
 
     @classmethod
-    def get_inheritance(cls, model):
-        return cls._inheritances[model]
+    def register_manager(cls, model, manager_name):
+        cls._manager_names[model] = manager_name

mayan/apps/acls/events.py

@@ -0,0 +1,16 @@
+from __future__ import absolute_import, unicode_literals
+
+from django.utils.translation import ugettext_lazy as _
+
+from mayan.apps.events.classes import EventTypeNamespace
+
+namespace = EventTypeNamespace(
+    label=_('Access control lists'), name='acls'
+)
+
+event_acl_created = namespace.add_event_type(
+    label=_('ACL created'), name='acl_created'
+)
+event_acl_edited = namespace.add_event_type(
+    label=_('ACL edited'), name='acl_edited'
+)

mayan/apps/acls/forms.py

@@ -0,0 +1,17 @@
+from __future__ import unicode_literals
+
+from django import forms
+from django.utils.translation import ugettext_lazy as _
+
+from mayan.apps.common.forms import FilteredSelectionForm
+
+from .models import AccessControlList
+
+
+class ACLCreateForm(FilteredSelectionForm, forms.ModelForm):
+    class Meta:
+        field_name = 'role'
+        fields = ('role',)
+        label = _('Role')
+        model = AccessControlList
+        widget_attributes = {'class': 'select2'}

mayan/apps/acls/icons.py

@@ -1,6 +1,12 @@
 from __future__ import absolute_import, unicode_literals
 
-from appearance.classes import Icon
+from mayan.apps.appearance.classes import Icon
+from mayan.apps.permissions.icons import icon_permission
 
+icon_acl_delete = Icon(driver_name='fontawesome', symbol='times')
 icon_acl_list = Icon(driver_name='fontawesome', symbol='lock')
-icon_acl_new = Icon(driver_name='fontawesome', symbol='plus')
+icon_acl_new = Icon(
+    driver_name='fontawesome-dual', primary_symbol='lock',
+    secondary_symbol='plus'
+)
+icon_acl_permissions = icon_permission

mayan/apps/acls/links.py

@@ -3,9 +3,11 @@ from __future__ import unicode_literals
 from django.apps import apps
 from django.utils.translation import ugettext_lazy as _
 
-from navigation import Link
+from mayan.apps.navigation.classes import Link
 
-from .icons import icon_acl_list, icon_acl_new
+from .icons import (
+    icon_acl_delete, icon_acl_list, icon_acl_new, icon_acl_permissions
+)
 from .permissions import permission_acl_view, permission_acl_edit
 
 
@@ -27,22 +29,22 @@ def get_kwargs_factory(variable_name):
     return get_kwargs
 
 
-link_acl_delete = Link(
-    args='resolved_object.pk', permissions=(permission_acl_edit,),
-    permissions_related='content_object', tags='dangerous', text=_('Delete'),
-    view='acls:acl_delete',
-)
-link_acl_list = Link(
-    icon_class=icon_acl_list, kwargs=get_kwargs_factory('resolved_object'),
-    permissions=(permission_acl_view,), text=_('ACLs'), view='acls:acl_list'
-)
 link_acl_create = Link(
     icon_class=icon_acl_new, kwargs=get_kwargs_factory('resolved_object'),
     permissions=(permission_acl_edit,), text=_('New ACL'),
     view='acls:acl_create'
 )
-link_acl_permissions = Link(
-    args='resolved_object.pk', permissions=(permission_acl_edit,),
-    permissions_related='content_object', text=_('Permissions'),
-    view='acls:acl_permissions',
+link_acl_delete = Link(
+    args='resolved_object.pk', icon_class=icon_acl_delete,
+    permissions=(permission_acl_edit,), tags='dangerous', text=_('Delete'),
+    view='acls:acl_delete'
+)
+link_acl_list = Link(
+    icon_class=icon_acl_list, kwargs=get_kwargs_factory('resolved_object'),
+    permissions=(permission_acl_view,), text=_('ACLs'), view='acls:acl_list'
+)
+link_acl_permissions = Link(
+    args='resolved_object.pk', icon_class=icon_acl_permissions,
+    permissions=(permission_acl_edit,),
+    text=_('Permissions'), view='acls:acl_permissions'
 )