Move Django and Celery settings

Django settings now reside in the smart settings app.
Celery settings now reside in the task manager app.

Signed-off-by: Roberto Rosario <roberto.rosario@mayan-edms.com>

.dockerignore

@@ -0,0 +1,10 @@
+.git
+.gitignore
+HISTORY.md
+mayan/media
+db.sqlite*
+docker
+!docker/rootfs
+./.*
+docs
+

.flake8

@@ -0,0 +1,3 @@
+[flake8]
+exclude = node_modules,static,templates
+ignore = E501, N801, N802, N804, N805, N806

.gitignore

@@ -3,26 +3,33 @@
 *.pyo
 *.sqlite
 *.sqlite3
-settings_local.py
-/celerybeat-schedule
-document_storage/
-/misc/mayan.geany
-mayan/media/document_cache/
-build/
-_build/
-gpg_home/
-/mayan/media/static/
-/whoosh_index/
-/fabfile_install
-/venv/
-.coverage
-/dist/
-.idea/
-static_collected/
 *egg-info*
-mayan/settings/local.py
-.vagrant
-.tox/
-coverage.xml
+.coverage
 .coverage.tox*
+.idea/
+*.swp
+.tox/
+.vagrant
+_build/
+/build/
+coverage.xml
+document_storage/
+gpg_home/
 htmlcov/
+mayan/media/
+mayan/media/document_cache/
+mayan/settings/local.py
+mayan/error.log
+settings_local.py
+static_collected/
+/celerybeat-schedule
+/fabfile_install
+/dist/
+/misc/mayan.geany
+/mayan/media/static/
+/venv/
+/venv3/
+/whoosh_index/
+google_fonts/
+node_modules/
+docs/build/

.gitlab-ci.yml

@@ -1,35 +1,226 @@
-image: python:2.7
-services:
-  - mysql
-  - postgres
-before_script:
-  - apt-get update -qq
-  - apt-get install -qq python-dev gcc tesseract-ocr tesseract-ocr-deu unpaper ghostscript libjpeg-dev libpng-dev libtiff-dev poppler-utils libreoffice
-variables:
-  POSTGRES_DB: "mayan_edms"
-  POSTGRES_PASSWORD: "postgres"
-  MYSQL_ALLOW_EMPTY_PASSWORD: "yes"
-  MYSQL_DATABASE: "mayan_edms"
-test:mysql:
+stages:
+  - test
+  - build_python
+  - build_docker
+  - build_documentation
+  - push_python
+  - push_docker
+  - push_documentation
+  - deploy
+
+job_docker_build:
+  stage: build_docker
+  image: docker:latest
+  services:
+    - docker:dind
+  before_script:
+    - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
   script:
-  - pip install -r requirements/testing.txt
-  - pip install -q mysql-python
-  - apt-get install -qq mysql-client
-  - mysql -h"$MYSQL_PORT_3306_TCP_ADDR" -P"$MYSQL_PORT_3306_TCP_PORT" -uroot -p"$MYSQL_ENV_MYSQL_ROOT_PASSWORD" -e "ALTER DATABASE $MYSQL_DATABASE CHARACTER SET utf8 COLLATE utf8_unicode_ci;"
-  - coverage run manage.py runtests --settings=mayan.settings.testing.gitlab-ci.db_mysql --nomigrations
-  - bash <(curl https://raw.githubusercontent.com/codecov/codecov-bash/master/codecov) -t $CODECOV_TOKEN
+    - docker build --pull -t "$CI_REGISTRY_IMAGE" -f docker/Dockerfile .
+    - VERSION=`cat docker/rootfs/version`
+    - docker tag "$CI_REGISTRY_IMAGE" "$CI_REGISTRY_IMAGE:$VERSION"
+    - docker run --rm "$CI_REGISTRY_IMAGE:$VERSION" run-tests
+    - docker push "$CI_REGISTRY_IMAGE:$VERSION"
+    - docker push "$CI_REGISTRY_IMAGE:latest"
+    - docker tag "$CI_REGISTRY_IMAGE:$VERSION" registry-1.docker.io/mayanedms/mayanedms:"$VERSION"
+    - docker tag "$CI_REGISTRY_IMAGE:$VERSION" registry-1.docker.io/mayanedms/mayanedms:latest
+    - docker logout
+  only:
+    - releases/all
+    - releases/docker
+
+job_docker_push:
+  stage: push_docker
+  image: docker:latest
+  services:
+    - docker:dind
+  before_script:
+    - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
+  script:
+    - VERSION=`cat docker/rootfs/version`
+    - docker pull "$CI_REGISTRY_IMAGE:$VERSION"
+    - docker tag "$CI_REGISTRY_IMAGE:$VERSION" registry-1.docker.io/mayanedms/mayanedms:"$VERSION"
+    - docker tag "$CI_REGISTRY_IMAGE:$VERSION" registry-1.docker.io/mayanedms/mayanedms:latest
+    - docker logout
+    - docker login -u "$DOCKER_HUB_USERNAME" -p "$DOCKER_HUB_PASSWORD" https://registry-1.docker.io
+    - docker push registry-1.docker.io/mayanedms/mayanedms:"$VERSION"
+    - docker push registry-1.docker.io/mayanedms/mayanedms:latest
+  only:
+    - releases/all
+    - releases/docker
+
+job_docker_nightly:
+  stage: build_docker
+  image: docker:latest
+  services:
+    - docker:dind
+  before_script:
+    - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
+  script:
+    - docker build --pull -t "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG" -f docker/Dockerfile .
+    - docker run --rm "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG" run-tests
+    - docker push "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG"
+  only:
+    - nightly
+    - staging
+
+job_documentation_build:
+  stage: build_documentation
+  image: ubuntu:18.04
+  artifacts:
+    expire_in: 1 hr
+    paths:
+    - docs
+  before_script:
+    - apt-get -qq update
+    - apt-get install -qq locales make python-pip enchant
+    - echo "LC_ALL=\"en_US.UTF-8\"" >> /etc/default/locale
+    - locale-gen en_US.UTF-8
+    - update-locale LANG=en_US.UTF-8
+    - export LC_ALL=en_US.UTF-8
+  script:
+    - pip install -r requirements/documentation.txt
+    - cd docs
+    - make html
+  only:
+    - releases/all
+    - releases/documentation
+
+job_documentation_push:
+  stage: push_documentation
+  image: ubuntu:18.04
+  before_script:
+    - mkdir -p ~/.ssh
+    - chmod 700 ~/.ssh
+    - echo "$DOCUMENTATION_SSH_KNOWN_HOSTS" > ~/.ssh/known_hosts
+    - chmod 644 ~/.ssh/known_hosts
+    - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client rsync -y )'
+    - eval $(ssh-agent -s)
+    - echo "$DOCUMENTATION_SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - > /dev/null
+  script:
+    - rsync -avzP --progress -e ssh docs/_build/html/* mayandocs@docs.mayan-edms.com:/home/rosarior/webapps/mayan_docs
+  only:
+    - releases/all
+    - releases/documentation
+
+job_build_python:
+  stage: build_python
+  image: ubuntu:18.04
+  artifacts:
+    expire_in: 1 hr
+    paths:
+    - dist
+  before_script:
+    - apt-get -qq update
+    - apt-get install -qq locales make python-pip
+  script:
+    - echo "LC_ALL=\"en_US.UTF-8\"" >> /etc/default/locale
+    - locale-gen en_US.UTF-8
+    - update-locale LANG=en_US.UTF-8
+    - export LC_ALL=en_US.UTF-8
+    - pip install -r requirements/build.txt
+    - make python-wheel
+  only:
+    - releases/all
+    - releases/python
+
+job_push_python:
+  stage: push_python
+  image: ubuntu:18.04
+  before_script:
+    - apt-get -qq update
+    - apt-get install -qq python-pip
+  script:
+    - pip install -r requirements/build.txt
+    - echo "$PYPIRC" > ~/.pypirc
+    - twine upload dist/* -r pypi
+  only:
+    - releases/all
+    - releases/python
+
+.test_base: &test_base
+  stage: test
+  image: ubuntu:18.04
+  cache:
+    paths:
+      - ~/.cache/pip/
+      - /var/cache/apt/archives/
+  before_script:
+    - apt-get -qq update
+    - apt-get install -qq locales
+    - echo "LC_ALL=\"en_US.UTF-8\"" >> /etc/default/locale
+    - locale-gen en_US.UTF-8
+    - update-locale LANG=en_US.UTF-8
+    - export LC_ALL=en_US.UTF-8
+    - apt-get install -qq curl exiftool gcc ghostscript gnupg1 graphviz libfuse2 libjpeg-dev libmagic1 libpng-dev libtiff-dev poppler-utils libreoffice poppler-utils python-dev python-pip tesseract-ocr tesseract-ocr-deu
+    - pip install -r requirements.txt -r requirements/testing-base.txt
+  only:
+    - releases/all
+    - releases/docker
+    - releases/python
+    - staging
+    - nightly
+
+test-mysql:
+  <<: *test_base
+  variables:
+    MYSQL_ALLOW_EMPTY_PASSWORD: "yes"
+    MYSQL_DATABASE: "mayan_edms"
+  services:
+    - mysql:8.0.3
+  script:
+    - apt-get install -qq libmysqlclient-dev mysql-client
+    - pip install mysqlclient
+    - mysql -h"$MYSQL_PORT_3306_TCP_ADDR" -P"$MYSQL_PORT_3306_TCP_PORT" -uroot -p"$MYSQL_ENV_MYSQL_ROOT_PASSWORD" -e "set global character_set_server=utf8mb4;"
+    - python manage.py test --mayan-apps --settings=mayan.settings.testing.gitlab-ci.db_mysql --nomigrations
   tags:
     - mysql
-test:postgres:
+
+test-postgres:
+  <<: *test_base
+  variables:
+    POSTGRES_DB: "mayan_edms"
+    POSTGRES_PASSWORD: "postgres"
+  services:
+    - postgres
   script:
-  - pip install -r requirements/testing.txt
-  - pip install -q psycopg2
-  - coverage run manage.py runtests --settings=mayan.settings.testing.gitlab-ci.db_postgres --nomigrations
-  - bash <(curl https://raw.githubusercontent.com/codecov/codecov-bash/master/codecov) -t $CODECOV_TOKEN
+    - apt-get install -qq libpq-dev
+    - pip install psycopg2
+    - python manage.py test --mayan-apps --settings=mayan.settings.testing.gitlab-ci.db_postgres --nomigrations
   tags:
     - postgres
-test:sqlite:
+
+test-sqlite:
+  <<: *test_base
   script:
-  - pip install -r requirements/testing.txt
-  - coverage run manage.py runtests --settings=mayan.settings.testing.gitlab-ci --nomigrations
-  - bash <(curl https://raw.githubusercontent.com/codecov/codecov-bash/master/codecov) -t $CODECOV_TOKEN
+    - python manage.py test --mayan-apps --settings=mayan.settings.testing.gitlab-ci --nomigrations
+
+deploy_demo:
+  environment:
+    name: demo
+  stage: deploy
+  before_script:
+    - mkdir -p ~/.ssh
+    - echo "${DEMO_SSH_KNOWN_HOST}" > ~/.ssh/known_hosts
+    - chmod 644 ~/.ssh/known_hosts
+    - echo "${DEMO_PRIVATE_KEY}" > ~/.ssh/id_rsa
+    - chmod 600 ~/.ssh/id_rsa
+  script:
+    - scp contrib/scripts/install/docker.sh root@$DEMO_IP_ADDRESS:~
+    - ssh root@$DEMO_IP_ADDRESS "DELETE_VOLUMES=true ./docker.sh"
+  when: manual
+
+deploy_staging:
+  environment:
+    name: staging
+  stage: deploy
+  before_script:
+    - mkdir -p ~/.ssh
+    - echo "${STAGING_SSH_KNOWN_HOST}" > ~/.ssh/known_hosts
+    - chmod 644 ~/.ssh/known_hosts
+    - echo "${STAGING_PRIVATE_KEY}" > ~/.ssh/id_rsa
+    - chmod 600 ~/.ssh/id_rsa
+  script:
+    - scp contrib/scripts/install/docker.sh root@$STAGING_IP_ADDRESS:~
+    - ssh root@$STAGING_IP_ADDRESS "DOCKER_MAYAN_IMAGE=registry.gitlab.com/mayan-edms/mayan-edms:staging ./docker.sh"
+  when: manual

.travis.yml

@@ -1,3 +1,5 @@
+dist: trusty
+sudo: required
 language: python
 python:
   - 2.7
@@ -8,7 +10,7 @@ env:
    - DB=sqlite
 before_install:
   - sudo apt-get update -qq
-  - sudo apt-get install -qq python-dev gcc tesseract-ocr tesseract-ocr-deu unpaper ghostscript libjpeg-dev libpng-dev poppler-utils libreoffice
+  - sudo apt-get install -qq python-dev python-pip gcc tesseract-ocr tesseract-ocr-deu ghostscript libjpeg-dev libpng-dev libtiff-dev poppler-utils libreoffice
 install:
   - "pip install -r requirements/testing.txt"
   - if [[ $DB == mysql ]]; then pip install -q mysql-python; fi
@@ -17,9 +19,9 @@ before_script:
   - mysql -e 'create database mayan_edms;'
   - psql -c 'create database mayan_edms;' -U postgres
 script:
-  - if [[ $DB == mysql ]]; then coverage run manage.py runtests --settings=mayan.settings.testing.travis.db_mysql --nomigrations; fi
-  - if [[ $DB == postgres ]]; then coverage run manage.py runtests --settings=mayan.settings.testing.travis.db_postgres --nomigrations; fi
-  - if [[ $DB == sqlite ]]; then coverage run manage.py runtests --settings=mayan.settings.testing.base --nomigrations; fi
+  - if [[ $DB == mysql ]]; then coverage run manage.py test --mayan-apps --settings=mayan.settings.testing.travis.db_mysql --nomigrations; fi
+  - if [[ $DB == postgres ]]; then coverage run manage.py test --mayan-apps --settings=mayan.settings.testing.travis.db_postgres --nomigrations; fi
+  - if [[ $DB == sqlite ]]; then coverage run manage.py test --mayan-apps --settings=mayan.settings.testing.base --nomigrations; fi
 after_success:
   - coveralls
 branches:

.tx/config

@@ -19,6 +19,18 @@ source_lang = en
 source_file = mayan/apps/authentication/locale/en/LC_MESSAGES/django.po
 type = PO
 
+[mayan-edms.autoadmin-2-0]
+file_filter = mayan/apps/autoadmin/locale/<lang>/LC_MESSAGES/django.po
+source_lang = en
+source_file = mayan/apps/autoadmin/locale/en/LC_MESSAGES/django.po
+type = PO
+
+[mayan-edms.cabinets-2-0]
+file_filter = mayan/apps/cabinets/locale/<lang>/LC_MESSAGES/django.po
+source_lang = en
+source_file = mayan/apps/cabinets/locale/en/LC_MESSAGES/django.po
+type = PO
+
 [mayan-edms.checkouts-2-0]
 file_filter = mayan/apps/checkouts/locale/<lang>/LC_MESSAGES/django.po
 source_lang = en
@@ -37,18 +49,24 @@ source_lang = en
 source_file = mayan/apps/converter/locale/en/LC_MESSAGES/django.po
 type = PO
 
+[mayan-edms.dashboards-2-0]
+file_filter = mayan/apps/dashboards/locale/<lang>/LC_MESSAGES/django.po
+source_lang = en
+source_file = mayan/apps/dashboards/locale/en/LC_MESSAGES/django.po
+type = PO
+
+[mayan-edms.dependencies-3-0]
+file_filter = mayan/apps/dependencies/locale/<lang>/LC_MESSAGES/django.po
+source_lang = en
+source_file = mayan/apps/dependencies/locale/en/LC_MESSAGES/django.po
+type = PO
+
 [mayan-edms.django_gpg-2-0]
 file_filter = mayan/apps/django_gpg/locale/<lang>/LC_MESSAGES/django.po
 source_lang = en
 source_file = mayan/apps/django_gpg/locale/en/LC_MESSAGES/django.po
 type = PO
 
-[mayan-edms.documents-2-0]
-file_filter = mayan/apps/documents/locale/<lang>/LC_MESSAGES/django.po
-source_lang = en
-source_file = mayan/apps/documents/locale/en/LC_MESSAGES/django.po
-type = PO
-
 [mayan-edms.document_comments-2-0]
 file_filter = mayan/apps/document_comments/locale/<lang>/LC_MESSAGES/django.po
 source_lang = en
@@ -61,6 +79,12 @@ source_lang = en
 source_file = mayan/apps/document_indexing/locale/en/LC_MESSAGES/django.po
 type = PO
 
+[mayan-edms.document_parsing-2-0]
+file_filter = mayan/apps/document_parsing/locale/<lang>/LC_MESSAGES/django.po
+source_lang = en
+source_file = mayan/apps/document_parsing/locale/en/LC_MESSAGES/django.po
+type = PO
+
 [mayan-edms.document_signatures-2-0]
 file_filter = mayan/apps/document_signatures/locale/<lang>/LC_MESSAGES/django.po
 source_lang = en
@@ -73,6 +97,12 @@ source_lang = en
 source_file = mayan/apps/document_states/locale/en/LC_MESSAGES/django.po
 type = PO
 
+[mayan-edms.documents-2-0]
+file_filter = mayan/apps/documents/locale/<lang>/LC_MESSAGES/django.po
+source_lang = en
+source_file = mayan/apps/documents/locale/en/LC_MESSAGES/django.po
+type = PO
+
 [mayan-edms.dynamic_search-2-0]
 file_filter = mayan/apps/dynamic_search/locale/<lang>/LC_MESSAGES/django.po
 source_lang = en
@@ -85,16 +115,10 @@ source_lang = en
 source_file = mayan/apps/events/locale/en/LC_MESSAGES/django.po
 type = PO
 
-[mayan-edms.folders-2-0]
-file_filter = mayan/apps/folders/locale/<lang>/LC_MESSAGES/django.po
+[mayan-edms.file_metadata-3-0]
+file_filter = mayan/apps/file_metadata/locale/<lang>/LC_MESSAGES/django.po
 source_lang = en
-source_file = mayan/apps/folders/locale/en/LC_MESSAGES/django.po
-type = PO
-
-[mayan-edms.installation-2-0]
-file_filter = mayan/apps/installation/locale/<lang>/LC_MESSAGES/django.po
-source_lang = en
-source_file = mayan/apps/installation/locale/en/LC_MESSAGES/django.po
+source_file = mayan/apps/file_metadata/locale/en/LC_MESSAGES/django.po
 type = PO
 
 [mayan-edms.linking-2-0]
@@ -127,10 +151,10 @@ source_lang = en
 source_file = mayan/apps/mirroring/locale/en/LC_MESSAGES/django.po
 type = PO
 
-[mayan-edms.navigation-2-0]
-file_filter = mayan/apps/navigation/locale/<lang>/LC_MESSAGES/django.po
+[mayan-edms.motd-2-0]
+file_filter = mayan/apps/motd/locale/<lang>/LC_MESSAGES/django.po
 source_lang = en
-source_file = mayan/apps/navigation/locale/en/LC_MESSAGES/django.po
+source_file = mayan/apps/motd/locale/en/LC_MESSAGES/django.po
 type = PO
 
 [mayan-edms.ocr-2-0]
@@ -145,6 +169,12 @@ source_lang = en
 source_file = mayan/apps/permissions/locale/en/LC_MESSAGES/django.po
 type = PO
 
+[mayan-edms.platform-2-0]
+file_filter = mayan/apps/platform/locale/<lang>/LC_MESSAGES/django.po
+source_lang = en
+source_file = mayan/apps/platform/locale/en/LC_MESSAGES/django.po
+type = PO
+
 [mayan-edms.rest_api-2-0]
 file_filter = mayan/apps/rest_api/locale/<lang>/LC_MESSAGES/django.po
 source_lang = en
@@ -164,9 +194,9 @@ source_file = mayan/apps/sources/locale/en/LC_MESSAGES/django.po
 type = PO
 
 [mayan-edms.statistics-2-0]
-file_filter = mayan/apps/statistics/locale/<lang>/LC_MESSAGES/django.po
+file_filter = mayan/apps/mayan_statistics/locale/<lang>/LC_MESSAGES/django.po
 source_lang = en
-source_file = mayan/apps/statistics/locale/en/LC_MESSAGES/django.po
+source_file = mayan/apps/mayan_statistics/locale/en/LC_MESSAGES/django.po
 type = PO
 
 [mayan-edms.storage-2-0]
@@ -181,6 +211,12 @@ source_lang = en
 source_file = mayan/apps/tags/locale/en/LC_MESSAGES/django.po
 type = PO
 
+[mayan-edms.task_manager-2-0]
+file_filter = mayan/apps/task_manager/locale/<lang>/LC_MESSAGES/django.po
+source_lang = en
+source_file = mayan/apps/task_manager/locale/en/LC_MESSAGES/django.po
+type = PO
+
 [mayan-edms.user_management-2-0]
 file_filter = mayan/apps/user_management/locale/<lang>/LC_MESSAGES/django.po
 source_lang = en

CONTRIBUTING.md

@@ -0,0 +1,79 @@
+Contributing to Mayan EDMS
+==========================
+
+Issues
+------
+
+- Before submitting a new issue, check for **open issues** first and join the
+discussion.
+- If your issue is not attended in a while, **be patient**, the core team is
+small and the codebase big.
+- Try to reproduce the issue using a **separate, clean installation**, sometimes
+the issue can be caused by an error in a configuration file and not with the
+code itself.
+- **Do not upload** sensitive, private or classified information or files with
+your issue. If the issue is triggered by a user file, create a dummy file with the
+same properties that can trigger the issue and upload that file instead.
+- Add steps that trigger the issue in a **repeatable manner**.
+- **Screenshots** go a long way in helping understand problems.
+- The issue must be related to the code only, do not open issues for problems
+with deployments, webservers, cloud providers, etc.
+- Do not open issues asking for **support or consulting**.
+
+Code
+----
+
+1. Complete and mail, or scan and email the corresponding Contributor Assignment Agreement: [Mayan EDMS Individual Contributor Assignment Agreement](https://gitlab.com/mayan-edms/mayan-edms/blob/master/docs/chapters/caa_individual.rst) or [Mayan EDMS Entity Contributor Assignment Agreement](https://gitlab.com/mayan-edms/mayan-edms/blob/master/docs/chapters/caa_entity.rst).
+1. Fork [the repository](http://gitlab.com/mayan-edms/mayan-edms).
+1. Choose the version for which you want to develop. The code is divided in the
+following branches:
+
+    a. **master:** Current stable version.
+    a. **series/X.Y:** There will be a **series** branch for each major version
+    release (1.0, 2.0). The **master** branch will be in sync with the latest
+    **series** branch.
+    a. **development:** Unstable branch where the development for the next minor or
+    major release is happening.
+    a. **feature/X:** These are branches dealing with a specific bug or feature
+    that is not yet ready to be merged.
+    a. The **master** and **series** branches are stable and not braking changes
+    must be submitted or merged. The **development** and **feature** branches
+    are unstable and should not be used in production.
+
+1. Start making your changes in your own separate branch.
+1. Write a test which shows that the bug was fixed or that the feature works as
+   expected.
+1. Sign your work. Your signature certifies your submission according to the
+   articles of the [Developer Certificate of Origin](https://gitlab.com/mayan-edms/mayan-edms/blob/master/DCO).
+   The sign-off should be in the form:
+
+    ````
+    Signed-off-by: John Doe <john.doe@example.com>
+    ````
+
+    You must use your real name and email, pseudonyms or anonymous contributions
+    are not allowed. If you set your user.name and user.email git configs, you can
+    sign your commit automatically with git commit -s.
+1. Submit a merge request for your changes.
+
+Feature requests
+----------------
+
+- Opening issues to start a discussion around a new feature or idea are welcomed.
+- Be explicit about your idea and provide a simple scenario, others need to really
+understand the concepts behind your idea if it is to be implemented.
+- Be realistic in your expectation and scope of your idea.
+
+Translations
+------------
+- Translations are handled using [Transifex](https://www.transifex.com/rosarior/mayan-edms/).
+- Join an existing team for the language to which you wish to contribute,
+if there is no team for your language, request for one to be created.
+- Don't request the creation of a new language and expect others to do all the
+work. If you request the creation of a new language try to do some of the
+translation. Language teams that don't see activity for a while are deleted.
+
+Code style
+----------
+- Refer to the [Development](https://docs.mayan-edms.com/topics/development.html)
+chapter for information and examples of the code style.

DCO

@@ -0,0 +1,36 @@
+Developer Certificate of Origin
+Version 1.1
+
+Copyright (C) 2004, 2006 The Linux Foundation and its contributors.
+660 York Street, Suite 102,
+San Francisco, CA 94110 USA
+
+Everyone is permitted to copy and distribute verbatim copies of this
+license document, but changing it is not allowed.
+
+
+Developer's Certificate of Origin 1.1
+
+By making a contribution to this project, I certify that:
+
+(a) The contribution was created in whole or in part by me and I
+    have the right to submit it under the open source license
+    indicated in the file; or
+
+(b) The contribution is based upon previous work that, to the best
+    of my knowledge, is covered under an appropriate open source
+    license and I have the right under that license to submit that
+    work with modifications, whether created in whole or in part
+    by me, under the same open source license (unless I am
+    permitted to submit under a different license), as indicated
+    in the file; or
+
+(c) The contribution was provided directly to me by some other
+    person who certified (a), (b) or (c) and I have not modified
+    it.
+
+(d) I understand and agree that this project and the contribution
+    are public and that a record of the contribution (including all
+    personal information I submit with it, including my sign-off) is
+    maintained indefinitely and may be redistributed consistent with
+    this project or the open source license(s) involved.

HISTORY_1_x.rst

@@ -0,0 +1,60 @@
+1.1.1 (2015-05-21)
+==================
+
+- Update to Django 1.6.11
+- Fix make_dist.sh script
+- Add test for issue #163
+- Activate tests for the sources app
+- Removal of the registration app
+- New simplified official project description
+- Improvements to the index admin interface
+- Removal of installation statistics gathering
+- Remove unused folder tag
+- Fix usage of ugettext to ugettext_lazy
+- Increase size of the lock name field
+- New style documentation
+
+
+1.1 (2015-02-10)
+================
+- Uses Celery for background tasks
+- Removal of the splash screen
+- Adds a home view with common function buttons
+- Support for sending and receiving documents via email
+- Removed custom logging app in favor of django-actvity-stream
+- Adds watch folders
+- Includes Vagrant file for unified development and testing environments
+- Per user locale profile (language and timezone)
+- Includes news document workflow app
+- Optional and required metadata types
+- Improved testings. Automated tests against SQLite, MySQL, PostgreSQL
+- Many new REST API endpoints added
+- Simplified text messages
+- Improved method for custom settings
+- Addition of CORS support to the REST API
+- Per document language setting instead of per installation language setting
+- Metadata validation and parsing support
+- Start of code updates towards Python 3 support
+- Simplified UI
+- Stable PDF previews generation
+- More technical documentation
+
+
+1.0 (2014-08-27)
+================
+- New home @ https://github.com/mayan-edms/mayan-edms
+- Updated to use Django 1.6
+- Translation updates
+- Custom model properties removal
+- Source code improvements
+- Removal of included 3rd party modules
+- Automatic testing and code coverage check
+- Update of required modules and libraries versions
+- Database connection leaks fixes
+- Support for deletion of detached signatures
+- Removal of Fabric based installations script
+- Pluggable OCR backends
+- OCR improvements
+- License change, Mayan EDMS in now licensed under the Apache 2.0 License
+- PyPI package, Mayan EDMS in now available on PyPI: https://pypi.python.org/pypi/mayan-edms/
+- New REST API

HISTORY_2_x.rst

@@ -0,0 +1,435 @@
+2.7.3 (2017-09-11)
+==================
+- Fix task manager queue list view. Thanks to LeVon Smoker for
+  the report.
+- Fix resolved link class URL mangling when the keep_query argument is
+  used. Thanks to Nick Douma (LordGaav) for the report and diagnostic
+  information. Fixes source navigation on the document upload wizard.
+
+2.7.2 (2017-09-06)
+==================
+- Fix new mailer creation view. GitLab issue #431.
+  Thanks to Robert Schöftner (@robert.schoeftner) for the
+  report and the solution.
+- Consolidate intial document created event and the first
+  document properties edited events. Preserve the user that
+  initially creates the document. GitLab issue #433. Thanks
+  to Jesaja Everling (@jeverling) for the report.
+- Sort the list of root cabinets. Thanks to Thomas Plotkowiak
+  for the request.
+- Sort the list of a document's cabinets.
+- Display a document's cabinet list in italics. GitLab issue #435.
+  Thanks to LeVon Smoker for the request.
+- Install mock by default to allow easier testing of deployed
+  instances.
+
+2.7.1 (2017-09-03)
+==================
+- Support unicode in URL querystring. GitLab issue #423.
+  Thanks to Gustavo Teixeira (@gsteixei) for the find.
+- Import errors during initialization are only ignored
+  if they are cause by a missing local.py. Thanks to
+  MacRobb Simpson for the report and solution.
+- Make sure the local.py created used unicode for strings
+  by default. GitLab issue #424. Thanks to Gustavo Teixeira
+  (@gsteixei) for the find.
+
+2.7 (2017-08-30)
+================
+- Add workaround for PDF with IndirectObject as the
+  rotation value. GitHub #261.
+- Add ACL list link with icon and use it for the document facet menu.
+- Fix mailing app permissions labels.
+- Add ACLs link and ACLs permissions to the mailer profile model.
+- Improve mailer URL regex.
+- Add ordering support to the SourceColumn class. GitLab issue #417.
+- Shows the cabinets in the document list. GitLab #417 @corneliusludmann
+- Add workaround for pycountry versions without the bibliographical key.
+  GitHub issue #250.
+- Skip UUID migration on Oracle backends. GitHub issue #251.
+- Allow changing the output format, DPI of the pdftoppm command, and
+  the output format of the converter via the CONVERTER_GRAPHICS_BACKEND_CONFIG
+  setting. GitHub issues #256 #257 GitLab issue #416.
+- Add support for workflow triggers.
+- Add support for workflow actions.
+- Add support for rendering workflows.
+- Add support for unbinding sub menus.
+- Fix mailing profile test view.
+- Disregard the last 3 dots that mark the end of the YAML document.
+- Add support for multiple dashboards.
+- Add support for removing dashboard widgets.
+- Convert document version view to item list view.
+- Add support for browsing individual document versions.
+- Add support for dropdown menus to the item list view template.
+- Add support for preserving the file extension when downloading a document
+  version. GitLab #415.
+- Split OCR app into OCR and parsing.
+- Remove Folders app.
+- Use the literal 'System' instead of the target name when
+  the action user in unknown.
+- Remove the view to submit all document for OCR.
+- When changing document types, don't delete the old metadata that is
+  also found in the new document type. GitLab issue #421.
+- Add tag attach and tag remove events.
+- Change the permission needed to attach and remove tags.
+- Add HTTP POST workflow state action.
+- Add access control grant workflow state action.
+- Beta Python 3 support.
+
+2.6.4 (2017-07-26)
+==================
+- Add missing replacements of reverse to resolve_url.
+
+2.6.3 (2017-07-25)
+==================
+- Add makefile target to launch a PostgreSQL container.
+- Use resolve_url instead of redirect to resolve the post login URL.
+- Make the intialsetup and performupgrade management tasks work
+  with signals to allow customization from 3rd party apps.
+- PEP8 cleanups.
+- Add tag_ids keyword argument to the Source.handle_upload
+  model method. GitLab issue #413.
+- Add overflow wrapping so wrap long titles in Firefox too.
+- Makes Roles searchable. GitLab issue #402.
+- Add line numbers to the debug and production loggers.
+  Add date and time to the production logger.
+- Add support for generating setup.py from a template. GitLab
+  #149 #200.
+- Add fade in animation to document images.
+
+2.6.2 (2017-07-19)
+==================
+- Fix deprecation warning to prepare upgrade to Django 1.11 and 2.0.
+- Fix document page zoom.
+- Add support to run tests against a MySQL, Postgres or Oracle container.
+- Improve tag widget customization by moving the markup to its own template.
+- Fix document page widget appearance in the document page list view.
+- Make document version order deterministic.
+- Allow total page number instrospection of encrypted PDF with non ASCII user properties. GitLab issue #411.
+- Oracle database compatibility update in the cabinets app. GitHub #258.
+
+2.6.1 (2017-07-18)
+==================
+- Fix issue when editing or removing metadata from multiple documents.
+
+2.6 (2017-07-18)
+================
+- Fix HTML mark up in window title. GitLab #397.
+- Add support for emailing documents to a recipient list. GitLab #396.
+- Backport metadata widget changes from @Macrobb. GitLab #377.
+- Make users and group searchable.
+- Add support for logging errors during in production mode.
+  Add COMMON_PRODUCTION_ERROR_LOG_PATH to control path of log file.
+  Defaults to mayan/error.log.
+- Add support logging request exceptions.
+- Add document list item view.
+- Sort setting by namespace label and by global name second.
+- Sort indexes by label.
+- Fix cabinets permission and access control checking.
+- The permission to add or remove documents to cabinets now applies to documents too.
+- Equalize dashboard widgets heights.
+- Switch the order of the DEFAULT_AUTHENTICATION_CLASSES of DRF. GitLab #400.
+- Backport document's version list view permission.
+- Improve code to unbind menu entries.
+- Renamed the document type permission namespace from "Document setup" to "Document types".
+- Add support for granting the document type edit, document type delete, and document type view
+  permissions to individual document type instances.
+- Improved tests by testing for accesses.
+- Increase the size of the mailing profile label field to 128 characters.
+
+2.5.2 (2017-07-08)
+==================
+- Improve new document creation signal handling.
+  Fixes issue with duplicate scanning at upload.
+
+2.5.1 (2017-07-08)
+==================
+- Update release target due to changes in PyPI.
+
+2.5 (2017-07-07)
+================
+- Add view to download a document's OCR text. GitLab #215
+- Add user configurable mailer. GitLab #286.
+- Use Toasts library for screen messages.
+- Reduce verbosity of some debug messages.
+- Add new lineart transformation.
+- Fix SANE source resolution field.
+- About and Profile menu reorganization.
+- PDF compatibility improvements.
+- Office document coversion improvements.
+- New metadata type setup UI.
+- Duplicated document scan support.
+- "Remember me" login support.
+- Forgotten password restore via email.
+- Document cache disabling.
+- Translation improvements.
+- Image loading improvements.
+- Lower JavaScript memory utilization.
+- HTML reponsive layout improvements.
+- Make document deletion a background task.
+- Unicode handling improvements.
+- Python3 compatilibyt improvements.
+- New screen messages using Toastr.
+
+2.4 (2017-06-23)
+================
+- Add Django-mathfilters.
+- Improve render of documents with no pages.
+- Add SANE scanner document source.
+- Added PDF orientation detection. GitLab issue #387.
+- Fix repeated permission list API URL. GitLab issue #389.
+- Fix role creation API endpoint not returning id. GitLab issue #390.
+- Make tags, metadata types and cabinets searchable via the dynamic search API. GitLab issue #344.
+- Add support for updating configuration options from environment variables.
+- Add purgelocks management command. GitLab issue #221.
+- Fix index rebuilding for multi value first levels. GitLab issue #391.
+- Truncate views titles via the APPEARANCE_MAXIMUM_TITLE_LENGTH setting. GitLab issue #217.
+- Add background task manager app. GitLab issue #132.
+- Add link to show a document's OCR errors. GitLab issue #291.
+
+2.3 (2017-06-08)
+================
+- Allow for bigger indexing expression templates.
+- Auto select checkbox when updating metadata values. GitLab issue #371.
+- Added support for passing the options allow-other and allow-root to the
+  FUSE index mirror. GitLab issue #385
+- Add support for check for the latest released version of Mayan from the
+  About menu.
+- Support for rebuilding specific indexes. GitLab issue #372.
+- Rewrite document indexing code to be faster and use less locking.
+- Use a predefined file path for the file lock.
+- Catch documents with not document version when displaying their thumbnails.
+- Document page navigation fix when using Mayan as a sub URL app.
+- Add support for indexing on workflow state changes.
+- Add search model list API endpoint.
+
+2.2 (2017-04-26)
+================
+- Remove the installation app (GitLab #301).
+- Add support for document page search
+- Remove recent searches feature
+- Remove dependency on the django-filetransfer library
+- Fix height calculation in resize transformation
+- Improve upgrade instructions
+- New image caching pipeline
+- New drop down menus for the documents, folders and tags app as well as for
+  the user links.
+- New Dashboard view
+- Moved licenses to their own module in every app
+- Update project to work with Django 1.10.4.
+- Tags are alphabetically ordered by label (GitLab #342).
+- Stop loading theme fonts from the web (GitLab #343).
+- Add support for attaching multiple tags (GitLab #307).
+- Integrate the Cabinets app.
+
+2.1.11 (2017-03-14)
+===================
+- Added a quick rename serializer to the document type API serializer.
+- Added per document type, workflow list API view.
+- Mayan EDMS was adopted a version 1.1 of the Linux Foundation Developer Certificate of Origin.
+- Added the detail url of a permission in the permission serializer.
+- Added endpoints for the ACL app API.
+- Implemented document workflows transition ACLs. GitLab issue #321.
+- Add document comments API endpoints. GitHub issue #249.
+- Add support for overriding the Celery class.
+- Changed the document upload view in source app to not use the HTTP referer
+  URL blindly, but instead recompose the URL using known view name. Needed
+  when integrating Mayan EDMS into other app via using iframes.
+- Addes size field to the document version serializer.
+- Removed the serializer from the deleted document restore API endpoint.
+- Added support for adding or editing document types to smart links via the
+  API.
+
+2.1.10 (2017-02-13)
+===================
+- Update Makefile to use twine for releases.
+- Add Makefile target to make test releases.
+
+2.1.9 (2017-02-13)
+==================
+- Update make file to Workaround long standing pypa wheel bug #99
+
+2.1.8 (2017-02-12)
+==================
+- Fixes in the trashed document API endpoints.
+- Improved tags API PUT and PATCH endpoints.
+- Bulk document adding when creating and editing tags.
+- The version of django-mptt is preserved in case mayan-cabinets is installed.
+- Add Django GPG API endpoints for singing keys.
+- Add API endpoints for the document states (workflows) app.
+- Add API endpoints for the messsage of the day (MOTD) app.
+- Add Smart link API endpoints.
+- Add writable versions of the Document and Document Type serializers (GitLab issues #348 and #349).
+- Close GitLab issue #310 "Metadata's lookup with chinese messages when new document"
+
+2.1.7 (2017-02-01)
+==================
+- Improved user management API endpoints.
+- Improved permissions API endpoints.
+- Improvements in the API tests of a few apps.
+- Addition Content type list API view to the common app.
+- Add API endpoints to the events app.
+- Enable the parser and validation fields of the metadata serializer.
+
+2.1.6 (2016-11-23)
+==================
+- Fix variable name typo in the rotation transformation class.
+- Update translations
+
+2.1.5 (2016-11-08)
+==================
+- Backport resize transformation math operation fix (GitLab #319).
+- Update Pillow to 3.1.2 (Security fix).
+- Backport zoom transformation performance improvement (GitLab #334).
+- Backport trash can navigation link resolution fix (GitLab #331).
+- Improve documentation regarding the use of GPG version 1 (GitLab #333).
+- Fix ACL create view HTML response type. (GitLab #335).
+- Expland staging folder and watch folder explanation.
+
+2.1.4 (2016-10-28)
+==================
+- Add missing link to the 2.1.3 release notes in the index file.
+- Improve TempfileCheckMixin.
+- Fix statistics namespace list display view.
+- Fix events list display view.
+- Update required Django version to 1.8.15.
+- Update required python-gnupg version to 0.3.9.
+- Improved orphaned temporary files test mixin.
+- Re-enable and improve GitLab CI MySQL testing.
+- Improved GPG handling.
+- New GPG backend system.
+- Minor documentation updates.
+
+2.1.3 (2016-06-29)
+==================
+- Add help message when initialsetup migration phase fails. Relates to GitLab issue #296.
+- Start using self.setdout instead of print as per documentation.
+- Fix GitLab issue #295, "When editing a user the top bar jumps to the name of the user".
+- Normalize handling of temporary file and directory creation.
+- Fix GitLab issue #309, "Temp files quickly filling-up my /tmp (1GB tmpfs)".
+- Explicitly check for residual temporary files in tests.
+- Add missing temporary file cleanup for office documents.
+- Fix file descriptor leak in the document signature download test.
+
+2.1.2 (2016-05-20)
+==================
+- Sort document languages and user profile locale language lists. GitLab issue #292.
+- Fix metadata lookup for {{ users }} and {{ group }}. Fixes GitLab #290.
+- Add Makefile for common development tasks
+
+2.1.1 (2016-05-17)
+==================
+- Fix navigation issue that make it impossible to add new sources. GitLab issue #288.
+- The Tesseract OCR backend now reports if the requested language file is missing. GitLab issue #289.
+- Ensure the automatic default index is created after the default document type.
+
+2.1 (2016-05-14)
+================
+- Upgrade to use Django 1.8.13. Issue #246.
+- Upgrade requirements.
+- Remove remaining references to Django's User model. GitLab issue #225
+- Rename 'Content' search box to 'OCR'.
+- Remove included login required middleware using django-stronghold instead (http://mikegrouchy.com/django-stronghold/).
+- Improve generation of success and error messages for class based views.
+- Remove ownership concept from folders.
+- Replace strip_spaces middleware with the spaceless template tag. GitLab issue #255
+- Deselect the update checkbox for optional metadata by default.
+- Silence all Django 1.8 model import warnings.
+- Implement per document type document creation permission. Closes GitLab issue #232.
+- Add icons to the document face menu links.
+- Increase icon to text spacing to 3px.
+- Make document type delete time period optional.
+- Fixed date locale handling in document properties, checkout and user detail views.
+- Add new permission: checkout details view.
+- Add HTML5 upload widget. Issue #162.
+- Add Message of the Day app. Issue #222
+- Update Document model's uuid field to use Django's native UUIDField class.
+- Add new split view index navigation
+- Newly uploaded documents appear in the Recent document list of the user.
+- Document indexes now have ACL support.
+- Remove the document index setup permission.
+- Status messages now display the object class on which they operate not just the word "Object".
+- More tests added.
+- Handle unicode filenames in staging folders.
+- Add staging file deletion permission.
+- New document_signature_view permission.
+- Add support for signing documents.
+- Instead of multiple keyservers only one keyserver is now supported.
+- Replace document type selection widget with an opened selection list.
+- Add mailing documentation chapter.
+- Add roadmap documentation chapter.
+- API updates.
+
+
+2.0.2 (2016-02-09)
+==================
+- Install testing dependencies when installing development dependencies.
+- Fix GitLab issue #250 "Empty optional lookup metadata trigger validation error".
+- Fix OCR API test.
+- Move metadata form value validation to .clean() method.
+- Only extract validation error messages from ValidationError exception instances.
+- Don't store empty metadata value if the update checkbox is not checked.
+- Add 2 second delay to document version tests to workaround MySQL limitation.
+- Strip HTML tags from the browser title.
+- Remove Docker and Docker Compose files.
+
+
+2.0.1 (2016-01-22)
+==================
+- Fix GitLab issue #243, "System allows a user to skip entering values for a required metadata field while uploading a new document"
+- Fix GitLab issue #245, "Add multiple metadata not possible"
+- Updated Vagrantfile to provision a production box too.
+
+
+2.0 (2015-12-04)
+================
+- New source homepage: https://gitlab.com/mayan-edms/mayan-edms
+- Update to Django 1.7
+- New Bootstrap Frontend UI
+- Easier theming and rebranding
+- Improved page navigation interface
+- Menu reorganization
+- Removal of famfam icon set
+- Improved document preview generation
+- Document submission for OCR changed to POST
+- New YAML based settings system
+- Removal of auto admin creation as separate app
+- Removal of dependencies
+- ACL system refactor
+- Object access control inheritance
+- Removal of anonymous user support
+- Metadata validators refactor
+- Trash can support
+- Retention policies
+- Support for sharing indexes as FUSE filesystems
+- Clickable preview images titles
+- Removal of eval
+- Smarter OCR, per page parsing or OCR fallback
+- Improve failure tolerance (not all Operational Errors are critical now)
+- RGB tags
+- Default document type and default document source
+- Link unbinding
+- Statistics refactor
+- Apps merge
+- New signals
+- Test improvements
+- Indexes recalculation after document creation too
+- Upgrade command
+- OCR data moved to ocr app from documents app
+- New internal document creation workflow return a document stub
+- Auto console debug logging during development and info during production
+- New class based and menu based navigation system
+- New class based transformations
+- Usage of Font Awesome icons set
+- Management command to remove obsolete permissions: `purgepermissions`
+- Normalization of 'title' and 'name' fields to 'label'
+- Improved API, now at version 1
+- Invert page title/project name order in browser title
+- Django's class based views pagination
+- Reduction of text strings
+- Removal of the CombinedSource class
+- Removal of default class ACLs
+- Removal of the ImageMagick and GraphicsMagick converter backends
+- Remove support for applying roles to new users automatically
+- Removal of the DOCUMENT_RESTRICTIONS_OVERRIDE permission
+- Removed the page_label field

LICENSE

@@ -1,4 +1,4 @@
-   Copyright 2011-2015 Roberto Rosario
+   Copyright 2011 Roberto Rosario
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

MANIFEST.in

@@ -1,3 +1,6 @@
-include README.rst LICENSE HISTORY.rst
-recursive-include mayan *.txt *.html *.css *.ico *.png *.jpg *.js *.po *.mo *.ttf *.woff *.woff2 LICENSE
-global-exclude mayan/settings/local.py mayan/settings/travis/* mayan/media/*
+include README.md LICENSE HISTORY.rst mayan/LICENSE
+recursive-include mayan/apps *.txt *.html *.css *.ico *.png *.jpg *.js *.mo *.ttf *.woff *.woff2 *.gif *.eot *.svg *.doc *.pdf *.tiff *.sig *.asc *.gpg *.zip *.tar *.gz *.bz2 *.tmpl
+global-exclude mayan/settings/local.py *.po
+prune mayan/apps/*/static/*/node_modules/*
+prune mayan/settings/travis/*
+prune mayan/media/*

Makefile

@@ -0,0 +1,313 @@
+.PHONY: clean-pyc clean-build
+
+help:
+	@echo "Usage: make <target>\n"
+	@awk 'BEGIN {FS = ":.*##"} /^[a-zA-Z_-]+:.*?## / { printf "  * %-40s -%s\n", $$1, $$2 }' $(MAKEFILE_LIST)|sort
+
+# Cleaning
+
+clean: ## Remove Python and build artifacts.
+clean: clean-build clean-pyc
+
+clean-build: ## Remove build artifacts.
+	rm -fr build/
+	rm -fr dist/
+	rm -fr *.egg-info
+
+clean-pyc: ## Remove Python artifacts.
+	find . -name '*.pyc' -exec rm -f {} +
+	find . -name '*.pyo' -exec rm -f {} +
+	find . -name '*~' -exec rm -f {} +
+
+
+# Testing
+
+test: clean-pyc
+test: ## MODULE=<python module name> - Run tests for a single app, module or test class.
+	./manage.py test $(MODULE) --settings=mayan.settings.testing.development --nomigrations $(ARGUMENTS)
+
+test-all: ## Run all tests.
+test-all: clean-pyc
+	./manage.py test --mayan-apps --settings=mayan.settings.testing.development --nomigrations $(ARGUMENTS)
+
+test-launch-postgres:
+	@docker rm -f test-postgres || true
+	@docker volume rm test-postgres || true
+	docker run -d --name test-postgres -p 5432:5432 -v test-postgres:/var/lib/postgresql/data healthcheck/postgres
+	sudo apt-get install -q libpq-dev
+	pip install psycopg2
+	while ! docker inspect --format='{{json .State.Health}}' test-postgres|grep 'Status":"healthy"'; do sleep 1; done
+
+test-with-postgres: ## MODULE=<python module name> - Run tests for a single app, module or test class against a Postgres database container.
+test-with-postgres: test-launch-postgres
+	./manage.py test $(MODULE) --settings=mayan.settings.testing.docker.db_postgres --nomigrations
+	@docker rm -f test-postgres || true
+	@docker volume rm test-postgres || true
+
+test-with-postgres-all: ## Run all tests against a Postgres database container.
+test-with-postgres-all: test-launch-postgres
+	./manage.py test --mayan-apps --settings=mayan.settings.testing.docker.db_postgres --nomigrations
+	@docker rm -f test-postgres || true
+	@docker volume rm test-postgres || true
+
+test-launch-mysql:
+	@docker rm -f test-mysql || true
+	@docker volume rm test-mysql || true
+	docker run -d --name test-mysql -p 3306:3306 -e MYSQL_ALLOW_EMPTY_PASSWORD=True -e MYSQL_DATABASE=mayan -v test-mysql:/var/lib/mysql healthcheck/mysql
+	sudo apt-get install -q libmysqlclient-dev mysql-client
+	pip install mysqlclient
+	while ! docker inspect --format='{{json .State.Health}}' test-mysql|grep 'Status":"healthy"'; do sleep 1; done
+	mysql -h 127.0.0.1 -P 3306 -uroot  -e "set global character_set_server=utf8mb4;"
+
+test-with-mysql: ## MODULE=<python module name> - Run tests for a single app, module or test class against a MySQL database container.
+test-with-mysql: test-launch-mysql
+	./manage.py test $(MODULE) --settings=mayan.settings.testing.docker.db_mysql --nomigrations
+	@docker rm -f test-mysql || true
+	@docker volume rm test-mysql || true
+
+
+test-with-mysql-all: ## Run all tests against a MySQL database container.
+test-with-mysql-all: test-launch-mysql
+	./manage.py test --mayan-apps --settings=mayan.settings.testing.docker.db_mysql --nomigrations
+	@docker rm -f test-mysql || true
+	@docker volume rm test-mysql || true
+
+test-launch-oracle:
+	@docker rm -f test-oracle || true
+	@docker volume rm test-oracle || true
+	docker run -d --name test-oracle -p 49160:22 -p 49161:1521 -e ORACLE_ALLOW_REMOTE=true -v test-oracle:/u01/app/oracle wnameless/oracle-xe-11g
+	# https://gist.github.com/kimus/10012910
+	pip install cx_Oracle
+	while ! nc -z 127.0.0.1 49161; do sleep 1; done
+	sleep 10
+
+test-with-oracle: ## MODULE=<python module name> - Run tests for a single app, module or test class against a Oracle database container.
+test-with-oracle: test-launch-oracle
+	./manage.py test $(MODULE) --settings=mayan.settings.testing.docker.db_oracle --nomigrations
+	@docker rm -f test-oracle || true
+	@docker volume rm test-oracle || true
+
+test-with-oracle-all: ## Run all tests against a Oracle database container.
+test-with-oracle-all: test-launch-oracle
+	./manage.py test --mayan-apps --settings=mayan.settings.testing.docker.db_oracle --nomigrations
+	@docker rm -f test-oracle || true
+	@docker volume rm test-oracle || true
+
+# Documentation
+
+docs-serve: ## Run the livehtml documentation generator.
+	cd docs;make livehtml
+
+docs-spellcheck: ## Spellcheck the documentation.
+	sphinx-build -b spelling -d docs/_build/ docs docs/_build/spelling
+
+# Translations
+
+translations-make: ## Refresh all translation files.
+	contrib/scripts/process_messages.py -m
+
+translations-compile: ## Compile all translation files.
+	contrib/scripts/process_messages.py -c
+
+translations-push: ## Upload all translation files to Transifex.
+	tx push -s
+
+translations-pull: ## Download all translation files from Transifex.
+	tx pull -f
+
+# Releases
+
+increase-version: ## Increase the version number of the entire project's files.
+	@VERSION=`grep "__version__ =" mayan/__init__.py| cut -d\' -f 2|./increase_version.py - $(PART)`; \
+	BUILD=`echo $$VERSION|awk '{split($$VERSION,a,"."); printf("0x%02d%02d%02d\n", a[1],a[2], a[3])}'`; \
+	sed -i -e "s/__build__ = 0x[0-9]*/__build__ = $${BUILD}/g" mayan/__init__.py; \
+	sed -i -e "s/__version__ = '[0-9\.]*'/__version__ = '$${VERSION}'/g" mayan/__init__.py; \
+	echo $$VERSION > docker/rootfs/version
+	make generate-setup
+
+python-test-release: ## Package (sdist and wheel) and upload to the PyPI test server.
+python-test-release: clean wheel
+	twine upload dist/* -r testpypi
+	@echo "Test with: pip install -i https://testpypi.python.org/pypi mayan-edms"
+
+python-release: ## Package (sdist and wheel) and upload a release.
+python-release: clean python-wheel
+	twine upload dist/* -r pypi
+
+python-sdist: ## Build the source distribution package.
+python-sdist: clean
+	python setup.py sdist
+	ls -l dist
+
+python-wheel: ## Build the wheel distribution package.
+python-wheel: clean python-sdist
+	pip wheel --no-index --no-deps --wheel-dir dist dist/*.tar.gz
+	ls -l dist
+
+python-release-test-via-docker-ubuntu: ## Package (sdist and wheel) and upload to the PyPI test server using an Ubuntu Docker builder.
+	docker run --rm --name mayan_release -v $(HOME):/host_home:ro -v `pwd`:/host_source -w /source ubuntu:16.04 /bin/bash -c "\
+	echo "LC_ALL=\"en_US.UTF-8\"" >> /etc/default/locale && \
+	locale-gen en_US.UTF-8 && \
+	update-locale LANG=en_US.UTF-8 && \
+	export LC_ALL=en_US.UTF-8 && \
+	cp -r /host_source/* . && \
+	apt-get update && \
+	apt-get install make python-pip -y && \
+	pip install -r requirements/build.txt && \
+	cp -r /host_home/.pypirc ~/.pypirc && \
+	make test-release"
+
+python-release-via-docker-ubuntu: ## Package (sdist and wheel) and upload to PyPI using an Ubuntu Docker builder.
+	docker run --rm --name mayan_release -v $(HOME):/host_home:ro -v `pwd`:/host_source -w /source ubuntu:16.04 /bin/bash -c "\
+	apt-get update && \
+	apt-get -y install locales && \
+	echo "LC_ALL=\"en_US.UTF-8\"" >> /etc/default/locale && \
+	locale-gen en_US.UTF-8 && \
+	update-locale LANG=en_US.UTF-8 && \
+	export LC_ALL=en_US.UTF-8 && \
+	cp -r /host_source/* . && \
+	apt-get install make python-pip -y && \
+	pip install -r requirements/build.txt && \
+	cp -r /host_home/.pypirc ~/.pypirc && \
+	make release"
+
+test-sdist-via-docker-ubuntu: ## Make an sdist package and test it using an Ubuntu Docker container.
+	docker run --rm --name mayan_sdist_test -v $(HOME):/host_home:ro -v `pwd`:/host_source -w /source ubuntu:16.04 /bin/bash -c "\
+	cp -r /host_source/* . && \
+	echo "LC_ALL=\"en_US.UTF-8\"" >> /etc/default/locale && \
+	locale-gen en_US.UTF-8 && \
+	update-locale LANG=en_US.UTF-8 && \
+	export LC_ALL=en_US.UTF-8 && \
+	apt-get update && \
+	apt-get install make python-pip libreoffice tesseract-ocr tesseract-ocr-deu poppler-utils -y && \
+	pip install -r requirements/development.txt && \
+	make sdist-test-suit \
+	"
+
+test-wheel-via-docker-ubuntu: ## Make a wheel package and test it using an Ubuntu Docker container.
+	docker run --rm --name mayan_wheel_test -v $(HOME):/host_home:ro -v `pwd`:/host_source -w /source ubuntu:16.04 /bin/bash -c "\
+	cp -r /host_source/* . && \
+	echo "LC_ALL=\"en_US.UTF-8\"" >> /etc/default/locale && \
+	locale-gen en_US.UTF-8 && \
+	update-locale LANG=en_US.UTF-8 && \
+	export LC_ALL=en_US.UTF-8 && \
+	apt-get update && \
+	apt-get install make python-pip libreoffice tesseract-ocr tesseract-ocr-deu poppler-utils -y && \
+	pip install -r requirements/development.txt && \
+	make wheel-test-suit \
+	"
+
+python-sdist-test-suit: sdist
+	rm -f -R _virtualenv
+	virtualenv _virtualenv
+	sh -c '\
+	. _virtualenv/bin/activate; \
+	pip install `ls dist/*.gz`; \
+	_virtualenv/bin/mayan-edms.py initialsetup; \
+	pip install mock==2.0.0; \
+	_virtualenv/bin/mayan-edms.py test --mayan-apps \
+	'
+
+python-wheel-test-suit: wheel
+	rm -f -R _virtualenv
+	virtualenv _virtualenv
+	sh -c '\
+	. _virtualenv/bin/activate; \
+	pip install `ls dist/*.whl`; \
+	_virtualenv/bin/mayan-edms.py initialsetup; \
+	pip install mock==2.0.0; \
+	_virtualenv/bin/mayan-edms.py test --mayan-apps \
+	'
+
+generate-setup: ## Create and update the setup.py file.
+generate-setup: generate-requirements
+	@./generate_setup.py
+	@echo "Complete."
+
+generate-requirements: ## Generate all requirements files from the project depedency declarations.
+	@./manage.py generaterequirements build > requirements/build.txt
+	@./manage.py generaterequirements development > requirements/development.txt
+	@./manage.py generaterequirements testing > requirements/testing-base.txt
+	@./manage.py generaterequirements production --exclude=django > requirements/base.txt
+	@./manage.py generaterequirements production --only=django > requirements/common.txt
+
+# Dev server
+
+runserver: ## Run the development server.
+	./manage.py runserver --settings=mayan.settings.development $(ADDRPORT)
+
+runserver_plus: ## Run the Django extension's development server.
+	./manage.py runserver_plus --settings=mayan.settings.development $(ADDRPORT)
+
+shell_plus: ## Run the shell_plus command.
+	./manage.py shell_plus --settings=mayan.settings.development
+
+test-with-docker-services-on: ## Launch and initialize production-like services using Docker (Postgres and Redis).
+	docker run -d --name redis -p 6379:6379 redis
+	docker run -d --name postgres -p 5432:5432 postgres
+	while ! nc -z 127.0.0.1 6379; do sleep 1; done
+	while ! nc -z 127.0.0.1 5432; do sleep 1; done
+	sleep 4
+	./manage.py initialsetup --settings=mayan.settings.staging.docker
+
+test-with-docker-services-off: ## Stop and delete the Docker production-like services.
+	docker stop postgres redis
+	docker rm postgres redis
+
+test-with-docker-frontend: ## Launch a front end instance that uses the production-like services.
+	./manage.py runserver --settings=mayan.settings.staging.docker
+
+test-with-docker-worker: ## Launch a worker instance that uses the production-like services.
+	./manage.py celery worker --settings=mayan.settings.staging.docker -B -l INFO -O fair
+
+docker-mysql-on: ## Launch and initialize a MySQL Docker container.
+	docker run -d --name mysql -p 3306:3306 -e MYSQL_ALLOW_EMPTY_PASSWORD=True -e MYSQL_DATABASE=mayan_edms mysql
+	while ! nc -z 127.0.0.1 3306; do sleep 1; done
+
+docker-mysql-off: ## Stop and delete the MySQL Docker container.
+	docker stop mysql
+	docker rm mysql
+
+docker-postgres-on: ## Launch and initialize a PostgreSQL Docker container.
+	docker run -d --name postgres -p 5432:5432 postgres
+	while ! nc -z 127.0.0.1 5432; do sleep 1; done
+
+docker-postgres-off: ## Stop and delete the PostgreSQL Docker container.
+	docker stop postgres
+	docker rm postgres
+
+
+# Security
+
+safety-check: ## Run a package safety check.
+	safety check
+
+
+# Other
+find-gitignores: ## Find stray .gitignore files.
+	@export FIND_GITIGNORES=`find -name '.gitignore'| wc -l`; \
+	if [ $${FIND_GITIGNORES} -gt 1 ] ;then echo "More than one .gitignore found."; fi
+
+python-build:
+	docker rm -f mayan-edms-build || true && \
+	docker run --rm --name mayan-edms-build -v $(HOME):/host_home:ro -v `pwd`:/host_source -w /source python:2-slim sh -c "\
+	rm /host_source/dist -R || true && \
+	mkdir /host_source/dist || true && \
+	export LC_ALL=C.UTF-8 && \
+	cp -r /host_source/* . && \
+	apt-get update && \
+	apt-get install -y make && \
+	pip install -r requirements/build.txt && \
+	make wheel && \
+	cp dist/* /host_source/dist/"
+
+check-readme: ## Checks validity of the README.rst file for PyPI publication.
+	python setup.py check -r -s
+
+check-missing-migrations: ## Make sure all models have proper migrations.
+	./manage.py makemigrations --dry-run --noinput --check
+
+setup-dev-environment: ## Bootstrap a virtualenv by install all dependencies to start developing.
+	pip install -r requirements.txt -r requirements/development.txt -r requirements/testing-base.txt -r requirements/documentation.txt -r requirements/build.txt
+
+-include docker/Makefile

README.md

@@ -0,0 +1,72 @@
+[![pypi][pypi]][pypi-url]
+[![builds][builds]][builds-url]
+[![coverage][cover]][cover-url]
+![python][python]
+![license][license]
+[![Docker pulls](https://img.shields.io/docker/pulls/mayanedms/mayanedms.svg?maxAge=3600)](https://hub.docker.com/r/mayanedms/mayanedms/)
+[![Docker Stars](https://img.shields.io/docker/stars/mayanedms/mayanedms.svg?maxAge=3600)](https://hub.docker.com/r/mayanedms/mayanedms/)
+[![Docker layers](https://images.microbadger.com/badges/image/mayanedms/mayanedms.svg)](https://microbadger.com/images/mayanedms/mayanedms)
+
+[pypi]: http://img.shields.io/pypi/v/mayan-edms.svg
+[pypi-url]: http://badge.fury.io/py/mayan-edms
+
+[builds]: https://gitlab.com/mayan-edms/mayan-edms/badges/master/build.svg
+[builds-url]: https://gitlab.com/mayan-edms/mayan-edms/pipelines
+
+[cover]: https://codecov.io/gitlab/mayan-edms/mayan-edms/coverage.svg?branch=master
+[cover-url]: https://codecov.io/gitlab/mayan-edms/mayan-edms?branch=master
+
+[python]: https://img.shields.io/pypi/pyversions/mayan-edms.svg
+[python-url]: https://img.shields.io/pypi/l/mayan-edms.svg?style=flat
+
+[license]: https://img.shields.io/pypi/l/mayan-edms.svg?style=flat
+[license-url]: https://img.shields.io/pypi/l/mayan-edms.svg?style=flat
+
+
+<div align="center">
+  <a href="http://www.mayan-edms.com">
+    <img width="200" heigth="200" src="https://gitlab.com/mayan-edms/mayan-edms/raw/master/docs/_static/mayan_logo.png">
+  </a>
+  <br>
+  <br>
+  <p>
+    Mayan EDMS is a document management system. Its main purpose is to store,
+    introspect, and categorize files, with a strong emphasis on preserving the
+    contextual and business information of documents. It can also OCR, preview,
+    label, sign, send, and receive thoses files. Other features of interest
+    are its workflow system, role based access control, and REST API.
+  <p>
+
+<p align="center">
+    <img width="400" src="https://gitlab.com/mayan-edms/mayan-edms/raw/master/docs/_static/overview.gif">
+</p>
+
+</div>
+
+<h2 align="center">Installation</h2>
+
+The easiest way to use Mayan EDMS is by using the official
+[Docker](https://www.docker.com/) image. Make sure Docker is properly installed
+and working before attempting to install Mayan EDMS.
+
+For the complete set of installation instructions visit the Mayan EDMS documentation
+at: https://docs.mayan-edms.com/topics/installation.html
+
+<h2 align="center">Hardware requirements</h2>
+
+- 2 Gigabytes of RAM (1 Gigabyte if OCR is turned off).
+- Multiple core CPU (64 bit, faster than 1 GHz recommended).
+
+<h2 align="center">Important links</h2>
+
+
+- [Homepage](http://www.mayan-edms.com)
+- [Documentation](https://docs.mayan-edms.com)
+- [Wiki](https://wiki.mayan-edms.com/)
+- [Contributing](https://gitlab.com/mayan-edms/mayan-edms/blob/master/CONTRIBUTING.md)
+- [Forum](https://forum.mayan-edms.com/)
+- [Source code, issues, bugs](https://gitlab.com/mayan-edms/mayan-edms)
+- [Plug-ins, other related projects](https://gitlab.com/mayan-edms/)
+- [Translations](https://www.transifex.com/rosarior/mayan-edms/)
+- [Videos](https://www.youtube.com/channel/UCJOOXHP1MJ9lVA7d8ZTlHPw)
+

README.rst

@@ -1,79 +1,60 @@
-|Build Status| |Coverage badge| |PyPI badge| |Installs badge| |License badge|
-
-|Logo|
-
-Description
------------
-
-Free Open Source Electronic Document Management System.
-
-`Website`_
-
-`Video demostration`_
-
-`Documentation`_
-
-`Translations`_
-
-`Mailing list (via Google Groups)`_
-
-|Animation|
-
-Support
--------
-Help support further improvements and development by donating at: https://www.patreon.com/siloraptor
-
-License
--------
-
-This project is open sourced under `Apache 2.0 License`_.
-
-Installation
-------------
-
-To install **Mayan EDMS**, simply do:
-
-.. code-block:: bash
-
-    $ virtualenv venv
-    $ source venv/bin/activate
-    $ pip install mayan-edms
-    $ mayan-edms.py initialsetup
-    $ mayan-edms.py runserver
-
-Point your browser to 127.0.0.1:8000 and use the automatically created admin
-account.
-
-Contribute
-----------
-
-- Check for open issues or open a fresh issue to start a discussion around a feature idea or a bug.
-- Fork `the repository`_ on GitLab to start making your changes to the **development** branch (or branch off of it).
-- Write a test which shows that the bug was fixed or that the feature works as expected.
-- Add yourself to the `contributors file`_.
-- Send a merge request.
+|pypi| |builds| |coverage| |python| |license| |docker_pulls| |docker_stars| |docker_layers|
 
 
-.. _Website: http://www.mayan-edms.com
-.. _Video demostration: http://bit.ly/pADNXv
-.. _Documentation: http://readthedocs.org/docs/mayan/en/latest/
-.. _Translations: https://www.transifex.com/projects/p/mayan-edms/
-.. _Mailing list (via Google Groups): http://groups.google.com/group/mayan-edms
-.. _Apache 2.0 License: https://www.apache.org/licenses/LICENSE-2.0.txt
-.. _`the repository`: http://gitlab.com/mayan-edms/mayan-edms
-.. _`contributors file`: https://gitlab.com/mayan-edms/mayan-edms/blob/master/docs/topics/contributors.rst
+.. image:: https://gitlab.com/mayan-edms/mayan-edms/raw/master/docs/_static/mayan_logo.png
+    :align: center
+    :width: 200
+    :height: 200
 
-.. |Build Status| image:: https://gitlab.com/ci/projects/6169/status.png?ref=master
-   :target: https://gitlab.com/ci/projects/6169?ref=master
-.. |Logo| image:: https://gitlab.com/mayan-edms/mayan-edms/raw/master/docs/_static/mayan_logo.png
-.. |Animation| image:: https://gitlab.com/mayan-edms/mayan-edms/raw/master/docs/_static/overview.gif
-.. |Installs badge| image:: http://img.shields.io/pypi/dm/mayan-edms.svg?style=flat
-   :target: https://crate.io/packages/mayan-edms/
-.. |PyPI badge| image:: http://img.shields.io/pypi/v/mayan-edms.svg?style=flat
+Mayan EDMS is a document management system. Its main purpose is to store,
+introspect, and categorize files, with a strong emphasis on preserving the
+contextual and business information of documents. It can also OCR, preview,
+label, sign, send, and receive thoses files. Other features of interest
+are its workflow system, role based access control, and REST API.
+
+.. image:: https://gitlab.com/mayan-edms/mayan-edms/raw/master/docs/_static/overview.gif
+    :align: center
+    :width: 300
+
+The easiest way to use Mayan EDMS is by using the official Docker_ image.
+Make sure Docker is properly installed and working before attempting to install
+Mayan EDMS.
+
+For the complete set of installation instructions visit the Mayan EDMS documentation
+at: https://docs.mayan-edms.com/topics/installation.html
+
+.. _Docker: https://www.docker.com/
+
+Hardware requirements
+
+- 2 Gigabytes of RAM (1 Gigabyte if OCR is turned off).
+- Multiple core CPU (64 bit, faster than 1 GHz recommended).
+
+
+Important links
+
+- `Homepage <http://www.mayan-edms.com>`__
+- `Documentation <https://docs.mayan-edms.com>`__
+- `Wiki <https://wiki.mayan-edms.com>`__
+- `Contributing <https://gitlab.com/mayan-edms/mayan-edms/blob/master/CONTRIBUTING.md>`__
+- `Forum <https://forum.mayan-edms.com>`__
+- `Source code, issues, bugs <https://gitlab.com/mayan-edms/mayan-edms>`__
+- `Plug-ins, other related projects <https://gitlab.com/mayan-edms/>`__
+- `Translations <https://www.transifex.com/rosarior/mayan-edms/>`__
+- `Videos <https://www.youtube.com/channel/UCJOOXHP1MJ9lVA7d8ZTlHPw>`__
+
+
+.. |pypi| image:: http://img.shields.io/pypi/v/mayan-edms.svg
    :target: http://badge.fury.io/py/mayan-edms
-.. |License badge| image:: http://img.shields.io/badge/license-Apache%202.0-green.svg?style=flat
-.. |Analytics| image:: https://ga-beacon.appspot.com/UA-52965619-2/mayan-edms/readme?pixel
-.. |Coverage badge| image:: https://codecov.io/gitlab/mayan-edms/mayan-edms/coverage.svg?branch=master
+.. |builds| image:: https://gitlab.com/mayan-edms/mayan-edms/badges/master/build.svg
+   :target: https://gitlab.com/mayan-edms/mayan-edms/pipelines
+.. |coverage| image:: https://codecov.io/gitlab/mayan-edms/mayan-edms/coverage.svg?branch=master
    :target: https://codecov.io/gitlab/mayan-edms/mayan-edms?branch=master
-
-|Analytics|
+.. |python| image:: https://img.shields.io/pypi/pyversions/mayan-edms.svg
+.. |license| image:: https://img.shields.io/pypi/l/mayan-edms.svg?style=flat
+.. |docker_pulls| image:: https://img.shields.io/docker/pulls/mayanedms/mayanedms.svg?maxAge=3600
+   :target: https://hub.docker.com/r/mayanedms/mayanedms/
+.. |docker_stars| image:: https://img.shields.io/docker/stars/mayanedms/mayanedms.svg?maxAge=3600
+   :target: https://hub.docker.com/r/mayanedms/mayanedms/
+.. |docker_layers| image:: https://images.microbadger.com/badges/image/mayanedms/mayanedms.svg
+   :target: https://microbadger.com/images/mayanedms/mayanedms

Vagrantfile

@@ -1,33 +0,0 @@
-# -*- mode: ruby -*-
-# vi: set ft=ruby :
-
-VAGRANTFILE_API_VERSION = "2"
-
-Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
-  config.vm.box = "ubuntu/trusty64"
-
-  config.vm.provider :lxc do |v, override|
-    override.vm.box = "fgrehm/trusty64-lxc"
-  end
-
-  config.vm.provider :virtualbox do |vb|
-    vb.customize ["modifyvm", :id, "--memory", "1024"]
-  end
-
-  # Development box
-  config.vm.define "development", autostart: false do |development|
-    development.vm.network "forwarded_port", guest: 8000, host: 8000
-    development.vm.synced_folder ".", "/mayan-edms-repository"
-    development.vm.provision :shell, :path => "contrib/scripts/install/development.sh", privileged: false
-
-    development.vm.provision "file", destination: "/home/vagrant/mayan-edms/mayan/settings/celery_redis.py", source: "contrib/settings/celery_redis.py"
-    development.vm.provision "file", destination: "/home/vagrant/mayan-edms/mayan_edms_worker.sh", source: "contrib/misc/mayan_edms_worker.sh"
-  end
-
-  # Production box
-  config.vm.define "production", autostart: false do |production|
-    production.vm.network "forwarded_port", guest: 80, host: 8080
-    production.vm.provision :shell, :path => "contrib/scripts/install/production.sh", privileged: true
-  end
-
-end

__init__.py.tmpl

@@ -0,0 +1,14 @@
+from __future__ import unicode_literals
+
+__title__ = 'Mayan EDMS'
+__version__ = '{{ version }}'
+__build__ = {{ build }}
+__build_string__ = '{{ build_string }}'
+__django_version__ = '1.11'
+__author__ = 'Roberto Rosario'
+__author_email__ = 'roberto.rosario@mayan-edms.com'
+__description__ = 'Free Open Source Electronic Document Management System'
+__license__ = 'Apache 2.0'
+__copyright_short__ = '2011 Roberto Rosario'
+__copyright__ = '{} {}'.format('Copyright', __copyright_short__)
+__website__ = 'https://www.mayan-edms.com'

contrib/misc/mayan_edms_worker.sh

@@ -1,3 +0,0 @@
-#!/bin/sh
-
-DJANGO_SETTINGS_MODULE='mayan.settings.celery_redis' celery -A mayan worker -l DEBUG -Ofair -B

contrib/nginx/mayan

@@ -1,22 +0,0 @@
-server {
-    listen 80;
-    server_name localhost;
-
-    location / {
-        include uwsgi_params;
-        uwsgi_pass unix:/usr/share/mayan-edms/uwsgi.sock;
-
-        client_max_body_size 30M;  # Increse if your plan to upload bigger documents
-        proxy_read_timeout 30s;  # Increase if your document uploads take more than 30 seconds
-    }
-
-    location /static {
-        alias /usr/share/mayan-edms/mayan/media/static;
-        expires 1h;
-    }
-
-    location /favicon.ico {
-        alias /usr/share/mayan-edms/mayan/media/static/appearance/images/favicon.ico;
-        expires 1h;
-    }
-}

contrib/scripts/PEP8_check.sh

@@ -1 +0,0 @@
-flake8 --ignore=E501 mayan/apps  | grep -v "F401 'models' imported but unused" | grep -v "F811 redefinition of unused 'models'"| grep -v "F401 'db' imported but unused" | grep -v "F812 list comprehension redefines 'user_id'" | grep -v "F812 list comprehension redefines 'document_id'" |less

contrib/scripts/docker/run-tests.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+export DEBIAN_FRONTEND=noninteractive
+
+apt-get update
+apt-get install -y --no-install-recommends tesseract-ocr-deu
+
+$MAYAN_PIP_BIN install -r ${MAYAN_INSTALL_DIR}/requirements-testing.txt
+
+$MAYAN_BIN test --mayan-apps --settings=mayan.settings.testing

contrib/scripts/install/development.sh

@@ -18,7 +18,7 @@ sudo apt-get -qq update
 sudo apt-get -y upgrade
 
 echo -e "\n -> Installing core binaries \n"
-sudo apt-get -y install git-core python-virtualenv gcc python-dev libjpeg-dev libpng-dev libtiff-dev tesseract-ocr poppler-utils unpaper libreoffice
+sudo apt-get -y install git-core python-virtualenv gcc python-dev libjpeg-dev libpng-dev libtiff-dev tesseract-ocr poppler-utils libreoffice
 
 echo -e "\n -> Cloning development branch of repository \n"
 git clone /mayan-edms-repository/ $INSTALLATION_DIRECTORY

contrib/scripts/install/docker.sh

@@ -0,0 +1,131 @@
+#!/bin/sh
+set -e
+
+# This script is meant for quick & easy install via:
+#   $ curl -fsSL get.mayan-edms.com -o get-mayan-edms.sh
+#   $ sh get-mayan-edms.sh
+#
+# NOTE: Make sure to verify the contents of the script
+#       you downloaded matches the contents of docker.sh
+#       located at https://gitlab.com/mayan-edms/mayan-edms/blob/master/contrib/scripts/install/docker.sh
+#       before executing.
+
+: ${VERBOSE:=true}
+: ${INSTALL_DOCKER:=false}
+: ${DELETE_VOLUMES:=false}
+: ${DATABASE_USER:=mayan}
+: ${DATABASE_NAME:=mayan}
+: ${DATABASE_PASSWORD:=mayanuserpass}
+: ${DOCKER_POSTGRES_IMAGE:=postgres:9.6}
+: ${DOCKER_POSTGRES_CONTAINER:=mayan-edms-postgres}
+: ${DOCKER_POSTGRES_VOLUME:=/docker-volumes/mayan-edms/postgres}
+: ${DOCKER_POSTGRES_PORT:=5432}
+: ${DOCKER_MAYAN_IMAGE:=mayanedms/mayanedms:latest}
+: ${DOCKER_MAYAN_CONTAINER:=mayan-edms}
+: ${DOCKER_MAYAN_VOLUME:=/docker-volumes/mayan-edms/media}
+
+cat << EOF
+
+███╗   ███╗ █████╗ ██╗   ██╗ █████╗ ███╗   ██╗
+████╗ ████║██╔══██╗╚██╗ ██╔╝██╔══██╗████╗  ██║
+██╔████╔██║███████║ ╚████╔╝ ███████║██╔██╗ ██║
+██║╚██╔╝██║██╔══██║  ╚██╔╝  ██╔══██║██║╚██╗██║
+██║ ╚═╝ ██║██║  ██║   ██║   ██║  ██║██║ ╚████║
+╚═╝     ╚═╝╚═╝  ╚═╝   ╚═╝   ╚═╝  ╚═╝╚═╝  ╚═══╝
+Docker deploy script
+
+NOTE: Make sure to verify the contents of this script
+matches the contents of docker.sh located at https://gitlab.com/mayan-edms/mayan-edms/blob/master/contrib/scripts/install/docker.sh before executing.
+
+EOF
+
+if [ "$VERBOSE" = true ]; then
+echo "Variable values to be used:"
+echo "---------------------------"
+echo "INSTALL_DOCKER: $INSTALL_DOCKER"
+echo "DELETE_VOLUMES: $DELETE_VOLUMES"
+echo "DATABASE_USER: $DATABASE_USER"
+echo "DATABASE_NAME: $DATABASE_NAME"
+echo "DATABASE_PASSWORD: $DATABASE_PASSWORD"
+echo "DOCKER_POSTGRES_IMAGE: $DOCKER_POSTGRES_IMAGE"
+echo "DOCKER_POSTGRES_CONTAINER: $DOCKER_POSTGRES_CONTAINER"
+echo "DOCKER_POSTGRES_VOLUME: $DOCKER_POSTGRES_VOLUME"
+echo "DOCKER_POSTGRES_PORT: $DOCKER_POSTGRES_PORT"
+echo "DOCKER_MAYAN_IMAGE: $DOCKER_MAYAN_IMAGE"
+echo "DOCKER_MAYAN_CONTAINER: $DOCKER_MAYAN_CONTAINER"
+echo "DOCKER_MAYAN_VOLUME: $DOCKER_MAYAN_VOLUME"
+echo "\nStarting in 10 seconds."
+sleep 10
+fi
+
+if [ "$INSTALL_DOCKER" = true ]; then
+    echo -n "* Installing Docker..."
+    curl -fsSL get.docker.com -o get-docker.sh >/dev/null
+    sh get-docker.sh >/dev/null 2>&1
+    rm get-docker.sh
+    echo "Done"
+fi
+
+if [ -z `which docker`  ]; then
+    echo "Docker is not installed. Rerun this script with the variable INSTALL_DOCKER set to true."
+    exit 1
+fi
+
+echo -n "* Removing existing Mayan EDMS and PostgreSQL containers (no data will be lost)..."
+true || docker stop $DOCKER_MAYAN_CONTAINER >/dev/null 2>&1
+true || docker rm $DOCKER_MAYAN_CONTAINER >/dev/null 2>&1
+true || docker stop $DOCKER_POSTGRES_CONTAINER >/dev/null 2>&1
+true || docker rm $DOCKER_POSTGRES_CONTAINER >/dev/null 2>&1
+echo "Done"
+
+if [ "$DELETE_VOLUMES" = true ]; then
+echo -n "* Deleting Docker volumes in 5 seconds (warning: this delete all document data)..."
+sleep 5
+true || rm DOCKER_MAYAN_VOLUME -Rf
+true || rm DOCKER_POSTGRES_VOLUME -Rf
+echo "Done"
+fi
+
+echo -n "* Pulling (downloading) the Mayan EDMS Docker image..."
+docker pull $DOCKER_MAYAN_IMAGE >/dev/null
+echo "Done"
+
+echo -n "* Pulling (downloading) the PostgreSQL Docker image..."
+docker pull $DOCKER_POSTGRES_IMAGE > /dev/null
+echo "Done"
+
+echo -n "* Deploying the PostgreSQL container..."
+docker run -d \
+--name $DOCKER_POSTGRES_CONTAINER \
+--restart=always \
+-p $DOCKER_POSTGRES_PORT:5432 \
+-e POSTGRES_USER=$DATABASE_USER \
+-e POSTGRES_DB=$DATABASE_NAME \
+-e POSTGRES_PASSWORD=$DATABASE_PASSWORD \
+-v $DOCKER_POSTGRES_VOLUME:/var/lib/postgresql/data \
+$DOCKER_POSTGRES_IMAGE >/dev/null
+echo "Done"
+
+echo -n "* Waiting for the PostgreSQL container to be ready (10 seconds)..."
+sleep 10
+echo "Done"
+
+echo -n "* Deploying Mayan EDMS container..."
+docker run -d \
+--name $DOCKER_MAYAN_CONTAINER \
+--restart=always \
+-p 80:8000 \
+-e MAYAN_DATABASE_ENGINE=django.db.backends.postgresql \
+-e MAYAN_DATABASE_HOST=172.17.0.1 \
+-e MAYAN_DATABASE_NAME=$DATABASE_NAME \
+-e MAYAN_DATABASE_PASSWORD=$DATABASE_PASSWORD \
+-e MAYAN_DATABASE_USER=$DATABASE_USER \
+-e MAYAN_DATABASE_PORT=$DOCKER_POSTGRES_PORT \
+-e MAYAN_DATABASE_CONN_MAX_AGE=0 \
+-v $DOCKER_MAYAN_VOLUME:/var/lib/mayan \
+$DOCKER_MAYAN_IMAGE >/dev/null
+echo "Done"
+
+echo -n "* Waiting for the Mayan EDMS container to be ready (might take a few minutes)..."
+while ! curl --output /dev/null --silent --head --fail http://localhost:80; do sleep 1 && echo -n .; done;
+echo "Done"

contrib/scripts/install/production.sh

@@ -20,7 +20,7 @@ apt-get -qq update
 apt-get -y upgrade
 
 echo -e "\n -> Installing core binaries \n"
-apt-get install nginx supervisor redis-server postgresql libpq-dev libjpeg-dev libmagic1 libpng-dev libreoffice libtiff-dev gcc ghostscript gpgv python-dev python-virtualenv tesseract-ocr unpaper poppler-utils -y
+apt-get install nginx supervisor redis-server postgresql libpq-dev libjpeg-dev libmagic1 libpng-dev libreoffice libtiff-dev gcc ghostscript gpgv python-dev python-virtualenv tesseract-ocr poppler-utils -y
 
 echo -e "\n -> Setting up virtualenv \n"
 rm -f ${INSTALLATION_DIRECTORY}
@@ -161,7 +161,7 @@ priority = 998
 EOF
 
 echo -e "\n -> Collecting the static files \n"
-mayan-edms.py collectstatic --noinput
+mayan-edms.py preparestatic --noinput
 
 echo -e "\n -> Making the installation directory readable and writable by the webserver user \n"
 chown www-data:www-data ${INSTALLATION_DIRECTORY} -R

contrib/scripts/make_dists.sh

@@ -1,4 +0,0 @@
-#!/bin/sh
-
-python setup.py sdist
-pip wheel --no-index --no-deps --wheel-dir dist dist/*.tar.gz

contrib/scripts/make_fabfile_tar.sh

@@ -1,3 +0,0 @@
-#!/bin/sh
-find 'fabfile' -depth -name '*.pyc' -exec rm {} \;
-tar -czvf contrib/fabfile.tar.gz fabfile

contrib/scripts/process_messages.py

@@ -1,26 +1,30 @@
 #!/usr/bin/env python
+
+from __future__ import print_function
+
 import os
 import optparse
 
 import sh
 
+
 APP_LIST = (
-    'acls', 'appearance', 'authentication', 'checkouts', 'common',
-    'converter', 'django_gpg', 'document_comments', 'document_indexing',
-    'document_signatures', 'document_states', 'documents', 'dynamic_search',
-    'events', 'folders', 'installation', 'linking', 'lock_manager', 'mailer',
-    'metadata', 'mirroring', 'navigation', 'ocr', 'permissions', 'rest_api',
-    'smart_settings', 'sources', 'statistics', 'storage', 'tags',
+    'acls', 'appearance', 'authentication', 'autoadmin', 'cabinets',
+    'checkouts', 'common', 'converter', 'dashboards', 'dependencies',
+    'django_gpg', 'document_comments', 'document_indexing',
+    'document_parsing', 'document_signatures', 'document_states',
+    'documents', 'dynamic_search', 'events', 'file_metadata', 'linking',
+    'lock_manager', 'mayan_statistics', 'mailer', 'metadata', 'mirroring',
+    'motd', 'navigation', 'ocr', 'permissions', 'platform', 'rest_api',
+    'smart_settings', 'sources', 'storage', 'tags', 'task_manager',
     'user_management'
 )
-LANGUAGE_LIST = (
-    'ar', 'bg', 'bs_BA', 'da', 'de_DE', 'en', 'es', 'fa', 'fr', 'hu', 'id',
-    'it', 'nl_NL', 'pl', 'pt', 'pt_BR', 'ro_RO', 'ru', 'sl_SI', 'vi_VN',
-    'zh_CN',
-)
 
-# Inactive translations
-# 'de_CH', 'hr_HR', 'lv', 'nb', 'sq', 'tr_TR', 'zh_TW'
+LANGUAGE_LIST = (
+    'ar', 'bg', 'bs_BA', 'cs', 'da_DK', 'de_DE', 'en', 'es', 'el', 'fa', 'fr',
+    'hu', 'id', 'it', 'lv', 'nl_NL', 'pl', 'pt', 'pt_BR', 'ro_RO', 'ru', 'sl_SI',
+    'tr_TR', 'vi_VN', 'zh',
+)
 
 makemessages = sh.Command('django-admin.py')
 makemessages = makemessages.bake('makemessages')
@@ -39,29 +43,29 @@ BASE_DIR = os.path.abspath(
 
 def process(command, app_list, language_list):
     if command == makemessages:
-        print 'Making messages'
+        print('Making messages')
     elif command == compilemessages:
-        print 'Compiling messages'
+        print('Compiling messages')
     elif command == pull_translations:
-        print 'Pulling translation files'
+        print('Pulling translation files')
     elif command == push_translations:
-        print 'Pushing translation files'
+        print('Pushing translation files')
 
     if command in [compilemessages, makemessages]:
         for app in app_list:
-            print 'Processing app: %s...' % app
+            print('Processing app: %s...' % app)
             app_path = os.path.join(BASE_DIR, 'apps', app)
             os.chdir(app_path)
             for lang in language_list:
-                print 'Doing language: %s' % lang
+                print('Doing language: %s' % lang)
                 command(locale=lang)
     elif command == pull_translations:
         for lang in language_list:
-            print 'Doing language: %s' % lang
+            print('Doing language: %s' % lang)
             command('-f', '-l', lang)
     elif command == push_translations:
         for lang in language_list:
-            print 'Doing language: %s' % lang
+            print('Doing language: %s' % lang)
             command('-s', '-l', lang)
 
 

contrib/scripts/statistics.py

@@ -0,0 +1,105 @@
+#!/usr/bin/env python
+
+from __future__ import print_function
+
+import os
+
+BASE_PATH = 'mayan/apps'
+
+
+def print_views_summary(module_filename):
+    with open(module_filename) as file_object:
+        print('    module:', module_filename)
+        count_class_based_views = 0
+        count_function_based_views = 0
+        for line in file_object:
+            if line.startswith('class') and 'View' in line:
+                count_class_based_views += 1
+
+            if line.startswith('def') and 'request' in line:
+                count_function_based_views += 1
+
+    print('      class based views: {}'.format(count_class_based_views))
+    print('      function based views: {}'.format(count_function_based_views))
+    return count_class_based_views, count_function_based_views
+
+
+def print_tests_summary(module_filename):
+    with open(module_filename) as file_object:
+        print('    module:', module_filename)
+        count_tests = 0
+        for line in file_object:
+            if line.startswith('    def test'):
+                count_tests += 1
+
+    print('      tests: {}'.format(count_tests))
+    return count_tests
+
+
+if __name__ == '__main__':
+    count_totals = {
+        'Apps': 0,
+        'Class based views': 0,
+        'Function based views': 0,
+        'Class based API views': 0,
+        'Function based API views': 0,
+        'Tests': 0,
+    }
+
+    for app_name in sorted(os.listdir(BASE_PATH)):
+        if app_name != '__init__.py':
+            count_totals['Apps'] += 1
+            print('\n\nApp name: {}'.format(app_name))
+            app_path = os.path.join(BASE_PATH, app_name)
+
+            print('\n  Views')
+            try:
+                module_filename = os.path.join(app_path, 'views.py')
+                count_class_based_views, count_function_based_views = print_views_summary(module_filename=module_filename)
+                count_totals['Class based views'] += count_class_based_views
+                count_totals['Function based views'] += count_function_based_views
+
+            except IOError:
+                # Check for multiple view files inside a view directory
+                try:
+                    module_path = os.path.join(app_path, 'views')
+                    for module_name in os.listdir(module_path):
+                        if not module_name.startswith('__init__.py') and not module_name.endswith('.pyc'):
+                            module_filename = os.path.join(module_path, module_name)
+                            count_class_based_views, count_function_based_views = print_views_summary(module_filename=module_filename)
+                            count_totals['Class based views'] += count_class_based_views
+                            count_totals['Function based views'] += count_function_based_views
+                except OSError:
+                    # No views directory, skip app
+                    print('    No views')
+
+            print('\n  API Views')
+            try:
+                module_filename = os.path.join(app_path, 'api_views.py')
+                count_class_based_views, count_function_based_views = print_views_summary(module_filename=module_filename)
+                count_totals['Class based API views'] += count_class_based_views
+                count_totals['Function based API views'] += count_function_based_views
+
+            except IOError:
+                # No API views directory, skip app
+                print('    No API views')
+
+            print('\n  Tests')
+            module_path = os.path.join(app_path, 'tests')
+            try:
+                for module_name in os.listdir(module_path):
+                    if not module_name.startswith('__init__.py') and not module_name.endswith('.pyc'):
+                        module_filename = os.path.join(module_path, module_name)
+                        if module_name.startswith('test'):
+                            count_tests = print_tests_summary(module_filename=module_filename)
+                            count_totals['Tests'] += count_tests
+
+            except OSError:
+                # No tests directory, skip app
+                print('    No tests')
+
+    print('-' * 10)
+
+    print('Totals:')
+    for key, value in count_totals.items():
+        print('  {}: {}'.format(key, value))

contrib/settings/celery_redis.py

@@ -1,5 +0,0 @@
-from .local import *
-
-CELERY_ALWAYS_EAGER = False
-BROKER_URL = 'redis://127.0.0.1:6379/0'
-CELERY_RESULT_BACKEND = 'redis://127.0.0.1:6379/0'

contrib/settings/ldap_connection_settings.py

@@ -0,0 +1,64 @@
+from __future__ import absolute_import
+
+import ldap
+from django_auth_ldap.config import LDAPSearch
+
+from .base import *  # NOQA
+from django.contrib.auth import get_user_model
+
+SECRET_KEY = '<your secret key>'
+
+# makes sure this works in Active Directory
+ldap.set_option(ldap.OPT_REFERRALS, 0)
+
+# This is the default, but I like to be explicit.
+AUTH_LDAP_ALWAYS_UPDATE_USER = True
+
+LDAP_USER_AUTO_CREATION = "False"
+LDAP_URL = "ldap://<your ldap server IP>:389/"
+LDAP_BASE_DN = "dc=paramatrix,dc=co,dc=in"
+LDAP_ADDITIONAL_USER_DN = "dc=people"
+LDAP_ADMIN_DN = ""
+LDAP_PASSWORD = ""
+
+AUTH_LDAP_SERVER_URI = LDAP_URL
+AUTH_LDAP_BIND_DN = LDAP_ADMIN_DN
+AUTH_LDAP_BIND_PASSWORD = LDAP_PASSWORD
+
+
+AUTH_LDAP_USER_SEARCH = LDAPSearch(
+    '%s,%s' % (LDAP_ADDITIONAL_USER_DN, LDAP_BASE_DN),
+    ldap.SCOPE_SUBTREE, '(uid=%(user)s)'
+)
+AUTH_LDAP_USER_ATTR_MAP = {
+    'first_name': 'cn',
+    'last_name': 'sn',
+    'email': 'mail'
+}
+AUTHENTICATION_BACKENDS = (
+    'django_auth_ldap.backend.LDAPBackend',
+    'mayan.settings.settings_local.EmailOrUsernameModelBackend',
+)
+
+
+class EmailOrUsernameModelBackend(object):
+    """
+    This is a ModelBacked that allows authentication with either a username or an email address.
+    """
+    def authenticate(self, username=None, password=None):
+        if '@' in username:
+            kwargs = {'email': username}
+        else:
+            kwargs = {'username': username}
+        try:
+            user = get_user_model().objects.get(**kwargs)
+            if user.check_password(password):
+                return user
+        except get_user_model().DoesNotExist:
+            return None
+
+    def get_user(self, username):
+        try:
+            return get_user_model().objects.get(pk=username)
+        except get_user_model().DoesNotExist:
+            return None

contrib/supervisor/mayan-celery.conf

@@ -1,26 +0,0 @@
-[program:mayan-worker]
-command = /usr/share/mayan-edms/bin/python /usr/share/mayan-edms/bin/mayan-edms.py celery --settings=mayan.settings.production worker -Ofair -l ERROR
-directory = /usr/share/mayan-edms
-user = www-data
-stdout_logfile = /var/log/mayan/worker-stdout.log
-stderr_logfile = /var/log/mayan/worker-stderr.log
-autostart = true
-autorestart = true
-startsecs = 10
-stopwaitsecs = 10
-killasgroup = true
-priority = 998
-
-[program:mayan-beat]
-command = /usr/share/mayan-edms/bin/python /usr/share/mayan-edms/bin/mayan-edms.py celery --settings=mayan.settings.production beat -l ERROR
-directory = /usr/share/mayan-edms
-user = www-data
-numprocs = 1
-stdout_logfile = /var/log/mayan/beat-stdout.log
-stderr_logfile = /var/log/mayan/beat-stderr.log
-autostart = true
-autorestart = true
-startsecs = 10
-stopwaitsecs = 1
-killasgroup = true
-priority = 998

contrib/supervisor/mayan-uwsgi.conf

@@ -1,6 +0,0 @@
-[program:mayan-uwsgi]
-command = /usr/share/mayan-edms/bin/uwsgi --ini /usr/share/mayan-edms/uwsgi.ini
-user = root
-autostart = true
-autorestart = true
-redirect_stderr = true

docker/Dockerfile

@@ -0,0 +1,157 @@
+# vim:set ft=dockerfile:
+
+####
+# BASE_IMAGE - Bare bones image with the base packages needed to run Mayan EDMS
+####
+
+FROM debian:9.8-slim as BASE_IMAGE
+
+LABEL maintainer="Roberto Rosario roberto.rosario@mayan-edms.com"
+
+ENV PYTHONUNBUFFERED=1 \
+    LC_ALL=C.UTF-8 \
+    PROJECT_INSTALL_DIR=/opt/mayan-edms
+
+# Debian package caching
+ARG APT_PROXY
+RUN set -x \
+&& if [ "${APT_PROXY}" ]; \
+    then echo "Acquire::http { Proxy \"http://${APT_PROXY}\"; };" > /etc/apt/apt.conf.d/01proxy \
+; fi \
+# Install base OS packages to run Mayan EDMS
+&& DEBIAN_FRONTEND=noninteractive \
+apt-get update \
+&& apt-get install -y --no-install-recommends \
+    exiftool \
+    ghostscript \
+    gpgv \
+    gnupg1 \
+    graphviz \
+    libfuse2 \
+    libmagic1 \
+    libmariadbclient18 \
+    libreoffice \
+    libpq5 \
+    poppler-utils \
+    redis-server \
+    sane-utils \
+    sudo \
+    supervisor \
+    tesseract-ocr \
+# Remove make and build dependencies
+&& apt-get remove make libproxy-tools libreoffice-avmedia-backend-vlc libvlc-bin libvlc5 libvlccore9 adwaita-icon-theme gsettings-desktop-schemas libgstreamer-plugins-base1.0-0 -y \
+&& apt-get autoremove -y --purge \
+# Add mayan user
+&& adduser mayan --disabled-password --disabled-login --no-create-home --gecos "" \
+# Pillow can't find zlib or libjpeg on aarch64 (ODROID C2)
+&& if [ "$(uname -m)" = "aarch64" ]; then \
+    ln -s /usr/lib/aarch64-linux-gnu/libz.so /usr/lib/ \
+    && ln -s /usr/lib/aarch64-linux-gnu/libjpeg.so /usr/lib/ \
+; fi \
+# Pillow can't find zlib or libjpeg on armv7l (ODROID HC1)
+&& if [ "$(uname -m)" = "armv7l" ]; then \
+    ln -s /usr/lib/arm-linux-gnueabihf/libz.so /usr/lib/ \
+    && ln -s /usr/lib/arm-linux-gnueabihf/libjpeg.so /usr/lib/ \
+; fi \
+# Discard data when Redis runs out of memory
+&& echo "maxmemory-policy allkeys-lru" >> /etc/redis/redis.conf \
+# Disable saving the Redis database
+echo "save \"\"" >> /etc/redis/redis.conf \
+# Only provision 1 database
+&& echo "databases 1" >> /etc/redis/redis.conf
+
+
+####
+# BUILDER_IMAGE - This image buildS the Python package and is discarded afterwards
+####
+
+# Reuse image
+FROM BASE_IMAGE as BUILDER_IMAGE
+
+WORKDIR /src
+
+# Copy the source files needed to build the Python package
+COPY --chown=mayan:mayan requirements /src/requirements
+COPY --chown=mayan:mayan \
+    HISTORY.rst \
+    LICENSE \
+    MANIFEST.in \
+    README.md \
+    README.rst \
+    setup.py \
+    /src/
+
+COPY --chown=mayan:mayan mayan /src/mayan
+
+# Install development packages needed to build the Python packages
+RUN DEBIAN_FRONTEND=noninteractive \
+apt-get install -y --no-install-recommends \
+    default-libmysqlclient-dev \
+    libffi-dev \
+    libjpeg-dev \
+    libpng-dev \
+    libpq-dev \
+    libtiff-dev \
+    zlib1g-dev \
+    libssl-dev \
+    g++ \
+    gcc \
+    python-dev \
+    python-virtualenv \
+&& mkdir -p "${PROJECT_INSTALL_DIR}" \
+&& chown -R mayan:mayan "${PROJECT_INSTALL_DIR}" \
+&& chown -R mayan:mayan /src
+
+USER mayan
+RUN python -m virtualenv "${PROJECT_INSTALL_DIR}" \
+&& . "${PROJECT_INSTALL_DIR}/bin/activate" \
+&& pip install --no-cache-dir --no-use-pep517 \
+    librabbitmq==1.6.1 \
+    mysql-python==1.2.5 \
+    psycopg2==2.7.3.2 \
+    redis==2.10.6 \
+# psutil is needed by ARM builds otherwise gevent and gunicorn fail to start
+&& UNAME=`uname -m` && if [ "${UNAME#*arm}" != $UNAME ]; then \
+    pip install --no-cache-dir --no-use-pep517 \
+    psutil==5.6.2 \
+; fi \
+# Install the Python packages needed to build Mayan EDMS
+&& pip install --no-cache-dir --no-use-pep517 -r /src/requirements/build.txt \
+# Build Mayan EDMS
+&& python setup.py sdist \
+# Install the built Mayan EDMS package
+&& pip install --no-cache-dir --no-use-pep517 dist/mayan* \
+# Install the static content
+&& mayan-edms.py installjavascript \
+&& MAYAN_STATIC_ROOT=${PROJECT_INSTALL_DIR}/static mayan-edms.py preparestatic --link --noinput
+
+COPY --chown=mayan:mayan requirements/testing-base.txt "${PROJECT_INSTALL_DIR}"
+
+####
+# Final image - BASE_IMAGE + Mayan install directory from the builder image
+####
+
+FROM BASE_IMAGE
+
+COPY --from=BUILDER_IMAGE --chown=mayan:mayan "${PROJECT_INSTALL_DIR}/" "${PROJECT_INSTALL_DIR}/"
+
+USER root
+
+COPY docker/rootfs /
+
+VOLUME ["/var/lib/mayan"]
+
+ENTRYPOINT ["entrypoint.sh"]
+
+EXPOSE 8000
+CMD ["mayan"]
+
+RUN ${PROJECT_INSTALL_DIR}/bin/mayan-edms.py platformtemplate supervisord_docker > /etc/supervisor/conf.d/mayan.conf \
+&& apt-get clean autoclean \
+&& apt-get autoremove --purge -y \
+&& rm -rf /var/lib/apt/lists/* \
+&& rm -f /var/cache/apt/archives/*.deb \
+# Remove temporary files owned by root from the platformtemplate step
+&& rm /tmp/* \
+# Keep displaying log messages to stdout
+&& find /var/log -type f | while read f; do echo -ne '' > $f; done;

docker/Makefile

@@ -0,0 +1,25 @@
+APT_PROXY ?= `/sbin/ip route|awk '/docker0/ { print $$9 }'`:3142
+IMAGE_VERSION ?= `cat docker/rootfs/version`
+CONSOLE_COLUMNS ?= `echo $$(tput cols)`
+CONSOLE_LINES ?= `echo $$(tput lines)`
+
+docker-build: ## Build a new image locally.
+	docker build -t mayanedms/mayanedms:$(IMAGE_VERSION) -f docker/Dockerfile .
+
+docker-build-with-proxy: ## Build a new image locally using an APT proxy as APT_PROXY.
+	docker build -t mayanedms/mayanedms:$(IMAGE_VERSION) -f docker/Dockerfile --build-arg APT_PROXY=$(APT_PROXY) .
+
+docker-shell: ## Launch a bash instance inside a running container. Pass the container name via DOCKER_CONTAINER.
+	docker exec -e TERM=$(TERM) -e "COLUMNS=$(CONSOLE_COLUMNS)" -e "LINES=$(CONSOLE_LINES)" -it $(DOCKER_CONTAINER) /bin/bash
+
+docker-test-container: ## Build and run a test container.
+docker-test-container: docker-build-with-proxy docker-test-cleanup
+	docker run -d --name test-mayan-edms -p 80:8000 -v test-mayan_data:/var/lib/mayan mayanedms/mayanedms:$(DOCKER_VERSION)
+
+docker-test-cleanup: ## Delete the test container and the test volume.
+	@docker rm -f test-mayan-edms || true
+	@docker volume rm test-mayan_data || true
+
+docker-test-all: ## Build and executed the test suite in a test container.
+docker-test-all: docker-build-with-proxy
+	docker run --rm run-tests

docker/docker-compose-development.yml

@@ -0,0 +1,72 @@
+version: '2.1'
+
+volumes:
+    broker:
+        driver: local
+    app:
+        driver: local
+    db:
+        driver: local
+    results:
+        driver: local
+
+services:
+  broker:
+    container_name: mayan-edms-broker
+    image: healthcheck/rabbitmq
+    environment:
+      RABBITMQ_DEFAULT_USER: mayan
+      RABBITMQ_DEFAULT_PASS: mayan
+      RABBITMQ_DEFAULT_VHOST: mayan
+    volumes:
+      - broker:/var/lib/rabbitmq
+  results:
+    container_name: mayan-edms-results
+    image: healthcheck/redis
+    volumes:
+      - results:/data
+  #db:
+  #  container_name: mayan-edms-db
+  #  image: healthcheck/mysql
+  #  environment:
+  #    MYSQL_DATABASE: mayan
+  #    MYSQL_PASSWORD: mayan-password
+  #    MYSQL_ROOT_PASSWORD: root-password
+  #    MYSQL_USER: mayan
+  #  volumes:
+  #    - db:/var/lib/mysql
+  db:
+    container_name: mayan-edms-db
+    image: healthcheck/postgres
+    environment:
+      POSTGRES_DB: mayan
+      POSTGRES_PASSWORD: mayan-password
+      POSTGRES_USER: mayan
+    volumes:
+      - db:/var/lib/postgresql/data
+  mayan-edms:
+    container_name: mayan-edms-app
+    image: mayan-edms/next
+    build:
+      context: ./
+      args:
+        - APT_PROXY=172.18.0.1:3142
+    depends_on:
+      broker:
+        condition: service_healthy
+      db:
+        condition: service_healthy
+      results:
+        condition: service_healthy
+    environment:
+      MAYAN_BROKER_URL: amqp://mayan:mayan@broker:5672/mayan
+      MAYAN_CELERY_RESULT_BACKEND: redis://results:6379/0
+      MAYAN_DATABASE_ENGINE: django.db.backends.postgresql
+      MAYAN_DATABASE_HOST: db
+      MAYAN_DATABASE_NAME: mayan
+      MAYAN_DATABASE_PASSWORD: mayan-password
+      MAYAN_DATABASE_USER: mayan
+    ports:
+      - "80:80"
+    volumes:
+      - app:/var/lib/mayan

docker/docker-compose.yml

@@ -0,0 +1,58 @@
+version: '2.1'
+
+volumes:
+    broker:
+        driver: local
+    app:
+        driver: local
+    db:
+        driver: local
+    results:
+        driver: local
+
+services:
+  broker:
+    container_name: mayan-edms-broker
+    image: healthcheck/rabbitmq
+    environment:
+      RABBITMQ_DEFAULT_USER: mayan
+      RABBITMQ_DEFAULT_PASS: mayan
+      RABBITMQ_DEFAULT_VHOST: mayan
+    volumes:
+      - broker:/var/lib/rabbitmq
+  results:
+    container_name: mayan-edms-results
+    image: healthcheck/redis
+    volumes:
+      - results:/data
+  db:
+    container_name: mayan-edms-db
+    image: healthcheck/postgres
+    environment:
+      POSTGRES_DB: mayan
+      POSTGRES_PASSWORD: mayan-password
+      POSTGRES_USER: mayan
+    volumes:
+      - db:/var/lib/postgresql/data
+  mayan-edms:
+    container_name: mayan-edms-app
+    image: mayanedms/mayanedms:latest
+    depends_on:
+      broker:
+        condition: service_healthy
+      db:
+        condition: service_healthy
+      results:
+        condition: service_healthy
+    environment:
+      MAYAN_BROKER_URL: amqp://mayan:mayan@broker:5672/mayan
+      MAYAN_CELERY_RESULT_BACKEND: redis://results:6379/0
+      MAYAN_DATABASE_ENGINE: django.db.backends.postgresql
+      MAYAN_DATABASE_HOST: db
+      MAYAN_DATABASE_NAME: mayan
+      MAYAN_DATABASE_PASSWORD: mayan-password
+      MAYAN_DATABASE_USER: mayan
+    ports:
+      - "80:8000"
+    volumes:
+      - app:/var/lib/mayan

docker/rootfs/usr/local/bin/entrypoint.sh

@@ -0,0 +1,128 @@
+#!/bin/sh
+
+set -e
+echo "mayan: starting entrypoint.sh"
+INSTALL_FLAG=/var/lib/mayan/system/SECRET_KEY
+CONCURRENCY_ARGUMENT=--concurrency=
+
+DEFAULT_USER_UID=1000
+DEFAULT_USER_GUID=1000
+
+export MAYAN_DEFAULT_BROKER_URL=redis://127.0.0.1:6379/0
+export MAYAN_DEFAULT_CELERY_RESULT_BACKEND=redis://127.0.0.1:6379/0
+
+export MAYAN_ALLOWED_HOSTS='["*"]'
+export MAYAN_BIN=/opt/mayan-edms/bin/mayan-edms.py
+export MAYAN_BROKER_URL=${MAYAN_BROKER_URL:-${MAYAN_DEFAULT_BROKER_URL}}
+export MAYAN_CELERY_RESULT_BACKEND=${MAYAN_CELERY_RESULT_BACKEND:-${MAYAN_DEFAULT_CELERY_RESULT_BACKEND}}
+export MAYAN_INSTALL_DIR=/opt/mayan-edms
+export MAYAN_PYTHON_BIN_DIR=/opt/mayan-edms/bin/
+export MAYAN_MEDIA_ROOT=/var/lib/mayan
+export MAYAN_SETTINGS_MODULE=${MAYAN_SETTINGS_MODULE:-mayan.settings.production}
+
+export MAYAN_GUNICORN_BIN=${MAYAN_PYTHON_BIN_DIR}gunicorn
+export MAYAN_GUNICORN_WORKERS=${MAYAN_GUNICORN_WORKERS:-2}
+export MAYAN_GUNICORN_TIMEOUT=${MAYAN_GUNICORN_TIMEOUT:-120}
+export MAYAN_PIP_BIN=${MAYAN_PYTHON_BIN_DIR}pip
+export MAYAN_STATIC_ROOT=${MAYAN_INSTALL_DIR}/static
+
+MAYAN_WORKER_FAST_CONCURRENCY=${MAYAN_WORKER_FAST_CONCURRENCY:-1}
+MAYAN_WORKER_MEDIUM_CONCURRENCY=${MAYAN_WORKER_MEDIUM_CONCURRENCY:-1}
+MAYAN_WORKER_SLOW_CONCURRENCY=${MAYAN_WORKER_SLOW_CONCURRENCY:-1}
+
+echo "mayan: changing uid/guid"
+usermod mayan -u ${MAYAN_USER_UID:-${DEFAULT_USER_UID}}
+groupmod mayan -g ${MAYAN_USER_GUID:-${DEFAULT_USER_GUID}}
+
+if [ "$MAYAN_WORKER_FAST_CONCURRENCY" -eq 0 ]; then
+    MAYAN_WORKER_FAST_CONCURRENCY=
+else
+    MAYAN_WORKER_FAST_CONCURRENCY="${CONCURRENCY_ARGUMENT}${MAYAN_WORKER_FAST_CONCURRENCY}"
+fi
+export MAYAN_WORKER_FAST_CONCURRENCY
+
+if [ "$MAYAN_WORKER_MEDIUM_CONCURRENCY" -eq 0 ]; then
+    MAYAN_WORKER_MEDIUM_CONCURRENCY=
+else
+    MAYAN_WORKER_MEDIUM_CONCURRENCY="${CONCURRENCY_ARGUMENT}${MAYAN_WORKER_MEDIUM_CONCURRENCY}"
+fi
+export MAYAN_WORKER_MEDIUM_CONCURRENCY
+
+if [ "$MAYAN_WORKER_SLOW_CONCURRENCY" -eq 0 ]; then
+    MAYAN_WORKER_SLOW_CONCURRENCY=
+else
+    MAYAN_WORKER_SLOW_CONCURRENCY="${CONCURRENCY_ARGUMENT}${MAYAN_WORKER_SLOW_CONCURRENCY}"
+fi
+export MAYAN_WORKER_SLOW_CONCURRENCY
+
+export CELERY_ALWAYS_EAGER=False
+export PYTHONPATH=$PYTHONPATH:$MAYAN_MEDIA_ROOT
+
+chown mayan:mayan /var/lib/mayan -R
+
+apt_get_install() {
+    apt-get -q update
+    apt-get install -y --force-yes --no-install-recommends --auto-remove "$@"
+    apt-get -q clean
+    rm -rf /var/lib/apt/lists/*
+}
+
+initialize() {
+    echo "mayan: initialize()"
+    su mayan -c "${MAYAN_BIN} initialsetup --force --no-javascript"
+}
+
+os_package_installs() {
+    echo "mayan: os_package_installs()"
+    if [ "${MAYAN_APT_INSTALLS}" ]; then
+        DEBIAN_FRONTEND=noninteractive apt_get_install $MAYAN_APT_INSTALLS
+    fi
+}
+
+pip_installs() {
+    echo "mayan: pip_installs()"
+    if [ "${MAYAN_PIP_INSTALLS}" ]; then
+        su mayan -c "${MAYAN_PIP_BIN} install $MAYAN_PIP_INSTALLS"
+    fi
+}
+
+start() {
+    echo "mayan: start()"
+    rm -rf /var/run/supervisor.sock
+    exec /usr/bin/supervisord -nc /etc/supervisor/supervisord.conf
+}
+
+upgrade() {
+    echo "mayan: upgrade()"
+    su mayan -c "${MAYAN_BIN} performupgrade --no-javascript"
+}
+
+os_package_installs || true
+pip_installs || true
+
+case "$1" in
+
+mayan) # Check if this is a new install, otherwise try to upgrade the existing
+       # installation on subsequent starts
+       if [ ! -f $INSTALL_FLAG ]; then
+           initialize
+       else
+           upgrade
+       fi
+       start
+       ;;
+
+run-tests) # Check if this is a new install, otherwise try to upgrade the existing
+           # installation on subsequent starts
+           if [ ! -f $INSTALL_FLAG ]; then
+               initialize
+           else
+               upgrade
+           fi
+           run-tests.sh
+           ;;
+
+*) su mayan -c "$@";
+   ;;
+
+esac

docker/rootfs/usr/local/bin/run-tests.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+export DEBIAN_FRONTEND=noninteractive
+
+apt-get update
+apt-get install -y --no-install-recommends gcc python-dev tesseract-ocr-deu
+
+su mayan -c "$MAYAN_PIP_BIN install -r ${MAYAN_INSTALL_DIR}/testing-base.txt"
+
+su mayan -c "$MAYAN_BIN test --mayan-apps --settings=mayan.settings.testing"

docker/rootfs/version

@@ -0,0 +1 @@
+3.2.6

docker/stack.yml

@@ -0,0 +1,32 @@
+version: '3.1'
+
+volumes:
+    app:
+        driver: local
+    db:
+        driver: local
+
+services:
+
+  db:
+    image: postgres
+    environment:
+      POSTGRES_DB: mayan
+      POSTGRES_PASSWORD: mayan-password
+      POSTGRES_USER: mayan
+    volumes:
+      - db:/var/lib/postgresql/data
+
+  app:
+    image: mayanedms/mayanedms:latest
+    ports:
+      - 80:8000
+    environment:
+      MAYAN_DATABASE_ENGINE: django.db.backends.postgresql
+      MAYAN_DATABASE_HOST: db
+      MAYAN_DATABASE_NAME: mayan
+      MAYAN_DATABASE_PASSWORD: mayan-password
+      MAYAN_DATABASE_USER: mayan
+      MAYAN_DATABASE_CONN_MAX_AGE: 0
+    volumes:
+      - app:/var/lib/mayan

docs/_static/css/custom.css

@@ -0,0 +1,5 @@
+.menuselection {
+    color: white;
+    background: #2c3e50;
+    padding: 3px;
+}

docs/_templates/donate.html

@@ -1,15 +1,16 @@
-<h3>Donate</h3>
-<p>Help support further improvements and development, via:
-    <ul>
-        <li>
-            - <a href="https://www.paypal.me/RobertoRosario">PayPal.Me</a>
-        </li>
-        <li>
-            - <a href="https://www.patreon.com/siloraptor">Patreon</a>
-        </li>
-        <li>- or Gumroad:
-            <script type="text/javascript" src="https://gumroad.com/js/gumroad.js"></script>
-            <a href="https://gumroad.com/l/UNApl" class="gumroad-button">Buy Mayan EDMS</a>
-        </li>
-    </ul>
+<h3>Support</h3>
+<p>
+    <a href='http://www.mayan-edms.com/providers/'>Consulting and support plans are available, click here</a>.
 </p>
+<hr />
+<p>
+    Or consider supporting Mayan EDMS by contributing to its development. (US tax payers, please note this contribution is not tax deductible).
+</p>
+<form action="https://www.paypal.com/cgi-bin/webscr" method="post" target="_top">
+    <input type="hidden" name="cmd" value="_s-xclick">
+    <input type="hidden" name="hosted_button_id" value="3PXN336XFXQNN">
+    <input type="image" src="https://www.paypalobjects.com/en_US/i/btn/btn_donateCC_LG.gif" border="0" name="submit" alt="PayPal - The safer, easier way to pay online!">
+    <img alt="" border="0" src="https://www.paypalobjects.com/en_US/i/scr/pixel.gif" width="1" height="1" style="display: none !important;">
+</form>
+
+<hr />

docs/chapters/acls.rst

@@ -1,21 +1,23 @@
-====================
+********************
 Access control lists
-====================
+********************
 
-Besides the permissions system explained in :doc:`permissions`, **Mayan EDMS**
-provides per object permission granting. This feature is used to grant a
-permission to a role, but this permission can only be executed for a limited
-number of objects (documents, folders, tags) instead of being effective
-system-wide.
+Besides the permissions system explained in :doc:`../chapters/permissions`,
+Mayan EDMS provides per object permission granting. This feature is used to
+grant a permission to a role, but this permission can only be executed for a
+limited number of objects (documents, folders, tags) instead of being
+effective system-wide.
 
 .. blockdiag::
 
    blockdiag {
+      default_shape = roundedbox
+
       document [ label = 'Document' ];
       role [ label = 'Role' ];
       permission [ label = 'Permission' ];
 
-       role -> document <- permission;
+       role -> permission -> document;
    }
 
 Example:
@@ -23,11 +25,13 @@ Example:
 .. blockdiag::
 
    blockdiag {
+      default_shape = roundedbox
+
       document [ label = '2015 Payroll report.txt', width=200 ];
       role [ label = 'Accountants' ];
       permission [ label = 'View document' ];
 
-      role -> document <- permission;
+      role -> permission -> document;
    }
 
 In this scenario only users in groups belonging to the ``Accountants`` role
@@ -36,18 +40,22 @@ would be able to view the ``2015 Payroll report.txt`` document.
 Inherited access control
 ========================
 
-It is also possible to grant a permission to a role for a specific document type (:doc:`document_types`).
-Under this scheme all users in groups belonging to that role will inherit that
-permission for all documents of that type.
+It is also possible to grant a permission to a role for a specific document
+type (:doc:`../chapters/document_types`). Under this scheme all users in
+groups belonging to that role will inherit that permission for all documents
+of that type.
 
 .. blockdiag::
 
    blockdiag {
+      default_shape = roundedbox
       document_type [ label = 'Document type' ];
       role [ label = 'Role' ];
       permission [ label = 'Permission' ];
+      documents [shape = "note", stacked];
 
-       role -> document_type <- permission;
+       role ->  permission -> document_type ;
+       document_type -> documents [folded, label = "inherit" ];
    }
 
 Example:
@@ -55,11 +63,14 @@ Example:
 .. blockdiag::
 
    blockdiag {
+      default_shape = roundedbox
       document_type [ label = 'Payroll reports', width=200 ];
       role [ label = 'Accountants' ];
       permission [ label = 'View document' ];
+      documents [shape = "note", stacked, label="payroll_report*.pdf" ];
 
-      role -> document_type <- permission;
+      role -> permission -> document_type ;
+       document_type -> documents [folded, label = "inherit" ];
    }
 
 The role ``Accountants`` is given the permission ``document view`` for the

docs/chapters/api.rst

@@ -0,0 +1,102 @@
+***
+API
+***
+
+Mayan EDMS provides an HTTP REST Application Program Interface (or API). This
+API allows integration with 3rd party software using simple HTTP requests.
+
+Several API authentication methods are provides: **Session**, **Token**,
+and **HTTP Basic**.
+
+The URL for the API can be found via the :menuselection:`Tools --> REST API
+menu. The API is also self-documenting. The live API documentation can be
+found in the :menuselection:`Tools --> API Documentation (Swagger)` menu for
+the Swagger version and in the
+:menuselection:`Tools --> API Documentation (ReDoc)` menu for the ReDoc version.
+
+The are a few ways to structure REST APIs. In the case of Mayan EDMS, API
+endpoints are structured by resource type. Examples:
+
+* /cabinets - To view or create new cabinets
+* /cabinets/<id> - To view the details, edit, or delete an existing cabinet.
+* /cabinets/<id>/documents - To view, add, or remove documents from an existing
+  cabinet.
+* /cabinets/<id>/documents/<id> - To view, add, or remove one document from an
+  existing cabinet.
+
+The API supports the HTTP verbs: **GET**, **POST**, **PUT**, **PATCH**,
+and **DELETE**.
+
+
+Example use
+===========
+
+Install Python Requests (http://docs.python-requests.org/en/master/)::
+
+    pip install requests
+
+Get a list of document types::
+
+    import requests
+
+    requests.get('http://127.0.0.1:8000/api/document_types/', auth=('username', 'password')).json()
+
+    {u'count': 1,
+     u'next': None,
+     u'previous': None,
+     u'results': [{u'delete_time_period': 30,
+       u'delete_time_unit': u'days',
+       u'documents_count': 12,
+       u'documents_url': u'http://127.0.0.1:8000/api/document_types/1/documents/',
+       u'filenames': [],
+       u'id': 1,
+       u'label': u'Default',
+       u'trash_time_period': None,
+       u'trash_time_unit': None,
+       u'url': u'http://127.0.0.1:8000/api/document_types/1/'}]}
+
+Upload a new document::
+
+    with open('test_document.pdf', mode='rb') as
+        requests.post('http://127.0.0.1:8000/api/documents/', auth=('username', 'password'), files={'file': file_object}, data={'document_type': 1}).json()
+
+    {u'description': u'',
+    u'document_type': 1,
+    u'id': 19,
+    u'label': u'test_document.pdf',
+    u'language': u'eng'}
+
+Use API tokens to avoid sending the username and password on every request. Obtain a token by making a POST request to ``/api/auth/token/obtain/?format=json``::
+
+    requests.post('http://127.0.0.1:8000/api/auth/token/obtain/?format=json', data={'username': 'username', 'password': 'password'}).json()
+
+    {u'token': u'4ccbc35b5eb327aa82dc3b7c9747b578900f02bb'}
+
+Add the API token to the request header::
+
+    headers = {'Authorization': 'Token 4ccbc35b5eb327aa82dc3b7c9747b578900f02bb'}
+
+    requests.get('http://127.0.0.1:8000/api/document_types/', headers=headers).json()
+
+    {u'description': u'',
+    u'document_type': 1,
+    u'id': 19,
+    u'label': u'test_document.pdf',
+    u'language': u'eng'}
+
+
+Use sessions to avoid having to add the headers on each request::
+
+    session = requests.Session()
+
+    headers = {'Authorization': 'Token 4ccbc35b5eb327aa82dc3b7c9747b578900f02bb'}
+
+    session.headers.update(headers)
+
+    session.get('http://127.0.0.1:8000/api/document_types/')
+
+    {u'description': u'',
+    u'document_type': 1,
+    u'id': 19,
+    u'label': u'test_document.pdf',
+    u'language': u'eng'}

docs/chapters/app_creation.rst

@@ -1,8 +1,8 @@
-============
+************
 App creation
-============
+************
 
-**Mayan EDMS** apps are essentially Django app with some extra code to register
+Mayan EDMS apps are essentially Django app with some extra code to register
 navigation, permissions and other relationships.
 
 
@@ -21,7 +21,7 @@ App modules
 
 - api_views.py
 
-  REST API views go here. **Mayan EDMS** uses Django REST Framework API view
+  REST API views go here. Mayan EDMS uses Django REST Framework API view
   classes.
 
 - apps.py
@@ -59,6 +59,17 @@ App modules
   corresponding signal in the ready() method of the MayanAppConfig subclass in
   apps.py
 
+- html_widgets.py
+
+  Classes to render an HTML widget. HTML widget are not the same as Django's
+  native form widgets. Form widgets only work as part of a form field.
+  HTML widgets are for use outside of forms, such as in a table cell.
+
+- licenses.py
+
+  This module outlines the license text of the third party content used in 
+  the app. It could be other Python libraries, JavaScript libraries, etc.  
+
 - links.py
 
   Defines the links to be used by the app. Import only from the navigation app
@@ -92,6 +103,11 @@ App modules
   This module acts as a shared memory space for the other modules of the app or
   other apps.
 
+- search.py
+
+  Search model definitions. Define which field of the app's models are
+  searchable.
+
 - serializers.py
 
   Hold Django REST Framework serializers used by the api_views.py module.
@@ -127,7 +143,7 @@ App modules
 
 - widgets.py
 
-  HTML widgets go here. This should be the only place with presentation
+  Custom form widgets go here. This should be the only place with presentation
   directives in the app (aside the templates).
 
 
@@ -135,7 +151,7 @@ Views
 =====
 
 The module common.generics provides custom generic class based views to be used.
-The basic views used to create, edit, view and delete objects in **Mayan EDMS**
+The basic views used to create, edit, view and delete objects in Mayan EDMS
 are: SingleObjectCreateView, SingleObjectDetailView, SingleObjectEditView,
 and SingleObjectListView
 

docs/chapters/backups.rst

@@ -0,0 +1,70 @@
+*******
+Backups
+*******
+
+To backup your install of Mayan EDMS just copy the actual document files and
+the database content. If you are using the default storage backend, the
+document files should be found in the ``media`` folder of your installation.
+
+To dump the content of your database manager refer to the documentation chapter
+regarding database data "dumping".
+
+Here is an example of how to perform a backup and a restore of a PostgreSQL
+database.
+
+To dump the database into an SQL text file::
+
+    pg_dump -h <host> -U <database user> -c <database name> -W > `date +%Y-%m-%d"_"%H-%M-%S`.sql
+
+Example::
+
+    pg_dump -h 127.0.0.1 -U mayan -c mayan -W > `date +%Y-%m-%d"_"%H-%M-%S`.sql
+
+To restore the database from the SQL text file::
+
+    psql -h <host> -U <database user> -d <database name> -W -f <sql dump file>
+
+Example::
+
+    psql -h 127.0.0.1 -U mayan -d mayan -W -f 2018-06-07_18-10-56.sql
+
+Here is an example of how to perform a backup and a restore of a PostgreSQL
+Docker container using a compressed dump file. A dump file is not compatible or
+can be used interchangeable with an SQL text file.
+
+To backup a PostgreSQL Docker container::
+
+    docker exec <container name> pg_dump -U <database user> -Fc -c <database name> > `date +%Y-%m-%d"_"%H-%M-%S`.dump
+
+Example::
+
+    docker exec mayan-edms-db pg_dump -U mayan -Fc -c mayan > `date +%Y-%m-%d"_"%H-%M-%S`.dump
+
+This will produce a compressed dump file with the current date and time as the filename.
+
+To restore a PostgreSQL Docker container::
+
+    docker exec -i <container name> pg_restore -U <database user> -d <database name> < <dump file>
+
+Since it is not possible to drop a currently open PostgreSQL database, this
+command must be used on a new and empty PostsgreSQL container.
+
+Example::
+
+    docker run -d \
+    --name mayan-edms-pg-new \
+    --restart=always \
+    -p 5432:5432 \
+    -e POSTGRES_USER=mayan \
+    -e POSTGRES_DB=mayan \
+    -e POSTGRES_PASSWORD=mayanuserpass \
+    -v /docker-volumes/mayan-edms/postgres-new:/var/lib/postgresql/data \
+    -d postgres:9.6
+
+    docker exec -i mayan-edms-pg-new pg_restore -U mayan -d mayan < 2018-06-07_17-09-34.dump
+
+More information at:
+
+ - PostgreSQL: https://www.postgresql.org/docs/current/static/backup.html
+ - MySQL: https://dev.mysql.com/doc/refman/5.7/en/mysqldump.html
+ - SQLite: Just copy the file ``mayan/media/db.sqlite3``

docs/chapters/caa_entity.rst

@@ -0,0 +1,233 @@
+**************************************************
+Mayan EDMS Entity Contributor Assignment Agreement
+**************************************************
+
+Thank you for your interest in contributing to Mayan EDMS ("We" or "Us").
+
+This contributor agreement ("Agreement") documents the rights granted by
+contributors to Us. To make this document effective, please print it, sign it
+(by copyright holder or authorized party) and send it to Us by email to
+caa@mayan-edms.com. This is a legally binding document, so please read it
+carefully before agreeing to it. The Agreement may cover more than one
+software project managed by Us.
+
+
+1. Definitions
+==============
+"You" means any Legal Entity on behalf of whom a Contribution has been received
+by Us. "Legal Entity" means an entity which is not a natural person.
+"Affiliates" means other Legal Entities that control, are controlled by, or
+under common control with that Legal Entity. For the purposes of this
+definition, "control" means (i) the power, direct or indirect, to cause the
+direction or management of such Legal Entity, whether by contract or otherwise,
+(ii) ownership of fifty percent (50%) or more of the outstanding shares or
+securities which vote to elect the management or other persons who direct such
+Legal Entity or (iii) beneficial ownership of such entity.
+
+"Contribution" means any work of authorship that is Submitted by You to Us in
+which You own or assert ownership of the Copyright. We cannot accept
+contributions for which you do not own the Copyright or for which you don't
+have the necessary legal power to transfer.
+
+"Copyright" means all rights protecting works of authorship owned or controlled
+by You or Your Affiliates, including copyright, moral and neighboring rights,
+as appropriate, for the full term of their existence including any extensions
+by You.
+
+"Material" means the work of authorship which is made available by Us to third
+parties. When this Agreement covers more than one software project, the
+Material means the work of authorship to which the Contribution was Submitted.
+After You Submit the Contribution, it may be included in the Material.
+
+"Submit" means any form of electronic, verbal, or written communication sent
+to Us or our representatives, including but not limited to electronic mailing
+lists, source code control systems, and issue tracking systems that are managed
+by, or on behalf of, Us for the purpose of discussing and improving the
+Material, but excluding communication that is conspicuously marked or otherwise
+designated in writing by You as "Not a Contribution."
+
+"Submission Date" means the date on which You Submit a Contribution to Us.
+
+"Effective Date" means the date You execute this Agreement or the date You
+first Submit a Contribution to Us, whichever is earlier.
+
+
+2. Grant of Rights
+==================
+
+2.1 Copyright Assignment
+------------------------
+
+a. At the time the Contribution is Submitted, You assign to Us all right, title,
+   and interest worldwide in all Copyright covering the Contribution; provided
+   that this transfer is conditioned upon compliance with Section 2.3.
+
+b. To the extent that any of the rights in Section 2.1(a) cannot be assigned by
+   You to Us, You grant to Us a perpetual, worldwide, exclusive, royalty-free,
+   transferable, irrevocable license under such non-assigned rights, with rights
+   to sublicense through multiple tiers of sublicensees, to practice such
+   non-assigned rights, including, but not limited to, the right to reproduce,
+   modify, display, perform and distribute the Contribution; provided that this
+   license is conditioned upon compliance with Section 2.3.
+
+c. To the extent that any of the rights in Section 2.1(a) can neither be
+   assigned nor licensed by You to Us, You irrevocably waive and agree never to
+   assert such rights against Us, any of our successors in interest, or any of
+   our licensees, either direct or indirect; provided that this agreement not
+   to assert is conditioned upon compliance with Section 2.3.
+
+d. Upon such transfer of rights to Us, the Contribution will be licenses under
+   the terms of the Material.
+
+2.2 Patent License
+------------------
+
+For patent claims including, without limitation, method, process, and apparatus
+claims which You or Your Affiliates own, control or have the right to grant,
+now or in the future, You grant to Us a perpetual, worldwide, non-exclusive,
+transferable, royalty-free, irrevocable patent license, with the right to
+sublicense these rights to multiple tiers of sublicensees, to make, have made,
+use, sell, offer for sale, import and otherwise transfer the Contribution and
+the Contribution in combination with the Material (and portions of such
+combination). This license is granted only to the extent that the exercise of
+the licensed rights infringes such patent claims; and provided that this license
+is conditioned upon compliance with Section 2.3.
+
+2.3 Outbound License
+--------------------
+
+As a condition on the grant of rights in Sections 2.1 and 2.2, We agree to
+license the Contribution only under the terms of the license or licenses which
+We are using on the Submission Date for the Material (including any rights to
+adopt any future version of a license if permitted).
+
+2.4 Moral Rights
+----------------
+
+If moral rights apply to the Contribution, to the maximum extent permitted by
+law, You waive and agree not to assert such moral rights against Us or our
+successors in interest, or any of our licensees, either direct or indirect.
+
+2.5 Our Rights
+--------------
+
+You acknowledge that We are not obligated to use Your Contribution as part of
+the Material and may decide to include any Contribution We consider appropriate.
+
+2.6 Reservation of Rights
+-------------------------
+
+Any rights not expressly assigned or licensed under this section are expressly
+reserved by You.
+
+
+3. Agreement
+============
+
+You confirm that:
+
+a. You have the legal authority to enter into this Agreement.
+
+b. You or Your Affiliates own the Copyright and patent claims covering the
+   Contribution which are required to grant the rights under Section 2.
+
+c. The grant of rights under Section 2 does not violate any grant of rights
+   which You or Your Affiliates have made to third parties.
+
+
+4. Disclaimer
+=============
+
+EXCEPT FOR THE EXPRESS WARRANTIES IN SECTION 3, THE CONTRIBUTION IS PROVIDED
+"AS IS". MORE PARTICULARLY, ALL EXPRESS OR IMPLIED WARRANTIES INCLUDING,
+WITHOUT LIMITATION, ANY IMPLIED WARRANTY OF MERCHANTABILITY, FITNESS FOR A
+PARTICULAR PURPOSE AND NON-INFRINGEMENT ARE EXPRESSLY DISCLAIMED BY YOU TO US
+AND BY US TO YOU. TO THE EXTENT THAT ANY SUCH WARRANTIES CANNOT BE DISCLAIMED,
+SUCH WARRANTY IS LIMITED IN DURATION TO THE MINIMUM PERIOD PERMITTED BY LAW.
+
+
+5. Consequential Damage Waiver
+==============================
+
+TO THE MAXIMUM EXTENT PERMITTED BY APPLICABLE LAW, IN NO EVENT WILL YOU OR US
+BE LIABLE FOR ANY LOSS OF PROFITS, LOSS OF ANTICIPATED SAVINGS, LOSS OF DATA,
+INDIRECT, SPECIAL, INCIDENTAL, CONSEQUENTIAL AND EXEMPLARY DAMAGES ARISING OUT
+OF THIS AGREEMENT REGARDLESS OF THE LEGAL OR EQUITABLE THEORY (CONTRACT, TORT
+OR OTHERWISE) UPON WHICH THE CLAIM IS BASED.
+
+
+6. Miscellaneous
+================
+
+6.1 Jurisdiction
+----------------
+
+This Agreement will be governed by and construed in accordance with the laws of
+Puerto Rico excluding its conflicts of law provisions. Under certain circumstances,
+the governing law in this section might be superseded by the United Nations
+Convention on Contracts for the International Sale of Goods ("UN Convention")
+and the parties intend to avoid the application of the UN Convention to this
+Agreement and, thus, exclude the application of the UN Convention in its
+entirety to this Agreement.
+
+6.2 Acceptance
+--------------
+This Agreement sets out the entire agreement between You and Us for Your
+Contributions to Us and overrides all other agreements or understandings.
+
+6.3 Third parties
+-----------------
+If You or We assign the rights or obligations received through this Agreement
+to a third party, as a condition of the assignment, that third party must
+agree in writing to abide by all the rights and obligations in the Agreement.
+
+6.4 Unmet responsibilities
+--------------------------
+The failure of either party to require performance by the other party of any
+provision of this Agreement in one situation shall not affect the right of a
+party to require such performance at any time in the future. A waiver of
+performance under a provision in one situation shall not be considered a
+waiver of the performance of the provision in the future or a waiver of the
+provision in its entirety.
+
+6.5 Continuation
+----------------
+If any provision of this Agreement is found void and unenforceable, such
+provision will be replaced to the extent possible with a provision that comes
+closest to the meaning of the original provision and which is enforceable.
+The terms and conditions set forth in this Agreement shall apply
+notwithstanding any failure of essential purpose of this Agreement or any
+limited remedy to the maximum extent possible under law.
+
+
+::
+
+
+    Name: _________________________________________________
+
+
+    Email: ________________________________________________
+
+
+    Address: ______________________________________________
+
+
+    Address (cont): _______________________________________
+
+
+    Country: ______________________________________________
+
+
+    Phone number: _________________________________________
+
+
+    Fax number: ___________________________________________
+
+
+
+    _______________________________________________________
+                             Signature
+
+
+
+Mayan EDMS Entity CAA Version 1.0 April 8, 2018

docs/chapters/caa_individual.rst

@@ -0,0 +1,234 @@
+******************************************************
+Mayan EDMS Individual Contributor Assignment Agreement
+******************************************************
+
+Thank you for your interest in contributing to Mayan EDMS ("We" or "Us").
+
+This contributor agreement ("Agreement") documents the rights granted by
+contributors to Us. To make this document effective, please print it, sign it
+(by copyright holder or authorized party) and send it to Us by email to
+caa@mayan-edms.com. This is a legally binding document, so please read it
+carefully before agreeing to it. The Agreement may cover more than one
+software project managed by Us.
+
+1. Definitions
+==============
+
+"You" means the individual who Submits a Contribution to Us.
+
+"Contribution" means any work of authorship that is Submitted by You to Us
+in which You own or assert ownership of the Copyright. We cannot accept
+contributions for which you do not own the Copyright or for which you don't
+have the necessary legal power to transfer.
+
+"Copyright" means all rights protecting works of authorship owned or
+controlled by You, including copyright, moral and neighboring rights,
+as appropriate, for the full term of their existence including any extensions
+by You.
+
+"Material" means the work of authorship which is made available by Us to third
+parties. When this Agreement covers more than one software project, the Material
+means the work of authorship to which the Contribution was Submitted. After
+You Submit the Contribution, it may be included in the Material.
+
+"Submit" means any form of electronic, verbal, or written communication
+sent to Us or our representatives, including but not limited to
+electronic mailing lists, source code control systems, and issue tracking systems
+that are managed by, or on behalf of, Us for the purpose of discussing and
+improving the Material, but excluding communication that is conspicuously marked
+or otherwise designated in writing by You as "Not a Contribution."
+
+"Submission Date" means the date on which You Submit a Contribution to Us.
+
+"Effective Date" means the date You execute this Agreement or the date You first
+Submit a Contribution to Us, whichever is earlier.
+
+2. Grant of Rights
+==================
+
+2.1 Copyright Assignment
+------------------------
+
+a. At the time the Contribution is Submitted, You assign to Us all right,
+   title, and interest worldwide in all Copyright covering the Contribution;
+   provided that this transfer is conditioned upon compliance with Section 2.3.
+
+b. To the extent that any of the rights in Section 2.1(a) cannot be assigned
+   by You to Us, You grant to Us a perpetual, worldwide, exclusive, royalty-free,
+   transferable, irrevocable license under such non-assigned rights, with rights
+   to sublicense through multiple tiers of sublicensees, to practice such
+   non-assigned rights, including, but not limited to, the right to reproduce,
+   modify, display, perform and distribute the Contribution; provided that
+   this license is conditioned upon compliance with Section 2.3.
+
+c. To the extent that any of the rights in Section 2.1(a) can neither be
+   assigned nor licensed by You to Us, You irrevocably waive and agree never to
+   assert such rights against Us, any of our successors in interest, or any of
+   our licensees, either direct or indirect; provided that this agreement not
+   to assert is conditioned upon compliance with Section 2.3.
+
+d. Upon such transfer of rights to Us, the Contribution will be licenses under
+   the terms of the Material.
+
+
+2.2 Patent License
+------------------
+
+For patent claims including, without limitation, method, process, and apparatus
+claims which You own, control or have the right to grant, now or in the future,
+You grant to Us a perpetual, worldwide, non-exclusive, transferable, royalty-free,
+irrevocable patent license, with the right to sublicense these rights to multiple
+tiers of sublicensees, to make, have made, use, sell, offer for sale, import and
+otherwise transfer the Contribution and the Contribution in combination with
+the Material (and portions of such combination). This license is granted only
+to the extent that the exercise of the licensed rights infringes such patent claims;
+and provided that this license is conditioned upon compliance with Section 2.3.
+
+
+2.3 Outbound License
+--------------------
+As a condition on the grant of rights in Sections 2.1 and 2.2, We agree to
+license the Contribution only under the terms of the license or licenses which
+We are using on the Submission Date for the Material (including any rights to
+adopt any future version of a license if permitted).
+
+
+2.4 Moral Rights
+----------------
+
+If moral rights apply to the Contribution, to the maximum extent permitted by law,
+You waive and agree not to assert such moral rights against Us or our successors
+in interest, or any of our licensees, either direct or indirect.
+
+
+2.5 Our Rights
+--------------
+
+You acknowledge that We are not obligated to use Your Contribution as part of
+the Material and may decide to include any Contribution We consider appropriate.
+
+
+2.6 Reservation of Rights
+-------------------------
+
+Any rights not expressly assigned or licensed under this section are expressly
+reserved by You.
+
+
+3. Agreement
+============
+
+You confirm that:
+
+a. You have the legal authority to enter into this Agreement.
+
+b. You own the Copyright and patent claims covering the Contribution which are
+   required to grant the rights under Section 2.
+
+c. The grant of rights under Section 2 does not violate any grant of rights
+   which You have made to third parties, including Your employer. If You are an
+   employee, You have had Your employer approve this Agreement or sign the Entity
+   version of this document. If You are less than eighteen years old, please have
+   Your parents or guardian sign the Agreement.
+
+
+4. Disclaimer
+=============
+
+EXCEPT FOR THE EXPRESS WARRANTIES IN SECTION 3, THE CONTRIBUTION IS PROVIDED
+"AS IS". MORE PARTICULARLY, ALL EXPRESS OR IMPLIED WARRANTIES INCLUDING, WITHOUT
+LIMITATION, ANY IMPLIED WARRANTY OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NON-INFRINGEMENT ARE EXPRESSLY DISCLAIMED BY YOU TO US AND BY US
+TO YOU. TO THE EXTENT THAT ANY SUCH WARRANTIES CANNOT BE DISCLAIMED, SUCH
+WARRANTY IS LIMITED IN DURATION TO THE MINIMUM PERIOD PERMITTED BY LAW.
+
+
+5. Consequential Damage Waiver
+==============================
+
+TO THE MAXIMUM EXTENT PERMITTED BY APPLICABLE LAW, IN NO EVENT WILL YOU OR US
+BE LIABLE FOR ANY LOSS OF PROFITS, LOSS OF ANTICIPATED SAVINGS, LOSS OF DATA,
+INDIRECT, SPECIAL, INCIDENTAL, CONSEQUENTIAL AND EXEMPLARY DAMAGES ARISING OUT
+OF THIS AGREEMENT REGARDLESS OF THE LEGAL OR EQUITABLE THEORY (CONTRACT, TORT
+OR OTHERWISE) UPON WHICH THE CLAIM IS BASED.
+
+
+6. Miscellaneous
+================
+
+6.1 Jurisdiction
+----------------
+
+This Agreement will be governed by and construed in accordance with the laws of
+Puerto Rico excluding its conflicts of law provisions. Under certain circumstances,
+the governing law in this section might be superseded by the United Nations
+Convention on Contracts for the International Sale of Goods ("UN Convention")
+and the parties intend to avoid the application of the UN Convention to this
+Agreement and, thus, exclude the application of the UN Convention in its
+entirety to this Agreement.
+
+6.2 Acceptance
+--------------
+
+This Agreement sets out the entire agreement between You and Us for Your
+Contributions to Us and overrides all other agreements or understandings.
+
+6.3 Third parties
+-----------------
+
+If You or We assign the rights or obligations received through this Agreement
+to a third party, as a condition of the assignment, that third party must
+agree in writing to abide by all the rights and obligations in the Agreement.
+
+6.4 Unmet responsibilities
+--------------------------
+
+The failure of either party to require performance by the other party of any
+provision of this Agreement in one situation shall not affect the right of a
+party to require such performance at any time in the future. A waiver of
+performance under a provision in one situation shall not be considered a
+waiver of the performance of the provision in the future or a waiver of the
+provision in its entirety.
+
+6.5 Continuation
+----------------
+
+If any provision of this Agreement is found void and unenforceable,
+such provision will be replaced to the extent possible with a provision that
+comes closest to the meaning of the original provision and which is enforceable.
+The terms and conditions set forth in this Agreement shall apply notwithstanding
+any failure of essential purpose of this Agreement or any limited remedy to
+the maximum extent possible under law.
+
+
+::
+
+
+    Name: _________________________________________________
+
+
+    Email: ________________________________________________
+
+
+    Address: ______________________________________________
+
+
+    Address (cont): _______________________________________
+
+
+    Country: ______________________________________________
+
+
+    Phone number: _________________________________________
+
+
+    Fax number: ___________________________________________
+
+
+
+    _______________________________________________________
+                             Signature
+
+
+
+Mayan EDMS Individual CAA Version 1.0 April 8, 2018

docs/chapters/cabinets.rst

@@ -0,0 +1,14 @@
+Cabinets
+========
+
+Cabinets are a multi-level method to organize documents. Each cabinet can
+contain documents as well as other sub level cabinets. Cabinets can
+be added from the :menuselection:`Cabinets --> Add new` menu.
+
+With a cabinet structure created, the next step is to add documents to them.
+To add documents to a cabinet, select the cabinet tab of a document, and
+select :menuselection:`Actions --> Add to cabinets` link.
+
+Documents can also be added in bulk to one or more cabinets from any view
+that lists documents.
+

docs/chapters/checkouts.rst

@@ -1,6 +1,6 @@
-=========
+*********
 Checkouts
-=========
+*********
 
 Checkouts are a way to block certain accesses or actions of a document for a
 period of time.

docs/chapters/database_conversion.rst

@@ -0,0 +1,103 @@
+*******************
+Database conversion
+*******************
+
+Version 3.1.x added a new management command to help convert data residing in
+an SQLite database to other database managers like PostgreSQL. Here is the
+conversion procedure.
+
+Direct install
+==============
+
+* Make a backup of your existing SQLite database and documents by copying the
+  ``/opt/mayan-edms/media`` folder.
+* :doc:`Upgrade to at least version 3.1.3. <../releases/3.1.3>`
+*  Migrate the existing SQLite database with the command ``performupgrade``::
+
+    sudo -u mayan MAYAN_MEDIA_ROOT=/opt/mayan-edms/media /opt/mayan-edms/bin/mayan-edms.py performupgrade
+
+* Install PostgreSQL::
+
+    sudo apt-get install postgresql libpq-dev
+
+* Provision a PostgreSQL database::
+
+    sudo -u postgres psql -c "CREATE USER mayan WITH password 'mayanuserpass';"
+    sudo -u postgres createdb -O mayan mayan
+
+* Install the Python client for PostgreSQL::
+
+    sudo -u mayan /opt/mayan-edms/bin/pip install --no-cache-dir --no-use-pep517 psycopg2==2.7.3.2
+
+* Copy the newly created fallback config file::
+
+    cp /opt/mayan-edms/media/config_backup.yml /opt/mayan-edms/media/config.yml
+
+* Edit the configuration file to add the entry for the PostgreSQL database and
+  rename the SQLite database to 'old'::
+
+    # Before
+    DATABASES:
+      default:
+        ATOMIC_REQUESTS: false
+        AUTOCOMMIT: true
+        CONN_MAX_AGE: 0
+        ENGINE: django.db.backends.sqlite3
+        HOST: ''
+        NAME: /opt/mayan-edms/media/db.sqlite3
+        OPTIONS: {}
+        PASSWORD: ''
+        PORT: ''
+        TEST: {CHARSET: null, COLLATION: null, MIRROR: null, NAME: null}
+        TIME_ZONE: null
+        USER: ''
+
+    # After
+    DATABASES:
+      old:
+        ATOMIC_REQUESTS: false
+        AUTOCOMMIT: true
+        CONN_MAX_AGE: 0
+        ENGINE: django.db.backends.sqlite3
+        HOST: ''
+        NAME: /opt/mayan-edms/media/db.sqlite3
+        OPTIONS: {}
+        PASSWORD: ''
+        PORT: ''
+        TEST: {CHARSET: null, COLLATION: null, MIRROR: null, NAME: null}
+        TIME_ZONE: null
+        USER: ''
+      default:
+        ATOMIC_REQUESTS: false
+        AUTOCOMMIT: true
+        CONN_MAX_AGE: 0
+        ENGINE: django.db.backends.postgresql
+        HOST: '127.0.0.1'
+        NAME: /opt/mayan-edms/media/db.sqlite3
+        OPTIONS: {}
+        PASSWORD: 'mayanuserpass'
+        PORT: ''
+        TEST: {CHARSET: null, COLLATION: null, MIRROR: null, NAME: null}
+        TIME_ZONE: null
+        USER: 'mayan'
+
+* Migrate the new database to create the empty tables::
+
+    sudo -u mayan MAYAN_DATABASE_ENGINE=django.db.backends.postgresql MAYAN_DATABASE_NAME=mayan MAYAN_DATABASE_PASSWORD=mayanuserpass MAYAN_DATABASE_USER=mayan MAYAN_DATABASE_HOST=127.0.0.1 MAYAN_MEDIA_ROOT=/opt/mayan-edms/media /opt/mayan-edms/bin/mayan-edms.py migrate
+
+* Convert the data in the SQLite and store it in the PostgreSQL database::
+
+    sudo -u mayan MAYAN_DATABASE_ENGINE=django.db.backends.postgresql MAYAN_DATABASE_NAME=mayan MAYAN_DATABASE_PASSWORD=mayanuserpass MAYAN_DATABASE_USER=mayan MAYAN_DATABASE_HOST=127.0.0.1 MAYAN_MEDIA_ROOT=/opt/mayan-edms/media /opt/mayan-edms/bin/mayan-edms.py convertdb --from=old --to=default --force
+
+* Update the supervisor config file to have Mayan EDMS run from the PostgreSQL database::
+
+    [supervisord]
+    environment=
+        <...>
+        MAYAN_DATABASE_ENGINE=django.db.backends.postgresql,
+        MAYAN_DATABASE_HOST=127.0.0.1,
+        MAYAN_DATABASE_NAME=mayan,
+        MAYAN_DATABASE_PASSWORD=mayanuserpass,
+        MAYAN_DATABASE_USER=mayan,
+        MAYAN_DATABASE_CONN_MAX_AGE=0,
+        <...>

docs/chapters/deploying.rst

@@ -0,0 +1,255 @@
+******************
+Direct deployments
+******************
+
+Mayan EDMS should be deployed like any other Django_ project and
+preferably using virtualenv_. Below are some ways to deploy and use Mayan EDMS.
+
+Being a Django_ and a Python_ project, familiarity with these technologies is
+recommended to better understand why Mayan EDMS does some of the things it
+does.
+
+Compilers and development libraries will be installed to compile runtime
+libraries. LibreOffice and Poppler utils will also be installed as they are
+used to convert document files. Supervisor (https://supervisord.org/), a
+Process Control System, will be used to monitor and keep all Mayan processes
+running.
+
+
+Basic deployment
+================
+This setup uses less memory and CPU resources at the expense of some speed.
+For another setup that offers more performance and scalability refer to the
+`Advanced deployment`_ below.
+
+1. Install binary dependencies:
+-------------------------------
+   If using a Debian_ or Ubuntu_ based Linux distribution, get the executable
+   requirements using::
+
+       sudo apt-get install g++ gcc ghostscript gnupg1 graphviz libfuse2 \
+       libjpeg-dev libmagic1 libpq-dev libpng-dev libreoffice libtiff-dev \
+       poppler-utils postgresql python-dev python-virtualenv redis-server \
+       sane-utils supervisor tesseract-ocr zlib1g-dev -y
+
+   .. note::
+
+       Platforms with the ARM CPU might also need additional requirements.
+       ::
+
+           sudo apt-get install libffi-dev libssl-dev -y
+
+
+2. Create the user account for the installation:
+------------------------------------------------
+   This will create an unprivileged user account that is also unable to login.
+   ::
+
+       sudo adduser mayan --disabled-password --disabled-login --no-create-home --gecos ""
+
+
+3. Create the parent directory where the project will be deployed:
+------------------------------------------------------------------
+   ``/opt/`` is a good choice as it is meant is for "software and add-on packages
+   that are not part of the default installation". (https://www.tldp.org/LDP/Linux-Filesystem-Hierarchy/html/opt.html)
+   ::
+
+       sudo mkdir /opt
+
+
+4. Create the Python virtual environment:
+-----------------------------------------
+   This will keep all the Python packages installed here isolated from the rest
+   of the Python packages in the system.
+   ::
+
+       sudo virtualenv /opt/mayan-edms
+
+
+5. Make the mayan user the owner of the installation directory:
+---------------------------------------------------------------
+   ::
+
+       sudo chown mayan:mayan /opt/mayan-edms -R
+
+
+6. Install Mayan EDMS from PyPI:
+--------------------------------
+   ::
+
+       sudo -u mayan /opt/mayan-edms/bin/pip install --no-cache-dir --no-use-pep517 mayan-edms
+
+
+7. Install the Python client for PostgreSQL and Redis:
+------------------------------------------------------
+   ::
+
+       sudo -u mayan /opt/mayan-edms/bin/pip install --no-cache-dir --no-use-pep517 psycopg2==2.7.3.2 redis==2.10.6
+
+   .. note::
+
+       Platforms with the ARM CPU might also need additional requirements.
+       ::
+
+           sudo -u mayan /opt/mayan-edms/bin/pip install --no-cache-dir --no-use-pep517 psutil==5.6.2
+
+
+8. Create the database for the installation:
+--------------------------------------------
+   ::
+
+       sudo -u postgres psql -c "CREATE USER mayan WITH password 'mayanuserpass';"
+       sudo -u postgres createdb -O mayan mayan
+
+
+9. Initialize the project:
+--------------------------
+   This step will create all the database structures, download static media files
+   like JavaScript libraries and HTML frameworks, and create and initial admin
+   account with a random password.
+
+   .. note::
+
+       For simplicity, the ``MAYAN_MEDIA_ROOT`` folder is set to be a subfolder
+       of the installation. If you want to keep your files separated from
+       the installation files, change the value of the ``MAYAN_MEDIA_ROOT``
+       variable in this and all subsequent steps. Be sure to first create the
+       folder and give ownership of it to the ``mayan`` user with the ``chown``
+       command.
+
+   .. warning::
+
+       If this step is interrupted, even if it is later resumed, will
+       cause the automatic admin user to no be created in some cases. Make sure all
+       environment variable and values are correct. If this happens, refer to the
+       troubleshooting chapters: :ref:`troubleshooting-autoadmin-account` and
+       :ref:`troubleshooting-admin-password`.
+
+   ::
+
+       sudo -u mayan MAYAN_DATABASE_ENGINE=django.db.backends.postgresql MAYAN_DATABASE_NAME=mayan \
+       MAYAN_DATABASE_PASSWORD=mayanuserpass MAYAN_DATABASE_USER=mayan \
+       MAYAN_DATABASE_HOST=127.0.0.1 MAYAN_MEDIA_ROOT=/opt/mayan-edms/media \
+       /opt/mayan-edms/bin/mayan-edms.py initialsetup
+
+
+10. Collect the static files:
+-----------------------------
+    This step merges and compressed static media files so they can be served more
+    effectively.
+
+    ::
+
+        sudo -u mayan MAYAN_MEDIA_ROOT=/opt/mayan-edms/media \
+        /opt/mayan-edms/bin/mayan-edms.py preparestatic --noinput
+
+
+11. Create the supervisor file at ``/etc/supervisor/conf.d/mayan.conf``:
+------------------------------------------------------------------------
+    ::
+
+        sudo MAYAN_DATABASE_ENGINE=django.db.backends.postgresql MAYAN_DATABASE_NAME=mayan \
+        MAYAN_DATABASE_PASSWORD=mayanuserpass MAYAN_DATABASE_USER=mayan \
+        MAYAN_DATABASE_HOST=127.0.0.1 MAYAN_MEDIA_ROOT=/opt/mayan-edms/media \
+        /opt/mayan-edms/bin/mayan-edms.py platformtemplate supervisord > /etc/supervisor/conf.d/mayan.conf
+
+
+12. Configure Redis:
+--------------------
+    Configure Redis to discard data when it runs out of memory, not save its
+    database and only keep 1 database:
+    ::
+
+        sudo echo "maxmemory-policy allkeys-lru" >> /etc/redis/redis.conf
+        sudo echo "save \"\"" >> /etc/redis/redis.conf
+        sudo echo "databases 1" >> /etc/redis/redis.conf
+        sudo systemctl restart redis
+
+13. Enable and restart the services [1_]:
+-----------------------------------------
+    ::
+
+        sudo systemctl enable supervisor
+        sudo systemctl restart supervisor
+
+
+14. Cleaning up:
+----------------
+    The following operating system dependencies are only needed during
+    installation and can be removed.
+    ::
+
+        sudo apt-get remove --purge libjpeg-dev libpq-dev libpng-dev libtiff-dev zlib1g-dev
+
+
+.. _deployment_advanced:
+
+Advanced deployment
+===================
+
+This variation uses RabbitMQ as the message broker. RabbitMQ consumes more
+memory but scales to thousands of messages per second. RabbitMQ messages are also
+persistent by default, this means that pending tasks are not lost in the case
+of a restart or power failure. The Gunicorn workers are increased to 3.
+
+
+1. Install RabbitMQ:
+--------------------
+
+   If using a Debian_ or Ubuntu_ based Linux distribution, get the executable
+   requirements using::
+
+       sudo apt-get install rabbitmq-server -y
+
+
+2. Install the Python client for RabbitMQ:
+------------------------------------------
+   ::
+
+       sudo -u mayan /opt/mayan-edms/bin/pip install --no-cache-dir --no-use-pep517 librabbitmq==2.0.0
+
+
+3. Create the RabbitMQ user and vhost:
+--------------------------------------
+   ::
+
+       sudo rabbitmqctl add_user mayan mayanrabbitmqpassword
+       sudo rabbitmqctl add_vhost mayan
+       sudo rabbitmqctl set_permissions -p mayan mayan ".*" ".*" ".*"
+
+
+4. Edit the supervisor file at ``/etc/supervisor/conf.d/mayan.conf``:
+---------------------------------------------------------------------
+   Replace (paying attention to the comma at the end)::
+
+       MAYAN_BROKER_URL="redis://127.0.0.1:6379/0",
+
+   with::
+
+       MAYAN_BROKER_URL="amqp://mayan:mayanrabbitmqpassword@localhost:5672/mayan",
+
+   increase the number of Gunicorn workers to 3 in the line (``-w 2`` section)::
+
+       command = /opt/mayan-edms/bin/gunicorn -w 2 mayan.wsgi --max-requests 1000 --max-requests-jitter 50 --worker-class gevent --bind 0.0.0.0:8000 --timeout 120
+
+   remove the concurrency limit (or increase it) of the fast worker (remove ``--concurrency=1``).
+
+
+5. Restart the services:
+------------------------
+   ::
+
+       sudo supervisorctl restart all
+
+
+
+
+[1]: https://bugs.launchpad.net/ubuntu/+source/supervisor/+bug/1594740
+
+.. _Debian: https://www.debian.org/
+.. _Django: https://www.djangoproject.com/
+.. _Python: https://www.python.org/
+.. _SQLite: https://www.sqlite.org/
+.. _Ubuntu: http://www.ubuntu.com/
+.. _virtualenv: http://www.virtualenv.org/en/latest/index.html
+.. _1: https://bugs.launchpad.net/ubuntu/+source/supervisor/+bug/1594740

docs/chapters/docker.rst

@@ -0,0 +1,450 @@
+============
+Docker image
+============
+
+How to use this image
+=====================
+
+.. _docker_install:
+
+Start a Mayan EDMS Docker image
+-------------------------------
+
+With Docker properly installed, proceed to download the Mayan EDMS image using
+the command::
+
+    docker pull mayanedms/mayanedms:<version>
+
+Instead of a specific version tag you may use then generic ``:latest`` tag
+to the get latest version available automatically. If you use the ``:latest``
+tag here, remember to do so in the next steps also.::
+
+    docker pull mayanedms/mayanedms:latest
+
+Then download version 9.6 of the Docker PostgreSQL image::
+
+    docker pull postgres:9.6
+
+Create and run a PostgreSQL container::
+
+    docker run -d \
+    --name mayan-edms-postgres \
+    --restart=always \
+    -p 5432:5432 \
+    -e POSTGRES_USER=mayan \
+    -e POSTGRES_DB=mayan \
+    -e POSTGRES_PASSWORD=mayanuserpass \
+    -v /docker-volumes/mayan-edms/postgres:/var/lib/postgresql/data \
+    -d postgres:9.6
+
+The PostgreSQL container will have one database named ``mayan``, with an user
+named ``mayan`` too, with a password of ``mayanuserpass``. The container will
+expose its internal 5432 port (PostgreSQL's default port) via the host's
+5432 port. The data of this container will reside on the host's
+``/docker-volumes/mayan-edms/postgres`` folder.
+
+Finally create and run a Mayan EDMS container::
+
+    docker run -d \
+    --name mayan-edms \
+    --restart=always \
+    -p 80:8000 \
+    -e MAYAN_DATABASE_ENGINE=django.db.backends.postgresql \
+    -e MAYAN_DATABASE_HOST=172.17.0.1 \
+    -e MAYAN_DATABASE_NAME=mayan \
+    -e MAYAN_DATABASE_PASSWORD=mayanuserpass \
+    -e MAYAN_DATABASE_USER=mayan \
+    -e MAYAN_DATABASE_CONN_MAX_AGE=0 \
+    -v /docker-volumes/mayan-edms/media:/var/lib/mayan \
+    mayanedms/mayanedms:<version>
+
+The Mayan EDMS container will connect to the PostgreSQL container via the
+``172.17.0.1`` IP address (the Docker host's default IP address). It will
+connect using the ``django.db.backends.postgresql`` database driver and
+connect to the ``mayan`` database using the ``mayan`` user with the password
+``mayanuserpass``. The container will keep connections to the database
+for up to 60 seconds in an attempt to reuse them increasing response time
+and reducing memory usage. The files of the container will be store in the
+host's ``/docker-volumes/mayan-edms/media`` folder. The container will
+expose its web service running on port 8000 on the host's port 80.
+
+The container will be available by browsing to ``http://localhost`` or to
+the IP address of the computer running the container.
+
+If another web server is running on port 80 use a different port in the
+``-p`` option. For example: ``-p 81:8000``.
+
+
+Using a dedicated Docker network
+--------------------------------
+
+Use this method to avoid having to expose PostreSQL port to the host's network
+or if you have other PostgreSQL instances but still want to use the default
+port of 5432 for this installation.
+
+Create the network::
+
+    docker network create mayan
+
+Launch the PostgreSQL container with the network option and remove the port
+binding (``-p 5432:5432``)::
+
+    docker run -d \
+    --name mayan-edms-postgres \
+    --network=mayan \
+    --restart=always \
+    -e POSTGRES_USER=mayan \
+    -e POSTGRES_DB=mayan \
+    -e POSTGRES_PASSWORD=mayanuserpass \
+    -v /docker-volumes/mayan-edms/postgres:/var/lib/postgresql/data \
+    -d postgres:9.6
+
+Launch the Mayan EDMS container with the network option and change the
+database hostname to the PostgreSQL container name (``mayan-edms-postgres``)
+instead of the IP address of the Docker host (``172.17.0.1``)::
+
+    docker run -d \
+    --name mayan-edms \
+    --network=mayan \
+    --restart=always \
+    -p 80:8000 \
+    -e MAYAN_DATABASE_ENGINE=django.db.backends.postgresql \
+    -e MAYAN_DATABASE_HOST=mayan-edms-postgres \
+    -e MAYAN_DATABASE_NAME=mayan \
+    -e MAYAN_DATABASE_PASSWORD=mayanuserpass \
+    -e MAYAN_DATABASE_USER=mayan \
+    -e MAYAN_DATABASE_CONN_MAX_AGE=0 \
+    -v /docker-volumes/mayan-edms/media:/var/lib/mayan \
+    mayanedms/mayanedms:<version>
+
+
+Stopping and starting the container
+-----------------------------------
+
+To stop the container use::
+
+    docker stop mayan-edms
+
+
+To start the container again::
+
+    docker start mayan-edms
+
+
+.. _docker_environment_variables:
+
+
+Environment Variables
+---------------------
+
+The Mayan EDMS image can be configure via environment variables.
+
+``MAYAN_DATABASE_ENGINE``
+
+Defaults to ``None``. This environment variable configures the database
+backend to use. If left unset, SQLite will be used. The database backends
+supported by this Docker image are:
+
+- ``'django.db.backends.postgresql'``
+- ``'django.db.backends.mysql'``
+- ``'django.db.backends.sqlite3'``
+
+When using the SQLite backend, the database file will be saved in the Docker
+volume. The SQLite database as used by Mayan EDMS is meant only for development
+or testing, never use it in production.
+
+``MAYAN_DATABASE_NAME``
+
+Defaults to 'mayan'. This optional environment variable can be used to define
+the database name that Mayan EDMS will connect to. For more information read
+the pertinent Django documentation page:
+:django-docs:`Connecting to the database <ref/databases/#connecting-to-the-database>`
+
+``MAYAN_DATABASE_USER``
+
+Defaults to 'mayan'. This optional environment variable is used to set the
+username that will be used to connect to the database. For more information
+read the pertinent Django documentation page:
+:django-docs:`Settings, USER <ref/settings/#user>`
+
+``MAYAN_DATABASE_PASSWORD``
+
+Defaults to ''. This optional environment variable is used to set the
+password that will be used to connect to the database. For more information
+read the pertinent Django documentation page:
+:django-docs:`Settings, PASSWORD <ref/settings/#password>`
+
+``MAYAN_DATABASE_HOST``
+
+Defaults to `None`. This optional environment variable is used to set the
+hostname that will be used to connect to the database. This can be the
+hostname of another container or an IP address. For more information read
+the pertinent Django documentation page:
+:django-docs:`Settings, HOST <ref/settings/#host>`
+
+``MAYAN_DATABASE_PORT``
+
+Defaults to `None`. This optional environment variable is used to set the
+port number to use when connecting to the database. An empty string means
+the default port. Not used with SQLite. For more information read the
+pertinent Django documentation page:
+:django-docs:`Settings, PORT <ref/settings/#port>`
+
+``MAYAN_BROKER_URL``
+
+This optional environment variable determines the broker that Celery will use
+to relay task messages between the frontend code and the background workers.
+For more information read the pertinent Celery Kombu documentation page: `Broker URL`_
+
+.. _Broker URL: http://kombu.readthedocs.io/en/latest/userguide/connections.html#connection-urls
+
+This Docker image supports using Redis and RabbitMQ as brokers.
+
+Caveat: If the `MAYAN_BROKER_URL` and `MAYAN_CELERY_RESULT_BACKEND` environment
+variables are specified, the built-in Redis server inside the container will
+be disabled.
+
+``MAYAN_CELERY_RESULT_BACKEND``
+
+This optional environment variable determines the results backend that Celery
+will use to relay result messages from the background workers to the frontend
+code. For more information read the pertinent Celery Kombu documentation page:
+`Task result backend settings`_
+
+.. _Task result backend settings: http://docs.celeryproject.org/en/3.1/configuration.html#celery-result-backend
+
+This Docker image supports using Redis and RabbitMQ as result backends.
+
+Caveat: If the `MAYAN_BROKER_URL` and `MAYAN_CELERY_RESULT_BACKEND` environment
+variables are specified, the built-in Redis server inside the container will
+be disabled.
+
+``MAYAN_SETTINGS_MODULE``
+
+Optional. Allows loading an alternate settings file.
+
+``MAYAN_DATABASE_CONN_MAX_AGE``
+
+Amount in seconds to keep a database connection alive. Allow reuse of database
+connections. For more information read the pertinent Django documentation
+page: :django-docs:`Settings, CONN_MAX_AGE <ref/settings/#conn-max-age>`
+According to new information Gunicorn's microthreads don't share connections
+and will exhaust the available Postgres connections available if a number
+other than 0 is used. Reference: https://serverfault.com/questions/635100/django-conn-max-age-persists-connections-but-doesnt-reuse-them-with-postgresq
+and https://github.com/benoitc/gunicorn/issues/996
+
+``MAYAN_GUNICORN_WORKERS``
+
+Optional. This environment variable controls the number of frontend workers
+that will be executed. If not specified the default is 2. For heavier loads,
+user a higher number. A formula recommended for this setting is the number
+of CPU cores + 1.
+
+``MAYAN_WORKER_FAST_CONCURRENCY``
+
+Optional. Changes the concurrency (number of child processes) of the Celery
+worker consuming the queues in the fast (low latency, short tasks) category.
+Default is 1. Use 0 to disable hardcoded concurrency and allow the Celery
+worker to launch its default number of child processes (equal to the number
+of CPUs detected).
+
+``MAYAN_WORKER_MEDIUM_CONCURRENCY``
+
+Optional. Changes the concurrency (number of child processes) of the Celery
+worker consuming the queues in the medium (medium latency, long running tasks)
+category. Default is 1. Use 0 to disable hardcoded concurrency and allow the
+Celery worker to launch its default number of child processes (equal to the
+number of CPUs detected).
+
+``MAYAN_WORKER_SLOW_CONCURRENCY``
+
+Optional. Changes the concurrency (number of child processes) of the Celery
+worker consuming the queues in the slow (high latency, very long running tasks)
+category. Default is 1. Use 0 to disable hardcoded concurrency and allow the
+Celery worker to launch its default number of child processes (equal to the
+number of CPUs detected).
+
+``MAYAN_USER_UID``
+
+Optional. Changes the UID of the ``mayan`` user internal to the Docker
+container. Defaults to 1000.
+
+``MAYAN_USER_GUID``
+
+Optional. Changes the GUID of the ``mayan`` user internal to the Docker
+container. Defaults to 1000.
+
+
+.. _docker-accessing-outside-data:
+
+Accessing outside data
+======================
+
+To use Mayan EDMS's staging folders or watch folders from Docker, the data
+for these source must be made accessible to the container. This is done by
+mounting the folders in the host computer to folders inside the container.
+This is necessary because Docker containers do not have access to host data
+on purpose. For example, to make a folder in the host accessible as a watch
+folder, add the following to the Docker command line when starting the
+container::
+
+    -v /opt/scanned_files:/scanned_files
+
+The command line would look like this::
+
+    docker run ... -v /opt/scanned_files:/scanned_files mayanedms/mayanedms:latest
+
+Now create a watch folder in Mayan EDMS using the path ``/scanned_files``
+and the documents from the host folder ``/opt/scanned_files`` will be
+automatically available. Use the same procedure to mount host folders to be
+used as staging folders. In this example ``/scanned_files`` was used as the
+container directory, but any path can be used as long as:
+
+- the path not an already existing path
+- the path is not used by any other program
+- the path is a single level path
+
+
+Performing backups
+==================
+
+To backup the existing data, stop the image and copy the content of the volume.
+For the example::
+
+    docker run -d --name mayan-edms --restart=always -p 80:8000 \
+    -v /docker-volumes/mayan:/var/lib/mayan \
+    -v /opt/scanned_files:/scanned_files mayanedms/mayanedms:latest
+
+That would be the ``/docker-volumes/mayan folder``::
+
+    sudo tar -zcvf backup.tar.gz /docker-volumes/mayan
+    sudo chown `whoami` backup.tar.gz
+
+If using an external PostgreSQL or MySQL database or database containers, these
+too need to be backed up using their respective procedures. A simple solution
+is to copy the entire database container volume after the container has
+been stopped.
+
+
+Restoring from a backup
+=======================
+
+Uncompress the backup archive in the original docker volume using::
+
+    sudo tar -xvzf backup.tar.gz -C /
+
+
+Upgrading
+=========
+
+Upgrading a Mayan EDMS Docker container is actually a matter of stopping and
+deleting the container, downloading the most recent version of the image and
+starting a container again. The container will take care of updating the
+database structure to the newest version if necessary.
+
+**IMPORTANT!** Do not delete the volume storing the data, only the container.
+
+Stop the container to be upgraded::
+
+    docker stop mayan-edms
+
+
+Remove the container::
+
+    docker rm mayan-edms
+
+
+Pull the new image version::
+
+    docker pull mayanedms/mayanedms:latest
+
+
+Start the container again with the new image version::
+
+    docker run -d --name mayan-edms --restart=always -p 80:8000 -v /docker-volumes/mayan:/var/lib/mayan mayanedms/mayanedms:latest
+
+
+Building the image
+==================
+
+Clone the repository with::
+
+    git clone https://gitlab.com/mayan-edms/mayan-edms.git
+
+Change to the directory of the cloned repository::
+
+    cd mayan-edms
+
+Execute Docker's build command using the provided makefile::
+
+    make docker-build
+
+Or using an APT cache to speed up the build::
+
+    make docker-build-with-proxy APT_PROXY=172.17.0.1:3142
+
+Replace the IP address `172.17.0.1` with the IP address of the computer
+running the APT proxy and caching service.
+
+
+Customizing the image
+=====================
+
+Simple method
+-------------
+
+If you just need to add a few Ubuntu or Python packages to your installation,
+you can use the following environment variables:
+
+``MAYAN_APT_INSTALLS``
+
+Specifies a list of Ubuntu .deb packages to be installed via APT when the
+container is first created. The installed packages are not lost when the image
+is stopped. Example: To install the Tesseract OCR language packs for German
+and Spanish add the following in your ``docker start`` command line::
+
+    -e MAYAN_APT_INSTALLS="tesseract-ocr-deu tesseract-ocr-spa"
+
+``MAYAN_PIP_INSTALLS``
+
+Specifies a list of Python packages to be installed via ``pip``. Packages will
+be downloaded from the Python Package Index (https://pypi.python.org) by
+default.
+
+
+Using Docker compose
+====================
+
+To deploy a complete production stack using the included Docker compose file
+execute::
+
+    docker-compose -f docker-compose.yml up -d
+
+This Docker compose file will provision four containers:
+
+- Postgres as the database
+- Redis as the Celery result storage
+- RabbitMQ as the Celery broker
+- Mayan EDMS using the above service containers
+
+To stop the stack use::
+
+    docker-compose -f docker-compose.yml stop
+
+The stack will also create four volumes to store the data of each container.
+These are:
+
+- mayan_app - The Mayan EDMS data container, normally called `mayan_data` when not using Docker compose.
+- mayan_broker - The broker volume, in this case RabbitMQ.
+- mayan_db - The database volume, in this case Postgres.
+- mayan_results - The celery result backend volume, in this case Redis.
+
+
+Nightly images
+==============
+The continuous integration pipeline used for testing development builds also
+produces a resulting Docker image. These are build automatically and their
+stability is not guaranteed. They should never be used in production.
+If you want to try out the Docker images the development uses or want a sneak
+peek at the new features being worked on checkout the container registry at:
+https://gitlab.com/mayan-edms/mayan-edms/container_registry

docs/chapters/document_types.rst

@@ -0,0 +1,88 @@
+Document types
+==============
+
+The basic unit of data in Mayan EDMS is the **document type**. A document
+type can be interpreted also as a document category, a document class, or a
+document template. Every other aspect of the system will rely or be tied to
+one or more document type. Create one document type for each type or class of
+document you intend to upload into Mayan EDMS.
+
+Document types need to be created before documents can be
+uploaded. It is not possible to upload documents without assigning them a
+document type.
+
+Document types usually mirror the type of physical, paper document they
+represent.
+
+Example document types:
+
+* Letter
+* Invoice
+* Timesheet
+* Blueprint
+
+.. blockdiag::
+
+   blockdiag {
+      default_shape = roundedbox
+
+      document_type [ label = 'each document type' ];
+      documents [ label = 'many documents' ];
+
+      document_type -> documents;
+   }
+
+
+Examples:
+
+.. blockdiag::
+
+   blockdiag {
+      default_shape = roundedbox
+
+      document_type [ label = 'Invoice' ];
+      documents_1 [ label = 'invoice_001.pdf', width=200 ];
+      documents_2 [ label = 'invoice_032.pdf', width=200 ];
+
+      document_type -> documents_1, documents_2;
+   }
+
+
+.. blockdiag::
+
+   blockdiag {
+      default_shape = roundedbox
+
+      document_type [ label = 'Receipts' ];
+      documents_1 [ label = 'groceries_18-01-11.pdf', width=200  ];
+      documents_2 [ label = 'car_payment-17-01-02.png', width=200  ];
+
+      document_type -> documents_1, documents_2;
+   }
+
+
+Settings and attributes are applied to document types and documents will
+inherit those settings and attributes based on the document type they were
+assigned when uploaded into Mayan EDMS. A document can only be of one
+type at a given moment, but if needed, the type of a document can be changed.
+Upon changing its type, the document will lose its previous settings and
+attributes, and will inherit the settings and attributes of its new type.
+
+
+Creating document types
+=======================
+
+.. admonition:: Permissions required
+    :class: warning
+
+    The "Create document types" permission is required for this action.
+
+
+#. Go to the :menuselection:`System --> Setup --> Document types` menu.
+#. From the :guilabel:`Actions` dropdown select :guilabel:`Create document type`.
+#. Enter a label to be shown to users when using this document type.
+#. Optional: Enter a period of time after which documents of this type will be moved to the trash.
+#. Optional: Select the unit of time for the period after which the document will be moved to the trash.
+#. Optional: Enter a period of time after which documents moved to the trash will be permanently deleted.
+#. Optional: Select the unit of time for the period of time after which the documents moved to the trash will be permanently deleted
+#. Press :guilabel:`Submit`.

docs/chapters/file_storage.rst

@@ -1,14 +1,15 @@
-============
+************
 File storage
-============
+************
 
-The files are stored and placed under **Mayan EDMS** "control" to avoid
+The files are stored and placed under Mayan EDMS "control" to avoid
 filename clashes each file gets renamed to its ``UUID`` (Universally Unique ID),
 without extension, and stored in a simple flat arrangement in a directory.
 
 .. blockdiag::
 
    blockdiag {
+      default_shape = roundedbox
       file [ label = 'mayan_1-1.pdf', width=120];
       document [ label = 'mayan/media/document_storage/ab6c1cfe-8a8f-4a30-96c9-f54f606b9248', width=450];
       file -> document [label = "upload"];
@@ -20,9 +21,16 @@ This doesn't stop access to the files but renaming, moving or updating
 directly them is not recommended because it would throw the database out
 of sync.
 
-Because **Mayan EDMS** components are as decoupled from each other as possible,
+Because Mayan EDMS components are as decoupled from each other as possible,
 storage in this case is decoupled and its behavior is controlled
 not by the project but by the ``Storage`` module class. All the other
 modules don't make any assumptions about how the actual document files are
 stored. This way files can be saved locally, over the network or even across
 the Internet and everything will still operate exactly the same.
+
+The default file storage backend: ``storage.backends.filebasedstorage.FileBasedStorage``
+is a simple backend that only supports paths and not IP addresses. In case you
+are interested in using remote volumes to store documents (NFS, SAMBA), first
+mount these volumes so that they appear as a directories to Mayan EDMS. For
+direct support for remote volumes a custom backend would be needed such as those
+provided by the Django Storages project (https://django-storages.readthedocs.org/en/latest/).

docs/chapters/index_examples.rst

@@ -0,0 +1,125 @@
+
+Index examples
+--------------
+
+Index of document types
+^^^^^^^^^^^^^^^^^^^^^^^
+
+.. admonition:: Permissions required
+    :class: warning
+
+    - The "Create new document indexes" permission is required for this action.
+    - The "Edit document indexes" permission is required for this action, globally
+      of via an ACL for a document index.
+    - The "Edit document types" permission is required for this action, globally
+      of via an ACL for a document type.
+
+
+This index will create one level for each document type in the system and place
+links to the document of each respective type.
+
+#. Go to the :menuselection:`System --> Setup --> Indexes` menu.
+#. Create a new index using :guilabel:`Actions` > :guilabel:`Create new`.
+#. Give it a label to describe it, and an internal name. The internal name is
+   used when referencing this index in other parts of the system.
+#. Press the :guilabel:`Template` link of the newly created index.
+#. Select :guilabel:`New child node` to create a new level in which the
+   following template code will be entered.
+   ::
+
+       {{ document.document_type }}
+
+#. Save the template.
+#. Click on :guilabel:`Document types` and associate this index with
+   existing document types in the system.
+#. Finally go to :menuselection:`Tools --> Rebuild indexes` to execute the
+   index template. The rebuild process is only necessary when changes are
+   made to the index templates. Otherwise they update automatically whenever
+   a new document is uploaded or existing documents properties are modified.
+#. A new index should appear under :menuselection:`Indexes` menu.
+
+You can also program different behavior based on the different document types,
+by use a comparison and a conditional statement. As the document_type itself
+is not a string you cannot directly use that for that comparison. You will have
+to use the label of the document type.
+
+For example::
+
+    {% if document.document_type.label == "Invoice" or document.document_type.label == "Letter" %}
+    Correspondence
+    {% else %}
+    {{ document.document_type }}
+    {% endif %}
+
+This will create and index level for each document type. Except for documents
+of types "Invoice" and "Letter", these will now go into the level "Correspondence".
+
+
+Index document by department, taken from the first character of the invoice number metadata
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Requires one index node with the template::
+
+    {% if document.metadata_value_of.invoice_number.0 == "A" %}Accounting
+    {% if document.metadata_value_of.invoice_number.0 == "H" %}Human Resources
+    {% endif %}
+
+
+Nested date index from a date contained in a metadata
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Assuming the metadata type is named **date_issued** with a date format
+of YYYY-MM-DD. The target is to have two levels: one for years and another
+sub level for months.
+
+**First level: Year**
+::
+
+    {{ document.metadata_value_of.date_issued|slice:"0:4" }}
+
+
+**Second level: Months**
+::
+
+    {{ document.metadata_value_of.date_issued|slice:"5:7" }}
+
+
+**Optional: Third level: Day**
+::
+
+    {{ document.metadata_value_of.date_issued|slice:"8:10" }}
+
+
+Index by OCR content
+^^^^^^^^^^^^^^^^^^^^
+
+This example indexes documents in a "quarterly report" level if they have the
+fragment “quarterly report” in the OCR text::
+
+    {% if "quarterly report" in document.latest_version.ocr_content|join:" "|lower %}Quarterly reports{% endif %}
+
+The same applies to text content extracted for the document::
+
+    {% if "quarterly report" in document.latest_version.content|join:" "|lower %}Quarterly reports{% endif %}
+
+
+
+Index documents not found in any cabinet
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+::
+
+    {% if document.cabinets.count == 0 %}No Cabinets{% endif %}
+
+
+Index documents not tagged
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+::
+
+    {% if document.tags.count == 0 %}No Tags{% endif %}
+
+
+Index documents specifically, by the year of a metadata field otherwise by their uploaded year
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+::
+
+    {% for tag in document.tags.all %}{% if tag.label == "Taxes" %}{% if document.metadata_value_of.tax_year|length_is:"4" %}{{ document.metadata_value_of.tax_year }}{% else %}{{ document.date_added|date:"Y" }}{% endif %}{% endif %}{% endfor %}

docs/chapters/indexes.rst

@@ -1,11 +1,10 @@
-=======
 Indexes
 =======
 
 Indexes are an automatic method to hierarchically organize documents in
-relation to their properties (:doc:`metadata`, label, MIME type, etc). To use
-indexes you need to first create an index template. Once created, associate
-the index to one or more :doc:`document_types`.
+relation to their properties (:doc:`../chapters/metadata`, label, MIME type,
+etc). To use indexes you need to first create an index template. Once created,
+associate the index to one or more :doc:`../chapters/document_types`.
 
 Index are hierarchical models so a tree template needs to be specified for them.
 This tree template will contain references to document metadata or properties
@@ -17,12 +16,15 @@ Example:
 - Metadata type: ``Product year``, associated as a required metadata for the document type ``Product sheet``.
 
 - Index: ``Product sheets per year``, and associated to the document type ``Product sheet``.
-- Index slug: ``product-sheets-per-year``. Slugs are internal unique identifiers that can be used by other **Mayan EDMS** modules to reference each index.
+- Index slug: ``product-sheets-per-year``. Slugs are internal unique identifiers that can be used by other Mayan EDMS modules to reference each index.
 - Index tree template as follows:
 
 .. blockdiag::
 
    blockdiag {
+      default_shape = roundedbox
+      span_width = 30;
+
       index [ label = 'Product sheets per year', width=180 ];
       root [ label = 'Root (Has document links? No)', width=450];
       level_2 [ label = '{{ document.metadata_value_of.product_year }} (Has document links? Yes)', width=450];
@@ -50,6 +52,8 @@ that will be generate based on the tree template would be as follows:
 .. blockdiag::
 
    blockdiag {
+      default_shape = roundedbox
+
       index [ label = 'Product sheets per year', width=180 ];
       year_1 [ label = '2001', width = 60 ];
       year_2 [ label = '2002', width = 60 ];
@@ -72,8 +76,11 @@ that will be generate based on the tree template would be as follows:
 
    }
 
+.. include:: ../chapters/index_examples.rst
+
+
 Mirroring
-=========
+---------
 
 Indexes can be exported as `FUSE <https://en.wikipedia.org/wiki/Filesystem_in_Userspace>`_
 filesystems. Using the management command ``mountindex`` we could export the
@@ -89,5 +96,20 @@ via the network with network file system software like
 `Samba <https://www.samba.org/>`_ or
 `NFS <https://en.wikipedia.org/wiki/Network_File_System>`_.
 
+.. blockdiag::
+
+    blockdiag {
+        orientation = portrait
+        span_width = 200;
+
+        index [ label = 'Product sheets per year', width=180 ];
+        block_device [ height = 100, label = "Block device\n(Hard drive)", shape = flowchart.database ];
+        network [ label = "Network", shape = cloud ];
+        user [ label = "Users", shape = actor ];
+
+        index -> block_device [ label = "mirroring", fontsize = 8 ];
+        block_device -> network -> user;
+   }
+
 Indexes and mirrored indexes are Read Only as they are generated as a result of
 prior activities like document uploads, metadata changes.

docs/chapters/languages.rst

@@ -0,0 +1,36 @@
+*********
+Languages
+*********
+
+The list of languages choices in the language dropdown used for documents is
+based on the current ISO 639 list. This list can be quite extensive. To reduce
+the number of languages available use the setting ``DOCUMENTS_LANGUAGE_CODES``,
+and set it to a nested list of abbreviations. This setting can be found in the
+:menuselection:`System --> Setup --> Settings --> Common` menu.
+
+For example, to reduce the list to just English and Spanish use
+::
+
+    DOCUMENTS_LANGUAGE_CODES = ('eng', 'spa')
+
+
+The default language to appear on the dropdown can also be configured using::
+
+    DOCUMENTS_LANGUAGE = 'spa'
+
+Use the correct ISO 639-3 language abbreviation (https://en.wikipedia.org/wiki/ISO_639)
+as this code is used in several subsystems in Mayan EDMS such as the OCR app
+to determine how to interpret the document.
+
+If using the Docker image, these settings can also be passed to the container
+as environment variables by prepending the ``MAYAN_`` suffix.
+
+Example::
+
+  -e MAYAN_DOCUMENTS_LANGUAGE_CODES='["eng", "spa"]'
+
+For more information check out the
+:ref:`environment variables <environment_variables>` chapter of the
+:doc:`../topics/settings` topic.
+
+

docs/chapters/mailing.rst

@@ -0,0 +1,22 @@
+*****************
+Mailing documents
+*****************
+
+Sending emails in Mayan EDMS is controlled by two different system depending on
+the type of email being sent. These are administrative emails like password
+reset emails and user emails sent from the application. To configure
+administrative email for things like password reset check the topic:
+:doc:`../topics/administration`
+
+Application emails
+==================
+
+To allow users to send emails or documents from within the web interface,
+Mayan EDMS provides its our own email system called Mailing Profiles.
+Mailing Profiles support access control per user role and can use different
+email backends. Mailing Profiles are created from the
+:menuselection:`System --> Setup` menu.
+
+Once created mailing profiles allow users to send email messages from
+within the user interface containing either an URL link to the document or
+the actual document as an attachment.

docs/chapters/metadata.rst

@@ -0,0 +1,99 @@
+********
+Metadata
+********
+
+Metadata is the name of the attribute of a document. The concept of metadata is
+divided in two: **metadata types** (size, color, distance) and **metadata values** for
+those types. Metadata types are defined in the setup menu and associated with
+document types. Then when a document is uploaded, a value for that metadata
+can be entered. There are two kinds of metadata type to document type relations:
+optional and required. When a metadata type is optional for a document type,
+it can be left blank for a document being uploaded and the upload will still
+be successful. On the other hand required metadata type must be given a value
+or it will not be possible to upload the document at hand.
+
+Examples of metadata type: Invoice number, color, employee id.
+
+The data entry of metadata types can be set to allow any value to be provided
+(the default) or a list of possible values can be entered in the ``Lookup``
+configuration option and users will be presented with a drop down list of options
+instead of the default text entry box.
+
+If metadata types are setup to allow any value to be entered a ``validation``
+option can be chosen to block the entry of invalid data. Metadata types also
+provide ``parsers`` which will not block the entry of data but are able to
+interpret and modify the value provided by the user to a conform to a specific
+format. An example of a provided parser is the date parser which will interpret
+and correct dates provided by users regardless of the format in which they are
+entered.
+
+
+Creating metadata types
+=======================
+
+.. admonition:: Permissions required
+    :class: warning
+
+    The "Create new metadata types" permission is required for this action.
+
+
+#. Go to the :menuselection:`System --> Setup --> Metadata types` menu.
+#. From the :guilabel:`Actions` dropdown select :guilabel:`Create new`.
+#. Provide a name to reference this metadata type in other parts of the system.
+#. Enter a label to be shown to users when using this metadata type.
+#. Optional: Enter a default value for the metadata type.
+#. Optional: Provide a comma separated list of options to restrict the data entry
+   when using this metadata type.
+#. Optional: Select a validator and a parser to validate and cleanup the data
+   entry when not using a predetermined list of values.
+#. Press :guilabel:`Submit`.
+
+
+Assigning a metadata type to a document type
+============================================
+
+.. admonition:: Permissions required
+    :class: warning
+
+    - The "Edit metadata types" permission is required for this action, globally or
+      via an ACL for a metadata type.
+    - Also the "Edit document type" permission
+      is required, globally or via an ACL for a document type.
+
+
+This action can be performed in two ways.
+
+Option 1: Via the metadata type view
+------------------------------------
+
+#. Go to the :menuselection:`System --> Setup --> Metadata types` menu.
+#. Click on the button :guilabel:`Document types` of the metadata type you which
+   to associate.
+#. From the list of existing document types press either:
+
+   - :guilabel:`None` if this metadata type will not be available for documents
+     of the type.
+   - :guilabel:`Optional` if this metadata type will be available and is
+     optional to provide a value for documents of the type.
+   - :guilabel:`Required` if this metadata type will be available and is
+     required to provide a value for documents of the type.
+
+#. Press :guilabel:`Save`.
+
+
+Option 2: Via the document type view
+------------------------------------
+
+#. Go to the :menuselection:`System --> Setup --> Document types` menu.
+#. Click on the button :guilabel:`Metadata types` of the metadata type you which
+   to associate.
+#. From the list of existing metadata types press either:
+
+   - :guilabel:`None` if this metadata type will not be available for documents
+     of the type.
+   - :guilabel:`Optional` if this metadata type will be available and is
+     optional to provide a value for documents of the type.
+   - :guilabel:`Required` if this metadata type will be available and is
+     required to provide a value for documents of the type.
+
+#. Press :guilabel:`Save`.

docs/chapters/object_storage.rst

@@ -0,0 +1,31 @@
+**************
+Object storage
+**************
+
+It is possible to use object storage instead of the default filesystem based
+storage. One such object storage system is Amazon S3 (Simple Storage Service)
+API compatible object storage. The following steps will configure Mayan EDMS
+to use a S3 style storage for documents.
+
+1. Install the django-storages and boto3 Python libraries.
+
+  * For the direct deployment method of installation use::
+
+        pip install django-storages boto3
+
+
+  * or if using the Docker image, add the following the command line that runs the container::
+
+    -e MAYAN_PIP_INSTALLS='django-storages boto3'
+
+
+2. From the web interface navigate to the :menuselection:`System --> Setup --> Setting --> Documents` menu.
+3. Locate the **DOCUMENTS_STORAGE_BACKEND** setting, press **Edit** and enter::
+
+    storages.backends.s3boto3.S3Boto3Storage
+
+4. Save and locate the setting **DOCUMENTS_STORAGE_BACKEND_ARGUMENTS**, press **Edit** and enter::
+
+    '{access_key: <your S3 access key>, secret_key: <your S3 secret key>, bucket_name: <S3 bucket name>}'
+
+5. Save and restart your Mayan EDMS installation for the setting to take effect.

docs/chapters/ocr_backend.rst

@@ -0,0 +1,26 @@
+***********
+OCR backend
+***********
+
+Mayan EDMS ships an OCR backend that uses the FLOSS engine Tesseract
+(https://github.com/tesseract-ocr/tesseract/), but it can
+use other engines. To support other engines crate a wrapper that subclasses the
+``OCRBackendBase`` class defined in mayan/apps/ocr/classes. This subclass should
+expose the ``execute`` method. For an example of how the Tesseract backend
+is implemented take a look at the file ``mayan/apps/ocr/backends/tesseract.py``
+
+Once you create you own backend, in your local.py settings add the option
+OCR_BACKEND and point it to your new OCR backend class path.
+
+The default value of OCR_BACKEND is ``"ocr.backends.tesseract.Tesseract"``
+
+To add support to OCR more languages when using Tesseract, install the
+corresponding language file. If using a Debian based OS, this command will
+display the available language files:
+
+    apt-cache search tesseract-ocr
+
+If using the Docker image, pass the environment variable MAYAN_APT_INSTALLS
+with the corresponding Tesseract language option. Example::
+
+    -e MAYAN_APT_INSTALLS='tesseract-ocr-deu'

docs/chapters/password_reset.rst

@@ -0,0 +1,38 @@
+**************
+Password reset
+**************
+
+To use the password reset feature, administrative emails need to be configured.
+These are sent by the system itself and not by the users. Their usage and
+configuration is different than the
+:doc:`email system used to share documents via email<../chapters/mailing>`.
+
+Sending administrative emails
+=============================
+
+To be able to send password reset emails configure the Django email settings
+via the :ref:`configuration file <configuration_file>`.
+
+Example::
+
+    EMAIL_BACKEND: django.core.mail.backends.smtp.EmailBackend
+    EMAIL_HOST: '<your smtp ip address or hostname>'
+    EMAIL_HOST_PASSWORD: '<your smtp password>'
+    EMAIL_HOST_USER: '<your smtp username>'
+    EMAIL_PORT: 25  # or 587 or your server's SMTP port
+    EMAIL_TIMEOUT:
+    EMAIL_USE_SSL: true
+    EMAIL_USE_TLS: false
+
+To change the reference URL in the password reset emails on in the
+default document mailing template modify the ``COMMON_PROJECT_URL`` setting.
+For information on the different ways to change a setting check the
+:doc:`../topics/settings` topic.
+
+To test the email settings use the management command ``sendtestemail``.
+Example::
+
+    mayan-edms.py sendtestemail myself@example.com
+
+
+

docs/chapters/permissions.rst

@@ -1,8 +1,8 @@
-===========
+***********
 Permissions
-===========
+***********
 
-**Mayan EDMS** provides very fine control over which actions users can
+Mayan EDMS provides very fine control over which actions users can
 perform. Action control works by allowing ``roles``, that are composed of
 ``groups`` of ``users`` to be granted a ``permission`` such that the holder of
 that permission can exercise it throughout the entire system.
@@ -10,6 +10,11 @@ that permission can exercise it throughout the entire system.
 .. blockdiag::
 
    blockdiag {
+      orientation = portrait
+      default_shape = roundedbox
+      span_width = 240;
+      span_height = 100;
+
       user [ label = 'Users' ];
       group [ label = 'Groups' ];
       role [ label = 'Roles' ];

docs/chapters/quick_labels.rst

@@ -0,0 +1,70 @@
+************
+Quick labels
+************
+
+Quick labels are predetermined filenames that allow the quick renaming of
+documents as they are uploaded or after they have been uploaded.
+
+Quick labels are added and associated to a document type.
+
+Example of quick label: Invoice, Receipt from X store, Purchase order.
+
+It is possible to preserve the file extension when using quick labels.
+Extensions are required for some operating system to be able to detect the
+correct file type to access the content.
+
+For example if a document file is named "file0001.pdf" and the quick label
+"Receipt from X store" is applied, the resulting document label will be
+"Receipt from X store.pdf".
+
+
+Creating quick labels
+=====================
+
+.. admonition:: Permissions required
+    :class: warning
+
+    The "Edit document types" permission is required for this action, either
+    globally or via an ACL for a document type.
+
+
+Since quick labels are associated with document types, creating quick labels
+must be done from the document type view.
+
+#. Go to the :menuselection:`System --> Setup --> Document types` menu.
+#. In the document type list, click on the :guilabel:`Quick labels` button of
+   the document type for which you wish to create a quick label.
+#. In the view titled "Quick labels for document type: <your document type>",
+   from the :guilabel:`Actions` dropdown select :guilabel:`Add quick label to document type`.
+#. At the quick label creation form enter the desired label and press :guilabel:`Save`.
+
+
+Using quick labels during upload
+================================
+
+#. Use the new document upload wizard from :menuselection:`Documents --> New document`.
+#. Select a document type and navigate to the penultimate step, where you have
+   the option to drag and drop files to upload.
+#. Select a an option from the :guilabel:`Quick document rename` dropdown.
+#. Optionally select the :guilabel:`Preserve extension` checkbox to keep the file
+   extension.
+#. Upload your documents.
+
+
+Using quick labels for existing documents
+=========================================
+
+.. admonition:: Permissions required
+    :class: warning
+
+    The "Edit document properties" permission is required for this action, either
+    globally or via an ACL for a document or document type.
+
+
+#. Navigate to the document preview view of the document to rename. Make sure
+   quick labels for the document type of the document select have been created.
+#. From the :guilabel:`Actions` dropdown select :guilabel:`Edit Properties`.
+#. Select a an option from the :guilabel:`Quick document rename` dropdown.
+#. Optionally select the :guilabel:`Preserve extension` checkbox to keep the file
+   extension.
+#. Press :guilabel:`Save` to rename the document.

docs/chapters/scaling_up.rst

@@ -0,0 +1,155 @@
+**********
+Scaling up
+**********
+
+The default installation method fits most use cases. If you use case requires
+more speed or capacity here are some suggestion that can help you improve the
+performance of your installation.
+
+
+Change the database manager
+===========================
+
+Use PostgreSQL or MySQL as the database manager.
+Tweak the memory setting of the database manager to increase memory allocation.
+More PostgreSQL specific examples are available in their wiki page:
+https://wiki.postgresql.org/wiki/Performance_Optimization
+
+
+Increase the number of Gunicorn workers
+=======================================
+
+The Gunicorn workers process HTTP requests and affect the speed at which the
+website responds.
+
+If you are using the Docker image, change the value of the
+``MAYAN_GUNICORN_WORKERS`` environment variable (check the Docker image chapter:
+:ref:`docker_environment_variables`). Normally this variable defaults to 2.
+Increase this number to match the number of CPU cores + 1.
+
+If you are using the direct deployment methods, change the line that reads::
+
+    command = /opt/mayan-edms/bin/gunicorn -w 2 mayan.wsgi --max-requests 500 --max-requests-jitter 50 --worker-class gevent --bind 0.0.0.0:8000 --timeout 120
+
+And increase the value of the ``-w 2`` argument. This line is found in the
+``[program:mayan-gunicorn]`` section of the supervisor configuration file.
+
+
+Background task processing
+==========================
+
+The Celery workers are system processes that take care of the background
+tasks requested by the frontend interactions like document image rendering
+and periodic tasks like OCR. There are several dozen tasks defined in the code.
+These tasks are divided into queues based on the app of the relationship
+between the tasks. The queues by default are divided into three groups
+based on the speed at which they need to be processed. The document page
+image rendering for example is categorized as a high volume, short duration
+task. The OCR is a high volume, long duration task. Email checking is a
+low volume, medium duration tasks. It is not advisable to have the same
+worker processing OCR to process image rendering too. If the worker is
+processing several OCR tasks it will not be able to provide fast images
+when an user is browsing the user interface. This is why by default the
+queues are split into 3 workers: fast, medium, and slow. Each worker will handle
+queues based on the latency required by each queue group.
+
+
+Optimizations
+-------------
+
+* Increase the number of workers and redistribute the queues among them
+  (only possible with direct deployments).
+* Launch more workers to service a queue. For example for faster document
+  image generation launch 2 workers to process the converter queue only
+  possible with direct deployments).
+* By default each worker process uses 1 thread. You can increase the thread
+  count of each worker process with the Docker environment options:
+
+  * ``MAYAN_WORKER_FAST_CONCURRENCY``
+  * ``MAYAN_WORKER_MEDIUM_CONCURRENCY``
+  * ``MAYAN_WORKER_SLOW_CONCURRENCY``
+
+* If using direct deployment, increase the value of the ``--concurrency=1``
+  argument of each worker in the supervisor file. You can also remove this
+  argument and let the Celery algorithm choose the number of threads to
+  launch. Usually this defaults to the number of CPU cores + 1.
+
+
+Change the message broker
+=========================
+Messages are the method of communication between front end interactive code
+and background tasks. In this regard messages can be thought as homologous
+to tasks requests. Improving how many messages can be sent, stored and
+sorted will impact the number of tasks the system can handle. To save on
+memory, the basic deployment method and the Docker image default to using
+Redis as a message broker. To increase capacity and reduce volatility of
+messages (pending tasks are not lost during shutdown) use RabbitMQ to
+shuffle messages.
+
+For direct installs refer to the :ref:`deployment_advanced` documentation
+section for the required changes.
+
+For the Docker image, launch a separate RabbitMQ container
+(https://hub.docker.com/_/rabbitmq/)::
+
+    docker run -d --name mayan-edms-rabbitmq -e RABBITMQ_DEFAULT_USER=mayan -e RABBITMQ_DEFAULT_PASS=mayanrabbitmqpassword -e RABBITMQ_DEFAULT_VHOST=mayan rabbitmq:3
+
+Pass the MAYAN_BROKER_URL environment variable (https://kombu.readthedocs.io/en/latest/userguide/connections.html#connection-urls)
+to the Mayan EDMS container so that it uses the RabbitMQ container the
+message broker::
+
+    -e MAYAN_BROKER_URL="amqp://mayan:mayanrabbitmqpassword@localhost:5672/mayan",
+
+When tasks finish, they leave behind a return status or the result of a
+calculation, these are stored for a while so that whoever requested the
+background task, is able retrieve the result. These results are stored in the
+result storage. By default a Redis server is launched inside the Mayan EDMS
+container. You can launch a separate Docker Redis container and tell the Mayan
+EDMS container to use this via the ``MAYAN_CELERY_RESULT_BACKEND`` environment
+variable. The format of this variable is explained here: http://docs.celeryproject.org/en/3.1/configuration.html#celery-result-backend
+
+
+Deployment type
+===============
+
+Docker provides a faster deployment and the overhead is not high on modern
+systems. It is however memory and CPU limited by default and you need to
+increase this limits. The settings to change the container resource limits
+are here: https://docs.docker.com/config/containers/resource_constraints/#limit-a-containers-access-to-memory
+
+For the best performance possible use the advanced deployment method on a
+host dedicated to serving only Mayan EDMS.
+
+
+Storage
+=======
+
+For best input and output speed use a block based local filesystem on an
+SSD drive for the ``/media`` sub folder. The location of the ``/media`` folder
+will be specified by the ``MEDIA_ROOT`` setting.
+
+If capacity is your bottom line, switch to an
+:doc:`object storage <../chapters/object_storage>` system.
+
+
+Use additional hosts
+====================
+
+When one host is not enough you can use multiple hosts and share the load.
+Make sure that all hosts share the ``/media`` folder as specified by the
+``MEDIA_ROOT`` setting, also the database, the broker, and the result storage.
+One setting that needs to be changed in this configuration is the lock
+manager backend.
+
+Resource locking is a technique to avoid two processes or tasks to modify
+the same resource at the same time causing a race condition. Mayan EDMS uses
+its own lock manager. By default the lock manager with use a simple file
+based lock backend ideal for single host installations. For multiple hosts
+installation the database backend must be used in other to coordinate the
+resource locks between the different hosts over a share data medium. This is
+accomplished by modifying the environment variable ``LOCK_MANAGER_BACKEND`` in
+both the direct deployment or the Docker image. Use the value
+``lock_manager.backends.model_lock.ModelLock`` to switch to the database
+resource lock backend. If you can also write your own lock manager backend
+for other data sharing mediums with better performance than a relational
+database like Redis, Memcached, Zoo Keeper.

docs/chapters/settings.rst

@@ -0,0 +1,113 @@
+********
+Settings
+********
+
+Mayan EDMS can be configure via environment variables or by setting files.
+
+
+.. _environment_variables:
+
+Via environment variables
+=========================
+
+To use environment variables, lookup the name of the setting you want to
+override in the "Settings" menu. The "Settings" menu is located inside the
+"Setup" main menu. To pass a value via an environment variable append
+``"MAYAN_"`` to the name of the settings option. For example, to change
+the number of documents displayed per page (COMMON_PAGINATE_BY, by default 40),
+use::
+
+    export MAYAN_COMMON_PAGINATE_BY=10
+
+Restart Mayan EDMS and the new value will take effect. The "Settings" menu
+can be used to verify if the overridden setting value is being interpreted
+correctly.
+
+
+.. _configuration_file:
+
+Via YAML configuration file
+===========================
+
+.. versionadded:: 3.1
+
+It is possible to modify the different settings by creating or editing the
+``media/config.yml`` file. This file is formatted in the YAML markup language (
+http://yaml.org/). Here is an example of what the looks like::
+
+    DOCUMENT_PARSING_AUTO_PARSING: true
+    DOCUMENT_PARSING_PDFTOTEXT_PATH: /usr/bin/pdftotext
+    EMAIL_BACKEND: django.core.mail.backends.smtp.EmailBackend
+    EMAIL_HOST: localhost
+    EMAIL_HOST_PASSWORD: ''
+    EMAIL_HOST_USER: ''
+    EMAIL_PORT: 25
+    EMAIL_TIMEOUT: null
+    EMAIL_USE_SSL: false
+    EMAIL_USE_TLS: false
+    FILE_UPLOAD_MAX_MEMORY_SIZE: 2621440
+    HOME_VIEW: common:home
+
+Every time Mayan EDMS is able to start correctly it will copy the ``config.yml``
+and create a backup copy in the same directory named ``config_backup.yml``.
+This file is used to revert to the last know configuration file known
+to be valid. You can revert manually by copy the file or by using the
+``revertsettings`` management command from the command line.
+
+
+Via Python settings files
+=========================
+
+Another way to configure Mayan EDMS is via Python-style, settings files.
+If Mayan EDMS was installed using the Python package a ``mayan_settings``
+folder will created for this purpose. If you installed Mayan EDMS
+according to the :doc:`../chapters/deploying` instructions provided in this
+documentation your ``mayan_settings`` folder should be located in the directory:
+``/usr/share/mayan-edms/mayan/media/mayan_settings``.
+
+If Mayan EDMS was installed using Docker, the ``mayan_settings`` folder
+will be found inside the install Docker volume. If you installed Mayan EDMS
+according to the :doc:`../chapters/docker` instructions provided in this
+documentation your ``mayan_settings`` folder should be located in the directory:
+``/docker-volumes/mayan/mayan_settings``.
+
+Create a file with any valid name and a ``.py`` extension in the
+``mayan_settings`` folder. The file must starts with a global import of
+``mayan.settings.production``. In the form::
+
+    from mayan.settings.production import *
+
+Now add the corresponding lines to override the default settings.
+In the settings file, it is not necessary to prepend the string ``MAYAN_`` to
+the setting option. For example, to change the number of documents displayed
+per page (COMMON_PAGINATE_BY, by default 40),
+use::
+
+    COMMON_PAGINATE_BY=10
+
+versus::
+
+    export MAYAN_COMMON_PAGINATE_BY=10
+
+when using the environment variable method.
+
+For this example let's assume the file was saved with the name ``mysettings.py``.
+
+The way used to tell Mayan EDMS to import this file will vary based on the
+installation method.
+
+For the :doc:`../chapters/deploying` method, the full import path will be
+``mayan.media.mayan_settings.mysettings`` and can be passed via the
+``--settings`` command line argument like this::
+
+    python manage.py runserver --settings=mayan.media.mayan_settings.mysettings
+
+or via the ``DJANGO_SETTINGS_MODULE`` environment variable like this::
+
+    export DJANGO_SETTINGS_MODULE=mayan.media.mayan_settings.mysettings
+
+For the :doc:`../chapters/docker` installation method, the full import path will be
+``mayan_settings.mysettings`` and can only be passed via the
+``MAYAN_SETTINGS_MODULE`` environment variable like this::
+
+    docker run <...> -e MAYAN_SETTINGS_MODULE=mayan_settings.mysettings

docs/chapters/signatures.rst

@@ -1,13 +1,33 @@
-===================
+*******************
 Document signatures
-===================
+*******************
 
-**Mayan EDMS** supports two types of document signatures: embedded and
+Mayan EDMS supports two types of document signatures: embedded and
 detached signatures. When a document with an embedded signature is
 uploaded, this signature is readily detected as part of the document
 inspection step. The status of the signature can be verified by accessing the
 signatures sections of a document.
 
+.. blockdiag::
+
+   blockdiag {
+
+      orientation = portrait
+      span_width = 240;
+
+      user [ shape = "actor" ];
+      key [ shape = flowchart.database, label = "Key" ];
+      document [ shape = "note", label = "Document" ];
+      document_signed [ label = "Signed\nDocument" ];
+
+
+      key -> user -> document -> document_signed;
+   }
+
+Signed documents are tamper-proof. That means that if any part of the document's
+file is edited (even just one character or one pixel), the document's
+signature becomes invalid alerting that an unauthorized change has occurred.
+
 Existing non signed documents can be signed in one of two ways:
 by downloading the document, signing it, and uploading the signed document
 as a new version of the existing one or by creating a detached signature for
@@ -21,4 +41,6 @@ keys no longer needed can also be deleted from this menu.
 
 Only `GNU Privacy Guard`_ signatures are support at the moment.
 
+Only version 1 of `GNU Privacy Guard`_ is supported at the moment.
+
 .. _`GNU Privacy Guard`: www.gnupg.org/

docs/chapters/smart_links.rst

@@ -1,4 +1,3 @@
-===========
 Smart links
 ===========
 
@@ -7,7 +6,7 @@ in their respective indexes. Smart links are useful when two documents are
 related somehow but are of different type or different hierarchical units.
 
 Example: A patient record can be related to a prescription drug information
-document, but they each belong to their own :doc:`indexes`.
+document, but they each belong to their own :doc:`../chapters/indexes`.
 
 Smart links are rule based, but don't create any organizational structure.
 Smart links just show the documents that match the rules as evaluated against

docs/chapters/sources.rst

@@ -0,0 +1,79 @@
+Sources
+=======
+
+Document sources define places from which documents can be uploaded or gathered.
+
+The current document sources supported are:
+
+- Web - ``HTML`` forms with a ``Browse`` button that will open the file dialog
+  when clicked to allow selection of files in the user's computer to be
+  uploaded as documents.
+- POP3 email - Provide the email, server and credential of a ``POP3`` based
+  email to be scanned periodically for email. The body of the email is uploaded
+  as a document and the attachments of the email are uploaded as separate
+  documents.
+- IMAP email - Same as the ``POP3`` email source but for email accounts using
+  the ``IMAP`` protocol.
+- Watch folder - A filesystem folder that is scanned periodically for files.
+  Any file in the watch folder is automatically uploaded.
+- Staging folder - Folder where networked attached scanned can save image
+  files. The files in these staging folders are scanned and a preview is
+  generated to help the process of upload. Staging folders and Watch folders
+  work in a similar way with the main difference being that Staging folders are
+  interactive while Watch folders are automatic; documents in a Watch folder
+  are uploaded periodically and documents in a Staging folder remain indefinitely
+  there until an user uploads them. A preview for files in a Staging folder is
+  also provided. An example of Staging folder use is when multiple people
+  are scanning documents but only one person must be allowed to upload those
+  documents. This one person examines the scans quality and decides what to
+  upload and what to reject and have re-scanned. Watch folders can be used
+  when the quality of the scans is irrelevant or when they will be known
+  to be of good quality, such as when receiving e-faxes as PDFs.
+
+.. blockdiag::
+
+    blockdiag {
+        mayan [shape = "roundedbox", label = "Mayan EDMS" ];
+        email_pop3 [shape = "mail", label = "e-mail (POP3)"];
+        email_imap [shape = "mail", label = "e-mail (IMAP)"];
+        staging [shape = "flowchart.database", label = "Staging folder" ];
+        watch [shape = "flowchart.database", label = "Watch folder" ];
+        automatic [shape = "box", label = "Automatic\n(via schedule)" ];
+        manual [shape = "actor", height=60, label = "Manual\n(user interaction)" ];
+        web [shape = "note", label = "Webform upload" ];
+
+        automatic -> mayan;
+        email_pop3 -> automatic;
+        email_imap -> automatic;
+        watch -> automatic;
+        manual -> mayan;
+        staging -> manual;
+        web -> manual;
+    }
+
+Document sources can be configure to allow document bundles to uploaded as
+compressed files which are decompressed and their content uploaded as separate
+documents. This feature is useful when migrating from another document
+manager system.
+
+
+.. note::
+
+    If you deployed a Mayan EDMS Docker container and want to use watched folders
+    or staging folder, refer to the Docker chapter :ref:`docker-accessing-outside-data`.
+
+
+Creating new sources
+====================
+
+.. admonition:: Permissions required
+    :class: warning
+
+    The "Create new document sources" permission is required for this action.
+
+
+#. Go to the :menuselection:`System --> Setup --> Sources` menu.
+#. From the :guilabel:`Actions` dropdown select the new type of source to create.
+#. Each source type will have different fields to customize its behavior. Enter
+   the required information in each field based on the help text provided.
+#. Press :guilabel:`Save`.

docs/chapters/tags.rst

@@ -0,0 +1,46 @@
+Tags
+====
+
+Tags are color coded properties that can be attached or removed from documents.
+Tags allow giving documents a binary property. Documents can also be tagged
+with more than one tag.
+
+Once tagged, documents can be searched by their tags. It is also possible to
+show all the documents tagged with a particular
+tag via the **Documents** link of each tag.
+
+To view, add, or remove tags from a specific document, go to the document view
+and click on the **Tags** tab. This view will show the current tags of a
+document. The **Actions** menu will now show two new options: one to attach
+more tags to the document and another to remove tags from the document.
+
+It is also possible to tag documents in bulk by selecting several documents
+from any view that displays a list documents and selecting the **Attach tag**
+or **Remove tags** from the dropdown menu.
+
+
+Creating tags
+=============
+
+.. admonition:: Permissions required
+    :class: warning
+
+    The "Create new tags" permission is required for this action.
+
+
+#. Go to the :menuselection:`Tags --> Create` menu.
+#. Enter a label to identify the tag.
+#. Select a color for the tag.
+#. Press :guilabel:`Submit`.
+
+
+View all tags
+=============
+
+.. admonition:: Permissions required
+    :class: warning
+
+    The "View tags" permission is required for this action, globally or via an
+    ACL for a tag.
+
+#. Go to the :menuselection:`Tags --> All` menu.

docs/chapters/transformations.rst

@@ -0,0 +1,21 @@
+***************
+Transformations
+***************
+
+Transformations are persistent manipulations to the previews of the stored
+documents. For example: a scanning equipment may only produce landscape PDFs.
+In this case a useful transformation for that document source would be to rotate
+all scanned documents by 270 degrees after being uploaded. By adding this
+transformation to the Mayan EDMS source that is connected to the scanner, all
+pages scanned via that source will inherit the transformation as they are
+created. The result is that whenever a document is uploaded from that scanner,
+it will appear in portrait orientation, instead of landscape orientation.
+
+Transformations can also be added to existing documents by clicking on a
+document's page and then clicking on "transformations". In this view the Actions
+menu will have a new option that reads "Create new transformation". Currently,
+the available transformations are: rotation, zoom, crop, and resize. Once the
+document image has been corrected, resubmit it for OCR for improved results.
+
+Transformations are not destructive and do not physically modify the document
+file, they just modify the document's graphical representation.

docs/chapters/translations.rst

@@ -1,6 +1,6 @@
-============
+************
 Translations
-============
+************
 
 Translations are handled online via the **Transifex** website:
 https://www.transifex.com/projects/p/mayan-edms/. To create a translation team

docs/chapters/upload_wizard.rst

@@ -0,0 +1,84 @@
+*************
+Upload wizard
+*************
+
+The steps needed to upgrade a document using form-tools' ``SessionWizard``
+were hardcoded in the ``source`` app. This made it very difficult to add or remove
+wizard steps.
+
+The steps of the wizard are now defined by a new class called
+``sources.wizard.WizardStep``. The existing steps to select a document type,
+enter metadata and tag the document, have been converted to function as
+``WizardSteps`` subclasses. The converted steps now live in
+
+``sources.wizards.WizardStepDocumentType``, ``tag.wizard_steps.WizardStepTags``,
+and ``metadata.wizard_steps.WizardStepMetadata``.
+
+The steps need to define the following methods:
+
+- ``done``: This method is execute when the wizard finished the last step
+  an enter the step where the actual file are uploaded. This steps is used
+  to encode form data into the URL query string that will be passed to the
+  document upload view for each file uploaded.
+
+- ``condition``: This method is used to display the step conditionally.
+  If this method return True it will be displayed during the upload wizard
+  execution. To skip the step, return False or None.
+
+- ``get_form_initial``: This method is used to return the initial data
+  for the step form. Use this method to set up initial values for the step's
+  form fields.
+
+- ``step_post_upload_process``: This method will be executed once the
+  document finishes uploading. Use this method to process the information
+  encoded in the URL querystring by the step's `done`` method.
+
+Once the ``WizardStep`` subclass is defined, it needs to be registered. This
+is done by calling the ``.register`` method of the ``WizardStep`` class with
+the subclass as the argument. Example::
+
+    WizardStep.register(WizardStepMetadata)
+
+This statement must be located after the subclass definition. Finally,
+the module defining the wizard step must be imported so that it is loaded
+with the rest of the code and enabled. The best place to do this is in the
+``.ready`` method of the apps' ``apps.py`` module. Example::
+
+    class TagsApp(MayanAppConfig):
+        has_rest_api = True
+        has_tests = True
+        name = 'tags'
+        verbose_name = _('Tags')
+
+        def ready(self):
+            super(TagsApp, self).ready()
+            from actstream import registry
+
+            from .wizard_steps import WizardStepTags  # NOQA
+
+The ``WizardStep`` class also allows for unregistering existing steps. This
+is accomplished by calling the ``.deregister`` method of the ``WizardStep``
+class and passing the subclass as the argument. This method should
+also be called inside the ``.ready`` method of an apps' ``apps.py``
+module. Example::
+
+
+    class TagsApp(MayanAppConfig):
+        has_rest_api = True
+        has_tests = True
+        name = 'tags'
+        verbose_name = _('Tags')
+
+        def ready(self):
+            super(TagsApp, self).ready()
+            from actstream import registry
+
+            from metadata.wizard_steps import WizardStepMetadata  # NOQA
+            from sources.wizards import WizardStep  # NOQA
+            from .wizard_steps import WizardStepTags  # NOQA
+
+            WizardStep.deregister(WizardStepTags)
+
+
+This will cause the tags assignment step to not be assigned to the upload
+wizard anymore.

docs/chapters/versioning.rst

@@ -0,0 +1,80 @@
+*******************
+Document versioning
+*******************
+
+Mayan EDMS has the ability to store different versions of the same
+document. A comment field is provided to allow users to summarize the new
+version changes in comparison with the previous one. If a new version was
+uploaded by mistake or such new version is no longer necessary the option to
+revert to a previous version of the document is provided.
+
+.. blockdiag::
+
+   blockdiag {
+      default_shape = roundedbox
+      orientation = portrait
+        node_width = 200;
+      version_1 [ label = "Version 1" ];
+      version_2 [ label = "Version 2" ];
+      document_1 [ label = "payroll_report.pdf" ];
+      document_2 [ label = "payroll_report_fixed.pdf" ];
+      upload_1 [ label = "payroll_report.pdf" ];
+      upload_2 [ label = "payroll_report_fixed.pdf" ];
+
+        upload_1 -> version_1 -> document_1;
+        upload_2 -> version_2 -> document_2;
+        document_1 -> document_2;
+   }
+
+Only the interactive document sources (:doc:`../chapters/sources`)
+(``Web`` and ``Staging folders``) are available to upload new document versions.
+
+There is no limit to the number of versions a document can have.
+
+.. blockdiag::
+
+    blockdiag {
+        default_shape = roundedbox
+        orientation = portrait
+        node_width = 200;
+
+        document [ label = "payroll_report.pdf" ];
+        versions [ label = "Versions", stacked ];
+
+        document -> versions;
+   }
+
+By default, the last version will be showed when working with the document
+but any of the versions can be inspected and viewed.
+
+
+View a document version list
+============================
+
+.. admonition:: Permissions required
+    :class: warning
+
+    The "View documents' versions list" permission is required for this action,
+    either globally or via an ACL for a document or document type.
+
+
+#. Navigate to the select document's preview view.
+#. Click on the sibebar's :guilabel:`Versions` button.
+
+
+Uploading a new document version
+================================
+
+.. admonition:: Permissions required
+    :class: warning
+
+    The "Create new document versions" permission is required for this action,
+    either globally or via an ACL for a document or document type.
+
+
+#. Navigate to the select document's versions list view.
+#. From the :guilabel:`Actions` dropdown select :guilabel:`Upload new version`.
+#. Optionally type a comment explaining the changes in the new version.
+#. Press the :guilabel:`Browse` button and select a new file.
+#. Press :guilabel:`Save` upload the new version.
+

docs/chapters/workflows.rst

@@ -0,0 +1,113 @@
+*********
+Workflows
+*********
+
+Introduction
+============
+
+Workflows provide a structured method for storing a sequence of states over
+which the a document will progress. Workflows keep track how a document has
+been processed so far.
+
+Workflows work by storing a series of states to help you know the "status"
+of a document. To move a workflow from one state to another, transitions are
+used.
+
+Transitions connect two different states and help provide context to know
+which state is possible to transition to, from a previous state. Transitions
+provide and order for the sequence of possible states changes.
+
+Transitions can be executed manually by users if they have the required access
+level as configure by the system administrator.
+
+.. blockdiag::
+   :caption: Example workflow. Circles represent states, rectangles represent transitions.
+
+      span_width = 30;
+
+      submitted [shape = circle, width = 60, height = 60];
+      approved [shape = circle, width = 60, height = 60];
+      rejected [shape = circle, width = 60, height = 60];
+      archived [shape = circle, width = 60, height = 60];
+
+      approve [label = approve];
+      reject [label = reject];
+      reopen [label = "re-open"];
+
+
+      submitted -> approve -> approved;
+      approved -> archived;
+      submitted -> reject -> rejected;
+      rejected -> archived;
+      archived -> reopen -> submitted;
+
+
+
+Workflows in Mayan EDMS are implemented as finite state machines
+(https://en.wikipedia.org/wiki/Finite-state_machine).
+
+Workflows are mainly used to represent business processes. But they can also be
+used an automation system to customized Mayan EDMS and have it perform some
+tasks automatically.
+
+
+Automation
+==========
+
+Besides being able to be executed manually by users, transitions can also be
+programmed execute automatically based on system events. This is called in
+Mayan EDMS transition triggering and is one of the tools available to
+automate business processes.
+
+For example:
+
+* Move a document from a "scanned" state to a "billed" state
+  when a tag is attached to the document.
+* Move a document from a "uploaded" state to a "OCR ready" state
+  when the OCR engine finishes processing the document.
+
+The other tool provided for process automation is being able to execute an
+action when a workflow state is reached or leaved. These are called state
+events.
+
+Some examples of state actions currently provided are:
+
+* Attach a tag to a document
+* Remove a tag from a document
+* Do an HTTP POST request to an external IP address
+* Edit the label or the description of a document.
+
+Other time more state actions are being added. Some state actions like the one
+that creates an HTTP POST request allow Mayan EDMS to trigger processes in
+external systems based on the state of a document. One example of this is
+triggering the billing process of an accounting system when an invoice is
+scanned in Mayan EDMS.
+
+Workflow state actions and transitions triggers are new features and are still
+evolving.
+
+Workflows allow translating business logic into a series of states. With the
+addition of state actions and transition triggers, the workflows in Mayan EDMS
+are no longer just informative but can be part of your actual business
+automation process.
+
+
+Implementation
+==============
+
+Internally, workflows are implemented as a finite state machines
+(https://en.wikipedia.org/wiki/Finite-state_machine). To make them simpler to
+use, workflow have been designed so that only one state can be the current
+active state for a workflow being executed. Another design decision is that
+every workflow needs at least one state marked as the initial state.
+
+
+Visualizations
+==============
+
+The graphical representation of a workflow is similar to a flowchart. The
+states are represented with circles. The transitions are represented with
+arrows. Circle with a double border represent the initial state of the workflow.
+
+To view the graphical representations of workflow use **Preview** button of
+the workflow in the setup view.

docs/conf.py

@@ -33,12 +33,13 @@ sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "_ext"))
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
 #extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode']
 #extensions = ["djangodocs", "sphinx.ext.intersphinx"]
-extensions = ['djangodocs', 'sphinxcontrib.blockdiag']
+extensions = [
+    'sphinx.ext.extlinks', 'sphinxcontrib.blockdiag', 'sphinxcontrib.spelling'
+]
 
 blockdiag_antialias = True
 blockdiag_html_image_format = "SVG"
 blockdiag_latex_image_format = "PDF"
-blockdiag_tex_image_format = "PDF"
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
@@ -54,7 +55,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = mayan.__title__
-copyright = mayan.__copyright__
+copyright = mayan.__copyright_short__
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
@@ -180,6 +181,9 @@ html_static_path = ['_static']
 # Output file base name for HTML help builder.
 htmlhelp_basename = 'MayanEDMSdoc'
 
+html_show_sourcelink = False
+
+html_show_sphinx = False
 
 # -- Options for LaTeX output --------------------------------------------------
 
@@ -225,28 +229,26 @@ latex_documents = [
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
 man_pages = [
-    ('index', 'mayanedms', 'Mayan EDMS Documentation',
+    ('index', 'mayanedms', '{} Documentation'.format(mayan.__title__),
      [mayan.__author__], 1)
 ]
 
+html_theme = 'sphinx_rtd_theme'
 
-# -- Custom options
-import alabaster
-
-html_theme_path = [alabaster.get_path()]
-extensions.append('alabaster')
-html_theme = 'alabaster'
-html_sidebars = {
-    '**': [
-        'about.html', 'donate.html', 'navigation.html', 'searchbox.html',
-    ]
-}
-html_theme_options = {
-    'description': mayan.__description__,
-    'github_button': False,
-    'travis_button': False,
-    'gratipay_user': 'rosarior',
-    'github_banner': False,
+# -- External links --
+extlinks = {
+    'django-docs': (
+        'https://docs.djangoproject.com/en/{}/%s'.format(
+            mayan.__django_version__
+        ), 'Django documentation section: '
+    ),
+    'github-issue': (
+        'https://github.com/mayan-edms/mayan-edms/issues/%s', 'GitHub issue #'
+    ),
+    'gitlab-issue': (
+        'https://gitlab.com/mayan-edms/mayan-edms/issues/%s', 'GitLab issue #'
+    )
 }
 
-html_logo = '_static/logo_pyramid_only.png'
+def setup(app):
+    app.add_stylesheet('css/custom.css')

docs/index.rst

@@ -1,24 +1,15 @@
 Welcome to Mayan EDMS!
 ======================
 
-**Mayan EDMS** is a `Free Open Source`_ `Electronic Document Management System`_, coded in
-the Python language using the Django_ web application framework and released
-under the `Apache 2.0 License`_. It provides an electronic vault or repository for electronic documents.
+.. image:: /_static/mayan_logo.png
+  :alt: Logo is a trademark of Roberto Rosario
+  :align: center
 
-The easiest way to install and try **Mayan EDMS** is by using a Debian based Linux distribution
-and installing it from PyPI with the following commands:
 
-.. code-block:: bash
-
-    sudo apt-get install libjpeg-dev libmagic1 libpng-dev libreoffice libtiff-dev gcc ghostscript gpgv python-dev python-virtualenv tesseract-ocr unpaper poppler-utils -y
-    virtualenv venv
-    source venv/bin/activate
-    pip install mayan-edms
-    mayan-edms.py initialsetup
-    mayan-edms.py runserver
-
-Point your browser to 127.0.0.1:8000 and use the automatically created admin
-account.
+Mayan EDMS is a `Free Open Source`_ `Electronic Document Management System`_,
+coded in the Python language using the Django_ web application framework and
+released under the `Apache 2.0 License`_. It provides an electronic vault or
+repository for electronic documents.
 
 .. image:: /_static/overview.gif
    :alt: Overview
@@ -26,20 +17,27 @@ account.
 .. toctree::
     :hidden:
 
-    Features <topics/features>
-    Installation <topics/installation>
-    Deploying <topics/deploying>
-    Release notes and upgrading <releases/index>
-    Concepts <topics/index>
-    Development <topics/development>
-    App creation <topics/app_creation>
-    Translations <topics/translations>
-    Contributors <topics/contributors>
-    Screenshots <topics/screenshots>
-    Licensing <topics/license>
-    FAQ <topics/faq>
-    Contact <topics/contact>
+    topics/features
+    topics/installation
+    releases/index
+    topics/getting_started
+    topics/adding_documents
+    topics/access_control
+    topics/categorization
+    topics/collaboration
+    topics/settings
+    topics/storage
+    topics/integration
+    topics/docker
+    topics/advanced
+    topics/administration
+    topics/troubleshooting
+    topics/development
+    topics/faq
+    topics/license
+    topics/contact
 
+.. _Docker: https://www.docker.com/
 .. _Django: http://www.djangoproject.com/
 .. _Free Open Source: http://en.wikipedia.org/wiki/Open_source
 .. _Electronic Document Management System: https://en.wikipedia.org/wiki/Document_management_system

docs/mercs/0001-merc-process.rst

@@ -0,0 +1,210 @@
+MERC 1: Purpose and Guidelines
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+:MERC: 1
+:Author: Michael Price
+:Status: Accepted
+:Type: Process
+:Created: 2018-02-17
+:Last-Modified: 2018-02-17
+
+.. contents:: Table of Contents
+   :depth: 3
+   :local:
+
+What is a MERC?
+===============
+
+A Mayan EDMS Request For Comment document or MERC document is a design
+document providing information to the Mayan EDMS community, or
+describing a new feature or process for Mayan EDMS. MERCs provide
+concise technical specifications of features, along with rationales.
+
+MERC Types
+==========
+
+There are three kinds of MERCs:
+
+1. A **Feature** MERC describes a new feature or implementation
+for Mayan EDMS. Most MERCs will be Feature MERCs.
+
+2. An **Informational** MERC describes a Mayan EDMS design issue, or
+provides general guidelines or information to the Mayan EDMS community,
+but does not propose a new feature. Informational MERCs do not
+necessarily represent a community consensus or recommendation, so users
+and implementers are free to ignore Informational MERCs or follow their
+advice.
+
+3. A **Process** MERC describes a process surrounding Mayan EDMS, or
+proposes a change to (or an event in) a process.  Process MERCs are
+like Feature MERCs but apply to areas other than the Mayan EDMS
+framework itself.  They may propose an implementation, but not to
+Mayan EDMS's codebase; they often require community consensus; unlike
+Informational MERCs, they are more than recommendations, and users
+are typically not free to ignore them.  Examples include
+procedures, guidelines, changes to the decision-making process, and
+changes to the tools or environment used in Mayan EDMS development.
+Any meta-MERC is also considered a Process MERC. (So this document
+is a Process MERC).
+
+MERC submission workflow
+========================
+
+Pre-proposal
+------------
+
+The MERC process begins with a new idea for Mayan EDMS. It is highly recommended
+that a single MERC contain a single key proposal or new idea. Small enhancements
+or patches usually don't need a MERC and follow Mayan EDMS's normal contribution
+process.
+
+MERCs should be focused on a single topic. If in doubt, split your MERC
+into several well-focused ones.
+
+Once the idea's been vetted, a draft MERC should be presented to the
+Mayan EDMS mailing list. This gives the author a chance to flesh out the
+draft MERC to make sure it's properly formatted, of high quality, and to address
+initial concerns about the proposal.
+
+The Core Developers will be responsible for accepting or rejecting the MERC proposal.
+
+
+Submitting the draft
+--------------------
+
+Following the discussion on Mayan EDMS mailing list, the proposal
+should be sent as a merge request to the Mayan EDMS repository. The draft must
+be written in MERC style; if it isn't the merge request may be rejected until proper
+formatting rules are followed.
+
+
+Implementation
+--------------
+
+Finally, once a MERC has been accepted, the implementation must be completed. In
+many cases some (or all) implementation will actually happen during the MERC
+process: Feature MERCs will often have fairly complete implementations before
+being reviewed. When the implementation is complete and incorporated
+into the main source code repository, the status will be changed to
+"Final".
+
+MERC format
+===========
+
+MERCs need to follow a common format and outline; this section describes
+that format.
+
+MERCs must be written in `reStructuredText <http://docutils.sourceforge.net/rst.html>`_
+(the same format as Mayan EDMS's documentation).
+
+Each MERC should have the following parts:
+
+#. A short descriptive title (e.g. "User document filters"), which is also reflected
+   in the MERC's filename (e.g. ``0002-user-document-filters.rst``).
+
+#. A preamble -- a rST `field list <http://docutils.sourceforge.net/docs/ref/rst/restructuredtext.html#field-lists>`_
+   containing metadata about the MERC, including the MERC number and so forth. See
+   `MERC Metadata`_ below for specific details.
+
+#. Abstract -- a short (~200 word) description of the technical issue
+   being addressed.
+
+#. Specification -- The technical specification should describe the syntax and
+   semantics of any new feature.  The specification should be detailed enough to
+   allow implementation -- that is, developers other than the author should
+   (given the right experience) be able to independently implement the feature,
+   given only the MERC.
+
+#. Motivation -- The motivation is critical for MERCs that want to add
+   substantial new features or materially refactor existing ones. It should
+   clearly explain why the existing solutions are inadequate to address the
+   problem that the MERC solves. MERC submissions without sufficient motivation
+   may be rejected outright.
+
+#. Rationale -- The rationale fleshes out the specification by describing what
+   motivated the design and why particular design decisions were made. It
+   should describe alternate designs that were considered and related work.
+
+   The rationale should provide evidence of consensus within the community and
+   discuss important objections or concerns raised during discussion.
+
+#. Backwards Compatibility -- All MERCs that introduce backwards
+   incompatibilities must include a section describing these incompatibilities
+   and their severity.  The MERC must explain how the author proposes to deal
+   with these incompatibilities. MERC submissions without a sufficient backwards
+   compatibility treatise may be rejected outright.
+
+#. Reference Implementation -- The reference implementation must be completed
+   before any MERC is given status "Final", but it need not be completed before
+   the MERC is accepted. While there is merit to the approach of reaching
+   consensus on the specification and rationale before writing code, the
+   principle of "rough consensus and running code" is still useful when it comes
+   to resolving many discussions of API details.
+
+   The final implementation must include tests and documentation, per Mayan EDMS
+   development guide.
+
+
+MERC Metadata
+-------------
+
+Each MERC must begin with some metadata given as an rST
+`field list <http://docutils.sourceforge.net/docs/ref/rst/restructuredtext.html#field-lists>`_.
+The headers must contain the following fields:
+
+``MERC``
+    The MERC number. In an initial merge request, this can be left out or given
+    as ``XXXX``; the reviewer who merges the pull request will assign the MERC
+    number.
+``Type``
+    ``Feature``, ``Informational``, or ``Process``
+``Status``
+    ``Draft``, ``Accepted``, ``Rejected``, ``Withdrawn``, ``Final``, or ``Superseded``
+``Created``
+    Original creation date of the MERC (in ``yyyy-mm-dd`` format)
+``Last-Modified``
+    Date the MERC was last modified (in ``yyyy-mm-dd`` format)
+``Author``
+    The MERC's author(s).
+``Implementation-Team``
+    The person/people who have committed to implementing this MERC
+``Requires``
+    If this MERC depends on another MERC being implemented first,
+    this should be a link to the required MERC.
+``Mayan EDMS-Version`` (optional)
+    For Feature MERCs, the version of Mayan EDMS (e.g. ``2.7.3``) that this
+    feature will be released in.
+``Replaces`` and ``Superseded-By`` (optional)
+    These fields indicate that a MERC has been rendered obsolete. The newer MERC
+    must have a ``Replaces`` header containing the number of the MERC that it
+    rendered obsolete; the older MERC has a ``Superseded-By`` header pointing to
+    the newer MERC.
+``Resolution`` (optional)
+    For MERCs that have been decided upon, this can be a link to the final
+    rationale for acceptance/rejection. It's also reasonable to simply update
+    the MERC with a "Resolution" section, in which case this header can be left
+    out.
+
+Auxiliary Files
+---------------
+
+MERCs may include auxiliary files such as diagrams.  Such files must be named
+``XXXX-descriptive-title.ext``, where "XXXX" is the MERC number,
+"descriptive-title" is a short slug indicating what the file contains, and
+"ext" is replaced by the actual file extension (e.g. "png").
+
+Reporting MERC Bugs, or Submitting MERC Updates
+===============================================
+
+How you report a bug, or submit a MERC update depends on several factors, such as
+the maturity of the MERC, the preferences of the MERC author, and the nature of
+your comments. For the early draft stages of the MERC, it's probably best to
+send your comments and changes directly to the MERC author. For more mature, or
+finished MERCs you can submit corrections as repository issues or merge requests
+against the git repository.
+
+When in doubt about where to send your changes, please check first with the MERC
+author and/or a core developer.
+
+MERC authors with git push privileges for the MERC repository can update the MERCs
+themselves.

docs/mercs/0002-test-writing.rst

@@ -0,0 +1,82 @@
+====================
+MERC 2: Test writing
+====================
+
+:MERC: 2
+:Author: Michael Price
+:Status: Accepted
+:Type: Feature
+:Created: 2018-02-22
+:Last-Modified: 2018-04-01
+
+.. contents:: Table of Contents
+   :depth: 3
+   :local:
+
+Abstract
+========
+
+This MERC proposes a standard methodology for writing tests for Mayan EDMS.
+
+Motivation
+==========
+
+Having a standard methodology for writing tests has the following advantages:
+
+1. Scaffolding can be reduced by providing the most frequently used
+   paradigms as methods or helper functions.
+2. Reduce the probabilities of errors slipping through poorly written tests.
+
+
+Specification
+=============
+
+1. Tests must test each view in at least two ways:
+
+    A. Object creations views must be tested with and without permissions.
+    B. Object detail, list and delete views must be tested with and without
+       object access.
+
+2. Tests must assert the status code of the response even
+   when the expected status is HTTP 200.
+3. The actual request performed must be enclosed in a private methods
+   so that the fail and pass tests use the same HTTP request.
+4. Test must verify that changes happened and didn't happened in the
+   database regardless of the return code. Even is an edit view returns
+   and error 4XX (404-Not found, 403-Forbidden, etc), the test must
+   ensure that the data was not indeed modified.
+5. All tests must use the test user created by the BaseAPITestCase and not
+   an super user unless absolutely required by the test.
+6. Each test must test just one thing.
+7. If a test object needs to be created before the execution of a request
+   this object must be created by a private method.
+
+Example:
+
+.. code-block:: python
+
+    def _request_tag_create(self):
+        return self.post(
+            viewname='rest_api:tag-list', data={
+                'label': TEST_TAG_LABEL, 'color': TEST_TAG_COLOR
+            }
+        )
+
+    def test_tag_create_view_no_permission(self):
+        response = self._request_tag_create()
+        self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
+        self.assertEqual(Tag.objects.count(), 0)
+
+    def test_tag_create_view_with_permission(self):
+        self.grant_permission(permission=permission_tag_create)
+        response = self._request_tag_create()
+        self.assertEqual(response.status_code, status.HTTP_201_CREATED)
+
+        tag = Tag.objects.first()
+        self.assertEqual(response.data['id'], tag.pk)
+        self.assertEqual(response.data['label'], TEST_TAG_LABEL)
+        self.assertEqual(response.data['color'], TEST_TAG_COLOR)
+
+        self.assertEqual(Tag.objects.count(), 1)
+        self.assertEqual(tag.label, TEST_TAG_LABEL)
+        self.assertEqual(tag.color, TEST_TAG_COLOR)

docs/mercs/0003-using-javascript-libraries.rst

@@ -0,0 +1,74 @@
+==================================
+MERC 3: Using JavaScript libraries
+==================================
+
+:MERC: 3
+:Author: Eric Riggs
+:Status: Accepted
+:Type: Feature
+:Created: 2018-03-08
+:Last-Modified: 2018-06-04
+
+.. contents:: Table of Contents
+   :depth: 3
+   :local:
+
+
+Abstract
+========
+
+This MERC proposes a standard way to use JavaScript libraries.
+
+Rationale
+=========
+
+Mayan EDMS uses several JavaScript libraries for user interface features.
+Currently, the libraries are not installed using any JavaScript package
+manager but copied uncompressed. Installing the libraries in this manner
+carries some disadvantages.
+
+Motivation
+==========
+
+The inclusion of the libraries in source form is required by many licenses
+if the library is not installed by a package manager in distributable form.
+There are several disadvantages with the current approach:
+
+1. Having the library in source form means that the entire weight of the
+   library's size carries over the overall size of the Mayan EDMS distribution files.
+   The justification for not doing this is the same as with the Python libraries
+   which are not copied with the code but downloaded upon installation.
+2. Upgrading the libraries means manually examining the version of the
+   installed in the project and manually searching, downloading, compressing
+   and adding the files to the repository.
+3. The source form of the libraries includes normal and minified versions
+   of the code and the accompanying CSS files. There is no define preference
+   and through the project both versions of the libraries are loaded
+   interchangeably. Using a packager manager the minified version would be
+   used of a pipeline to minify the installed libraries should be added.
+
+Backwards Compatibility
+=======================
+
+There are no backwards compatibility issues with this proposal.
+
+
+Specification
+=============
+
+Changes needed:
+
+1. Python based JavaScript package manager. Alternatively a Python wrapper
+   for a JavaScript package manager could be used.
+2. Package manifest for the JavaScript libraries used.
+3. Installation pipeline to install the JavaScript libraries during the
+   installation and setup of the project.
+
+References:
+
+- https://github.com/JDeuce/powser
+- https://github.com/javrasya/version-manager
+- https://github.com/inveniosoftware-attic/setuptools-bower
+- https://pypi.python.org/pypi/django-bower-cache/0.5.0
+- http://django-pipeline.readthedocs.io/en/latest/index.html
+- https://github.com/nvbn/django-bower

docs/mercs/0004-support-forum.rst

@@ -0,0 +1,46 @@
+=====================
+MERC 4: Support forum
+=====================
+
+:MERC: 4
+:Author: Michael Price
+:Status: Accepted
+:Type: Process
+:Created: 2018-02-27
+:Last-Modified: 2018-09-04
+
+.. contents:: Table of Contents
+   :depth: 3
+   :local:
+
+Abstract
+========
+
+This MERC proposes the move of the official community meeting place
+from Google Group to a self hosted forum platform.
+
+Motivation
+==========
+
+Google Groups is not a proper forum platform and as such is lacking many
+features that would increase participation. Google Groups has not had any
+significant update in years and the age of the platform is visible. There
+are no official mobile apps for Google Groups, no responsible template
+is not provided. There are not integration options. These factors
+contribute to the problems of knowledge dilution and one time
+participation being experienced in the Mayan EDMS community.
+
+Other concerns for moving the community to a self hosted forum solution
+are that Google Groups presents a single point of failure by relying on
+a free product hosted by a commercial company with no continuation plan
+for the product. Google Groups doesn't provide archiving features and
+the current archive solution relies on other third party services, one
+of which (GMANE) has stopped working. Recent user privacy and censorship
+issues regarding Google, reinforce the need for a self hosted solution.
+
+
+Specification
+=============
+
+Platform chosen was phpBB. Factors were: written in PHP, ease of installation,
+compatible with several database managers, mature, extensive development history.

docs/mercs/0005-explicit-arguments.rst

@@ -0,0 +1,149 @@
+==========================
+MERC 5: Explicit arguments
+==========================
+
+:MERC: 5
+:Author: Roberto Rosario
+:Status: Accepted
+:Type: Feature
+:Created: 2018-12-30
+:Last-Modified: 2018-12-31
+
+.. contents:: Table of Contents
+   :depth: 3
+   :local:
+
+
+Abstract
+========
+
+This MERC proposes the adoption of a new methodology when performing calls.
+It seeks to reduce the use of positional arguments in favor of keyword
+arguments in as many places as possible.
+
+
+Motivation
+==========
+
+As the project grows, legibility of code becomes more important. Keyword
+argument help document the use of services, classes and functions. Refactors
+that affect the interface of services are also easier to find and update and
+fix. Positional argument can cause a call to continue working as long as the
+datatype of the argument remains the same. Usage of keyword arguments will
+automatically raise and error that will prevent such situations. Keyword
+argument further eliminate the relevance of position or the arguments, and
+the arguments can be sorted alphabetically for easier visual scanning or by
+semantic significance improving code readability.
+
+
+Specification
+=============
+
+Adoption of this MERC will require an audit of existing calls and the use
+of the method proposed for new calls. Every call regardless of the type or
+origin of the source callable will name each argument used. By type it is
+meant: classes, functions, methods. Origin means: local from the project,
+from the framework, third party libraries or the standard library.
+
+
+Backwards Compatibility
+=======================
+
+No backwards compatibility issues are expected. New errors arising from the use
+if keyword arguments could be interpreted as existing latent issues that
+have not been uncovered.
+
+
+Reference Implementation
+========================
+
+Example:
+
+Before:
+
+.. code-block:: python
+
+    from mayan.apps.common.classes import Template
+
+    Template(
+        'menu_main', 'appearance/menu_main.html'
+    )
+
+
+After:
+
+.. code-block:: python
+
+    from mayan.apps.common.classes import Template
+
+    Template(
+        name='menu_main', template_name='appearance/menu_main.html'
+    )
+
+
+When calls use a mixture or positional and keyword arguments, the keywords
+arguments can only be found after the positional arguments. Complete use
+of keyword arguments allow the reposition of arguments for semantic
+purposes.
+
+Example:
+
+Before:
+
+.. code-block:: python
+
+    from django.conf.urls import url
+
+    from .views import AboutView, HomeView, RootView
+
+    urlpatterns = [
+        url(r'^$', RootView.as_view(), name='root'),
+        url(r'^home/$', HomeView.as_view(), name='home'),
+        url(r'^about/$', AboutView.as_view(), name='about_view'),
+    ]
+
+
+After:
+
+.. code-block:: python
+
+    from django.conf.urls import url
+
+    from .views import AboutView, HomeView, RootView
+
+    urlpatterns = [
+        url(regex=r'^$', name='root', view=RootView.as_view()),
+        url(regex=r'^home/$', name='home', view=HomeView.as_view()),
+        url(regex=r'^about/$', name='about_view', view=AboutView.as_view()),
+    ]
+
+
+Keyword arguments should also be used for callables that pass those to others
+down the line like Django's ``reverse`` function. Any change to the name of
+the ``pk`` URL parameter will raise an exception in this code alerting to
+any possible incompatible use.
+
+
+Example:
+
+.. code-block:: python
+
+    def get_absolute_url(self):
+        return reverse(
+            viewname='documents:document_preview', kwargs={'pk': self.pk}
+        )
+
+
+This becomes even more important when multiple URL parameters are used. Since
+the API documentation is auto generated from the code itself, it would make
+sense to rename the first URL parameter from ``pk`` to ``document_pk``. Such
+change will cause all address to view resolutions to break forcing their
+update and allowing all consumers' interface usage to remain synchronized to the
+callable's interface.
+
+.. code-block:: python
+
+    url(
+        regex=r'^documents/(?P<pk>[0-9]+)/versions/(?P<document_version_pk>[0-9]+)/pages/(?P<document_page_pk>[0-9]+)/image/$',
+        name='documentpage-image', view=APIDocumentPageImageView.as_view()
+    ),

docs/mercs/0006-lower-information-disclose.rst

@@ -0,0 +1,81 @@
+==================================
+MERC 6: Lower information disclose
+==================================
+
+:MERC: 6
+:Author: Michael Price
+:Status: Accepted
+:Type: Feature
+:Created: 2018-12-30
+:Last-Modified: 2018-12-31
+
+.. contents:: Table of Contents
+   :depth: 3
+   :local:
+
+Abstract
+========
+
+This MERC proposes the use of errors that don't disclose the existence of a
+resource in the event that the requester doesn't have the required credentials.
+
+Motivation
+==========
+
+When an user tries to perform an action like opening a view to a document for
+which the required permission is missing, a permission required or access
+denied error is presented. This is semantically correct, but from the stand
+point of security it is still failing because it is letting the user know
+that such document exists in the first place. This MERC proposes changing the
+error message for existing resource to one that doesn't divulge any information
+to unauthorized parties, like "Not Found".
+
+Specification
+=============
+
+Out of the 4 basic CRUD operations, Read, Update and Delete should return an
+HTTP 404 error instead of an HTTP 403 error. Only the Create operation will
+continue returning the current HTTP 403 error, unless it is creating a
+new resource that is related to an existing resource.
+
+Since most view use the internal custom CRUD classes making a change to the
+``ObjectPermissionCheckMixin`` class to raise an HTTP 404 on object access
+failure will fulfill the proposal of this MERC.
+
+Adding the ``object_permission_raise_404`` class attribute and setting it
+to default to False will allow fulfilling the goal of this MERC while
+keeping the existing functionality intact.
+
+
+Example:
+
+.. code-block:: python
+
+    class ObjectPermissionCheckMixin(object):
+        """
+        If object_permission_raise_404 is True an HTTP 404 error will be raised
+        instead of the normal 403.
+        """
+        object_permission = None
+        object_permission_raise_404 = False
+
+        def get_permission_object(self):
+            return self.get_object()
+
+        def dispatch(self, request, *args, **kwargs):
+            if self.object_permission:
+                try:
+                    AccessControlList.objects.check_access(
+                        permissions=self.object_permission, user=request.user,
+                        obj=self.get_permission_object(),
+                        related=getattr(self, 'object_permission_related', None)
+                    )
+                except PermissionDenied:
+                    if self.object_permission_raise_404:
+                        raise Http404
+                    else:
+                        raise
+
+            return super(
+                ObjectPermissionCheckMixin, self
+            ).dispatch(request, *args, **kwargs)

docs/mercs/index.rst

@@ -0,0 +1,55 @@
+*****
+MECRs
+*****
+
+Mayan EDMS Request For Comment documents index.
+
+.. contents:: Table of Contents
+   :depth: 2
+   :local:
+
+Status
+======
+
+Accepted
+--------
+
+.. toctree::
+   :maxdepth: 1
+
+   ../mercs/0001-merc-process
+   ../mercs/0002-test-writing
+   ../mercs/0003-using-javascript-libraries
+   ../mercs/0005-explicit-arguments
+   ../mercs/0006-lower-information-disclose
+
+Draft
+-----
+
+.. toctree::
+   :maxdepth: 1
+
+   ../mercs/merging-roles-and-groups
+
+
+Type
+====
+
+Process
+-------
+.. toctree::
+   :maxdepth: 1
+
+   ../mercs/0001-merc-process
+   ../mercs/0004-support-forum
+
+
+Feature
+-------
+.. toctree::
+   :maxdepth: 1
+
+   ../mercs/0002-test-writing
+   ../mercs/0003-using-javascript-libraries
+   ../mercs/0005-explicit-arguments
+   ../mercs/0006-lower-information-disclose

docs/mercs/merging-roles-and-groups.rst

@@ -0,0 +1,67 @@
+===============================
+MERC XX: Unify Roles and Groups
+===============================
+
+:MERC: XX
+:Author: Michael Price
+:Status: Draft
+:Type: Feature
+:Created: 2018-02-27
+:Last-Modified: 2018-02-27
+
+.. contents:: Table of Contents
+   :depth: 3
+   :local:
+
+Abstract
+========
+
+This MERC proposes the merging of the Roles and Group models.
+
+Rationale
+=========
+
+Mayan EDMS uses Groups as units of users that are meant to mirror an
+organization's actual user hierarchy. Roles are used as permission units.
+
+Separation of concerns is a concept Mayan EDMS executes very successfully
+but when it comes to the Roles/Groups relationship that execution causes
+overheads without providing advantages in the day to day operations.
+
+In reality there is almost a 1 to 1 correlation between Roles and Groups.
+Other permissions systems already use Groups as permission units without
+disadvantages. An example of this is LDAP and its commercial counterpart
+Active Directory.
+
+Motivation
+==========
+
+Merging the Role and Group model will reduce some complexity when initially
+setting up Mayan EDMS. The merge allows removing a Mayan EDMS model in
+favor of using a native Django model for the same task.
+
+Merging the Role and Group models will also provide a speed boost in every
+permission check and queryset filtering. These checks are nested in nature.
+Since the access checks are performed for every view and for every link
+in the view the performance gain should be substantial.
+
+Backwards Compatibility
+=======================
+
+To avoid loss of role configuration a data migration will be needed to
+convert existing roles to groups.
+
+
+Specification
+=============
+
+Changes needed:
+
+1. Data migration to convert existing roles to groups.
+2. Prepend or append an identifier to the migrated roles.
+3. Intermediate model to map permissions to a group. This will substitute
+   the Role model's permissions many to many field.
+4. Update the ``AccessControlList`` models roles field to point to the group
+   models.
+5. Update the role checks in the ``check_access`` and ``restrict_queryset``
+   ``AccessControlList`` model manager methods.

docs/releases/0.10.rst

@@ -16,36 +16,36 @@ Version 0.10
     * LibreOffice (https://www.libreoffice.org/)
     * unoconv [version 0.5] (https://github.com/dagwieers/unoconv)
 
-* The new office documents converter won't convert files with the extension 
-  .docx because these files are recognized as zip files instead.  This 
+* The new office documents converter won't convert files with the extension
+  .docx because these files are recognized as zip files instead.  This
   is an issue of the libmagic library.
 
-* New configuration option added ``CONVERTER_UNOCONV_USE_PIPE`` that controls 
-  how unoconv handles the communication with LibreOffice.  The default of 
-  ``True`` causes unoconv to use **pipes**, this approach is slower than using 
+* New configuration option added ``CONVERTER_UNOCONV_USE_PIPE`` that controls
+  how unoconv handles the communication with LibreOffice.  The default of
+  ``True`` causes unoconv to use **pipes**, this approach is slower than using
   **TCP/IP** ports but it is more stable.
-  
-* Initial `REST` `API` that exposes documents properties and one method, this 
-  new `API` is used by the new smart document widget and requires the 
+
+* Initial `REST` `API` that exposes documents properties and one method, this
+  new `API` is used by the new smart document widget and requires the
   package ``djangorestframework``, users must issue a::
-  
+
   $ pip install -r requirements/production.txt
-  
+
   to install this new requirement.
-  
+
 * MIME type detection and caching performance updates.
 * Updated the included version of ``jQuery`` to 1.7
 * Updated the included version of ``JqueryAsynchImageLoader`` to 0.9.7
-* Document image serving response now specifies a MIME type for increased 
+* Document image serving response now specifies a MIME type for increased
   browser compatibility.
 * Small change in the scheduler that increases stability.
 * Russian translation updates Sergey Glita
-* Improved and generalized the OCR queue locking mechanism, this should 
-  eliminate any posibility of race conditions between Mayan EDMS OCR nodes.
+* Improved and generalized the OCR queue locking mechanism, this should
+  eliminate any possibility of race conditions between Mayan EDMS OCR nodes.
 * Added support for signals to the OCR queue, this results in instant OCR
   processing upon submittal of a document to the OCR queue, this works in
   addition to the current polling processing which eliminates the
-  posibility of stale documents in the OCR queue.
+  possibility of stale documents in the OCR queue.
 * Added multiple document OCR submit link
-* Re enabled tesseract language specific OCR processing and added a one
+* Re enabled Tesseract language specific OCR processing and added a one
   (1) time language neutral retry for failed language specific OCR

docs/releases/0.11.rst

@@ -17,11 +17,11 @@ Version 0.11
 * Added a view to delete the document image cache, useful when switching
   converter backends or doing diagnostics.
 * Added South to the requirements.
-* Merged documents' filename and extension database fiels into a single
+* Merged documents' filename and extension database fields into a single
   filename field, filename are store as uploaded not manipulation is done
-  Users with existing data must install South and run the appropiate
+  Users with existing data must install South and run the appropriate
   migrate commands::
-  
+
     $ pip install -r requirements/production.txt
     $ ./manager syncdb
     $ ./manage.py migrate documents 0001 --fake
@@ -30,5 +30,5 @@ Version 0.11
 * Added new office document mimetype
   * application/vnd.ms-office
 * Fixed documents not saving the file encoding
-* Removed extra slash in ajax-loader.gif URL fixes #15, thanks to 
+* Removed extra slash in ajax-loader.gif URL fixes #15, thanks to
   IHLeanne for finding this one

docs/releases/0.12.1.rst

@@ -1,13 +1,12 @@
-================================
-Mayan EDMS v0.12.1 release notes
-================================
+Version 0.12.1
+==============
 
 *May 2012*
 
 This is the first maintenance release of the 0.12 series.
 
 Overview
-========
+--------
 
 While bug fixes and minor feature were the focus for this release, some
 bigger changes were included because of their importance.  The parsing of
@@ -19,68 +18,81 @@ environments where users don't have the ability to install OS level
 binaries.
 
 
-What's new in Mayan EDMS v0.12.1
-================================
+Changes
+-------
 
 Fabric file (fabfile)
-~~~~~~~~~~~~~~~~~~~~~
+^^^^^^^^^^^^^^^^^^^^^
+
 A Fabric file is included to help users not very familiar with Ubuntu,
-Python and Django install **Mayan EDMS**, or for system administrators
+Python and Django install Mayan EDMS, or for system administrators
 looking to automate the install whether in local or remote systems.
-At the moment the fabfile will install **Mayan EDMS** in the same configurations
+At the moment the fabfile will install Mayan EDMS in the same configurations
 listed in this documentation, that is:  (Ubuntu/Debian/Fedora) + virtualenv + Apache + MySQL.
 Feel free to submit your configuration settings and files for different databases,
 webserver or Linux distribution.  More configurations will be added to
 the fabfile as more are tested.
 
+
 Documentation update
-~~~~~~~~~~~~~~~~~~~~
+^^^^^^^^^^^^^^^^^^^^
+
 The installation instructions were updated to include the installation of
 the libpng-dev and libjpeg-dev libraries as well as the installation of
-the poppler-utils package.  An additional step to help users test their
-new installation of **Mayan EDMS** was also added.
+the ``poppler-utils`` package.  An additional step to help users test their
+new installation of Mayan EDMS was also added.
+
 
 Translations
-~~~~~~~~~~~~
+^^^^^^^^^^^^
+
 The Italian translation has been synchronized with the source files at
 Transifex and finished to %100 completion.
 
+
 Usability improvements
-~~~~~~~~~~~~~~~~~~~~~~
+^^^^^^^^^^^^^^^^^^^^^^
+
 The index instance view now feature the same multi document action
 buttons (Submit to OCR, delete, download, etc) as the mail and recent
 document views.
 
+
 Better office document conversion
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 A new method of converting office documents has been implemented, this
 new method doesn't require the use of the command line utility ``UNOCONV``.
 If this new method proves to work better than previous solutions the use
 of ``UNOCONV`` may be deprecated in the future.  The conversion method
-adds just one new configuration option: `CONVERTER_LIBREOFFICE_PATH`
-which defaults to '/usr/bin/libreoffice'.
+adds just one new configuration option: ``CONVERTER_LIBREOFFICE_PATH``
+which defaults to ``/usr/bin/libreoffice``.
+
 
 Better PDF text parsing
-~~~~~~~~~~~~~~~~~~~~~~~
+^^^^^^^^^^^^^^^^^^^^^^^
+
 Brian E. submitted a patch to use the Poppler package pdftotext utility to
 extract text from PDF files.  This is now the default method Mayan EDMS
 will execute to try to extract text from a PDF and failing that will
 fallback to the previous method.  This change add a new configuration
-option: `OCR_PDFTOTEXT_PATH` to specify the location of the ``pdftotext``
-executable, it defaults to '/usr/bin/pdftotext'.  Be sure to install the
+option: ``OCR_PDFTOTEXT_PATH`` to specify the location of the ``pdftotext``
+executable, it defaults to ``/usr/bin/pdftotext``.  Be sure to install the
 ``poppler-utils`` os package to take advantage of this new parser.
 
+
 Changed defaults
-~~~~~~~~~~~~~~~~
+^^^^^^^^^^^^^^^^
+
 The OCR queue is now active by default when first created during the
-``syncdb`` phase and the `OCR_AUTOMATIC_OCR` option now defaults
+``syncdb`` phase and the ``OCR_AUTOMATIC_OCR`` option now defaults
 to ``True``.  These two changes are made to reduce the steps required for
 new users to start enjoying the benefits of automatic text extraction from
 uploaded documents without having to read the documentation and have a more
 functional default install.
 
+
 Upgrading from a previous version
-=================================
+---------------------------------
 
 Start off by adding the new requirements::
 
@@ -104,13 +116,17 @@ The upgrade procedure is now complete.
 
 
 Backward incompatible changes
-=============================
+-----------------------------
+
 * None
 
+
 Bugs fixed
-==========
-* Issue #25 "Office document conversion error"
+----------
+:github-issue:`25` "Office document conversion error"
+
+
+Removals
+--------
 
-Stuff removed
-=============
 * None