Merge branch 'versions/minor' into features/redactions_upstream_merge

Signed-off-by: Roberto Rosario <roberto.rosario@mayan-edms.com>

.gitlab-ci.yml

@@ -63,6 +63,7 @@ job_docker_nightly:
   only:
     - nightly
     - staging
+    - /^clients\/.+$/
 
 job_documentation_build:
   stage: build_documentation
@@ -160,6 +161,7 @@ job_push_python:
     - releases/python
     - staging
     - nightly
+    - /^clients\/.+$/
 
 test-mysql:
   <<: *test_base

.tx/config

@@ -115,6 +115,12 @@ source_lang = en
 source_file = mayan/apps/events/locale/en/LC_MESSAGES/django.po
 type = PO
 
+[mayan-edms.file_caching-3-0]
+file_filter = mayan/apps/file_caching/locale/<lang>/LC_MESSAGES/django.po
+source_lang = en
+source_file = mayan/apps/file_caching/locale/en/LC_MESSAGES/django.po
+type = PO
+
 [mayan-edms.file_metadata-3-0]
 file_filter = mayan/apps/file_metadata/locale/<lang>/LC_MESSAGES/django.po
 source_lang = en
@@ -222,3 +228,10 @@ file_filter = mayan/apps/user_management/locale/<lang>/LC_MESSAGES/django.po
 source_lang = en
 source_file = mayan/apps/user_management/locale/en/LC_MESSAGES/django.po
 type = PO
+
+[mayan-edms.weblink-3-0]
+file_filter = mayan/apps/weblinks/locale/<lang>/LC_MESSAGES/django.po
+source_lang = en
+source_file = mayan/apps/weblinks/locale/en/LC_MESSAGES/django.po
+type = PO
+

CHANGES_BC.rst

@@ -0,0 +1,10 @@
+- Use Select2 widget for the document type selection form.
+- Update source column matching to be additive and not exclusive.
+- Add two columns to show the number of documents per workflow and
+  workflow state.
+- Sort module.
+- Add link to sort individual indexes.
+- Support exclusions from source columns.
+- Improve source column exclusion. Improve for model subclasses in partial querysets.
+- Add sortable index instance label column.
+- Add rectangle drawing transformation.

HISTORY.rst

@@ -36,14 +36,47 @@
 - Remove encapsulate helper.
 - Add support for menu inheritance.
 - Emphasize source column labels.
+- Backport file cache manager app.
+- Convert document image cache to use file cache manager app.
+  Add setting DOCUMENTS_CACHE_MAXIMUM_SIZE defaults to 500 MB.
+- Replace djcelery and replace it with django-celery-beat.
+- Update Celery to version 4.3.0
+  Thanks to Jakob Haufe (@sur5r) and Jesaja Everling (@jeverling)
+  for much of the research and code updates.
+- Support wildcard MIME type associations for the file metadata drivers.
+- Rename MAYAN_GUID to MAYAN_GID
+- Update Gunicorn to use sync workers.
+- Include devpi-server as a development dependency.
+- Update default Docker stack file.
+- Remove Redis from the Docker image.
+- Add Celery flower to the Docker image.
+- Allow PIP proxying to the Docker image during build.
+- Default Celery worker concurrency to 0 (auto).
+- Set DJANGO_SETTINGS_MODULE environment variable to make it
+  available to sub processes.
+- Add entrypoint commands to run single workers, single gunicorn
+  or single celery commands like "flower".
+- Add platform template to return queues for a worker.
+- Update the EXIFTOOL driver to run for all documents
+  regardless of MIME type.
+- Remove task inspection from task manager app.
+- Move pagination navigation inside the toolbar.
+- Remove document image clear link and view.
+  This is now handled by the file caching app.
+- Add web links app.
+- Add support to display column help text
+  as a tooltip.
+- Update numeric dashboard widget to display
+  thousand commas.
+- Add support for disabling document pages.
 
 3.2.6 (2019-07-10)
 ==================
-* Remove the smart settings app * import.
-* Encode settings YAML before hashing.
-* Fix document icon used in the workflow runtime links.
-* Add trashed date time label.
-* Fix thumbnail generation issue. GitLab issue #637.
+- Remove the smart settings app * import.
+- Encode settings YAML before hashing.
+- Fix document icon used in the workflow runtime links.
+- Add trashed date time label.
+- Fix thumbnail generation issue. GitLab issue #637.
   Thanks to Giacomo Cariello (@giacomocariello) for the report
   and the merge request fixing the issue.
 
@@ -95,6 +128,7 @@
 ==================
 - Add support for disabling the random primary key
   test mixin.
+- Add a reusable task to upload documents.
 - Fix mailing profile log columns mappings.
   GitLab issue #626. Thanks to Jesaja Everling (@jeverling)
   for the report.

Makefile

@@ -18,7 +18,7 @@ clean-pyc: ## Remove Python artifacts.
 	find . -name '*.pyc' -exec rm -f {} +
 	find . -name '*.pyo' -exec rm -f {} +
 	find . -name '*~' -exec rm -f {} +
-
+	find . -name '__pycache__' -exec rm -R -f {} + 
 
 # Testing
 
@@ -234,10 +234,10 @@ generate-requirements: ## Generate all requirements files from the project deped
 # Dev server
 
 runserver: ## Run the development server.
-	./manage.py runserver --settings=mayan.settings.development $(ADDRPORT)
+	./manage.py runserver --nothreading --settings=mayan.settings.development $(ADDRPORT)
 
 runserver_plus: ## Run the Django extension's development server.
-	./manage.py runserver_plus --settings=mayan.settings.development $(ADDRPORT)
+	./manage.py runserver_plus --nothreading --settings=mayan.settings.development $(ADDRPORT)
 
 shell_plus: ## Run the shell_plus command.
 	./manage.py shell_plus --settings=mayan.settings.development
@@ -258,7 +258,7 @@ test-with-docker-frontend: ## Launch a front end instance that uses the producti
 	./manage.py runserver --settings=mayan.settings.staging.docker
 
 test-with-docker-worker: ## Launch a worker instance that uses the production-like services.
-	./manage.py celery worker --settings=mayan.settings.staging.docker -B -l INFO -O fair
+	DJANGO_SETTINGS_MODULE=mayan.settings.staging.docker ./manage.py celery worker -A mayan -B -l INFO -O fair
 
 docker-mysql-on: ## Launch and initialize a MySQL Docker container.
 	docker run -d --name mysql -p 3306:3306 -e MYSQL_ALLOW_EMPTY_PASSWORD=True -e MYSQL_DATABASE=mayan_edms mysql

contrib/scripts/process_messages.py

@@ -13,11 +13,12 @@ APP_LIST = (
     'checkouts', 'common', 'converter', 'dashboards', 'dependencies',
     'django_gpg', 'document_comments', 'document_indexing',
     'document_parsing', 'document_signatures', 'document_states',
-    'documents', 'dynamic_search', 'events', 'file_metadata', 'linking',
-    'lock_manager', 'mailer', 'mayan_statistics', 'metadata', 'mirroring',
-    'motd', 'navigation', 'ocr', 'permissions', 'platform', 'rest_api',
-    'smart_settings', 'sources', 'storage', 'tags', 'task_manager',
-    'user_management'
+    'documents', 'dynamic_search', 'events', 'file_caching',
+    'file_metadata', 'linking', 'lock_manager', 'mailer',
+    'mayan_statistics', 'metadata', 'mirroring', 'motd', 'navigation',
+    'ocr', 'permissions', 'platform', 'rest_api', 'smart_settings',
+    'sources', 'storage', 'tags', 'task_manager', 'user_management',
+    'weblinks'
 )
 
 LANGUAGE_LIST = (

docker/Dockerfile

@@ -4,7 +4,7 @@
 # BASE_IMAGE - Bare bones image with the base packages needed to run Mayan EDMS
 ####
 
-FROM debian:9.8-slim as BASE_IMAGE
+FROM debian:10.0-slim as BASE_IMAGE
 
 LABEL maintainer="Roberto Rosario roberto.rosario@mayan-edms.com"
 
@@ -22,6 +22,7 @@ RUN set -x \
 && DEBIAN_FRONTEND=noninteractive \
 apt-get update \
 && apt-get install -y --no-install-recommends \
+    ca-certificates \
     exiftool \
     ghostscript \
     gpgv \
@@ -29,11 +30,11 @@ apt-get update \
     graphviz \
     libfuse2 \
     libmagic1 \
-    libmariadbclient18 \
+    libmariadb3 \
     libreoffice \
     libpq5 \
     poppler-utils \
-    redis-server \
+    python3-distutils \
     sane-utils \
     sudo \
     supervisor \
@@ -52,22 +53,20 @@ apt-get update \
 && if [ "$(uname -m)" = "armv7l" ]; then \
     ln -s /usr/lib/arm-linux-gnueabihf/libz.so /usr/lib/ \
     && ln -s /usr/lib/arm-linux-gnueabihf/libjpeg.so /usr/lib/ \
-; fi \
-# Discard data when Redis runs out of memory
-&& echo "maxmemory-policy allkeys-lru" >> /etc/redis/redis.conf \
-# Disable saving the Redis database
-echo "save \"\"" >> /etc/redis/redis.conf \
-# Only provision 1 database
-&& echo "databases 1" >> /etc/redis/redis.conf
-
+; fi
 
 ####
-# BUILDER_IMAGE - This image buildS the Python package and is discarded afterwards
+# BUILDER_IMAGE - This image builds the Python package and is discarded afterwards
+#                 only the build artifact is carried over to the next image.
 ####
 
 # Reuse image
 FROM BASE_IMAGE as BUILDER_IMAGE
 
+# Python libraries caching
+ARG PIP_INDEX_URL
+ARG PIP_TRUSTED_HOST
+
 WORKDIR /src
 
 # Copy the source files needed to build the Python package
@@ -96,31 +95,32 @@ apt-get install -y --no-install-recommends \
     libssl-dev \
     g++ \
     gcc \
-    python-dev \
-    python-virtualenv \
+    python3-dev \
+    python3-venv \
 && mkdir -p "${PROJECT_INSTALL_DIR}" \
 && chown -R mayan:mayan "${PROJECT_INSTALL_DIR}" \
 && chown -R mayan:mayan /src
 
 USER mayan
-RUN python -m virtualenv "${PROJECT_INSTALL_DIR}" \
+RUN python3 -m venv "${PROJECT_INSTALL_DIR}" \
 && . "${PROJECT_INSTALL_DIR}/bin/activate" \
-&& pip install --no-cache-dir --no-use-pep517 \
-    librabbitmq==1.6.1 \
-    mysql-python==1.2.5 \
-    psycopg2==2.7.3.2 \
-    redis==2.10.6 \
+&& pip install --no-cache-dir \
+    librabbitmq==2.0.0 \
+    mysqlclient==1.4.2.post1 \
+    psycopg2==2.8.3 \
+    redis==3.2.1 \
+    flower==0.9.3 \
 # psutil is needed by ARM builds otherwise gevent and gunicorn fail to start
 && UNAME=`uname -m` && if [ "${UNAME#*arm}" != $UNAME ]; then \
-    pip install --no-cache-dir --no-use-pep517 \
+    pip install --no-cache-dir \
     psutil==5.6.2 \
 ; fi \
 # Install the Python packages needed to build Mayan EDMS
-&& pip install --no-cache-dir --no-use-pep517 -r /src/requirements/build.txt \
+&& pip install --no-cache-dir -r /src/requirements/build.txt \
 # Build Mayan EDMS
-&& python setup.py sdist \
+&& python3 setup.py sdist \
 # Install the built Mayan EDMS package
-&& pip install --no-cache-dir --no-use-pep517 dist/mayan* \
+&& pip install --no-cache-dir dist/mayan* \
 # Install the static content
 && mayan-edms.py installdependencies \
 && MAYAN_STATIC_ROOT=${PROJECT_INSTALL_DIR}/static mayan-edms.py preparestatic --link --noinput
@@ -128,7 +128,7 @@ RUN python -m virtualenv "${PROJECT_INSTALL_DIR}" \
 COPY --chown=mayan:mayan requirements/testing-base.txt "${PROJECT_INSTALL_DIR}"
 
 ####
-# Final image - BASE_IMAGE + Mayan install directory from the builder image
+# Final image - BASE_IMAGE + BUILDER_IMAGE artifact (Mayan install directory)
 ####
 
 FROM BASE_IMAGE
@@ -144,7 +144,7 @@ VOLUME ["/var/lib/mayan"]
 ENTRYPOINT ["entrypoint.sh"]
 
 EXPOSE 8000
-CMD ["mayan"]
+CMD ["run_all"]
 
 RUN ${PROJECT_INSTALL_DIR}/bin/mayan-edms.py platformtemplate supervisord_docker > /etc/supervisor/conf.d/mayan.conf \
 && apt-get clean autoclean \

docker/Makefile

@@ -1,4 +1,9 @@
-APT_PROXY ?= `/sbin/ip route|awk '/docker0/ { print $$9 }'`:3142
+HOST_IP = `/sbin/ip route|awk '/docker0/ { print $$9 }'`
+
+APT_PROXY ?= $(HOST_IP):3142
+PIP_INDEX_URL ?= http://$(HOST_IP):3141/root/pypi/+simple/
+PIP_TRUSTED_HOST ?= $(HOST_IP)
+
 IMAGE_VERSION ?= `cat docker/rootfs/version`
 CONSOLE_COLUMNS ?= `echo $$(tput cols)`
 CONSOLE_LINES ?= `echo $$(tput lines)`
@@ -7,7 +12,7 @@ docker-build: ## Build a new image locally.
 	docker build -t mayanedms/mayanedms:$(IMAGE_VERSION) -f docker/Dockerfile .
 
 docker-build-with-proxy: ## Build a new image locally using an APT proxy as APT_PROXY.
-	docker build -t mayanedms/mayanedms:$(IMAGE_VERSION) -f docker/Dockerfile --build-arg APT_PROXY=$(APT_PROXY) .
+	docker build -t mayanedms/mayanedms:$(IMAGE_VERSION) -f docker/Dockerfile --build-arg APT_PROXY=$(APT_PROXY) --build-arg PIP_INDEX_URL=$(PIP_INDEX_URL) --build-arg PIP_TRUSTED_HOST=$(PIP_TRUSTED_HOST) --build-arg HTTP_PROXY=$(HTTP_PROXY) --build-arg HTTPS_PROXY=$(HTTPS_PROXY) .
 
 docker-shell: ## Launch a bash instance inside a running container. Pass the container name via DOCKER_CONTAINER.
 	docker exec -e TERM=$(TERM) -e "COLUMNS=$(CONSOLE_COLUMNS)" -e "LINES=$(CONSOLE_LINES)" -it $(DOCKER_CONTAINER) /bin/bash
@@ -23,3 +28,13 @@ docker-test-cleanup: ## Delete the test container and the test volume.
 docker-test-all: ## Build and executed the test suite in a test container.
 docker-test-all: docker-build-with-proxy
 	docker run --rm run-tests
+
+docker-compose-build:
+	docker-compose -f docker/docker-compose.yml -p mayan-edms build
+
+docker-compose-build-with-proxy:
+	docker-compose -f docker/docker-compose.yml -p mayan-edms build --build-arg APT_PROXY=$(APT_PROXY) --build-arg PIP_INDEX_URL=$(PIP_INDEX_URL) --build-arg PIP_TRUSTED_HOST=$(PIP_TRUSTED_HOST) --build-arg HTTP_PROXY=$(HTTP_PROXY) --build-arg HTTPS_PROXY=$(HTTPS_PROXY)
+
+docker-compose-up:
+	docker-compose -f docker/docker-compose.yml -p mayan-edms up
+

docker/docker-compose-development.yml

@@ -1,72 +0,0 @@
-version: '2.1'
-
-volumes:
-    broker:
-        driver: local
-    app:
-        driver: local
-    db:
-        driver: local
-    results:
-        driver: local
-
-services:
-  broker:
-    container_name: mayan-edms-broker
-    image: healthcheck/rabbitmq
-    environment:
-      RABBITMQ_DEFAULT_USER: mayan
-      RABBITMQ_DEFAULT_PASS: mayan
-      RABBITMQ_DEFAULT_VHOST: mayan
-    volumes:
-      - broker:/var/lib/rabbitmq
-  results:
-    container_name: mayan-edms-results
-    image: healthcheck/redis
-    volumes:
-      - results:/data
-  #db:
-  #  container_name: mayan-edms-db
-  #  image: healthcheck/mysql
-  #  environment:
-  #    MYSQL_DATABASE: mayan
-  #    MYSQL_PASSWORD: mayan-password
-  #    MYSQL_ROOT_PASSWORD: root-password
-  #    MYSQL_USER: mayan
-  #  volumes:
-  #    - db:/var/lib/mysql
-  db:
-    container_name: mayan-edms-db
-    image: healthcheck/postgres
-    environment:
-      POSTGRES_DB: mayan
-      POSTGRES_PASSWORD: mayan-password
-      POSTGRES_USER: mayan
-    volumes:
-      - db:/var/lib/postgresql/data
-  mayan-edms:
-    container_name: mayan-edms-app
-    image: mayan-edms/next
-    build:
-      context: ./
-      args:
-        - APT_PROXY=172.18.0.1:3142
-    depends_on:
-      broker:
-        condition: service_healthy
-      db:
-        condition: service_healthy
-      results:
-        condition: service_healthy
-    environment:
-      MAYAN_BROKER_URL: amqp://mayan:mayan@broker:5672/mayan
-      MAYAN_CELERY_RESULT_BACKEND: redis://results:6379/0
-      MAYAN_DATABASE_ENGINE: django.db.backends.postgresql
-      MAYAN_DATABASE_HOST: db
-      MAYAN_DATABASE_NAME: mayan
-      MAYAN_DATABASE_PASSWORD: mayan-password
-      MAYAN_DATABASE_USER: mayan
-    ports:
-      - "80:80"
-    volumes:
-      - app:/var/lib/mayan

docker/docker-compose.yml

@@ -1,58 +1,130 @@
-version: '2.1'
+version: '3.7'
 
-volumes:
-    broker:
-        driver: local
-    app:
-        driver: local
-    db:
-        driver: local
-    results:
-        driver: local
+networks:
+  mayan-bridge:
+    driver: bridge
 
 services:
-  broker:
-    container_name: mayan-edms-broker
-    image: healthcheck/rabbitmq
-    environment:
-      RABBITMQ_DEFAULT_USER: mayan
-      RABBITMQ_DEFAULT_PASS: mayan
-      RABBITMQ_DEFAULT_VHOST: mayan
-    volumes:
-      - broker:/var/lib/rabbitmq
-  results:
-    container_name: mayan-edms-results
-    image: healthcheck/redis
-    volumes:
-      - results:/data
-  db:
-    container_name: mayan-edms-db
-    image: healthcheck/postgres
-    environment:
-      POSTGRES_DB: mayan
-      POSTGRES_PASSWORD: mayan-password
-      POSTGRES_USER: mayan
-    volumes:
-      - db:/var/lib/postgresql/data
-  mayan-edms:
-    container_name: mayan-edms-app
-    image: mayanedms/mayanedms:latest
+  app:
+    build:
+      context: ..
+      dockerfile: ./docker/Dockerfile
     depends_on:
-      broker:
-        condition: service_healthy
-      db:
-        condition: service_healthy
-      results:
-        condition: service_healthy
-    environment:
-      MAYAN_BROKER_URL: amqp://mayan:mayan@broker:5672/mayan
-      MAYAN_CELERY_RESULT_BACKEND: redis://results:6379/0
-      MAYAN_DATABASE_ENGINE: django.db.backends.postgresql
-      MAYAN_DATABASE_HOST: db
-      MAYAN_DATABASE_NAME: mayan
-      MAYAN_DATABASE_PASSWORD: mayan-password
-      MAYAN_DATABASE_USER: mayan
+      - postgresql
+      - redis
+      # Enable to use RabbitMQ
+      #- rabbitmq
+    environment: &mayan_env
+      # Enable to use RabbitMQ
+      # MAYAN_CELERY_BROKER_URL: amqp://mayan:mayanrabbitpass@broker:5672/mayan
+      # Disable Redis Broker to use RabbitMQ as Broker
+      MAYAN_CELERY_BROKER_URL: redis://redis:6379/1
+      MAYAN_CELERY_RESULT_BACKEND: redis://redis:6379/0
+      MAYAN_DATABASES: "{'default':{'ENGINE':'django.db.backends.postgresql','NAME':'mayan','PASSWORD':'mayandbpass','USER':'mayan','HOST':'postgresql'}}"
+    image: mayanedms/mayanedms:3.2.6
+    networks:
+      - mayan-bridge
     ports:
       - "80:8000"
+    restart: unless-stopped
     volumes:
-      - app:/var/lib/mayan
+      - /docker-volumes/mayan-edms/media:/var/lib/mayan
+
+  postgresql:
+    environment:
+      POSTGRES_DB: mayan
+      POSTGRES_PASSWORD: mayandbpass
+      POSTGRES_USER: mayan
+    image: postgres:9.6
+    networks:
+      - mayan-bridge
+    restart: unless-stopped
+    volumes:
+      - /docker-volumes/mayan-edms/postgres:/var/lib/postgresql/data
+
+  redis:
+    command:
+      - redis-server
+      - --databases
+      - "2"
+      - --maxmemory-policy
+      - allkeys-lru
+      - --save
+      - ""
+    image: redis:5.0
+    networks:
+      - mayan-bridge
+    restart: unless-stopped
+
+  # Optional services
+
+  # celery_flower:
+  #   command:
+  #     - run_celery
+  #     - flower
+  #   depends_on:
+  #     - postgresql
+  #     - redis
+  #     # Enable to use RabbitMQ
+  #     # - rabbitmq
+  #   environment:
+  #     <<: *mayan_env
+  #   image: mayanedms/mayanedms:3.2.6
+  #   networks:
+  #     - mayan-bridge
+  #   ports:
+  #     - "5555:5555"
+  #   restart: unless-stopped
+
+  # Enable to use RabbitMQ
+  # rabbitmq:
+  #   container_name: mayan-edms-rabbitmq
+  #   image: healthcheck/rabbitmq
+  #   environment:
+  #     RABBITMQ_DEFAULT_USER: mayan
+  #     RABBITMQ_DEFAULT_PASS: mayanrabbitpass
+  #     RABBITMQ_DEFAULT_VHOST: mayan
+  #   networks:
+  #     - mayan-bridge
+  #   restart: unless-stopped
+  #   volumes:
+  #     - /docker-volumes/mayan-edms/rabbitmq:/var/lib/rabbitmq
+
+  # Enable to run stand alone workers
+  # worker_fast:
+  #   command:
+  #     - run_worker
+  #     - fast
+  #   depends_on:
+  #     - postgresql
+  #     - redis
+  #     # Enable to use RabbitMQ
+  #     # - rabbitmq
+  #   environment:
+  #     <<: *mayan_env
+  #   image: mayanedms/mayanedms:3.2.6
+  #   networks:
+  #     - mayan-bridge
+  #   restart: unless-stopped
+  #   volumes:
+  #     - /docker-volumes/mayan-edms/media:/var/lib/mayan
+
+  # Enable to run stand frontend gunicorn
+  # frontend:
+  #   command:
+  #     - run_frontend
+  #   depends_on:
+  #     - postgresql
+  #     - redis
+  #     # Enable to use RabbitMQ
+  #     # - rabbitmq
+  #   environment:
+  #     <<: *mayan_env
+  #   image: mayanedms/mayanedms:3.2.6
+  #   networks:
+  #     - mayan-bridge
+  #   ports:
+  #     - "81:8000"
+  #   restart: unless-stopped
+  #   volumes:
+  #     - /docker-volumes/mayan-edms/media:/var/lib/mayan

docker/rootfs/usr/local/bin/entrypoint.sh

@@ -1,4 +1,7 @@
-#!/bin/sh
+#!/bin/bash
+
+# Use bash and not sh to support argument slicing "${@:2}"
+# sh defaults to dash instead of bash.
 
 set -e
 echo "mayan: starting entrypoint.sh"
@@ -6,19 +9,15 @@ INSTALL_FLAG=/var/lib/mayan/system/SECRET_KEY
 CONCURRENCY_ARGUMENT=--concurrency=
 
 DEFAULT_USER_UID=1000
-DEFAULT_USER_GUID=1000
-
-export MAYAN_DEFAULT_BROKER_URL=redis://127.0.0.1:6379/0
-export MAYAN_DEFAULT_CELERY_RESULT_BACKEND=redis://127.0.0.1:6379/0
+DEFAULT_USER_GID=1000
 
 export MAYAN_ALLOWED_HOSTS='["*"]'
 export MAYAN_BIN=/opt/mayan-edms/bin/mayan-edms.py
-export MAYAN_BROKER_URL=${MAYAN_BROKER_URL:-${MAYAN_DEFAULT_BROKER_URL}}
-export MAYAN_CELERY_RESULT_BACKEND=${MAYAN_CELERY_RESULT_BACKEND:-${MAYAN_DEFAULT_CELERY_RESULT_BACKEND}}
 export MAYAN_INSTALL_DIR=/opt/mayan-edms
 export MAYAN_PYTHON_BIN_DIR=/opt/mayan-edms/bin/
 export MAYAN_MEDIA_ROOT=/var/lib/mayan
 export MAYAN_SETTINGS_MODULE=${MAYAN_SETTINGS_MODULE:-mayan.settings.production}
+export DJANGO_SETTINGS_MODULE=${MAYAN_SETTINGS_MODULE}
 
 export MAYAN_GUNICORN_BIN=${MAYAN_PYTHON_BIN_DIR}gunicorn
 export MAYAN_GUNICORN_WORKERS=${MAYAN_GUNICORN_WORKERS:-2}
@@ -26,13 +25,9 @@ export MAYAN_GUNICORN_TIMEOUT=${MAYAN_GUNICORN_TIMEOUT:-120}
 export MAYAN_PIP_BIN=${MAYAN_PYTHON_BIN_DIR}pip
 export MAYAN_STATIC_ROOT=${MAYAN_INSTALL_DIR}/static
 
-MAYAN_WORKER_FAST_CONCURRENCY=${MAYAN_WORKER_FAST_CONCURRENCY:-1}
-MAYAN_WORKER_MEDIUM_CONCURRENCY=${MAYAN_WORKER_MEDIUM_CONCURRENCY:-1}
-MAYAN_WORKER_SLOW_CONCURRENCY=${MAYAN_WORKER_SLOW_CONCURRENCY:-1}
-
-echo "mayan: changing uid/guid"
-usermod mayan -u ${MAYAN_USER_UID:-${DEFAULT_USER_UID}}
-groupmod mayan -g ${MAYAN_USER_GUID:-${DEFAULT_USER_GUID}}
+MAYAN_WORKER_FAST_CONCURRENCY=${MAYAN_WORKER_FAST_CONCURRENCY:-0}
+MAYAN_WORKER_MEDIUM_CONCURRENCY=${MAYAN_WORKER_MEDIUM_CONCURRENCY:-0}
+MAYAN_WORKER_SLOW_CONCURRENCY=${MAYAN_WORKER_SLOW_CONCURRENCY:-0}
 
 if [ "$MAYAN_WORKER_FAST_CONCURRENCY" -eq 0 ]; then
     MAYAN_WORKER_FAST_CONCURRENCY=
@@ -55,11 +50,9 @@ else
 fi
 export MAYAN_WORKER_SLOW_CONCURRENCY
 
-export CELERY_ALWAYS_EAGER=False
+# Allow importing of user setting modules
 export PYTHONPATH=$PYTHONPATH:$MAYAN_MEDIA_ROOT
 
-chown mayan:mayan /var/lib/mayan -R
-
 apt_get_install() {
     apt-get -q update
     apt-get install -y --force-yes --no-install-recommends --auto-remove "$@"
@@ -67,9 +60,9 @@ apt_get_install() {
     rm -rf /var/lib/apt/lists/*
 }
 
-initialize() {
-    echo "mayan: initialize()"
-    su mayan -c "${MAYAN_BIN} initialsetup --force --no-javascript"
+initialsetup() {
+    echo "mayan: initialsetup()"
+    su mayan -c "${MAYAN_BIN} initialsetup --force --no-dependencies"
 }
 
 os_package_installs() {
@@ -86,43 +79,71 @@ pip_installs() {
     fi
 }
 
-start() {
+run_all() {
     echo "mayan: start()"
     rm -rf /var/run/supervisor.sock
     exec /usr/bin/supervisord -nc /etc/supervisor/supervisord.conf
 }
 
-upgrade() {
-    echo "mayan: upgrade()"
-    su mayan -c "${MAYAN_BIN} performupgrade --no-javascript"
+performupgrade() {
+    echo "mayan: performupgrade()"
+    su mayan -c "${MAYAN_BIN} performupgrade --no-dependencies"
+}
+
+make_ready() {
+    # Check if this is a new install, otherwise try to upgrade the existing
+    # installation on subsequent starts
+    if [ ! -f $INSTALL_FLAG ]; then
+        initialsetup
+    else
+        performupgrade
+    fi
+}
+
+set_uid_guid() {
+    echo "mayan: changing uid/guid"
+    usermod mayan -u ${MAYAN_USER_UID:-${DEFAULT_USER_UID}}
+    groupmod mayan -g ${MAYAN_USER_GID:-${DEFAULT_USER_GID}}
 }
 
 os_package_installs || true
 pip_installs || true
+chown mayan:mayan /var/lib/mayan -R
 
 case "$1" in
 
-mayan) # Check if this is a new install, otherwise try to upgrade the existing
-       # installation on subsequent starts
-       if [ ! -f $INSTALL_FLAG ]; then
-           initialize
-       else
-           upgrade
-       fi
-       start
+run_initialsetup)
+    initialsetup
     ;;
 
-run-tests) # Check if this is a new install, otherwise try to upgrade the existing
-           # installation on subsequent starts
-           if [ ! -f $INSTALL_FLAG ]; then
-               initialize
-           else
-               upgrade
-           fi
-           run-tests.sh
+run_performupgrade)
+    performupgrade
     ;;
 
-*) su mayan -c "$@";
+run_all)
+    make_ready
+    run_all
+    ;;
+
+run_celery)
+    run_celery.sh "${@:2}"
+    ;;
+
+run_frontend)
+    run_frontend.sh
+    ;;
+
+run_tests)
+    make_ready
+    run_tests.sh
+    ;;
+
+run_worker)
+    run_worker.sh "${@:2}"
+    ;;
+
+*)
+    su mayan -c "$@"
     ;;
 
 esac

docker/rootfs/usr/local/bin/run_celery.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+# Use -A and not --app. Both are the same but behave differently
+# -A can be located before the command while --app cannot.
+su mayan -c "${MAYAN_PYTHON_BIN_DIR}celery -A mayan $@"

docker/rootfs/usr/local/bin/run_frontend.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+MAYAN_GUNICORN_MAX_REQUESTS=${MAYAN_GUNICORN_MAX_REQUESTS:-500}
+MAYAN_GUNICORN_MAX_REQUESTS_JITTERS=${MAYAN_GUNICORN_MAX_REQUESTS_JITTERS:-50}
+MAYAN_GUNICORN_WORKER_CLASS=${MAYAN_GUNICORN_WORKER_CLASS:-sync}
+
+su mayan -c "${MAYAN_PYTHON_BIN_DIR}gunicorn -w ${MAYAN_GUNICORN_WORKERS} mayan.wsgi --max-requests ${MAYAN_GUNICORN_MAX_REQUESTS} --max-requests-jitter ${MAYAN_GUNICORN_MAX_REQUESTS_JITTERS} --worker-class ${MAYAN_GUNICORN_WORKER_CLASS} --bind 0.0.0.0:8000 --timeout ${MAYAN_GUNICORN_TIMEOUT}"

docker/rootfs/usr/local/bin/run_worker.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+QUEUE_LIST=`MAYAN_WORKER_NAME=$1 su mayan -c "${MAYAN_PYTHON_BIN_DIR}mayan-edms.py platformtemplate worker_queues"`
+
+# Use -A and not --app. Both are the same but behave differently
+# -A can be located before the command while --app cannot.
+# Pass ${@:2} to allow overriding the defaults arguments
+su mayan -c "${MAYAN_PYTHON_BIN_DIR}celery -A mayan worker -Ofair -l ERROR -Q $QUEUE_LIST ${@:2}"

docker/stack.yml

@@ -9,24 +9,32 @@ volumes:
 services:
 
   db:
-    image: postgres
     environment:
       POSTGRES_DB: mayan
-      POSTGRES_PASSWORD: mayan-password
+      POSTGRES_PASSWORD: mayandbpass
       POSTGRES_USER: mayan
+    image: postgres
     volumes:
       - db:/var/lib/postgresql/data
 
   app:
+    environment:
+      MAYAN_CELERY_BROKER_URL: redis://redis:6379/1
+      MAYAN_CELERY_RESULT_BACKEND: redis://redis:6379/0
+      MAYAN_DATABASES: "{'default':{'ENGINE':'django.db.backends.postgresql','NAME':'mayan','PASSWORD':'mayandbpass','USER':'mayan','HOST':'db'}}"
     image: mayanedms/mayanedms:latest
     ports:
       - 80:8000
-    environment:
-      MAYAN_DATABASE_ENGINE: django.db.backends.postgresql
-      MAYAN_DATABASE_HOST: db
-      MAYAN_DATABASE_NAME: mayan
-      MAYAN_DATABASE_PASSWORD: mayan-password
-      MAYAN_DATABASE_USER: mayan
-      MAYAN_DATABASE_CONN_MAX_AGE: 0
     volumes:
       - app:/var/lib/mayan
+
+  redis:
+    command:
+      - redis-server
+      - --databases
+      - "2"
+      - --maxmemory-policy
+      - allkeys-lru
+      - --save
+      - ""
+    image: redis:5.0

docs/chapters/deploying.rst

@@ -220,11 +220,11 @@ of a restart or power failure. The Gunicorn workers are increased to 3.
 ---------------------------------------------------------------------
    Replace (paying attention to the comma at the end)::
 
-       MAYAN_BROKER_URL="redis://127.0.0.1:6379/0",
+       MAYAN_CELERY_BROKER_URL="redis://127.0.0.1:6379/0",
 
    with::
 
-       MAYAN_BROKER_URL="amqp://mayan:mayanrabbitmqpassword@localhost:5672/mayan",
+       MAYAN_CELERY_BROKER_URL="amqp://mayan:mayanrabbitmqpassword@localhost:5672/mayan",
 
    increase the number of Gunicorn workers to 3 in the line (``-w 2`` section)::
 

docs/chapters/docker.rst

@@ -49,12 +49,7 @@ Finally create and run a Mayan EDMS container::
     --name mayan-edms \
     --restart=always \
     -p 80:8000 \
-    -e MAYAN_DATABASE_ENGINE=django.db.backends.postgresql \
-    -e MAYAN_DATABASE_HOST=172.17.0.1 \
-    -e MAYAN_DATABASE_NAME=mayan \
-    -e MAYAN_DATABASE_PASSWORD=mayanuserpass \
-    -e MAYAN_DATABASE_USER=mayan \
-    -e MAYAN_DATABASE_CONN_MAX_AGE=0 \
+    -e MAYAN_DATABASES="{'default':{'ENGINE':'django.db.backends.postgresql','NAME':'mayan','PASSWORD':'mayanuserpass','USER':'mayan','HOST':'172.17.0.1'}}" \
     -v /docker-volumes/mayan-edms/media:/var/lib/mayan \
     mayanedms/mayanedms:<version>
 
@@ -108,12 +103,7 @@ instead of the IP address of the Docker host (``172.17.0.1``)::
     --network=mayan \
     --restart=always \
     -p 80:8000 \
-    -e MAYAN_DATABASE_ENGINE=django.db.backends.postgresql \
-    -e MAYAN_DATABASE_HOST=mayan-edms-postgres \
-    -e MAYAN_DATABASE_NAME=mayan \
-    -e MAYAN_DATABASE_PASSWORD=mayanuserpass \
-    -e MAYAN_DATABASE_USER=mayan \
-    -e MAYAN_DATABASE_CONN_MAX_AGE=0 \
+    -e MAYAN_DATABASES="{'default':{'ENGINE':'django.db.backends.postgresql','NAME':'mayan','PASSWORD':'mayanuserpass','USER':'mayan','HOST':'mayan-edms-postgres'}}" \
     -v /docker-volumes/mayan-edms/media:/var/lib/mayan \
     mayanedms/mayanedms:<version>
 
@@ -137,102 +127,14 @@ To start the container again::
 Environment Variables
 ---------------------
 
-The Mayan EDMS image can be configure via environment variables.
-
-``MAYAN_DATABASE_ENGINE``
-
-Defaults to ``None``. This environment variable configures the database
-backend to use. If left unset, SQLite will be used. The database backends
-supported by this Docker image are:
-
-- ``'django.db.backends.postgresql'``
-- ``'django.db.backends.mysql'``
-- ``'django.db.backends.sqlite3'``
-
-When using the SQLite backend, the database file will be saved in the Docker
-volume. The SQLite database as used by Mayan EDMS is meant only for development
-or testing, never use it in production.
-
-``MAYAN_DATABASE_NAME``
-
-Defaults to 'mayan'. This optional environment variable can be used to define
-the database name that Mayan EDMS will connect to. For more information read
-the pertinent Django documentation page:
-:django-docs:`Connecting to the database <ref/databases/#connecting-to-the-database>`
-
-``MAYAN_DATABASE_USER``
-
-Defaults to 'mayan'. This optional environment variable is used to set the
-username that will be used to connect to the database. For more information
-read the pertinent Django documentation page:
-:django-docs:`Settings, USER <ref/settings/#user>`
-
-``MAYAN_DATABASE_PASSWORD``
-
-Defaults to ''. This optional environment variable is used to set the
-password that will be used to connect to the database. For more information
-read the pertinent Django documentation page:
-:django-docs:`Settings, PASSWORD <ref/settings/#password>`
-
-``MAYAN_DATABASE_HOST``
-
-Defaults to `None`. This optional environment variable is used to set the
-hostname that will be used to connect to the database. This can be the
-hostname of another container or an IP address. For more information read
-the pertinent Django documentation page:
-:django-docs:`Settings, HOST <ref/settings/#host>`
-
-``MAYAN_DATABASE_PORT``
-
-Defaults to `None`. This optional environment variable is used to set the
-port number to use when connecting to the database. An empty string means
-the default port. Not used with SQLite. For more information read the
-pertinent Django documentation page:
-:django-docs:`Settings, PORT <ref/settings/#port>`
-
-``MAYAN_BROKER_URL``
-
-This optional environment variable determines the broker that Celery will use
-to relay task messages between the frontend code and the background workers.
-For more information read the pertinent Celery Kombu documentation page: `Broker URL`_
-
-.. _Broker URL: http://kombu.readthedocs.io/en/latest/userguide/connections.html#connection-urls
-
-This Docker image supports using Redis and RabbitMQ as brokers.
-
-Caveat: If the `MAYAN_BROKER_URL` and `MAYAN_CELERY_RESULT_BACKEND` environment
-variables are specified, the built-in Redis server inside the container will
-be disabled.
-
-``MAYAN_CELERY_RESULT_BACKEND``
-
-This optional environment variable determines the results backend that Celery
-will use to relay result messages from the background workers to the frontend
-code. For more information read the pertinent Celery Kombu documentation page:
-`Task result backend settings`_
-
-.. _Task result backend settings: http://docs.celeryproject.org/en/3.1/configuration.html#celery-result-backend
-
-This Docker image supports using Redis and RabbitMQ as result backends.
-
-Caveat: If the `MAYAN_BROKER_URL` and `MAYAN_CELERY_RESULT_BACKEND` environment
-variables are specified, the built-in Redis server inside the container will
-be disabled.
+The common set of settings can also be modified via environment variables when
+using the Docker image. In addition to the common set of settings, some Docker
+image specific environment variables are available.
 
 ``MAYAN_SETTINGS_MODULE``
 
 Optional. Allows loading an alternate settings file.
 
-``MAYAN_DATABASE_CONN_MAX_AGE``
-
-Amount in seconds to keep a database connection alive. Allow reuse of database
-connections. For more information read the pertinent Django documentation
-page: :django-docs:`Settings, CONN_MAX_AGE <ref/settings/#conn-max-age>`
-According to new information Gunicorn's microthreads don't share connections
-and will exhaust the available Postgres connections available if a number
-other than 0 is used. Reference: https://serverfault.com/questions/635100/django-conn-max-age-persists-connections-but-doesnt-reuse-them-with-postgresq
-and https://github.com/benoitc/gunicorn/issues/996
-
 ``MAYAN_GUNICORN_WORKERS``
 
 Optional. This environment variable controls the number of frontend workers
@@ -269,12 +171,21 @@ number of CPUs detected).
 Optional. Changes the UID of the ``mayan`` user internal to the Docker
 container. Defaults to 1000.
 
-``MAYAN_USER_GUID``
+``MAYAN_USER_GID``
 
-Optional. Changes the GUID of the ``mayan`` user internal to the Docker
+Optional. Changes the GID of the ``mayan`` user internal to the Docker
 container. Defaults to 1000.
 
 
+Included drivers
+----------------
+
+The Docker image supports using Redis and RabbitMQ as result backends. For
+databases, the image includes support for PostgreSQL and MySQL/MariaDB.
+Support for additional brokers or databases may be added using the
+``MAYAN_APT_INSTALL`` environment variable.
+
+
 .. _docker-accessing-outside-data:
 
 Accessing outside data
@@ -442,6 +353,7 @@ These are:
 
 Nightly images
 ==============
+
 The continuous integration pipeline used for testing development builds also
 produces a resulting Docker image. These are build automatically and their
 stability is not guaranteed. They should never be used in production.

docs/chapters/scaling_up.rst

@@ -94,11 +94,11 @@ For the Docker image, launch a separate RabbitMQ container
 
     docker run -d --name mayan-edms-rabbitmq -e RABBITMQ_DEFAULT_USER=mayan -e RABBITMQ_DEFAULT_PASS=mayanrabbitmqpassword -e RABBITMQ_DEFAULT_VHOST=mayan rabbitmq:3
 
-Pass the MAYAN_BROKER_URL environment variable (https://kombu.readthedocs.io/en/latest/userguide/connections.html#connection-urls)
+Pass the MAYAN_CELERY_BROKER_URL environment variable (https://kombu.readthedocs.io/en/latest/userguide/connections.html#connection-urls)
 to the Mayan EDMS container so that it uses the RabbitMQ container the
 message broker::
 
-    -e MAYAN_BROKER_URL="amqp://mayan:mayanrabbitmqpassword@localhost:5672/mayan",
+    -e MAYAN_CELERY_BROKER_URL="amqp://mayan:mayanrabbitmqpassword@localhost:5672/mayan",
 
 When tasks finish, they leave behind a return status or the result of a
 calculation, these are stored for a while so that whoever requested the

docs/chapters/sources.rst

@@ -15,7 +15,8 @@ The current document sources supported are:
 - IMAP email - Same as the ``POP3`` email source but for email accounts using
   the ``IMAP`` protocol.
 - Watch folder - A filesystem folder that is scanned periodically for files.
-  Any file in the watch folder is automatically uploaded.
+  Any file in the watch folder is automatically uploaded. When the upload for a
+  file is completed, the file is removed from source folder.
 - Staging folder - Folder where networked attached scanned can save image
   files. The files in these staging folders are scanned and a preview is
   generated to help the process of upload. Staging folders and Watch folders

docs/releases/3.2.3.rst

@@ -19,7 +19,6 @@ Changes
   GitLab issue #625. Thanks to Jesaja Everling (@jeverling)
   for the report and the research.
 
-
 Removals
 --------
 

docs/releases/3.3.rst

@@ -49,6 +49,41 @@ Changes
 - Remove encapsulate helper.
 - Add support for menu inheritance.
 - Emphasize source column labels.
+- Backport file cache manager app.
+- Convert document image cache to use file cache manager app.
+  Add setting DOCUMENTS_CACHE_MAXIMUM_SIZE defaults to 500 MB.
+- Update Celery to version 4.3.0. Settings changed:
+  MAYAN_BROKER_URL to MAYAN_CELERY_BROKER_URL,
+  MAYAN_CELERY_ALWAYS_EAGER to MAYAN_CELERY_TASK_ALWAYS_EAGER.
+- Replace djcelery and replace it with django-celery-beat.
+- Update Celery to version 4.3.0 with 55e9b2263cbdb9b449361412fd18d8ee0a442dd3
+  from versions/next, code from GitLab issue #594 and GitLab merge request !55.
+  Thanks to Jakob Haufe (@sur5r) and Jesaja Everling (@jeverling)
+  for much of the research and code updates.
+- Support wildcard MIME type associations for the file metadata drivers.
+- Rename MAYAN_GUID to MAYAN_GID
+- Update Gunicorn to use sync workers.
+- Include devpi-server as a development dependency.
+- Update default Docker stack file.
+- Remove Redis from the Docker image.
+- Add Celery flower to the Docker image.
+- Allow PIP proxying to the Docker image during build.
+- Default Celery worker concurrency to 0 (auto).
+- Set DJANGO_SETTINGS_MODULE environment variable to make it
+  available to sub processes.
+- Add entrypoint commands to run single workers, single gunicorn
+  or single celery commands like "flower".
+- Add platform template to return queues for a worker.
+- Remove task inspection from task manager app.
+- Move pagination navigation inside the toolbar.
+- Remove document image clear link and view.
+  This is now handled by the file caching app.
+- Add web links app.
+- Add support to display column help text
+  as a tooltip.
+- Update numeric dashboard widget to display
+  thousand commas.
+- Add support for disabling document pages.
 
 Removals
 --------
@@ -160,7 +195,13 @@ Backward incompatible changes
 Bugs fixed or issues closed
 ---------------------------
 
+- :gitlab-issue:`526` RuntimeWarning: Never call result.get() within a task!
 - :gitlab-issue:`532` Workflow preview isn't updated right after transitions are modified
+- :gitlab-issue:`540` hint-outdated/update documentation
+- :gitlab-issue:`594` 3.2b1: Unable to install/run under Python 3.5/3.6/3.7
 - :gitlab-issue:`634` Failing docker entrypoint when using secret config
+- :gitlab-issue:`635` Build a docker image for Python3
+- :gitlab-issue:`644` Update sane-utils package in docker image.
+
 
 .. _PyPI: https://pypi.python.org/pypi/mayan-edms/

mayan/__init__.py

@@ -3,7 +3,7 @@ from __future__ import unicode_literals
 __title__ = 'Mayan EDMS'
 __version__ = '3.2.6'
 __build__ = 0x030206
-__build_string__ = 'v3.2.6_Wed Jul 10 03:18:15 2019 -0400'
+__build_string__ = 'v3.2.6-68-gab601f9180_Wed Jul 17 04:30:11 2019 -0400'
 __django_version__ = '1.11'
 __author__ = 'Roberto Rosario'
 __author_email__ = 'roberto.rosario@mayan-edms.com'

mayan/apps/acls/classes.py

@@ -12,6 +12,7 @@ logger = logging.getLogger(__name__)
 class ModelPermission(object):
     _functions = {}
     _inheritances = {}
+    _manager_names = {}
     _registry = {}
 
     @classmethod
@@ -97,6 +98,24 @@ class ModelPermission(object):
     def get_inheritance(cls, model):
         return cls._inheritances[model]
 
+    @classmethod
+    def get_manager(cls, model):
+        try:
+            manager_name = cls.get_manager_name(model=model)
+        except KeyError:
+            manager_name = None
+
+        if manager_name:
+            manager = getattr(model, manager_name)
+        else:
+            manager = model._meta.default_manager
+
+        return manager
+
+    @classmethod
+    def get_manager_name(cls, model):
+        return cls._manager_names[model]
+
     @classmethod
     def register_function(cls, model, function):
         cls._functions[model] = function
@@ -104,3 +123,7 @@ class ModelPermission(object):
     @classmethod
     def register_inheritance(cls, model, related):
         cls._inheritances[model] = related
+
+    @classmethod
+    def register_manager(cls, model, manager_name):
+        cls._manager_names[model] = manager_name

mayan/apps/acls/managers.py

@@ -200,14 +200,11 @@ class AccessControlListManager(models.Manager):
 
         return result
 
-    def check_access(self, obj, permissions, user, manager=None):
+    def check_access(self, obj, permissions, user):
         # Allow specific managers for models that have more than one
         # for example the Document model when checking for access for a trashed
         # document.
 
-        if manager:
-            source_queryset = manager.all()
-        else:
         meta = getattr(obj, '_meta', None)
 
         if not meta:
@@ -219,9 +216,10 @@ class AccessControlListManager(models.Manager):
             )
             return True
         else:
-                source_queryset = obj._meta.default_manager.all()
+            manager = ModelPermission.get_manager(model=obj._meta.model)
+            source_queryset = manager.all()
 
-        restricted_queryset = obj._meta.default_manager.none()
+        restricted_queryset = manager.none()
         for permission in permissions:
             # Default relationship betweens permissions is OR
             # TODO: Add support for AND relationship

mayan/apps/appearance/static/appearance/css/base.css

@@ -98,14 +98,10 @@ hr {
     min-height: 120px;
     padding-bottom: 1px;
     padding-top: 20px;
-    text-shadow: 1px 1px 1px rgba(0, 0, 0, 0.3);
+    text-shadow: 1px 1px 3px rgba(0, 0, 0, 1);
     white-space: normal;
 }
 
-.btn-block .fa {
-    text-shadow: 1px 1px 1px rgba(0, 0, 0, 0.3);
-}
-
 .radio ul li {
     list-style-type:none;
 }
@@ -115,14 +111,10 @@ a i {
 }
 
 .dashboard-widget {
-    box-shadow: 1px 1px 1px rgba(0, 0, 0, 0.3);
+    box-shadow: 1px 1px 3px rgba(0, 0, 0, 0.7);
     border: 1px solid black;
 }
 
-.dashboard-widget .panel-heading i {
-    text-shadow: 1px 1px 1px rgba(0, 0, 0, 0.3);
-}
-
 .dashboard-widget-icon {
     font-size: 200%;
 }
@@ -220,6 +212,18 @@ a i {
     font-weight: bold;
 }
 
+.panel-highlighted {
+    box-shadow: 0px 0px 3px #18bc9c, 10px 10px 20px #000000;
+}
+
+.panel-highlighted:hover {
+    box-shadow: 0px 0px 3px #18bc9c, 10px 10px 20px #000000, 0px 0px 8px #000000;
+}
+
+.panel-item:not(.panel-highlighted):hover  {
+    box-shadow: 0px 0px 8px #000000;
+}
+
 /* Content */
 @media (min-width:1200px) {
     .container-fluid {
@@ -249,14 +253,6 @@ a i {
     margin: auto;
 }
 
-.thin_border {
-    border: 1px solid black;
-    display: block;
-    margin-left: auto;
-    margin-right: auto;
-}
-
-
 .thin_border-thumbnail {
   display: block;
   max-width: 100%;
@@ -266,6 +262,14 @@ a i {
   margin: auto;
 }
 
+/* Must go after .thin_border-thumbnail */
+.thin_border {
+    border: 1px solid black;
+    display: inline;
+    margin-left: 0px;
+    margin-right: 0px;
+}
+
 #ajax-spinner {
     position: fixed;
     top: 16px;
@@ -536,5 +540,20 @@ a i {
 }
 
 .navbar-fixed-top {
-    box-shadow: 0px 3px 5px rgba(0, 0, 0, 0.5);
+    box-shadow: 0px 3px 3px rgba(0, 0, 0, 0.4);
+}
+
+.toolbar {
+    border: 1px solid rgba(0, 0, 0, 0.1);
+    box-shadow: 1px 1px 2px rgba(0, 0, 0, .3);
+    margin-bottom: 10px;
+    padding-bottom: 8px;
+    padding-left: 12px;
+    padding-right: 15px;
+    padding-top: 8px;
+}
+
+#body-plain {
+    padding-top: 0px;
+    margin-top: 10px;
 }

mayan/apps/appearance/static/appearance/js/base.js

@@ -6,7 +6,8 @@ var MayanAppClass = MayanApp;
 
 var partialNavigation = new PartialNavigation({
     initialURL: initialURL,
-    disabledAnchorClasses: ['disabled'],
+    disabledAnchorClasses: [
+        'btn-multi-item-action', 'disabled', 'pagination-disabled'
+    ],
     excludeAnchorClasses: ['fancybox', 'new_window', 'non-ajax'],
-    formBeforeSerializeCallbacks: [MayanApp.MultiObjectFormProcess],
 });

mayan/apps/appearance/static/appearance/js/mayan_app.js

@@ -17,28 +17,34 @@ class MayanApp {
 
     // Class methods and variables
 
-    static MultiObjectFormProcess ($form, options) {
-        /*
-         * ajaxForm callback to add the external item checkboxes to the
-         * submitted form
-         */
+    static countChecked() {
+        var checkCount = $('.check-all-slave:checked').length;
 
-        if ($form.hasClass('form-multi-object-action')) {
-            // Turn form data into an object
-            var formArray = $form.serializeArray().reduce(function (obj, item) {
-                obj[item.name] = item.value;
-                return obj;
-            }, {});
+        if (checkCount) {
+            $('#multi-item-title').hide();
+            $('#multi-item-actions').show();
+        } else {
+            $('#multi-item-title').show();
+            $('#multi-item-actions').hide();
+        }
+    }
 
-            // Add all checked checkboxes to the form data
-            $('.form-multi-object-action-checkbox:checked').each(function() {
-                var $this = $(this);
-                formArray[$this.attr('name')] = $this.attr('value');
+    static setupMultiItemActions () {
+        $('body').on('change', '.check-all-slave', function () {
+            MayanApp.countChecked();
         });
 
-            // Set the form data as the data to send
-            options.data = formArray;
-        }
+        $('body').on('click', '.btn-multi-item-action', function (event) {
+            var id_list = [];
+            $('.check-all-slave:checked').each(function (index, value) {
+                //Split the name (ie:"pk_200") and extract only the ID
+                id_list.push(value.name.split('_')[1]);
+            });
+            event.preventDefault();
+            partialNavigation.setLocation(
+                $(this).attr('href') + '?id_list=' + id_list.join(',')
+            );
+        });
     }
 
     static setupNavBarState () {
@@ -166,10 +172,10 @@ class MayanApp {
         var self = this;
 
         this.setupAJAXSpinner();
-        this.setupAutoSubmit();
         this.setupFormHotkeys();
         this.setupFullHeightResizing();
         this.setupItemsSelector();
+        MayanApp.setupMultiItemActions();
         this.setupNavbarCollapse();
         MayanApp.setupNavBarState();
         this.setupNewWindowAnchor();
@@ -177,6 +183,7 @@ class MayanApp {
             value.app = self;
             app.doRefreshAJAXMenu(value);
         });
+        this.setupPanelSelection();
         partialNavigation.initialize();
     }
 
@@ -200,14 +207,6 @@ class MayanApp {
         });
     }
 
-    setupAutoSubmit () {
-        $('body').on('change', '.select-auto-submit', function () {
-            if ($(this).val()) {
-                $(this.form).trigger('submit');
-            }
-        });
-    }
-
     setupFormHotkeys () {
         $('body').on('keypress', '.form-hotkey-enter', function (e) {
             if ((e.which && e.which == 13) || (e.keyCode && e.keyCode == 13)) {
@@ -238,9 +237,22 @@ class MayanApp {
         app.lastChecked = null;
 
         $('body').on('click', '.check-all', function (event) {
+            var $this = $(this);
             var checked = $(event.target).prop('checked');
             var $checkBoxes = $('.check-all-slave');
 
+            if (checked === undefined) {
+                checked = $this.data('checked');
+                checked = !checked;
+                $this.data('checked', checked);
+
+                if (checked) {
+                    $this.find('[data-fa-i2svg]').addClass($this.data('icon-checked')).removeClass($this.data('icon-unchecked'));
+                } else {
+                    $this.find('[data-fa-i2svg]').addClass($this.data('icon-unchecked')).removeClass($this.data('icon-checked'));
+                }
+            }
+
             $checkBoxes.prop('checked', checked);
             $checkBoxes.trigger('change');
         });
@@ -286,6 +298,58 @@ class MayanApp {
         });
     }
 
+   setupPanelSelection () {
+        var app = this;
+
+        // Setup panel highlighting on check
+        $('body').on('change', '.check-all-slave', function (event) {
+            var checked = $(event.target).prop('checked');
+            if (checked) {
+                $(this).closest('.panel-item').addClass('panel-highlighted');
+            } else {
+                $(this).closest('.panel-item').removeClass('panel-highlighted');
+            }
+        });
+
+        $('body').on('click', '.panel-item', function (event) {
+            var $this = $(this);
+            var targetSrc = $(event.target).prop('src');
+            var targetHref = $(event.target).prop('href');
+            var targetIsButton = event.target.tagName === 'BUTTON';
+            var lastChecked = null;
+
+            if ((targetSrc === undefined) && (targetHref === undefined) && (targetIsButton === false)) {
+                var $checkbox = $this.find('.check-all-slave');
+                var checked = $checkbox.prop('checked');
+
+                if (checked) {
+                    $checkbox.prop('checked', '');
+                    $checkbox.trigger('change');
+                } else {
+                    $checkbox.prop('checked', 'checked');
+                    $checkbox.trigger('change');
+                }
+
+                if(!app.lastChecked) {
+                    app.lastChecked = $checkbox;
+                }
+
+                if (event.shiftKey) {
+                    var $checkBoxes = $('.check-all-slave');
+
+                    var start = $checkBoxes.index($checkbox);
+                    var end = $checkBoxes.index(app.lastChecked);
+
+                    $checkBoxes.slice(
+                        Math.min(start, end), Math.max(start, end) + 1
+                    ).prop('checked', app.lastChecked.prop('checked')).trigger('change');
+                }
+                app.lastChecked = $checkbox;
+                window.getSelection().removeAllRanges();
+            }
+        });
+    }
+
     setupScrollView () {
         $('.scrollable').scrollview();
     }

mayan/apps/appearance/templates/appearance/base.html

@@ -136,6 +136,9 @@
             },
         {% endfor %}
     ];
+    $(function () {
+      $('[data-toggle="tooltip"]').tooltip();
+    })
 </script>
 {% block javascript %}{% endblock %}
 

mayan/apps/appearance/templates/appearance/base_plain.html

@@ -33,7 +33,7 @@
             }
         </script>
     </head>
-    <body>
+    <body id="body-plain">
         {% block content_plain %}{% endblock %}
 
         <script src="{% static 'appearance/node_modules/jquery/dist/jquery.min.js' %}" type="text/javascript"></script>

mayan/apps/appearance/templates/appearance/generic_list_items_subtemplate.html

@@ -11,41 +11,9 @@
                 {% include 'appearance/no_results.html' %}
             </div>
         {% else %}
-            <h4>
-                {% if page_obj %}
-                    {% if page_obj.paginator.num_pages != 1 %}
-                        {% blocktrans with page_obj.start_index as start and page_obj.end_index as end and page_obj.paginator.object_list|length as total and page_obj.number as page_number and page_obj.paginator.num_pages as total_pages %}Total ({{ start }} - {{ end }} out of {{ total }}) (Page {{ page_number }} of {{ total_pages }}){% endblocktrans %}
-                    {% else %}
-                        {% blocktrans with page_obj.paginator.object_list|length as total %}Total: {{ total }}{% endblocktrans %}
-                    {% endif %}
-                {% else %}
-                    {% blocktrans with object_list|length as total %}Total: {{ total }}{% endblocktrans %}
-                {% endif %}
-            </h4>
-            <hr>
+            {% include "appearance/list_header.html" %}
+            {% navigation_resolve_menu name='multi item' sort_results=True source=object_list.0 as links_multi_menus_results %}
             <div class="well center-block">
-                <div class="clearfix">
-                    <div class="pull-right">
-                        <form action="{% url 'common:multi_object_action_view' %}" class="form-multi-object-action" method="get">
-                            {% if object_list %}
-                                {% if not hide_multi_item_actions %}
-                                    {% get_multi_item_links_form object_list %}
-                                {% endif %}
-                                {% if multi_item_actions %}
-                                    <fieldset style="margin-top: -10px;">
-                                        <input class="check-all" type="checkbox"/>&nbsp;
-                                        {{ multi_item_form }}
-                                    </fieldset>
-                                {% endif %}
-                            {% endif %}
-                        </form>
-                    </div>
-                </div>
-
-                {% if object_list %}
-                    <hr style="border-bottom: 1px solid lightgrey;">
-                {% endif %}
-
                 <div class="row row-items">
                     {% for object in object_list %}
                         <div class="{{ column_class|default:'col-xs-12 col-sm-4 col-md-3 col-lg-2' }}">
@@ -53,9 +21,9 @@
                                 <div class="panel-heading">
                                     <div class="form-group">
                                         <div class="checkbox">
-                                            <label for="id_indexes_0">
-                                                {% if multi_item_actions %}
-                                                    <input class="form-multi-object-action-checkbox check-all-slave checkbox" type="checkbox" name="pk_{{ object.pk }}" />
+                                            <label for="id_indexes_0" style="cursor: auto;">
+                                                {% if links_multi_menus_results %}
+                                                    <input class="form-multi-object-action-checkbox check-all-slave checkbox" name="pk_{{ object.pk }}" style="cursor: pointer;" type="checkbox" />
                                                 {% endif %}
 
                                                 <span style="color: white; word-break: break-all; overflow-wrap: break-word;">
@@ -68,13 +36,8 @@
                                                     {% else %}
                                                         {% navigation_get_source_columns source=object only_identifier=True as source_column %}
                                                         {% navigation_source_column_resolve column=source_column as column_value %}
-
-                                                        {% if source_column.is_attribute_absolute_url or source_column.is_object_absolute_url %}
-                                                            <a href="{% navigation_source_column_get_absolute_url source_column=source_column obj=object %}">{{ column_value }}</a>
-                                                        {% else %}
                                                         {{ column_value }}
                                                     {% endif %}
-                                                    {% endif %}
                                                 </span>
                                             </label>
                                         </div>
@@ -82,7 +45,6 @@
 
                                 </div>
                                 <div class="panel-body">
-
                                     {% if not hide_columns %}
                                         {% navigation_get_source_columns source=object exclude_identifier=True as source_columns %}
                                         {% for column in source_columns %}
@@ -136,7 +98,6 @@
                         </div>
                     {% endfor %}
                 </div>
-                {% include 'pagination/pagination.html' %}
             </div>
         {% endif %}
     </div>

mayan/apps/appearance/templates/appearance/generic_list_subtemplate.html

@@ -1,6 +1,7 @@
 {% load i18n %}
 {% load static %}
 
+{% load appearance_tags %}
 {% load common_tags %}
 {% load navigation_tags %}
 
@@ -11,44 +12,16 @@
                 {% include 'appearance/no_results.html' %}
             </div>
         {% else %}
-            <h4>
-                {% if page_obj %}
-                    {% if page_obj.paginator.num_pages != 1 %}
-                        {% blocktrans with page_obj.start_index as start and page_obj.end_index as end and page_obj.paginator.object_list|length as total and page_obj.number as page_number and page_obj.paginator.num_pages as total_pages %}Total ({{ start }} - {{ end }} out of {{ total }}) (Page {{ page_number }} of {{ total_pages }}){% endblocktrans %}
-                    {% else %}
-                        {% blocktrans with page_obj.paginator.object_list|length as total %}Total: {{ total }}{% endblocktrans %}
-                    {% endif %}
-                {% else %}
-                    {% blocktrans with object_list|length as total %}Total: {{ total }}{% endblocktrans %}
-                {% endif %}
-            </h4>
-            <hr>
-
+            {% include "appearance/list_header.html" %}
+            {% navigation_resolve_menu name='multi item' sort_results=True source=object_list.0 as links_multi_menus_results %}
             <div class="well center-block">
-                <div class="clearfix">
-                    <div class="pull-right">
-                        <form action="{% url 'common:multi_object_action_view' %}" class="form-multi-object-action" method="get">
-                            {% if object_list %}
-                                {% if not hide_multi_item_actions %}
-                                    {% get_multi_item_links_form object_list %}
-                                {% endif %}
-                                {% if multi_item_actions %}
-                                    <fieldset style="margin-top: -10px; margin-bottom: 10px;">
-                                        {{ multi_item_form }}
-                                    </fieldset>
-                                {% endif %}
-                            {% endif %}
-                        </form>
-                    </div>
-                </div>
-
                 <div class="table-responsive">
                     <table class="table table-condensed table-striped">
                         <tbody>
                             {% if not hide_header %}
                                 <tr>
-                                    {% if multi_item_actions %}
-                                        <th class="first"><input class="checkbox check-all" type="checkbox" /></th>
+                                    {% if links_multi_menus_results %}
+                                        <th class="first"></th>
                                     {% endif %}
 
                                     {% if not hide_object %}
@@ -58,30 +31,40 @@
                                         {% if source_column %}
                                             <th>
                                                 {% if source_column.is_sortable %}
-                                                    <a href="{% navigation_get_sort_field_querystring column=source_column %}">{{ source_column.label }}
+                                                    <a href="{% navigation_get_sort_field_querystring column=source_column %}">{{ source_column.label }}</a>
                                                     {% if source_column.get_sort_field == sort_field %}
                                                         {% if icon_sort %}{{ icon_sort.render }}{% endif %}
                                                     {% endif %}
-                                                    </a>
                                                 {% else %}
                                                     {{ source_column.label }}
                                                 {% endif %}
+
+                                                {% if source_column.help_text %}
+                                                    <span data-toggle="tooltip" data-placement="bottom" title="{{ source_column.help_text }}">
+                                                    {% get_icon icon_path='mayan.apps.navigation.icons.icon_source_column_help_text' %}
+                                                    </span>
+                                                {% endif %}
                                             </th>
                                         {% endif %}
                                     {% endif %}
 
                                     {% if not hide_columns %}
                                         {% navigation_get_source_columns source=object_list exclude_identifier=True as source_columns %}
-                                        {% for column in source_columns %}
+                                        {% for source_column in source_columns %}
                                             <th>
-                                                {% if column.is_sortable %}
-                                                    <a href="{% navigation_get_sort_field_querystring column=column %}">{{ column.label }}
-                                                    {% if column.get_sort_field == sort_field %}
+                                                {% if source_column.is_sortable %}
+                                                    <a href="{% navigation_get_sort_field_querystring column=source_column %}">{{ source_column.label }}</a>
+                                                    {% if source_column.get_sort_field == sort_field %}
                                                         {% if icon_sort %}{{ icon_sort.render }}{% endif %}
                                                     {% endif %}
-                                                    </a>
                                                 {% else %}
-                                                    {{ column.label }}
+                                                    {{ source_column.label }}
+                                                {% endif %}
+
+                                                {% if source_column.help_text %}
+                                                    <span data-toggle="tooltip" data-placement="bottom" title="{{ source_column.help_text }}">
+                                                    {% get_icon icon_path='mayan.apps.navigation.icons.icon_source_column_help_text' %}
+                                                    </span>
                                                 {% endif %}
                                             </th>
                                         {% endfor %}
@@ -99,9 +82,9 @@
                             {% for object in object_list %}
                                 <tr>
 
-                                    {% if multi_item_actions %}
+                                    {% if links_multi_menus_results %}
                                         <td>
-                                            <input type="checkbox" class="form-multi-object-action-checkbox check-all-slave checkbox" name="pk_{{ object.pk }}" value="" />
+                                            <input class="form-multi-object-action-checkbox check-all-slave checkbox" name="pk_{{ object.pk }}" type="checkbox" value="" />
                                         </td>
                                     {% endif %}
 
@@ -112,11 +95,7 @@
                                         {% navigation_source_column_resolve column=source_column as column_value %}
                                         {% if column_value %}
                                             <td>
-                                                {% if source_column.is_attribute_absolute_url or source_column.is_object_absolute_url %}
-                                                    <a href="{% navigation_source_column_get_absolute_url source_column=source_column obj=object %}">{{ column_value }}</a>
-                                                {% else %}
                                                 {{ column_value }}
-                                                {% endif %}
                                             </td>
                                         {% endif %}
                                     {% endif %}
@@ -170,7 +149,6 @@
                         </tbody>
                     </table>
                 </div>
-                {% include 'pagination/pagination.html' %}
             </div>
         {% endif %}
     </div>

mayan/apps/appearance/templates/appearance/list_header.html

@@ -0,0 +1,28 @@
+{% load i18n %}
+{% load static %}
+
+{% load common_tags %}
+{% load navigation_tags %}
+
+{% if object_list %}
+    <h4>
+        {% if page_obj %}
+            {% if page_obj.paginator.num_pages != 1 %}
+                {% blocktrans with page_obj.start_index as start and page_obj.end_index as end and page_obj.paginator.object_list|length as total and page_obj.number as page_number and page_obj.paginator.num_pages as total_pages %}Total ({{ start }} - {{ end }} out of {{ total }}) (Page {{ page_number }} of {{ total_pages }}){% endblocktrans %}
+            {% else %}
+                {% blocktrans with page_obj.paginator.object_list|length as total %}Total: {{ total }}{% endblocktrans %}
+            {% endif %}
+        {% else %}
+            {% blocktrans with object_list|length as total %}Total: {{ total }}{% endblocktrans %}
+        {% endif %}
+    </h4>
+    <hr>
+
+    {% if not hide_multi_item_actions %}
+        {% navigation_resolve_menu name='multi item' sort_results=True source=object_list.0 as links_multi_menus_results %}
+    {% endif %}
+{% endif %}
+
+<div class="clearfix">
+    {% include 'appearance/list_toolbar.html' %}
+</div>

mayan/apps/appearance/templates/appearance/list_toolbar.html

@@ -0,0 +1,90 @@
+{% load i18n %}
+
+{% load common_tags %}
+{% load navigation_tags %}
+
+{% if is_paginated or links_multi_menus_results %}
+    <div class="well center-block toolbar">
+{% endif %}
+
+    {% if links_multi_menus_results %}
+        <div class="pull-left">
+            <div class="btn-toolbar" role="toolbar" style="margin-right: 10px;">
+                <div class="btn-group">
+                        <a class="btn btn-default btn-sm check-all" data-checked=false data-icon-checked="fa fa-check-square" data-icon-unchecked="far fa-square" title="{% trans 'Select/Deselect all' %}">
+                            <i class="far fa-square"></i>
+                        </a>
+                </div>
+            </div>
+        </div>
+    {% endif %}
+
+    {% if is_paginated %}
+        <div class="pull-left">
+            <div class="btn-toolbar" role="toolbar">
+                <div class="btn-group">
+                    {% if page_obj.has_previous %}
+                        <a class="btn btn-default btn-sm" href="?{{ page_obj.previous_page_number.querystring }}">&lsaquo;&lsaquo;</a>
+                    {% else %}
+                        <a class="btn btn-default btn-sm disabled" href="#">&lsaquo;&lsaquo;</a>
+                    {% endif %}
+
+                    {% for page in page_obj.pages %}
+
+                        {% if page %}
+
+                            {% ifequal page page_obj.number %}
+                                <a class="active btn btn-default btn-sm pagination-disabled" href="#">{{ page }}</a>
+                            {% else %}
+                                <a class="btn btn-default btn-sm" href="?{{ page.querystring }}">{{ page }}</a>
+                            {% endifequal %}
+                        {% else %}
+                            <a class="btn btn-default btn-sm disabled" href="#">...</a>
+                        {% endif %}
+
+                    {% endfor %}
+                    {% if page_obj.has_next %}
+                        <a class="btn btn-default btn-sm" href="?{{ page_obj.next_page_number.querystring }}">&rsaquo;&rsaquo;</a>
+                    {% else %}
+                        <a class="btn btn-default btn-sm disabled" href="#">&rsaquo;&rsaquo;</a>
+                    {% endif %}
+                </div>
+            </div>
+        </div>
+    {% endif %}
+
+    {% if links_multi_menus_results %}
+        <p class="pull-right" id="multi-item-title" style="line-height: 16px; padding-top: 8px;">{% trans 'Select items to activate bulk actions. Use Shift + click to select many.' %}</p>
+
+        <div class="pull-right btn-group" id="multi-item-actions" style="display: none;">
+            <button aria-expanded="true" class="btn btn-danger btn-sm dropdown-toggle" data-toggle="dropdown" type="button">
+                {% trans 'Bulk actions' %}
+                <span class="caret"></span>
+                <span class="sr-only">{% trans 'Toggle Dropdown' %}</span>
+            </button>
+            <ul class="dropdown-menu" role="menu">
+                {% for multi_item_menu_results in links_multi_menus_results %}
+                    {% for link_group in multi_item_menu_results.link_groups %}
+                        {% with link_group.links as object_navigation_links %}
+                        {% with 'true' as as_li %}
+                        {% with 'true' as hide_active_anchor %}
+                        {% with 'btn-sm btn-multi-item-action' as link_classes %}
+                            {% include 'navigation/generic_navigation.html' %}
+                        {% endwith %}
+                        {% endwith %}
+                        {% endwith %}
+                        {% endwith %}
+
+                    {% endfor %}
+                    {% if not forloop.last and link_group %}
+                        <li class="divider"></li>
+                    {% endif %}
+                {% endfor %}
+            </ul>
+        </div>
+    {% endif %}
+
+{% if is_paginated or links_multi_menus_results %}
+    <div class="clearfix"></div>
+    </div>
+{% endif %}

mayan/apps/appearance/templates/pagination/pagination.html

@@ -11,7 +11,7 @@
             {% if page %}
 
                 {% ifequal page page_obj.number %}
-                    <li class="active"><a href="#">{{ page }}</a></li>
+                    <li class="active"><a class="pagination-disabled" href="#">{{ page }}</a></li>
                 {% else %}
                     <li><a href="?{{ page.querystring }}">{{ page }}</a></li>
                 {% endifequal %}

mayan/apps/authentication/templates/authentication/login.html

@@ -17,7 +17,7 @@
     {% motd %}
 
     <div class="row">
-        <div class="col-xs-10 col-xs-offset-1 col-sm-8 col-sm-offset-2 col-md-6 col-md-offset-3 col-lg-4 col-lg-offset-4">
+        <div class="col-xs-12 col-sm-10 col-sm-offset-1 col-md-10 col-md-offset-1 col-lg-6 col-lg-offset-3">
             <div class="panel panel-primary">
                 <div class="panel-heading">
                     <h3 class="panel-title">&nbsp;</h3>

mayan/apps/authentication/tests/test_views.py

@@ -14,7 +14,6 @@ from django.utils.http import urlunquote_plus
 from mayan.apps.common.tests import GenericViewTestCase
 from mayan.apps.smart_settings.classes import Namespace
 from mayan.apps.user_management.permissions import permission_user_edit
-from mayan.apps.user_management.tests.mixins import UserTestMixin
 from mayan.apps.user_management.tests.literals import TEST_USER_PASSWORD_EDITED
 
 from ..settings import setting_maximum_session_length
@@ -262,7 +261,7 @@ class UserLoginTestCase(GenericViewTestCase):
         self.assertEqual(response.redirect_chain, [(TEST_REDIRECT_URL, 302)])
 
 
-class UserViewTestCase(UserTestMixin, UserPasswordViewTestMixin, GenericViewTestCase):
+class UserViewTestCase(UserPasswordViewTestMixin, GenericViewTestCase):
     def test_user_set_password_view_no_access(self):
         self._create_test_user()
 

mayan/apps/autoadmin/templates/autoadmin/credentials.html

@@ -4,7 +4,7 @@
 
 {% if autoadmin_properties.account %}
     <div class="row">
-        <div class="col-xs-10 col-xs-offset-1 col-sm-8 col-sm-offset-2 col-md-6 col-md-offset-3 col-lg-4 col-lg-offset-4">
+        <div class="col-xs-12 col-sm-10 col-sm-offset-1 col-md-10 col-md-offset-1 col-lg-6 col-lg-offset-3">
             <br>
             <div class="panel panel-primary">
                 <div class="panel-heading">

mayan/apps/cabinets/migrations/0002_auto_20190729_0236.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.22 on 2019-07-29 02:36
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('cabinets', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='cabinet',
+            name='label',
+            field=models.CharField(help_text='A short text used to identify the cabinet.', max_length=128, verbose_name='Label'),
+        ),
+    ]

mayan/apps/cabinets/models.py

@@ -32,7 +32,10 @@ class Cabinet(MPTTModel):
         blank=True, db_index=True, null=True, on_delete=models.CASCADE,
         related_name='children', to='self'
     )
-    label = models.CharField(max_length=128, verbose_name=_('Label'))
+    label = models.CharField(
+        help_text=_('A short text used to identify the cabinet.'),
+        max_length=128, verbose_name=_('Label')
+    )
     documents = models.ManyToManyField(
         blank=True, related_name='cabinets', to=Document,
         verbose_name=_('Documents')

mayan/apps/cabinets/urls.py

@@ -12,55 +12,62 @@ from .views import (
     CabinetDeleteView, CabinetDetailView, CabinetEditView, CabinetListView,
 )
 
-urlpatterns = [
+urlpatterns_cabinets = [
     url(
-        regex=r'^list/$', view=CabinetListView.as_view(), name='cabinet_list'
+        regex=r'^cabinets/$', view=CabinetListView.as_view(), name='cabinet_list'
     ),
     url(
-        regex=r'^(?P<pk>\d+)/child/add/$', view=CabinetChildAddView.as_view(),
-        name='cabinet_child_add'
-    ),
-    url(
-        regex=r'^create/$', view=CabinetCreateView.as_view(),
+        regex=r'^cabinets/create/$', view=CabinetCreateView.as_view(),
         name='cabinet_create'
     ),
     url(
-        regex=r'^(?P<pk>\d+)/edit/$', view=CabinetEditView.as_view(),
-        name='cabinet_edit'
+        regex=r'^cabinets/(?P<pk>\d+)/children/add/$', view=CabinetChildAddView.as_view(),
+        name='cabinet_child_add'
     ),
     url(
-        regex=r'^(?P<pk>\d+)/delete/$', view=CabinetDeleteView.as_view(),
+        regex=r'^cabinets/(?P<pk>\d+)/delete/$', view=CabinetDeleteView.as_view(),
         name='cabinet_delete'
     ),
     url(
-        regex=r'^(?P<pk>\d+)/$', view=CabinetDetailView.as_view(),
-        name='cabinet_view'
+        regex=r'^cabinets/(?P<pk>\d+)/edit/$', view=CabinetEditView.as_view(),
+        name='cabinet_edit'
     ),
     url(
-        regex=r'^document/(?P<pk>\d+)/cabinet/add/$',
+        regex=r'^cabinets/(?P<pk>\d+)/$', view=CabinetDetailView.as_view(),
+        name='cabinet_view'
+    ),
+]
+
+urlpatterns_documents_cabinets = [
+    url(
+        regex=r'^documents/(?P<pk>\d+)/cabinets/add/$',
         view=DocumentAddToCabinetView.as_view(), name='document_cabinet_add'
     ),
     url(
-        regex=r'^document/multiple/cabinet/add/$',
+        regex=r'^documents/multiple/cabinets/add/$',
         view=DocumentAddToCabinetView.as_view(),
         name='document_multiple_cabinet_add'
     ),
     url(
-        regex=r'^document/(?P<pk>\d+)/cabinet/remove/$',
+        regex=r'^documents/(?P<pk>\d+)/cabinets/remove/$',
         view=DocumentRemoveFromCabinetView.as_view(),
         name='document_cabinet_remove'
     ),
     url(
-        regex=r'^document/multiple/cabinet/remove/$',
+        regex=r'^documents/multiple/cabinets/remove/$',
         view=DocumentRemoveFromCabinetView.as_view(),
         name='multiple_document_cabinet_remove'
     ),
     url(
-        regex=r'^document/(?P<pk>\d+)/cabinet/list/$',
+        regex=r'^documents/(?P<pk>\d+)/cabinets/$',
         view=DocumentCabinetListView.as_view(), name='document_cabinet_list'
     ),
 ]
 
+urlpatterns = []
+urlpatterns.extend(urlpatterns_cabinets)
+urlpatterns.extend(urlpatterns_documents_cabinets)
+
 api_urls = [
     url(
         regex=r'^cabinets/(?P<pk>[0-9]+)/documents/(?P<document_pk>[0-9]+)/$',

mayan/apps/checkouts/migrations/0008_checkedoutdocument.py

@@ -0,0 +1,26 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.20 on 2019-07-25 04:52
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('documents', '0050_auto_20190725_0451'),
+        ('checkouts', '0007_auto_20180310_1715'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='CheckedOutDocument',
+            fields=[
+            ],
+            options={
+                'proxy': True,
+                'indexes': [],
+            },
+            bases=('documents.document',),
+        ),
+    ]

mayan/apps/checkouts/tests/test_views.py

@@ -315,43 +315,6 @@ class DocumentCheckoutViewTestCase(
             response=response, text=self.test_document.label, status_code=200
         )
 
-    def test_document_check_out_new_version(self):
-        """
-        Gitlab issue #231
-        User shown option to upload new version of a document even though it
-        is blocked by checkout - v2.0.0b2
-
-        Expected results:
-            - Link to upload version view should not resolve
-            - Upload version view should reject request
-        """
-        self._create_test_case_superuser()
-        self._check_out_test_document()
-        self.login_superuser()
-
-        response = self.post(
-            viewname='sources:upload_version', kwargs={
-                'document_pk': self.test_document.pk
-            }, follow=True
-        )
-
-        self.assertContains(
-            response=response, text='blocked from uploading',
-            status_code=200
-        )
-
-        response = self.get(
-            viewname='documents:document_version_list', kwargs={
-                'pk': self.test_document.pk
-            }, follow=True
-        )
-
-        # Needed by the url view resolver
-        response.context.current_app = None
-        resolved_link = link_document_version_upload.resolve(context=response.context)
-
-        self.assertEqual(resolved_link, None)
-
     def test_document_check_in_forcefull_view_no_permission(self):
         # Gitlab issue #237
         # Forcefully checking in a document by a user without adequate
@@ -388,3 +351,47 @@ class DocumentCheckoutViewTestCase(
         )
         self.assertEqual(response.status_code, 302)
         self.assertFalse(self.test_document.is_checked_out())
+
+
+class NewVersionBlockViewTestCase(
+    DocumentCheckoutTestMixin, DocumentCheckoutViewTestMixin,
+    GenericDocumentViewTestCase):
+
+    def test_document_check_out_new_version(self):
+        """
+        Gitlab issue #231
+        User shown option to upload new version of a document even though it
+        is blocked by checkout - v2.0.0b2
+
+        Expected results:
+            - Link to upload version view should not resolve
+            - Upload version view should reject request
+        """
+        self._create_test_case_superuser()
+        self._check_out_test_document()
+        self.login_superuser()
+
+        response = self.post(
+            viewname='sources:upload_version', kwargs={
+                'document_pk': self.test_document.pk
+            }, follow=True
+        )
+
+        self.assertContains(
+            response=response, text='blocked from uploading',
+            status_code=200
+        )
+
+        response = self.get(
+            viewname='documents:document_version_list', kwargs={
+                'pk': self.test_document.pk
+            }, follow=True
+        )
+
+        # Needed by the url view resolver
+        response.context.current_app = None
+        resolved_link = link_document_version_upload.resolve(
+            context=response.context
+        )
+
+        self.assertEqual(resolved_link, None)

mayan/apps/common/dependencies.py

@@ -61,102 +61,9 @@ PythonDependency(
         SOFTWARE.
     ''', module=__name__, name='PyYAML', version_string='==5.1.1'
 )
-PythonDependency(
-    copyright_text='''
-        Copyright (c) 2015 Ask Solem & contributors.  All rights reserved.
-        Copyright (c) 2012-2014 GoPivotal, Inc.  All rights reserved.
-        Copyright (c) 2009, 2010, 2011, 2012 Ask Solem, and individual contributors.  All rights reserved.
-
-        Celery is licensed under The BSD License (3 Clause, also known as
-        the new BSD license).  The license is an OSI approved Open Source
-        license and is GPL-compatible(1).
-
-        The license text can also be found here:
-        http://www.opensource.org/licenses/BSD-3-Clause
-
-        License
-        =======
-
-        Redistribution and use in source and binary forms, with or without
-        modification, are permitted provided that the following conditions are met:
-        * Redistributions of source code must retain the above copyright
-        notice, this list of conditions and the following disclaimer.
-        * Redistributions in binary form must reproduce the above copyright
-        notice, this list of conditions and the following disclaimer in the
-        documentation and/or other materials provided with the distribution.
-        * Neither the name of Ask Solem, nor the
-        names of its contributors may be used to endorse or promote products
-        derived from this software without specific prior written permission.
-
-        THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-        AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
-        THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-        PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL Ask Solem OR CONTRIBUTORS
-        BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-        CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-        SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-        INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-        CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-        ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-        POSSIBILITY OF SUCH DAMAGE.
-
-        Documentation License
-        =====================
-
-        The documentation portion of Celery (the rendered contents of the
-        "docs" directory of a software distribution or checkout) is supplied
-        under the Creative Commons Attribution-Noncommercial-Share Alike 3.0
-        United States License as described by
-        http://creativecommons.org/licenses/by-nc-sa/3.0/us/
-
-        Footnotes
-        =========
-        (1) A GPL-compatible license makes it possible to
-        combine Celery with other software that is released
-        under the GPL, it does not mean that we're distributing
-        Celery under the GPL license.  The BSD license, unlike the GPL,
-        let you distribute a modified version without making your
-        changes open source.
-    ''', module=__name__, name='celery', version_string='==3.1.24'
-)
-PythonDependency(
-    copyright_text='''
-        Copyright (c) 2012-2013 GoPivotal, Inc.  All Rights Reserved.
-        Copyright (c) 2009-2012 Ask Solem.  All Rights Reserved.
-        All rights reserved.
-
-        Redistribution and use in source and binary forms, with or without
-        modification, are permitted provided that the following conditions are met:
-
-        * Redistributions of source code must retain the above copyright notice,
-        this list of conditions and the following disclaimer.
-        * Redistributions in binary form must reproduce the above copyright
-        notice, this list of conditions and the following disclaimer in the
-        documentation and/or other materials provided with the distribution.
-
-        Neither the name of Ask Solem nor the names of its contributors may be used
-        to endorse or promote products derived from this software without specific
-        prior written permission.
-
-        THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-        AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
-        THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-        PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
-        BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-        CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-        SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-        INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-        CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-        ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-        POSSIBILITY OF SUCH DAMAGE.
-    ''', module=__name__, name='django-celery', version_string='==3.2.1'
-)
 PythonDependency(
     module=__name__, name='django-downloadview', version_string='==1.9'
 )
-PythonDependency(
-    module=__name__, name='django-environ', version_string='==0.4.5'
-)
 PythonDependency(
     module=__name__, name='django-formtools', version_string='==2.1'
 )
@@ -383,6 +290,10 @@ PythonDependency(
     module=__name__, environment=environment_development, name='Werkzeug',
     version_string='==0.15.4'
 )
+PythonDependency(
+    module=__name__, environment=environment_development, name='devpi-server',
+    version_string='==5.0.0'
+)
 PythonDependency(
     environment=environment_development, module=__name__,
     name='django-debug-toolbar', version_string='==1.11'

mayan/apps/common/http.py

@@ -2,6 +2,7 @@ from __future__ import unicode_literals
 
 from django.http import QueryDict
 from django.utils.encoding import force_bytes
+from django.utils.six import PY3
 
 
 class URL(object):
@@ -20,9 +21,7 @@ class URL(object):
 
     def to_string(self):
         if self._args.keys():
-            query = force_bytes(
-                '?{}'.format(self._args.urlencode())
-            )
+            query = '?{}'.format(self._args.urlencode())
         else:
             query = ''
 
@@ -31,6 +30,9 @@ class URL(object):
         else:
             path = ''
 
-        result = force_bytes('{}{}'.format(path, query))
+        result = '{}{}'.format(path, query)
 
+        if PY3:
             return result
+        else:
+            return force_bytes(result)

mayan/apps/common/management/commands/initialsetup.py

@@ -28,8 +28,8 @@ class Command(management.BaseCommand):
         )
 
         parser.add_argument(
-            '--no-javascript', action='store_true', dest='no_javascript',
-            help='Don\'t download the JavaScript dependencies.',
+            '--no-dependencies', action='store_true', dest='no_dependencies',
+            help='Don\'t download dependencies.',
         )
 
     def initialize_system(self, force=False):
@@ -88,9 +88,9 @@ class Command(management.BaseCommand):
         self.initialize_system(force=options.get('force', False))
         pre_initial_setup.send(sender=self)
 
-        if not options.get('no_javascript', False):
+        if not options.get('no_dependencies', False):
             management.call_command(
-                command_name='installjavascript', interactive=False
+                command_name='installdependencies', interactive=False
             )
 
         management.call_command(

mayan/apps/common/management/commands/literals.py

@@ -1,10 +0,0 @@
-from __future__ import unicode_literals
-
-SETTING_FILE_TEMPLATE = '''
-from __future__ import absolute_import, unicode_literals
-
-from .base import *
-
-SECRET_KEY = '{0}'
-
-'''

mayan/apps/common/management/commands/performupgrade.py

@@ -11,8 +11,8 @@ class Command(management.BaseCommand):
 
     def add_arguments(self, parser):
         parser.add_argument(
-            '--no-javascript', action='store_true', dest='no_javascript',
-            help='Don\'t download the JavaScript dependencies.',
+            '--no-dependencies', action='store_true', dest='no_dependencies',
+            help='Don\'t download dependencies.',
         )
 
     def handle(self, *args, **options):
@@ -25,9 +25,9 @@ class Command(management.BaseCommand):
                 )
             )
 
-        if not options.get('no_javascript', False):
+        if not options.get('no_dependencies', False):
             management.call_command(
-                command_name='installjavascript', interactive=False
+                command_name='installdependencies', interactive=False
             )
 
         try:

mayan/apps/common/management/commands/purgeperiodictasks.py

@@ -2,7 +2,7 @@ from __future__ import unicode_literals
 
 from django.core import management
 
-from djcelery.models import IntervalSchedule, PeriodicTask
+from django_celery_beat.models import IntervalSchedule, PeriodicTask
 
 
 class Command(management.BaseCommand):

mayan/apps/common/mixins.py

@@ -1,6 +1,7 @@
 from __future__ import unicode_literals
 
 from django.contrib import messages
+from django.contrib.contenttypes.models import ContentType
 from django.core.exceptions import ImproperlyConfigured, PermissionDenied
 from django.http import Http404, HttpResponseRedirect
 from django.shortcuts import get_object_or_404
@@ -8,6 +9,7 @@ from django.urls import reverse
 from django.utils.translation import ungettext, ugettext_lazy as _
 from django.views.generic.detail import SingleObjectMixin
 
+from mayan.apps.acls.classes import ModelPermission
 from mayan.apps.acls.models import AccessControlList
 from mayan.apps.permissions import Permission
 
@@ -17,6 +19,28 @@ from .literals import PK_LIST_SEPARATOR
 from .settings import setting_home_view
 
 
+class ContentTypeViewMixin(object):
+    """
+    This mixin makes it easier for views to retrieve a content type from
+    the URL pattern.
+    """
+    content_type_url_kw_args = {
+        'app_label': 'app_label',
+        'model_name': 'model'
+    }
+
+    def get_content_type(self):
+        return get_object_or_404(
+            klass=ContentType,
+            app_label=self.kwargs[
+                self.content_type_url_kw_args['app_label']
+            ],
+            model=self.kwargs[
+                self.content_type_url_kw_args['model_name']
+            ]
+        )
+
+
 class DeleteExtraDataMixin(object):
     """
     Mixin to populate the extra data needed for delete views
@@ -103,7 +127,15 @@ class ExternalObjectMixin(object):
                 'get_external_object_queryset() method.'
             )
 
-        return self.external_object_queryset or self.external_object_class.objects.all()
+        queryset = self.external_object_queryset
+
+        if not queryset:
+            manager = ModelPermission.get_manager(
+                model=self.external_object_class
+            )
+            queryset = manager.all()
+
+        return queryset
 
     def get_external_object_queryset_filtered(self):
         queryset = self.get_external_object_queryset()
@@ -118,6 +150,20 @@ class ExternalObjectMixin(object):
         return queryset
 
 
+class ExternalContentTypeObjectMixin(ContentTypeViewMixin, ExternalObjectMixin):
+    """
+    Mixin to retrieve an external object by content type from the URL pattern.
+    """
+    external_object_pk_url_kwarg = 'object_id'
+
+    def get_external_object_queryset(self):
+        content_type = self.get_content_type()
+        self.external_object_class = content_type.model_class()
+        return super(
+            ExternalContentTypeObjectMixin, self
+        ).get_external_object_queryset()
+
+
 class FormExtraKwargsMixin(object):
     """
     Mixin that allows a view to pass extra keyword arguments to forms
@@ -250,9 +296,9 @@ class ObjectActionMixin(object):
 
     def get_success_message(self, count):
         return ungettext(
-            self.success_message,
-            self.success_message_plural,
-            count
+            singular=self.success_message,
+            plural=self.success_message_plural,
+            number=count
         ) % {
             'count': count,
         }
@@ -271,14 +317,15 @@ class ObjectActionMixin(object):
                 pass
             except ActionError:
                 messages.error(
-                    self.request, self.error_message % {'instance': instance}
+                    message=self.error_message % {'instance': instance},
+                    request=self.request
                 )
             else:
                 self.action_count += 1
 
         messages.success(
-            self.request,
-            self.get_success_message(count=self.action_count)
+            message=self.get_success_message(count=self.action_count),
+            request=self.request
         )
 
 

mayan/apps/common/tests/test_views.py

@@ -4,7 +4,6 @@ from django.contrib.auth import get_user_model
 from django.contrib.contenttypes.models import ContentType
 
 from mayan.apps.acls.classes import ModelPermission
-from mayan.apps.user_management.tests.mixins import UserTestMixin
 
 from ..models import ErrorLogEntry
 from ..permissions_runtime import permission_error_log_view
@@ -13,7 +12,7 @@ from .base import GenericViewTestCase
 from .literals import TEST_ERROR_LOG_ENTRY_RESULT
 
 
-class CommonViewTestCase(UserTestMixin, GenericViewTestCase):
+class CommonViewTestCase(GenericViewTestCase):
     def _request_about_view(self):
         return self.get(viewname='common:about_view')
 

mayan/apps/common/urls.py

@@ -1,7 +1,7 @@
 from __future__ import unicode_literals
 
 from django.conf.urls import url
-from django.views.i18n import JavaScriptCatalog, set_language
+from django.views.i18n import JavaScriptCatalog
 
 from .api_views import (
     APIContentTypeList, APITemplateDetailView, APITemplateListView
@@ -10,30 +10,10 @@ from .views import (
     AboutView, CurrentUserLocaleProfileDetailsView,
     CurrentUserLocaleProfileEditView, FaviconRedirectView, HomeView,
     LicenseView, ObjectErrorLogEntryListClearView, ObjectErrorLogEntryListView,
-    RootView, SetupListView, ToolsListView, multi_object_action_view
+    RootView, SetupListView, ToolsListView
 )
 
-urlpatterns = [
-    url(regex=r'^$', view=RootView.as_view(), name='root'),
-    url(regex=r'^home/$', view=HomeView.as_view(), name='home'),
-    url(regex=r'^about/$', view=AboutView.as_view(), name='about_view'),
-    url(regex=r'^license/$', view=LicenseView.as_view(), name='license_view'),
-    url(
-        regex=r'^object/multiple/action/$', view=multi_object_action_view,
-        name='multi_object_action_view'
-    ),
-    url(regex=r'^setup/$', view=SetupListView.as_view(), name='setup_list'),
-    url(regex=r'^tools/$', view=ToolsListView.as_view(), name='tools_list'),
-    url(
-        regex=r'^user/locale/$',
-        view=CurrentUserLocaleProfileDetailsView.as_view(),
-        name='current_user_locale_profile_details'
-    ),
-    url(
-        regex=r'^user/locale/edit/$',
-        view=CurrentUserLocaleProfileEditView.as_view(),
-        name='current_user_locale_profile_edit'
-    ),
+urlpatterns_error_logs = [
     url(
         regex=r'^object/(?P<app_label>[-\w]+)/(?P<model>[-\w]+)/(?P<object_id>\d+)/errors/$',
         view=ObjectErrorLogEntryListView.as_view(), name='object_error_list'
@@ -45,7 +25,20 @@ urlpatterns = [
     ),
 ]
 
-urlpatterns += [
+urlpatterns_user_locale = [
+    url(
+        regex=r'^user/locale/$',
+        view=CurrentUserLocaleProfileDetailsView.as_view(),
+        name='current_user_locale_profile_details'
+    ),
+    url(
+        regex=r'^user/locale/edit/$',
+        view=CurrentUserLocaleProfileEditView.as_view(),
+        name='current_user_locale_profile_edit'
+    ),
+]
+
+urlpatterns_misc = [
     url(
         regex=r'^favicon\.ico$', view=FaviconRedirectView.as_view()
     ),
@@ -53,11 +46,21 @@ urlpatterns += [
         regex=r'^jsi18n/(?P<packages>\S+?)/$', view=JavaScriptCatalog.as_view(),
         name='javascript_catalog'
     ),
-    url(
-        regex=r'^set_language/$', view=set_language, name='set_language'
-    ),
 ]
 
+urlpatterns = [
+    url(regex=r'^$', view=RootView.as_view(), name='root'),
+    url(regex=r'^home/$', view=HomeView.as_view(), name='home'),
+    url(regex=r'^about/$', view=AboutView.as_view(), name='about_view'),
+    url(regex=r'^license/$', view=LicenseView.as_view(), name='license_view'),
+    url(regex=r'^setup/$', view=SetupListView.as_view(), name='setup_list'),
+    url(regex=r'^tools/$', view=ToolsListView.as_view(), name='tools_list'),
+]
+
+urlpatterns.extend(urlpatterns_error_logs)
+urlpatterns.extend(urlpatterns_misc)
+urlpatterns.extend(urlpatterns_user_locale)
+
 api_urls = [
     url(
         regex=r'^content_types/$', view=APIContentTypeList.as_view(),

mayan/apps/common/views.py

@@ -1,15 +1,11 @@
 from __future__ import absolute_import, unicode_literals
 
-from json import dumps
-
 from django.conf import settings
 from django.contrib import messages
 from django.contrib.contenttypes.models import ContentType
-from django.http import HttpResponseRedirect
 from django.shortcuts import get_object_or_404
-from django.urls import reverse, reverse_lazy
+from django.urls import reverse_lazy
 from django.utils import timezone, translation
-from django.utils.http import urlencode
 from django.utils.translation import ugettext_lazy as _
 from django.views.generic import RedirectView
 
@@ -220,67 +216,3 @@ class ToolsListView(SimpleView):
                 'These modules are used to do system maintenance.'
             )
         }
-
-
-def multi_object_action_view(request):
-    """
-    Proxy view called first when using a multi object action, which
-    then redirects to the appropriate specialized view
-    """
-    next = request.POST.get(
-        'next', request.GET.get(
-            'next', request.META.get(
-                'HTTP_REFERER', reverse(setting_home_view.value)
-            )
-        )
-    )
-
-    action = request.GET.get('action', None)
-    id_list = ','.join(
-        [key[3:] for key in request.GET.keys() if key.startswith('pk_')]
-    )
-    items_property_list = [
-        (key[11:]) for key in request.GET.keys() if key.startswith('properties_')
-    ]
-
-    if not action:
-        messages.error(
-            message=_('No action selected.'), request=request
-        )
-        return HttpResponseRedirect(
-            redirect_to=request.META.get(
-                'HTTP_REFERER', reverse(setting_home_view.value)
-            )
-        )
-
-    if not id_list and not items_property_list:
-        messages.error(
-            message=_('Must select at least one item.'),
-            request=request
-        )
-        return HttpResponseRedirect(
-            redirect_to=request.META.get(
-                'HTTP_REFERER', reverse(setting_home_view.value)
-            )
-        )
-
-    # Separate redirects to keep backwards compatibility with older
-    # functions that don't expect a properties_list parameter
-    if items_property_list:
-        return HttpResponseRedirect(
-            redirect_to='%s?%s' % (
-                action,
-                urlencode(
-                    {
-                        'items_property_list': dumps(items_property_list),
-                        'next': next
-                    }
-                )
-            )
-        )
-    else:
-        return HttpResponseRedirect(
-            redirect_to='%s?%s' % (
-                action, urlencode({'id_list': id_list, 'next': next})
-            )
-        )

mayan/apps/converter/classes.py

@@ -10,7 +10,6 @@ import sh
 
 from django.utils.translation import ugettext_lazy as _
 
-from mayan.apps.common.serialization import yaml_load
 from mayan.apps.mimetype.api import get_mimetype
 from mayan.apps.storage.settings import setting_temporary_directory
 from mayan.apps.storage.utils import (
@@ -147,7 +146,7 @@ class ConverterBase(object):
                 logger.error('Exception launching Libre Office; %s', exception)
                 raise
             finally:
-                fs_cleanup(libreoffice_home_directory)
+                fs_cleanup(filename=libreoffice_home_directory)
 
             # LibreOffice return a PDF file with the same name as the input
             # provided but with the .pdf extension.
@@ -181,7 +180,7 @@ class ConverterBase(object):
             shutil.copyfileobj(
                 fsrc=converted_file_object, fdst=temporary_converted_file_object
             )
-        fs_cleanup(converted_file_path)
+        fs_cleanup(filename=converted_file_path)
         temporary_converted_file_object.seek(0)
         return temporary_converted_file_object
 

mayan/apps/converter/migrations/0014_auto_20190626_1904.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.20 on 2019-06-26 19:04
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('converter', '0013_auto_20180823_2353'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='transformation',
+            name='name',
+            field=models.CharField(choices=[('crop', 'Crop: left, top, right, bottom'), ('draw_rectangle', 'Draw rectangle: left, top, right, bottom, fillcolor, outlinecolor, outlinewidth'), ('draw_rectangle_percent', 'Draw rectangle (percents coordinates): left, top, right, bottom, fillcolor, outlinecolor, outlinewidth'), ('flip', 'Flip'), ('gaussianblur', 'Gaussian blur: radius'), ('lineart', 'Line art'), ('mirror', 'Mirror'), ('resize', 'Resize: width, height'), ('rotate', 'Rotate: degrees, fillcolor'), ('rotate180', 'Rotate 180 degrees'), ('rotate270', 'Rotate 270 degrees'), ('rotate90', 'Rotate 90 degrees'), ('unsharpmask', 'Unsharp masking: radius, percent, threshold'), ('zoom', 'Zoom: percent')], max_length=128, verbose_name='Name'),
+        ),
+    ]

mayan/apps/converter/tests/test_transformations.py

@@ -121,7 +121,7 @@ class TransformationTestCase(GenericDocumentTestCase):
             arguments={'top': '10'}
         )
 
-        self.assertTrue(document_page.generate_image().startswith('page'))
+        self.assertTrue(document_page.generate_image())
 
     def test_crop_transformation_invalid_arguments(self):
         self._silence_logger(name='mayan.apps.converter.managers')
@@ -132,8 +132,7 @@ class TransformationTestCase(GenericDocumentTestCase):
             obj=document_page, transformation=TransformationCrop,
             arguments={'top': 'x', 'left': '-'}
         )
-
-        self.assertTrue(document_page.generate_image().startswith('page'))
+        self.assertTrue(document_page.generate_image())
 
     def test_crop_transformation_non_valid_range_arguments(self):
         self._silence_logger(name='mayan.apps.converter.managers')
@@ -145,7 +144,7 @@ class TransformationTestCase(GenericDocumentTestCase):
             arguments={'top': '-1000', 'bottom': '100000000'}
         )
 
-        self.assertTrue(document_page.generate_image().startswith('page'))
+        self.assertTrue(document_page.generate_image())
 
     def test_crop_transformation_overlapping_ranges_arguments(self):
         self._silence_logger(name='mayan.apps.converter.managers')
@@ -162,7 +161,7 @@ class TransformationTestCase(GenericDocumentTestCase):
             arguments={'left': '1000', 'right': '10000'}
         )
 
-        self.assertTrue(document_page.generate_image().startswith('page'))
+        self.assertTrue(document_page.generate_image())
 
     def test_lineart_transformations(self):
         document_page = self.test_document.pages.first()
@@ -172,7 +171,7 @@ class TransformationTestCase(GenericDocumentTestCase):
             arguments={}
         )
 
-        self.assertTrue(document_page.generate_image().startswith('page'))
+        self.assertTrue(document_page.generate_image())
 
     def test_rotate_transformations(self):
         document_page = self.test_document.pages.first()
@@ -182,18 +181,18 @@ class TransformationTestCase(GenericDocumentTestCase):
             arguments={}
         )
 
-        self.assertTrue(document_page.generate_image().startswith('page'))
+        self.assertTrue(document_page.generate_image())
 
         Transformation.objects.add_to_object(
             obj=document_page, transformation=TransformationRotate180,
             arguments={}
         )
 
-        self.assertTrue(document_page.generate_image().startswith('page'))
+        self.assertTrue(document_page.generate_image())
 
         Transformation.objects.add_to_object(
             obj=document_page, transformation=TransformationRotate270,
             arguments={}
         )
 
-        self.assertTrue(document_page.generate_image().startswith('page'))
+        self.assertTrue(document_page.generate_image())

mayan/apps/converter/transformations.py

@@ -3,7 +3,7 @@ from __future__ import unicode_literals
 import hashlib
 import logging
 
-from PIL import Image, ImageColor, ImageFilter
+from PIL import Image, ImageColor, ImageDraw, ImageFilter
 
 from django.utils.translation import string_concat, ugettext_lazy as _
 from django.utils.encoding import force_bytes
@@ -151,6 +151,208 @@ class TransformationCrop(BaseTransformation):
         return self.image.crop((left, top, right, bottom))
 
 
+class TransformationDrawRectangle(BaseTransformation):
+    arguments = (
+        'left', 'top', 'right', 'bottom', 'fillcolor', 'outlinecolor',
+        'outlinewidth'
+    )
+    label = _('Draw rectangle')
+    name = 'draw_rectangle'
+
+    def execute_on(self, *args, **kwargs):
+        super(TransformationDrawRectangle, self).execute_on(*args, **kwargs)
+
+        try:
+            left = int(self.left or '0')
+        except ValueError:
+            left = 0
+
+        try:
+            top = int(self.top or '0')
+        except ValueError:
+            top = 0
+
+        try:
+            right = int(self.right or '0')
+        except ValueError:
+            right = 0
+
+        try:
+            bottom = int(self.bottom or '0')
+        except ValueError:
+            bottom = 0
+
+        if left < 0:
+            left = 0
+
+        if left > self.image.size[0] - 1:
+            left = self.image.size[0] - 1
+
+        if top < 0:
+            top = 0
+
+        if top > self.image.size[1] - 1:
+            top = self.image.size[1] - 1
+
+        if right < 0:
+            right = 0
+
+        if right > self.image.size[0] - 1:
+            right = self.image.size[0] - 1
+
+        if bottom < 0:
+            bottom = 0
+
+        if bottom > self.image.size[1] - 1:
+            bottom = self.image.size[1] - 1
+
+        # Invert right value
+        # Pillow uses left, top, right, bottom to define a viewport
+        # of real coordinates
+        # We invert the right and bottom to define a viewport
+        # that can crop from the right and bottom borders without
+        # having to know the real dimensions of an image
+        right = self.image.size[0] - right
+        bottom = self.image.size[1] - bottom
+
+        if left > right:
+            left = right - 1
+
+        if top > bottom:
+            top = bottom - 1
+
+        logger.debug(
+            'left: %f, top: %f, right: %f, bottom: %f', left, top, right,
+            bottom
+        )
+
+        fillcolor_value = getattr(self, 'fillcolor', None)
+        if fillcolor_value:
+            fill_color = ImageColor.getrgb(fillcolor_value)
+        else:
+            fill_color = 0
+
+        outlinecolor_value = getattr(self, 'outlinecolor', None)
+        if outlinecolor_value:
+            outline_color = ImageColor.getrgb(outlinecolor_value)
+        else:
+            outline_color = None
+
+        outlinewidth_value = getattr(self, 'outlinewidth', None)
+        if outlinewidth_value:
+            outline_width = int(outlinewidth_value)
+        else:
+            outline_width = 0
+
+        draw = ImageDraw.Draw(self.image)
+        draw.rectangle(
+            (left, top, right, bottom), fill=fill_color, outline=outline_color,
+            width=outline_width
+        )
+
+        return self.image
+
+
+class TransformationDrawRectanglePercent(BaseTransformation):
+    arguments = (
+        'left', 'top', 'right', 'bottom', 'fillcolor', 'outlinecolor',
+        'outlinewidth'
+    )
+    label = _('Draw rectangle (percents coordinates)')
+    name = 'draw_rectangle_percent'
+
+    def execute_on(self, *args, **kwargs):
+        super(TransformationDrawRectanglePercent, self).execute_on(*args, **kwargs)
+
+        try:
+            left = float(self.left or '0')
+        except ValueError:
+            left = 0
+
+        try:
+            top = float(self.top or '0')
+        except ValueError:
+            top = 0
+
+        try:
+            right = float(self.right or '0')
+        except ValueError:
+            right = 0
+
+        try:
+            bottom = float(self.bottom or '0')
+        except ValueError:
+            bottom = 0
+
+        if left < 0:
+            left = 0
+
+        if left > 100:
+            left = 100
+
+        if top < 0:
+            top = 0
+
+        if top > 100:
+            top = 100
+
+        if right < 0:
+            right = 0
+
+        if right > 100:
+            right = 100
+
+        if bottom < 0:
+            bottom = 0
+
+        if bottom > 100:
+            bottom = 100
+
+        logger.debug(
+            'left: %f, top: %f, right: %f, bottom: %f', left, top, right,
+            bottom
+        )
+
+        fillcolor_value = getattr(self, 'fillcolor', None)
+        if fillcolor_value:
+            fill_color = ImageColor.getrgb(fillcolor_value)
+        else:
+            fill_color = 0
+
+        outlinecolor_value = getattr(self, 'outlinecolor', None)
+        if outlinecolor_value:
+            outline_color = ImageColor.getrgb(outlinecolor_value)
+        else:
+            outline_color = None
+
+        outlinewidth_value = getattr(self, 'outlinewidth', None)
+        if outlinewidth_value:
+            outline_width = int(outlinewidth_value)
+        else:
+            outline_width = 0
+
+        left = left / 100.0 * self.image.size[0]
+        top = top / 100.0 * self.image.size[1]
+
+        # Invert right value
+        # Pillow uses left, top, right, bottom to define a viewport
+        # of real coordinates
+        # We invert the right and bottom to define a viewport
+        # that can crop from the right and bottom borders without
+        # having to know the real dimensions of an image
+
+        right = self.image.size[0] - (right / 100.0 * self.image.size[0])
+        bottom = self.image.size[1] - (bottom / 100.0 * self.image.size[1])
+
+        draw = ImageDraw.Draw(self.image)
+        draw.rectangle(
+            (left, top, right, bottom), fill=fill_color, outline=outline_color,
+            width=outline_width
+        )
+
+        return self.image
+
+
 class TransformationFlip(BaseTransformation):
     arguments = ()
     label = _('Flip')
@@ -316,6 +518,10 @@ class TransformationZoom(BaseTransformation):
 
 
 BaseTransformation.register(transformation=TransformationCrop)
+BaseTransformation.register(transformation=TransformationDrawRectangle)
+BaseTransformation.register(
+    transformation=TransformationDrawRectanglePercent
+)
 BaseTransformation.register(transformation=TransformationFlip)
 BaseTransformation.register(transformation=TransformationGaussianBlur)
 BaseTransformation.register(transformation=TransformationLineArt)

mayan/apps/converter/urls.py

@@ -9,19 +9,19 @@ from .views import (
 
 urlpatterns = [
     url(
-        regex=r'^create_for/(?P<app_label>[-\w]+)/(?P<model>[-\w]+)/(?P<object_id>\d+)/$',
-        view=TransformationCreateView.as_view(), name='transformation_create'
-    ),
-    url(
-        regex=r'^list_for/(?P<app_label>[-\w]+)/(?P<model>[-\w]+)/(?P<object_id>\d+)/$',
+        regex=r'^object/(?P<app_label>[-\w]+)/(?P<model>[-\w]+)/(?P<object_id>\d+)/transformations/$',
         view=TransformationListView.as_view(), name='transformation_list'
     ),
     url(
-        regex=r'^delete/(?P<pk>\d+)/$', view=TransformationDeleteView.as_view(),
+        regex=r'^object/(?P<app_label>[-\w]+)/(?P<model>[-\w]+)/(?P<object_id>\d+)/transformations/create/$',
+        view=TransformationCreateView.as_view(), name='transformation_create'
+    ),
+    url(
+        regex=r'^transformations/(?P<pk>\d+)/delete/$', view=TransformationDeleteView.as_view(),
         name='transformation_delete'
     ),
     url(
-        regex=r'^edit/(?P<pk>\d+)/$', view=TransformationEditView.as_view(),
+        regex=r'^transformations/(?P<pk>\d+)/edit/$', view=TransformationEditView.as_view(),
         name='transformation_edit'
     ),
 ]

mayan/apps/dashboards/classes.py

@@ -1,5 +1,6 @@
 from __future__ import unicode_literals
 
+from django.contrib.humanize.templatetags.humanize import intcomma
 from django.template import loader
 
 
@@ -85,7 +86,8 @@ class DashboardWidgetNumeric(BaseDashboardWidget):
 
     def get_context(self):
         return {
-            'count': self.count,
+            'count': intcomma(value=self.count),
+            'count_raw': self.count,
             'icon_class': self.icon_class,
             'label': self.label,
             'link': self.link,

mayan/apps/dashboards/templates/dashboards/numeric_widget.html

@@ -1,5 +1,7 @@
 {% load i18n %}
 
+{% load appearance_tags %}
+
 <div class="col-xs-12 col-sm-6 col-md-4 col-lg-3 match-height">
     <div class="panel panel-secondary dashboard-widget">
         <div class="panel-heading">
@@ -9,7 +11,7 @@
                         <i class="dashboard-widget-icon {{ icon }}"></i>
                     {% elif icon_class %}
                         <div class="dashboard-widget-icon">
-                            {{ icon_class.render }}
+                            {% appearance_icon_render icon_class enable_shadow=True %}
                         </div>
                     {% endif %}
                 </div>

mayan/apps/django_gpg/urls.py

@@ -11,35 +11,35 @@ from .views import (
 
 urlpatterns = [
     url(
-        regex=r'^(?P<pk>\d+)/$', view=KeyDetailView.as_view(),
+        regex=r'^keys/(?P<pk>\d+)/$', view=KeyDetailView.as_view(),
         name='key_detail'
     ),
     url(
-        regex=r'^(?P<pk>\d+)/delete/$', view=KeyDeleteView.as_view(),
+        regex=r'^keys/(?P<pk>\d+)/delete/$', view=KeyDeleteView.as_view(),
         name='key_delete'
     ),
     url(
-        regex=r'^(?P<pk>\d+)/download/$', view=KeyDownloadView.as_view(),
+        regex=r'^keys/(?P<pk>\d+)/download/$', view=KeyDownloadView.as_view(),
         name='key_download'
     ),
     url(
-        regex=r'^list/private/$', view=PrivateKeyListView.as_view(),
+        regex=r'^keys/private/$', view=PrivateKeyListView.as_view(),
         name='key_private_list'
     ),
     url(
-        regex=r'^list/public/$', view=PublicKeyListView.as_view(),
+        regex=r'^keys/public/$', view=PublicKeyListView.as_view(),
         name='key_public_list'
     ),
     url(
-        regex=r'^upload/$', view=KeyUploadView.as_view(), name='key_upload'
+        regex=r'^keys/upload/$', view=KeyUploadView.as_view(), name='key_upload'
     ),
-    url(regex=r'^query/$', view=KeyQueryView.as_view(), name='key_query'),
+    url(regex=r'^keys/query/$', view=KeyQueryView.as_view(), name='key_query'),
     url(
-        regex=r'^query/results/$', view=KeyQueryResultView.as_view(),
+        regex=r'^keys/query/results/$', view=KeyQueryResultView.as_view(),
         name='key_query_results'
     ),
     url(
-        regex=r'^receive/(?P<key_id>.+)/$', view=KeyReceive.as_view(),
+        regex=r'^keys/receive/(?P<key_id>.+)/$', view=KeyReceive.as_view(),
         name='key_receive'
     ),
 ]

mayan/apps/document_comments/urls.py

@@ -11,25 +11,25 @@ from .views import (
 
 urlpatterns = [
     url(
-        regex=r'^(?P<pk>\d+)/comment/add/$',
+        regex=r'^documents/(?P<pk>\d+)/comments/$',
+        view=DocumentCommentListView.as_view(), name='comments_for_document'
+    ),
+    url(
+        regex=r'^documents/(?P<pk>\d+)/comments/add/$',
         view=DocumentCommentCreateView.as_view(), name='comment_add'
     ),
     url(
-        regex=r'^comment/(?P<pk>\d+)/delete/$',
+        regex=r'^comments/(?P<pk>\d+)/delete/$',
         view=DocumentCommentDeleteView.as_view(), name='comment_delete'
     ),
     url(
-        regex=r'^comment/(?P<pk>\d+)/$',
+        regex=r'^comments/(?P<pk>\d+)/$',
         view=DocumentCommentDetailView.as_view(), name='comment_details'
     ),
     url(
-        regex=r'^comment/(?P<pk>\d+)/edit/$',
+        regex=r'^comments/(?P<pk>\d+)/edit/$',
         view=DocumentCommentEditView.as_view(), name='comment_edit'
     ),
-    url(
-        regex=r'^(?P<pk>\d+)/comment/list/$',
-        view=DocumentCommentListView.as_view(), name='comments_for_document'
-    ),
 ]
 
 api_urls = [

mayan/apps/document_indexing/tests/test_views.py

@@ -196,3 +196,36 @@ class IndexToolsViewTestCase(
 
         # An instance root exists
         self.assertTrue(self.test_index.instance_root.pk)
+
+    def test_index_rebuild_view_no_permission(self):
+        self._create_test_index()
+
+        self.test_index.node_templates.create(
+            parent=self.test_index.template_root,
+            expression=TEST_INDEX_TEMPLATE_DOCUMENT_LABEL_EXPRESSION,
+            link_documents=True
+        )
+
+        response = self._request_test_index_rebuild_view()
+        self.assertEqual(response.status_code, 404)
+
+        self.assertEqual(IndexInstanceNode.objects.count(), 0)
+
+    def test_index_rebuild_view_with_access(self):
+        self._create_test_index()
+
+        self.test_index.node_templates.create(
+            parent=self.test_index.template_root,
+            expression=TEST_INDEX_TEMPLATE_DOCUMENT_LABEL_EXPRESSION,
+            link_documents=True
+        )
+
+        self.grant_access(
+            obj=self.test_index,
+            permission=permission_document_indexing_rebuild
+        )
+
+        response = self._request_test_index_rebuild_view()
+        self.assertEqual(response.status_code, 302)
+
+        self.assertNotEqual(IndexInstanceNode.objects.count(), 0)

mayan/apps/document_indexing/urls.py

@@ -15,72 +15,81 @@ from .views import (
     TemplateNodeCreateView, TemplateNodeDeleteView, TemplateNodeEditView
 )
 
-urlpatterns = [
+urlpatterns_indexes = [
     url(
-        regex=r'^setup/document_types/(?P<pk>\d+)/index_templates/$',
+        regex=r'^document_types/(?P<pk>\d+)/index_templates/$',
         view=DocumentTypeIndexesView.as_view(),
         name='document_type_index_templates'
     ),
     url(
-        regex=r'^setup/index/list/$', view=SetupIndexListView.as_view(),
+        regex=r'^indexes/$', view=SetupIndexListView.as_view(),
         name='index_setup_list'
     ),
     url(
-        regex=r'^setup/index/create/$', view=SetupIndexCreateView.as_view(),
+        regex=r'^indexes/create/$', view=SetupIndexCreateView.as_view(),
         name='index_setup_create'
     ),
     url(
-        regex=r'^setup/index/(?P<pk>\d+)/edit/$',
-        view=SetupIndexEditView.as_view(), name='index_setup_edit'
-    ),
-    url(
-        regex=r'^setup/index/(?P<pk>\d+)/delete/$',
+        regex=r'^indexes/(?P<pk>\d+)/delete/$',
         view=SetupIndexDeleteView.as_view(), name='index_setup_delete'
     ),
     url(
-        regex=r'^setup/index/(?P<pk>\d+)/template/$',
-        view=SetupIndexTreeTemplateListView.as_view(), name='index_setup_view'
+        regex=r'^indexes/(?P<pk>\d+)/edit/$',
+        view=SetupIndexEditView.as_view(), name='index_setup_edit'
     ),
     url(
-        regex=r'^setup/index/(?P<pk>\d+)/document_types/$',
+        regex=r'^indexes/(?P<pk>\d+)/document_types/$',
         view=SetupIndexDocumentTypesView.as_view(),
         name='index_setup_document_types'
     ),
     url(
-        regex=r'^setup/index/(?P<pk>\d+)/rebuild/$',
+        regex=r'^indexes/(?P<pk>\d+)/rebuild/$',
         view=SetupIndexRebuildView.as_view(), name='index_setup_rebuild'
     ),
     url(
-        regex=r'^setup/template/node/(?P<pk>\d+)/create/child/$',
+        regex=r'^indexes/(?P<pk>\d+)/nodes/$',
+        view=SetupIndexTreeTemplateListView.as_view(), name='index_setup_view'
+    ),
+    url(
+        regex=r'^indexes/nodes/(?P<pk>\d+)/children/create/$',
         view=TemplateNodeCreateView.as_view(), name='template_node_create'
     ),
     url(
-        regex=r'^setup/template/node/(?P<pk>\d+)/edit/$',
-        view=TemplateNodeEditView.as_view(), name='template_node_edit'
-    ),
-    url(
-        regex=r'^setup/template/node/(?P<pk>\d+)/delete/$',
+        regex=r'^indexes/nodes/(?P<pk>\d+)/delete/$',
         view=TemplateNodeDeleteView.as_view(), name='template_node_delete'
     ),
-
     url(
-        regex=r'^index/list/$', view=IndexListView.as_view(), name='index_list'
+        regex=r'^indexes/nodes/(?P<pk>\d+)/edit/$',
+        view=TemplateNodeEditView.as_view(), name='template_node_edit'
+    ),
+]
+
+urlpatterns_index_instances = [
+    url(
+        regex=r'^index_instances/$', view=IndexListView.as_view(), name='index_list'
     ),
     url(
-        regex=r'^instance/node/(?P<pk>\d+)/$',
+        regex=r'^index_instances/nodes/(?P<pk>\d+)/$',
         view=IndexInstanceNodeView.as_view(), name='index_instance_node_view'
     ),
+    url(
+        regex=r'^documents/(?P<pk>\d+)/index_instances/$',
+        view=DocumentIndexNodeListView.as_view(), name='document_index_list'
+    ),
+]
 
+urlpatterns_tools = [
     url(
         regex=r'^indexes/rebuild/$', view=IndexesRebuildView.as_view(),
         name='rebuild_index_instances'
     ),
-    url(
-        regex=r'^list/for/document/(?P<pk>\d+)/$',
-        view=DocumentIndexNodeListView.as_view(), name='document_index_list'
-    ),
 ]
 
+urlpatterns = []
+urlpatterns.extend(urlpatterns_indexes)
+urlpatterns.extend(urlpatterns_index_instances)
+urlpatterns.extend(urlpatterns_tools)
+
 api_urls = [
     url(
         regex=r'^indexes/node/(?P<pk>[0-9]+)/documents/$',

mayan/apps/document_parsing/apps.py

@@ -86,7 +86,7 @@ class DocumentParsingApp(MayanAppConfig):
         )
 
         ModelField(
-            model=Document, name='versions__pages__content__content'
+            model=Document, name='versions__version_pages__content__content'
         )
 
         ModelPermission.register(
@@ -118,7 +118,7 @@ class DocumentParsingApp(MayanAppConfig):
         )
 
         document_search.add_model_field(
-            field='versions__pages__content__content', label=_('Content')
+            field='versions__version_pages__content__content', label=_('Content')
         )
 
         document_page_search.add_model_field(

mayan/apps/document_parsing/links.py

@@ -10,6 +10,11 @@ from .permissions import (
     permission_parse_document
 )
 
+
+def is_document_page_disabled(context):
+    return not context['resolved_object'].enabled
+
+
 link_document_content = Link(
     args='resolved_object.id',
     icon_class_path='mayan.apps.document_parsing.icons.icon_document_content',
@@ -17,7 +22,7 @@ link_document_content = Link(
     view='document_parsing:document_content'
 )
 link_document_page_content = Link(
-    args='resolved_object.id',
+    args='resolved_object.id', conditional_disable=is_document_page_disabled,
     icon_class_path='mayan.apps.document_parsing.icons.icon_document_content',
     permissions=(permission_content_view,), text=_('Content'),
     view='document_parsing:document_page_content'

mayan/apps/document_signatures/urls.py

@@ -12,37 +12,37 @@ from .views import (
 
 urlpatterns = [
     url(
-        regex=r'^(?P<pk>\d+)/details/$',
+        regex=r'^signatures/(?P<pk>\d+)/details/$',
         view=DocumentVersionSignatureDetailView.as_view(),
         name='document_version_signature_details'
     ),
     url(
-        regex=r'^signature/(?P<pk>\d+)/download/$',
+        regex=r'^signatures/(?P<pk>\d+)/download/$',
         view=DocumentVersionSignatureDownloadView.as_view(),
         name='document_version_signature_download'
     ),
     url(
-        regex=r'^document/version/(?P<pk>\d+)/signatures/list/$',
+        regex=r'^documents/versions/(?P<pk>\d+)/signatures/$',
         view=DocumentVersionSignatureListView.as_view(),
         name='document_version_signature_list'
     ),
     url(
-        regex=r'^documents/version/(?P<pk>\d+)/signature/detached/upload/$',
+        regex=r'^documents/versions/(?P<pk>\d+)/signatures/detached/upload/$',
         view=DocumentVersionSignatureUploadView.as_view(),
         name='document_version_signature_upload'
     ),
     url(
-        regex=r'^documents/version/(?P<pk>\d+)/signature/detached/create/$',
+        regex=r'^documents/versions/(?P<pk>\d+)/signatures/detached/create/$',
         view=DocumentVersionDetachedSignatureCreateView.as_view(),
         name='document_version_signature_detached_create'
     ),
     url(
-        regex=r'^documents/version/(?P<pk>\d+)/signature/embedded/create/$',
+        regex=r'^documents/versions/(?P<pk>\d+)/signatures/embedded/create/$',
         view=DocumentVersionEmbeddedSignatureCreateView.as_view(),
         name='document_version_signature_embedded_create'
     ),
     url(
-        regex=r'^signature/(?P<pk>\d+)/delete/$',
+        regex=r'^signatures/(?P<pk>\d+)/delete/$',
         view=DocumentVersionSignatureDeleteView.as_view(),
         name='document_version_signature_delete'
     ),

mayan/apps/document_states/api_views.py

@@ -27,7 +27,6 @@ from .serializers import (
 )
 
 from .settings import settings_workflow_image_cache_time
-from .storages import storage_workflowimagecache
 from .tasks import task_generate_workflow_image
 
 
@@ -204,7 +203,8 @@ class APIWorkflowImageView(generics.RetrieveAPIView):
         )
 
         cache_filename = task.get(timeout=WORKFLOW_IMAGE_TASK_TIMEOUT)
-        with storage_workflowimagecache.open(cache_filename) as file_object:
+        cache_file = self.get_object().cache_partition.get_file(filename=cache_filename)
+        with cache_file.open() as file_object:
             response = HttpResponse(file_object.read(), content_type='image')
             if '_hash' in request.GET:
                 patch_cache_control(

mayan/apps/document_states/apps.py

@@ -1,7 +1,7 @@
 from __future__ import unicode_literals
 
 from django.apps import apps
-from django.db.models.signals import post_save
+from django.db.models.signals import post_migrate, post_save
 from django.utils.translation import ugettext_lazy as _
 
 from mayan.apps.acls.classes import ModelPermission
@@ -25,7 +25,8 @@ from .classes import DocumentStateHelper, WorkflowAction
 from .events import event_workflow_created, event_workflow_edited
 from .dependencies import *  # NOQA
 from .handlers import (
-    handler_index_document, handler_launch_workflow, handler_trigger_transition
+    handler_create_workflow_image_cache, handler_index_document,
+    handler_launch_workflow, handler_trigger_transition
 )
 from .html_widgets import WorkflowLogExtraDataWidget, widget_transition_events
 from .links import (
@@ -328,6 +329,17 @@ class DocumentStatesApp(MayanAppConfig):
                 link_workflow_template_preview
             ), sources=(Workflow,)
         )
+
+        menu_list_facet.unbind_links(
+            links=(
+                link_acl_list, link_events_for_object,
+                link_object_event_types_user_subcriptions_list,
+                link_workflow_template_document_types,
+                link_workflow_template_state_list, link_workflow_template_transition_list,
+                link_workflow_template_preview
+            ), sources=(WorkflowRuntimeProxy,)
+        )
+
         menu_list_facet.bind_links(
             links=(
                 link_document_type_workflow_templates,
@@ -441,6 +453,10 @@ class DocumentStatesApp(MayanAppConfig):
 
         # Index updating
 
+        post_migrate.connect(
+            dispatch_uid='workflows_handler_create_workflow_image_cache',
+            receiver=handler_create_workflow_image_cache,
+        )
         post_save.connect(
             dispatch_uid='workflows_handler_index_document_save',
             receiver=handler_index_document,

mayan/apps/document_states/handlers.py

@@ -6,6 +6,22 @@ from django.utils.translation import ugettext_lazy as _
 from mayan.apps.document_indexing.tasks import task_index_document
 from mayan.apps.events.classes import EventType
 
+from .literals import (
+    WORKFLOW_IMAGE_CACHE_STORAGE_INSTANCE_PATH, WORKFLOW_IMAGE_CACHE_NAME
+)
+from .settings import setting_workflow_image_cache_maximum_size
+
+
+def handler_create_workflow_image_cache(sender, **kwargs):
+    Cache = apps.get_model(app_label='file_caching', model_name='Cache')
+    Cache.objects.update_or_create(
+        defaults={
+            'label': _('Workflow images'),
+            'storage_instance_path': WORKFLOW_IMAGE_CACHE_STORAGE_INSTANCE_PATH,
+            'maximum_size': setting_workflow_image_cache_maximum_size.value,
+        }, name=WORKFLOW_IMAGE_CACHE_NAME,
+    )
+
 
 def handler_index_document(sender, **kwargs):
     task_index_document.apply_async(

mayan/apps/document_states/literals.py

@@ -2,6 +2,8 @@ from __future__ import unicode_literals
 
 from django.utils.translation import ugettext_lazy as _
 
+DEFAULT_WORKFLOW_IMAGE_CACHE_MAXIMUM_SIZE = 50 * 2 ** 20  # 50 Megabytes
+
 FIELD_TYPE_CHOICE_CHAR = 1
 FIELD_TYPE_CHOICE_INTEGER = 2
 FIELD_TYPE_CHOICES = (
@@ -30,4 +32,6 @@ WORKFLOW_ACTION_WHEN_CHOICES = (
     (WORKFLOW_ACTION_ON_ENTRY, _('On entry')),
     (WORKFLOW_ACTION_ON_EXIT, _('On exit')),
 )
+WORKFLOW_IMAGE_CACHE_NAME = 'workflow_images'
+WORKFLOW_IMAGE_CACHE_STORAGE_INSTANCE_PATH = 'mayan.apps.document_states.storages.storage_workflowimagecache'
 WORKFLOW_IMAGE_TASK_TIMEOUT = 60

mayan/apps/document_states/models.py

@@ -6,24 +6,23 @@ import logging
 
 from furl import furl
 from graphviz import Digraph
-import yaml
-try:
-    from yaml import CSafeLoader as SafeLoader
-except ImportError:
-    from yaml import SafeLoader
 
+from django.apps import apps
 from django.conf import settings
 from django.core import serializers
 from django.core.exceptions import PermissionDenied, ValidationError
-from django.core.files.base import ContentFile
 from django.db import IntegrityError, models, transaction
 from django.db.models import F, Max, Q
 from django.urls import reverse
-from django.utils.encoding import force_text, python_2_unicode_compatible
+from django.utils.encoding import (
+    force_bytes, force_text, python_2_unicode_compatible
+)
+from django.utils.functional import cached_property
 from django.utils.module_loading import import_string
 from django.utils.translation import ugettext_lazy as _
 
 from mayan.apps.acls.models import AccessControlList
+from mayan.apps.common.serialization import yaml_load
 from mayan.apps.common.validators import YAMLValidator, validate_internal_name
 from mayan.apps.documents.models import Document, DocumentType
 from mayan.apps.documents.permissions import permission_document_view
@@ -33,11 +32,11 @@ from .error_logs import error_log_state_actions
 from .events import event_workflow_created, event_workflow_edited
 from .literals import (
     FIELD_TYPE_CHOICES, WIDGET_CLASS_CHOICES, WORKFLOW_ACTION_WHEN_CHOICES,
-    WORKFLOW_ACTION_ON_ENTRY, WORKFLOW_ACTION_ON_EXIT
+    WORKFLOW_ACTION_ON_ENTRY, WORKFLOW_ACTION_ON_EXIT,
+    WORKFLOW_IMAGE_CACHE_NAME
 )
 from .managers import WorkflowManager
 from .permissions import permission_workflow_transition
-from .storages import storage_workflowimagecache
 
 logger = logging.getLogger(__name__)
 
@@ -74,18 +73,36 @@ class Workflow(models.Model):
     def __str__(self):
         return self.label
 
-    def generate_image(self):
-        cache_filename = '{}-{}'.format(self.id, self.get_hash())
-        image = self.render()
+    @cached_property
+    def cache(self):
+        Cache = apps.get_model(app_label='file_caching', model_name='Cache')
+        return Cache.objects.get(name=WORKFLOW_IMAGE_CACHE_NAME)
 
-        # Since open "wb+" doesn't create files, check if the file
-        # exists, if not then create it
-        if not storage_workflowimagecache.exists(cache_filename):
-            storage_workflowimagecache.save(
-                name=cache_filename, content=ContentFile(content='')
+    @cached_property
+    def cache_partition(self):
+        partition, created = self.cache.partitions.get_or_create(
+            name='{}'.format(self.pk)
+        )
+        return partition
+
+    def delete(self, *args, **kwargs):
+        self.cache_partition.delete()
+        return super(Workflow, self).delete(*args, **kwargs)
+
+    def generate_image(self):
+        cache_filename = '{}'.format(self.get_hash())
+
+        if self.cache_partition.get_file(filename=cache_filename):
+            logger.debug(
+                'workflow cache file "%s" found', cache_filename
+            )
+        else:
+            logger.debug(
+                'workflow cache file "%s" not found', cache_filename
             )
 
-        with storage_workflowimagecache.open(cache_filename, mode='wb+') as file_object:
+            image = self.render()
+            with self.cache_partition.create_file(filename=cache_filename) as file_object:
                 file_object.write(image)
 
         return cache_filename
@@ -109,12 +126,16 @@ class Workflow(models.Model):
             Workflow.objects.filter(pk=self.pk)
         ) + list(
             WorkflowState.objects.filter(workflow__pk=self.pk)
+        ) + list(
+            WorkflowStateAction.objects.filter(state__workflow__pk=self.pk)
         ) + list(
             WorkflowTransition.objects.filter(workflow__pk=self.pk)
         )
 
         return hashlib.sha256(
+            force_bytes(
                 serializers.serialize('json', objects_lists)
+            )
         ).hexdigest()
 
     def get_initial_state(self):
@@ -465,7 +486,7 @@ class WorkflowTransitionField(models.Model):
         return self.label
 
     def get_widget_kwargs(self):
-        return yaml.load(stream=self.widget_kwargs, Loader=SafeLoader)
+        return yaml_load(stream=self.widget_kwargs)
 
 
 @python_2_unicode_compatible

mayan/apps/document_states/settings.py

@@ -7,8 +7,20 @@ from django.utils.translation import ugettext_lazy as _
 
 from mayan.apps.smart_settings.classes import Namespace
 
+from .literals import DEFAULT_WORKFLOW_IMAGE_CACHE_MAXIMUM_SIZE
+from .utils import callback_update_workflow_image_cache_size
+
 namespace = Namespace(label=_('Workflows'), name='document_states')
 
+setting_workflow_image_cache_maximum_size = namespace.add_setting(
+    global_name='WORKFLOW_IMAGE_CACHE_MAXIMUM_SIZE',
+    default=DEFAULT_WORKFLOW_IMAGE_CACHE_MAXIMUM_SIZE,
+    help_text=_(
+        'The threshold at which the WORKFLOW_IMAGE_CACHE_STORAGE_BACKEND will '
+        'start deleting the oldest workflow image cache files. Specify the '
+        'size in bytes.'
+    ), post_edit_function=callback_update_workflow_image_cache_size
+)
 settings_workflow_image_cache_time = namespace.add_setting(
     global_name='WORKFLOWS_IMAGE_CACHE_TIME', default='31556926',
     help_text=_(

mayan/apps/document_states/tests/test_models.py

@@ -0,0 +1,11 @@
+from __future__ import unicode_literals
+
+from mayan.apps.common.tests import BaseTestCase
+
+from .mixins import WorkflowTestMixin
+
+
+class WorkflowModelTestCase(WorkflowTestMixin, BaseTestCase):
+    def test_workflow_template_preview(self):
+        self._create_test_workflow()
+        self.assertTrue(self.test_workflow.get_api_image_url())

mayan/apps/document_states/urls.py

@@ -225,7 +225,7 @@ urlpatterns_workflow_transition_fields = [
     ),
 ]
 
-urlpatterns = [
+urlpatterns_tools = [
     url(
         regex=r'^tools/workflows/launch/$',
         view=ToolLaunchWorkflows.as_view(),
@@ -233,6 +233,8 @@ urlpatterns = [
     ),
 ]
 
+urlpatterns = []
+urlpatterns.extend(urlpatterns_tools)
 urlpatterns.extend(urlpatterns_workflow_instances)
 urlpatterns.extend(urlpatterns_workflow_runtime_proxies)
 urlpatterns.extend(urlpatterns_workflow_states)

mayan/apps/document_states/utils.py

@@ -0,0 +1,12 @@
+from __future__ import unicode_literals
+
+from django.apps import apps
+
+from .literals import WORKFLOW_IMAGE_CACHE_NAME
+
+
+def callback_update_workflow_image_cache_size(setting):
+    Cache = apps.get_model(app_label='file_caching', model_name='Cache')
+    cache = Cache.objects.get(name=WORKFLOW_IMAGE_CACHE_NAME)
+    cache.maximum_size = setting.value
+    cache.save()

mayan/apps/document_states/views/__init__.py

@@ -1 +0,0 @@
-

mayan/apps/document_states/views/workflow_template_state_views.py

@@ -1,53 +1,30 @@
 from __future__ import absolute_import, unicode_literals
 
-from django.contrib import messages
-from django.db import transaction
 from django.http import Http404, HttpResponseRedirect
 from django.shortcuts import get_object_or_404
 from django.template import RequestContext
-from django.urls import reverse, reverse_lazy
+from django.urls import reverse
 from django.utils.translation import ugettext_lazy as _
 
 from mayan.apps.common.generics import (
-    AddRemoveView, ConfirmView, FormView, SingleObjectCreateView,
-    SingleObjectDeleteView, SingleObjectDetailView,
+    FormView, SingleObjectCreateView, SingleObjectDeleteView,
     SingleObjectDynamicFormCreateView, SingleObjectDynamicFormEditView,
     SingleObjectEditView, SingleObjectListView
 )
 from mayan.apps.common.mixins import ExternalObjectMixin
-from mayan.apps.documents.events import event_document_type_edited
-from mayan.apps.documents.models import DocumentType
-from mayan.apps.documents.permissions import permission_document_type_edit
-from mayan.apps.events.classes import EventType
-from mayan.apps.events.models import StoredEventType
 
 from ..classes import WorkflowAction
-from ..events import event_workflow_edited
 from ..forms import (
-    WorkflowActionSelectionForm, WorkflowForm, WorkflowPreviewForm,
-    WorkflowStateActionDynamicForm, WorkflowStateForm, WorkflowTransitionForm,
-    WorkflowTransitionTriggerEventRelationshipFormSet
-)
-from ..icons import (
-    icon_workflow_template_list, icon_workflow_state, icon_workflow_state_action,
-    icon_workflow_transition, icon_workflow_transition_field
+    WorkflowActionSelectionForm, WorkflowStateActionDynamicForm,
+    WorkflowStateForm
 )
+from ..icons import icon_workflow_state, icon_workflow_state_action
 from ..links import (
-    link_workflow_template_create, link_workflow_template_state_create,
+    link_workflow_template_state_create,
     link_workflow_template_state_action_selection,
-    link_workflow_template_transition_create,
-    link_workflow_template_transition_field_create,
 )
-from ..models import (
-    Workflow, WorkflowState, WorkflowStateAction, WorkflowTransition,
-    WorkflowTransitionField
-)
-from ..permissions import (
-    permission_workflow_create, permission_workflow_delete,
-    permission_workflow_edit, permission_workflow_tools,
-    permission_workflow_view,
-)
-from ..tasks import task_launch_all_workflows
+from ..models import Workflow, WorkflowState, WorkflowStateAction
+from ..permissions import permission_workflow_edit, permission_workflow_view
 
 
 class WorkflowTemplateStateActionCreateView(SingleObjectDynamicFormCreateView):

mayan/apps/document_states/views/workflow_template_transition_views.py

@@ -1,53 +1,28 @@
 from __future__ import absolute_import, unicode_literals
 
 from django.contrib import messages
-from django.db import transaction
-from django.http import Http404, HttpResponseRedirect
-from django.shortcuts import get_object_or_404
 from django.template import RequestContext
-from django.urls import reverse, reverse_lazy
+from django.urls import reverse
 from django.utils.translation import ugettext_lazy as _
 
 from mayan.apps.common.generics import (
-    AddRemoveView, ConfirmView, FormView, SingleObjectCreateView,
-    SingleObjectDeleteView, SingleObjectDetailView,
-    SingleObjectDynamicFormCreateView, SingleObjectDynamicFormEditView,
+    FormView, SingleObjectCreateView, SingleObjectDeleteView,
     SingleObjectEditView, SingleObjectListView
 )
 from mayan.apps.common.mixins import ExternalObjectMixin
-from mayan.apps.documents.events import event_document_type_edited
-from mayan.apps.documents.models import DocumentType
-from mayan.apps.documents.permissions import permission_document_type_edit
 from mayan.apps.events.classes import EventType
 from mayan.apps.events.models import StoredEventType
 
-from ..classes import WorkflowAction
-from ..events import event_workflow_edited
 from ..forms import (
-    WorkflowActionSelectionForm, WorkflowForm, WorkflowPreviewForm,
-    WorkflowStateActionDynamicForm, WorkflowStateForm, WorkflowTransitionForm,
-    WorkflowTransitionTriggerEventRelationshipFormSet
-)
-from ..icons import (
-    icon_workflow_template_list, icon_workflow_state, icon_workflow_state_action,
-    icon_workflow_transition, icon_workflow_transition_field
+    WorkflowTransitionForm, WorkflowTransitionTriggerEventRelationshipFormSet
 )
+from ..icons import icon_workflow_transition, icon_workflow_transition_field
 from ..links import (
-    link_workflow_template_create, link_workflow_template_state_create,
-    link_workflow_template_state_action_selection,
     link_workflow_template_transition_create,
     link_workflow_template_transition_field_create,
 )
-from ..models import (
-    Workflow, WorkflowState, WorkflowStateAction, WorkflowTransition,
-    WorkflowTransitionField
-)
-from ..permissions import (
-    permission_workflow_create, permission_workflow_delete,
-    permission_workflow_edit, permission_workflow_tools,
-    permission_workflow_view,
-)
-from ..tasks import task_launch_all_workflows
+from ..models import Workflow, WorkflowTransition, WorkflowTransitionField
+from ..permissions import permission_workflow_edit, permission_workflow_view
 
 
 class WorkflowTemplateTransitionCreateView(ExternalObjectMixin, SingleObjectCreateView):

mayan/apps/document_states/views/workflow_template_views.py

@@ -2,46 +2,23 @@ from __future__ import absolute_import, unicode_literals
 
 from django.contrib import messages
 from django.db import transaction
-from django.http import Http404, HttpResponseRedirect
-from django.shortcuts import get_object_or_404
 from django.template import RequestContext
-from django.urls import reverse, reverse_lazy
+from django.urls import reverse_lazy
 from django.utils.translation import ugettext_lazy as _
 
 from mayan.apps.common.generics import (
-    AddRemoveView, ConfirmView, FormView, SingleObjectCreateView,
-    SingleObjectDeleteView, SingleObjectDetailView,
-    SingleObjectDynamicFormCreateView, SingleObjectDynamicFormEditView,
-    SingleObjectEditView, SingleObjectListView
+    AddRemoveView, ConfirmView, SingleObjectCreateView, SingleObjectDeleteView,
+    SingleObjectDetailView, SingleObjectEditView, SingleObjectListView
 )
-from mayan.apps.common.mixins import ExternalObjectMixin
 from mayan.apps.documents.events import event_document_type_edited
 from mayan.apps.documents.models import DocumentType
 from mayan.apps.documents.permissions import permission_document_type_edit
-from mayan.apps.events.classes import EventType
-from mayan.apps.events.models import StoredEventType
 
-from ..classes import WorkflowAction
 from ..events import event_workflow_edited
-from ..forms import (
-    WorkflowActionSelectionForm, WorkflowForm, WorkflowPreviewForm,
-    WorkflowStateActionDynamicForm, WorkflowStateForm, WorkflowTransitionForm,
-    WorkflowTransitionTriggerEventRelationshipFormSet
-)
-from ..icons import (
-    icon_workflow_template_list, icon_workflow_state, icon_workflow_state_action,
-    icon_workflow_transition, icon_workflow_transition_field
-)
-from ..links import (
-    link_workflow_template_create, link_workflow_template_state_create,
-    link_workflow_template_state_action_selection,
-    link_workflow_template_transition_create,
-    link_workflow_template_transition_field_create,
-)
-from ..models import (
-    Workflow, WorkflowState, WorkflowStateAction, WorkflowTransition,
-    WorkflowTransitionField
-)
+from ..forms import WorkflowForm, WorkflowPreviewForm
+from ..icons import icon_workflow_template_list
+from ..links import link_workflow_template_create
+from ..models import Workflow
 from ..permissions import (
     permission_workflow_create, permission_workflow_delete,
     permission_workflow_edit, permission_workflow_tools,

mayan/apps/documents/api_views.py

@@ -36,7 +36,6 @@ from .serializers import (
     WritableDocumentTypeSerializer, WritableDocumentVersionSerializer
 )
 from .settings import settings_document_page_image_cache_time
-from .storages import storage_documentimagecache
 from .tasks import task_generate_document_page_image
 
 logger = logging.getLogger(__name__)
@@ -165,7 +164,7 @@ class APIDocumentPageImageView(generics.RetrieveAPIView):
 
         AccessControlList.objects.check_access(
             obj=document, permissions=(permission_required,),
-            user=self.request.user, manager=Document.passthrough
+            user=self.request.user
         )
         return document
 
@@ -175,7 +174,7 @@ class APIDocumentPageImageView(generics.RetrieveAPIView):
         )
 
     def get_queryset(self):
-        return self.get_document_version().pages.all()
+        return self.get_document_version().pages_all.all()
 
     def get_serializer(self, *args, **kwargs):
         return None
@@ -205,11 +204,13 @@ class APIDocumentPageImageView(generics.RetrieveAPIView):
         )
 
         cache_filename = task.get(timeout=DOCUMENT_IMAGE_TASK_TIMEOUT)
-        with storage_documentimagecache.open(cache_filename) as file_object:
+        cache_file = self.get_object().cache_partition.get_file(filename=cache_filename)
+        with cache_file.open() as file_object:
             response = HttpResponse(file_object.read(), content_type='image')
             if '_hash' in request.GET:
                 patch_cache_control(
-                    response, max_age=settings_document_page_image_cache_time.value
+                    response=response,
+                    max_age=settings_document_page_image_cache_time.value
                 )
             return response
 

mayan/apps/documents/apps.py

@@ -1,6 +1,6 @@
 from __future__ import absolute_import, unicode_literals
 
-from django.db.models.signals import post_delete
+from django.db.models.signals import post_delete, post_migrate
 from django.utils.translation import ugettext_lazy as _
 
 from mayan.apps.acls.classes import ModelPermission
@@ -43,11 +43,11 @@ from .events import (
     event_document_view
 )
 from .handlers import (
-    handler_create_default_document_type, handler_remove_empty_duplicates_lists,
-    handler_scan_duplicates_for,
+    handler_create_default_document_type, handler_create_document_cache,
+    handler_remove_empty_duplicates_lists, handler_scan_duplicates_for
 )
 from .links import (
-    link_clear_image_cache, link_document_clear_transformations,
+    link_document_clear_transformations,
     link_document_clone_transformations, link_document_delete,
     link_document_document_type_edit, link_document_download,
     link_document_duplicates_list, link_document_edit,
@@ -60,6 +60,8 @@ from .links import (
     link_document_multiple_download, link_document_multiple_favorites_add,
     link_document_multiple_favorites_remove, link_document_multiple_restore,
     link_document_multiple_trash, link_document_multiple_update_page_count,
+    link_document_page_disable, link_document_page_multiple_disable,
+    link_document_page_enable, link_document_page_multiple_enable,
     link_document_page_navigation_first, link_document_page_navigation_last,
     link_document_page_navigation_next, link_document_page_navigation_previous,
     link_document_page_return, link_document_page_rotate_left,
@@ -100,6 +102,11 @@ from .widgets import (
 )
 
 
+def is_document_page_enabled(context):
+    return context['object'].enabled
+
+
+
 class DocumentsApp(MayanAppConfig):
     app_namespace = 'documents'
     app_url = 'documents'
@@ -214,12 +221,21 @@ class DocumentsApp(MayanAppConfig):
         ModelPermission.register_inheritance(
             model=Document, related='document_type',
         )
+        ModelPermission.register_manager(
+            model=Document, manager_name='passthrough'
+        )
         ModelPermission.register_inheritance(
             model=DocumentPage, related='document_version__document',
         )
+        ModelPermission.register_manager(
+            model=DocumentPage, manager_name='passthrough'
+        )
         ModelPermission.register_inheritance(
             model=DocumentPageResult, related='document_version__document',
         )
+        ModelPermission.register_manager(
+            model=DocumentPageResult, manager_name='passthrough'
+        )
         ModelPermission.register_inheritance(
             model=DocumentTypeFilename, related='document_type',
         )
@@ -262,13 +278,21 @@ class DocumentsApp(MayanAppConfig):
         # DocumentPage
         SourceColumn(
             attribute='get_label', is_identifier=True,
-            is_object_absolute_url=True, source=DocumentPage
+            is_object_absolute_url=True, source=DocumentPage,
+            widget_condition=is_document_page_enabled
         )
         SourceColumn(
             func=lambda context: document_page_thumbnail_widget.render(
                 instance=context['object']
             ), label=_('Thumbnail'), source=DocumentPage
         )
+        SourceColumn(
+            attribute='enabled', include_label=True, source=DocumentPage,
+            widget=TwoStateWidget
+        )
+        SourceColumn(
+            attribute='page_number', include_label=True, source=DocumentPage
+        )
 
         SourceColumn(
             attribute='get_label', is_identifier=True,
@@ -377,7 +401,7 @@ class DocumentsApp(MayanAppConfig):
 
         menu_setup.bind_links(links=(link_document_type_setup,))
         menu_tools.bind_links(
-            links=(link_clear_image_cache, link_duplicated_document_scan)
+            links=(link_duplicated_document_scan,)
         )
 
         # Document type links
@@ -503,6 +527,16 @@ class DocumentsApp(MayanAppConfig):
                 link_document_page_navigation_last
             ), sources=(DocumentPage,)
         )
+        menu_multi_item.bind_links(
+            links=(
+                link_document_page_multiple_disable,
+                link_document_page_multiple_enable
+            ), sources=(DocumentPage,)
+        )
+        menu_object.bind_links(
+            links=(link_document_page_disable, link_document_page_enable),
+            sources=(DocumentPage,)
+        )
         menu_list_facet.bind_links(
             links=(link_transformation_list,), sources=(DocumentPage,)
         )
@@ -527,6 +561,10 @@ class DocumentsApp(MayanAppConfig):
             dispatch_uid='handler_create_default_document_type',
             receiver=handler_create_default_document_type
         )
+        post_migrate.connect(
+            dispatch_uid='documents_handler_create_document_cache',
+            receiver=handler_create_document_cache,
+        )
         post_version_upload.connect(
             dispatch_uid='handler_scan_duplicates_for',
             receiver=handler_scan_duplicates_for

mayan/apps/documents/handlers.py

@@ -1,8 +1,13 @@
 from __future__ import unicode_literals
 
 from django.apps import apps
+from django.utils.translation import ugettext_lazy as _
 
-from .literals import DEFAULT_DOCUMENT_TYPE_LABEL
+from .literals import (
+    DEFAULT_DOCUMENT_TYPE_LABEL, DOCUMENT_CACHE_STORAGE_INSTANCE_PATH,
+    DOCUMENT_IMAGES_CACHE_NAME
+)
+from .settings import setting_document_cache_maximum_size
 from .signals import post_initial_document_type
 from .tasks import task_clean_empty_duplicate_lists, task_scan_duplicates_for
 
@@ -21,6 +26,17 @@ def handler_create_default_document_type(sender, **kwargs):
         )
 
 
+def handler_create_document_cache(sender, **kwargs):
+    Cache = apps.get_model(app_label='file_caching', model_name='Cache')
+    Cache.objects.update_or_create(
+        defaults={
+            'label': _('Document images'),
+            'storage_instance_path': DOCUMENT_CACHE_STORAGE_INSTANCE_PATH,
+            'maximum_size': setting_document_cache_maximum_size.value,
+        }, name=DOCUMENT_IMAGES_CACHE_NAME,
+    )
+
+
 def handler_scan_duplicates_for(sender, instance, **kwargs):
     task_scan_duplicates_for.apply_async(
         kwargs={'document_id': instance.document.pk}

mayan/apps/documents/icons.py

@@ -12,8 +12,6 @@ icon_document_type = Icon(
 
 icon_menu_documents = Icon(driver_name='fontawesome', symbol='book')
 
-icon_clear_image_cache = Icon(driver_name='fontawesome', symbol='file-image')
-
 icon_dashboard_document_types = icon_document_type
 icon_dashboard_documents_in_trash = Icon(
     driver_name='fontawesome', symbol='trash-alt'
@@ -27,8 +25,6 @@ icon_dashboard_new_documents_this_month = Icon(
 icon_dashboard_total_document = Icon(
     driver_name='fontawesome', symbol='book'
 )
-
-
 icon_document_quick_download = Icon(
     driver_name='fontawesome', symbol='download'
 )
@@ -106,6 +102,14 @@ icon_favorite_document_remove = Icon(
     secondary_symbol='minus'
 )
 
+# Document pages
+
+icon_document_page_disable = Icon(
+    driver_name='fontawesomecss', css_classes='far fa-eye-slash'
+)
+icon_document_page_enable = Icon(
+    driver_name='fontawesomecss', css_classes='far fa-eye'
+)
 icon_document_page_navigation_first = Icon(
     driver_name='fontawesome', symbol='step-backward'
 )

mayan/apps/documents/links.py

@@ -8,7 +8,7 @@ from mayan.apps.converter.permissions import (
 from mayan.apps.navigation.classes import Link
 
 from .icons import (
-    icon_clear_image_cache, icon_document_list_recent_access,
+    icon_document_list_recent_access,
     icon_recent_added_document_list, icon_document_page_navigation_first,
     icon_document_page_navigation_last, icon_document_page_navigation_next,
     icon_document_page_navigation_previous, icon_document_page_return,
@@ -19,14 +19,14 @@ from .icons import (
     icon_duplicated_document_list, icon_duplicated_document_scan
 )
 from .permissions import (
-    permission_document_delete, permission_document_download,
-    permission_document_properties_edit, permission_document_print,
-    permission_document_restore, permission_document_tools,
-    permission_document_version_revert, permission_document_view,
-    permission_document_trash, permission_document_type_create,
-    permission_document_type_delete, permission_document_type_edit,
-    permission_document_type_view, permission_empty_trash,
-    permission_document_version_view
+    permission_document_delete, permission_document_edit,
+    permission_document_download, permission_document_properties_edit,
+    permission_document_print, permission_document_restore,
+    permission_document_tools, permission_document_version_revert,
+    permission_document_view, permission_document_trash,
+    permission_document_type_create, permission_document_type_delete,
+    permission_document_type_edit, permission_document_type_view,
+    permission_empty_trash, permission_document_version_view
 )
 from .settings import setting_zoom_max_level, setting_zoom_min_level
 
@@ -43,11 +43,11 @@ def is_not_current_version(context):
 
 
 def is_first_page(context):
-    return context['resolved_object'].page_number <= 1
+    return context['resolved_object'].siblings.first() == context['resolved_object']
 
 
 def is_last_page(context):
-    return context['resolved_object'].page_number >= context['resolved_object'].document_version.pages.count()
+    return context['resolved_object'].siblings.last() == context['resolved_object']
 
 
 def is_max_zoom(context):
@@ -58,6 +58,14 @@ def is_min_zoom(context):
     return context['zoom'] <= setting_zoom_min_level.value
 
 
+def is_document_page_enabled(context):
+    return context['resolved_object'].enabled
+
+
+def is_document_page_disabled(context):
+    return not context['resolved_object'].enabled
+
+
 # Facet
 link_document_preview = Link(
     args='resolved_object.id',
@@ -264,22 +272,37 @@ link_document_list_deleted = Link(
     text=_('Trash can'), view='documents:document_list_deleted'
 )
 
-# Tools
-link_clear_image_cache = Link(
-    icon_class=icon_clear_image_cache,
-    description=_(
-        'Clear the graphics representations used to speed up the documents\' '
-        'display and interactive transformations results.'
-    ), permissions=(permission_document_tools,),
-    text=_('Clear document image cache'),
-    view='documents:document_clear_image_cache'
-)
 link_trash_can_empty = Link(
     permissions=(permission_empty_trash,), text=_('Empty trash'),
     view='documents:trash_can_empty'
 )
 
 # Document pages
+
+link_document_page_disable = Link(
+    condition=is_document_page_enabled,
+    icon_class_path='mayan.apps.documents.icons.icon_document_page_disable',
+    kwargs={'pk': 'resolved_object.id'},
+    permissions=(permission_document_edit,), text=_('Disable page'),
+    view='documents:document_page_disable'
+)
+link_document_page_multiple_disable = Link(
+    icon_class_path='mayan.apps.documents.icons.icon_document_page_disable',
+    text=_('Disable pages'),
+    view='documents:document_page_multiple_disable'
+)
+link_document_page_enable = Link(
+    condition=is_document_page_disabled,
+    icon_class_path='mayan.apps.documents.icons.icon_document_page_enable',
+    kwargs={'pk': 'resolved_object.id'},
+    permissions=(permission_document_edit,), text=_('Enable page'),
+    view='documents:document_page_enable'
+)
+link_document_page_multiple_enable = Link(
+    icon_class_path='mayan.apps.documents.icons.icon_document_page_enable',
+    text=_('Enable pages'),
+    view='documents:document_page_multiple_enable'
+)
 link_document_page_navigation_first = Link(
     args='resolved_object.pk', conditional_disable=is_first_page,
     icon_class=icon_document_page_navigation_first,
@@ -323,6 +346,7 @@ link_document_page_rotate_right = Link(
     text=_('Rotate right'), view='documents:document_page_rotate_right',
 )
 link_document_page_view = Link(
+    conditional_disable=is_document_page_disabled,
     icon_class_path='mayan.apps.documents.icons.icon_document_page_view',
     permissions=(permission_document_view,), text=_('Page image'),
     view='documents:document_page_view', args='resolved_object.pk'

mayan/apps/documents/literals.py

@@ -9,6 +9,7 @@ CHECK_TRASH_PERIOD_INTERVAL = 60
 DELETE_STALE_STUBS_INTERVAL = 60 * 10  # 10 minutes
 DEFAULT_DELETE_PERIOD = 30
 DEFAULT_DELETE_TIME_UNIT = TIME_DELTA_UNIT_DAYS
+DEFAULT_DOCUMENTS_CACHE_MAXIMUM_SIZE = 500 * 2 ** 20  # 500 Megabytes
 DEFAULT_DOCUMENTS_HASH_BLOCK_SIZE = 65535
 DEFAULT_LANGUAGE = 'eng'
 DEFAULT_LANGUAGE_CODES = (
@@ -30,8 +31,11 @@ DEFAULT_LANGUAGE_CODES = (
 DEFAULT_ZIP_FILENAME = 'document_bundle.zip'
 DEFAULT_DOCUMENT_TYPE_LABEL = _('Default')
 DOCUMENT_IMAGE_TASK_TIMEOUT = 120
+DOCUMENT_IMAGES_CACHE_NAME = 'document_images'
+DOCUMENT_CACHE_STORAGE_INSTANCE_PATH = 'mayan.apps.documents.storages.storage_documentimagecache'
 STUB_EXPIRATION_INTERVAL = 60 * 60 * 24  # 24 hours
 UPDATE_PAGE_COUNT_RETRY_DELAY = 10
+UPLOAD_NEW_DOCUMENT_RETRY_DELAY = 10
 UPLOAD_NEW_VERSION_RETRY_DELAY = 10
 
 PAGE_RANGE_ALL = 'all'

mayan/apps/documents/managers.py

@@ -22,28 +22,9 @@ class DocumentManager(models.Manager):
 
     def get_queryset(self):
         return TrashCanQuerySet(
-            self.model, using=self._db
+            model=self.model, using=self._db
         ).filter(in_trash=False).filter(is_stub=False)
 
-    def invalidate_cache(self):
-        for document in self.model.objects.all():
-            document.invalidate_cache()
-
-
-class DocumentPageCachedImage(models.Manager):
-    def get_by_natural_key(self, filename, document_page_natural_key):
-        DocumentPage = apps.get_model(
-            app_label='documents', model_name='DocumentPage'
-        )
-        try:
-            document_page = DocumentPage.objects.get_by_natural_key(
-                *document_page_natural_key
-            )
-        except DocumentPage.DoesNotExist:
-            raise self.model.DoesNotExist
-
-        return self.get(document_page__pk=document_page.pk, filename=filename)
-
 
 class DocumentPageManager(models.Manager):
     def get_by_natural_key(self, page_number, document_version_natural_key):
@@ -57,6 +38,11 @@ class DocumentPageManager(models.Manager):
 
         return self.get(document_version__pk=document_version.pk, page_number=page_number)
 
+    def get_queryset(self):
+        return models.QuerySet(
+            model=self.model, using=self._db
+        ).filter(enabled=True)
+
 
 class DocumentTypeManager(models.Manager):
     def check_delete_periods(self):

mayan/apps/documents/migrations/0049_auto_20190715_0454.py

@@ -0,0 +1,37 @@
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+from ..storages import storage_documentimagecache
+
+
+def operation_clear_old_cache(apps, schema_editor):
+    DocumentPageCachedImage = apps.get_model(
+        'documents', 'DocumentPageCachedImage'
+    )
+
+    for cached_image in DocumentPageCachedImage.objects.using(schema_editor.connection.alias).all():
+        # Delete each cached image directly since the model doesn't exists and
+        # will not trigger the physical deletion of the stored file
+        storage_documentimagecache.delete(cached_image.filename)
+        cached_image.delete()
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('documents', '0048_auto_20190711_0544'),
+    ]
+
+    operations = [
+        migrations.RunPython(
+            code=operation_clear_old_cache,
+            reverse_code=migrations.RunPython.noop
+        ),
+        migrations.RemoveField(
+            model_name='documentpagecachedimage',
+            name='document_page',
+        ),
+        migrations.DeleteModel(
+            name='DocumentPageCachedImage',
+        ),
+    ]

mayan/apps/documents/migrations/0050_auto_20190725_0451.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.20 on 2019-07-25 04:51
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('documents', '0049_auto_20190715_0454'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='document',
+            name='language',
+            field=models.CharField(blank=True, default='eng', help_text='The dominant language in the document.', max_length=8, verbose_name='Language'),
+        ),
+    ]

mayan/apps/documents/migrations/0051_documentpage_enabled.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.22 on 2019-07-29 07:03
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('documents', '0050_auto_20190725_0451'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='documentpage',
+            name='enabled',
+            field=models.BooleanField(default=True, verbose_name='Enabled'),
+        ),
+    ]

mayan/apps/documents/models/document_models.py

@@ -136,10 +136,6 @@ class Document(models.Model):
         if latest_version:
             return latest_version.get_api_image_url(*args, **kwargs)
 
-    def invalidate_cache(self):
-        for document_version in self.versions.all():
-            document_version.invalidate_cache()
-
     @property
     def is_in_trash(self):
         return self.in_trash
@@ -240,6 +236,18 @@ class Document(models.Model):
     def page_count(self):
         return self.latest_version.page_count
 
+    @property
+    def pages_all(self):
+        try:
+            return self.latest_version.pages_all
+        except AttributeError:
+            # Document has no version yet
+            DocumentPage = apps.get_model(
+                app_label='documents', model_name='DocumentPage'
+            )
+
+            return DocumentPage.objects.none()
+
     @property
     def pages(self):
         try:

mayan/apps/documents/models/document_page_models.py

@@ -4,13 +4,14 @@ import logging
 
 from furl import furl
 
-from django.core.files.base import ContentFile
 from django.db import models
 from django.urls import reverse
 from django.utils.encoding import force_text, python_2_unicode_compatible
+from django.utils.functional import cached_property
 from django.utils.translation import ugettext_lazy as _
 
 from mayan.apps.converter.literals import DEFAULT_ZOOM_LEVEL, DEFAULT_ROTATION
+
 from mayan.apps.converter.models import Transformation
 from mayan.apps.converter.transformations import (
     BaseTransformation, TransformationResize, TransformationRotate,
@@ -18,17 +19,16 @@ from mayan.apps.converter.transformations import (
 )
 from mayan.apps.converter.utils import get_converter_class
 
-from ..managers import DocumentPageCachedImage, DocumentPageManager
+from ..managers import DocumentPageManager
 from ..settings import (
     setting_disable_base_image_cache, setting_disable_transformed_image_cache,
     setting_display_width, setting_display_height, setting_zoom_max_level,
     setting_zoom_min_level
 )
-from ..storages import storage_documentimagecache
 
 from .document_version_models import DocumentVersion
 
-__all__ = ('DocumentPage', 'DocumentPageCachedImage', 'DocumentPageResult')
+__all__ = ('DocumentPage', 'DocumentPageResult')
 logger = logging.getLogger(__name__)
 
 
@@ -38,15 +38,17 @@ class DocumentPage(models.Model):
     Model that describes a document version page
     """
     document_version = models.ForeignKey(
-        on_delete=models.CASCADE, related_name='pages', to=DocumentVersion,
+        on_delete=models.CASCADE, related_name='version_pages', to=DocumentVersion,
         verbose_name=_('Document version')
     )
+    enabled = models.BooleanField(default=True, verbose_name=_('Enabled'))
     page_number = models.PositiveIntegerField(
         db_index=True, default=1, editable=False,
         verbose_name=_('Page number')
     )
 
     objects = DocumentPageManager()
+    passthrough = models.Manager()
 
     class Meta:
         ordering = ('page_number',)
@@ -56,12 +58,15 @@ class DocumentPage(models.Model):
     def __str__(self):
         return self.get_label()
 
-    @property
-    def cache_filename(self):
-        return 'page-cache-{}'.format(self.uuid)
+    @cached_property
+    def cache_partition(self):
+        partition, created = self.document_version.cache.partitions.get_or_create(
+            name=self.uuid
+        )
+        return partition
 
     def delete(self, *args, **kwargs):
-        self.invalidate_cache()
+        self.cache_partition.delete()
         super(DocumentPage, self).delete(*args, **kwargs)
 
     def detect_orientation(self):
@@ -80,29 +85,24 @@ class DocumentPage(models.Model):
 
     def generate_image(self, *args, **kwargs):
         transformation_list = self.get_combined_transformation_list(*args, **kwargs)
-
-        cache_filename = '{}-{}'.format(
-            self.cache_filename, BaseTransformation.combine(transformation_list)
-        )
+        combined_cache_filename = BaseTransformation.combine(transformation_list)
 
         # Check is transformed image is available
-        logger.debug('transformations cache filename: %s', cache_filename)
+        logger.debug('transformations cache filename: %s', combined_cache_filename)
 
-        if not setting_disable_transformed_image_cache.value and storage_documentimagecache.exists(cache_filename):
+        if not setting_disable_transformed_image_cache.value and self.cache_partition.get_file(filename=combined_cache_filename):
             logger.debug(
-                'transformations cache file "%s" found', cache_filename
+                'transformations cache file "%s" found', combined_cache_filename
             )
         else:
             logger.debug(
-                'transformations cache file "%s" not found', cache_filename
+                'transformations cache file "%s" not found', combined_cache_filename
             )
             image = self.get_image(transformations=transformation_list)
-            with storage_documentimagecache.open(cache_filename, 'wb+') as file_object:
+            with self.cache_partition.create_file(filename=combined_cache_filename) as file_object:
                 file_object.write(image.getvalue())
 
-            self.cached_images.create(filename=cache_filename)
-
-        return cache_filename
+        return combined_cache_filename
 
     def get_absolute_url(self):
         return reverse(
@@ -159,7 +159,6 @@ class DocumentPage(models.Model):
             zoom_level = setting_zoom_max_level.value
 
         # Generate transformation hash
-
         transformation_list = []
 
         # Stored transformations first
@@ -186,13 +185,15 @@ class DocumentPage(models.Model):
         return transformation_list
 
     def get_image(self, transformations=None):
-        cache_filename = self.cache_filename
+        cache_filename = 'base_image'
         logger.debug('Page cache filename: %s', cache_filename)
 
-        if not setting_disable_base_image_cache.value and storage_documentimagecache.exists(cache_filename):
+        cache_file = self.cache_partition.get_file(filename=cache_filename)
+
+        if not setting_disable_base_image_cache.value and cache_file:
             logger.debug('Page cache file "%s" found', cache_filename)
 
-            with storage_documentimagecache.open(cache_filename) as file_object:
+            with cache_file.open() as file_object:
                 converter = get_converter_class()(
                     file_object=file_object
                 )
@@ -200,8 +201,8 @@ class DocumentPage(models.Model):
                 converter.seek_page(page_number=0)
 
                 # This code is also repeated below to allow using a context
-                # manager with storage_documentimagecache.open and close it
-                # automatically.
+                # manager with cache_file.open and close it automatically.
+                # Apply runtime transformations
                 for transformation in transformations:
                     converter.transform(transformation=transformation)
 
@@ -218,14 +219,11 @@ class DocumentPage(models.Model):
 
                     page_image = converter.get_page()
 
-                    # Since open "wb+" doesn't create files, check if the file
-                    # exists, if not then create it
-                    if not storage_documentimagecache.exists(cache_filename):
-                        storage_documentimagecache.save(name=cache_filename, content=ContentFile(content=''))
-
-                    with storage_documentimagecache.open(cache_filename, 'wb+') as file_object:
+                    # Since open "wb+" doesn't create files, create it explicitly
+                    with self.cache_partition.create_file(filename=cache_filename) as file_object:
                         file_object.write(page_image.getvalue())
 
+                    # Apply runtime transformations
                     for transformation in transformations:
                         converter.transform(transformation=transformation)
 
@@ -236,14 +234,8 @@ class DocumentPage(models.Model):
                     'Error creating page cache file "%s"; %s',
                     cache_filename, exception
                 )
-                storage_documentimagecache.delete(cache_filename)
                 raise
 
-    def invalidate_cache(self):
-        storage_documentimagecache.delete(self.cache_filename)
-        for cached_image in self.cached_images.all():
-            cached_image.delete()
-
     @property
     def is_in_trash(self):
         return self.document.is_in_trash
@@ -254,7 +246,7 @@ class DocumentPage(models.Model):
         ) % {
             'document': force_text(self.document),
             'page_num': self.page_number,
-            'total_pages': self.document_version.pages.count()
+            'total_pages': self.document_version.pages_all.count()
         }
     get_label.short_description = _('Label')
 
@@ -277,38 +269,6 @@ class DocumentPage(models.Model):
         return '{}-{}'.format(self.document_version.uuid, self.pk)
 
 
-class DocumentPageCachedImage(models.Model):
-    document_page = models.ForeignKey(
-        on_delete=models.CASCADE, related_name='cached_images',
-        to=DocumentPage, verbose_name=_('Document page')
-    )
-    datetime = models.DateTimeField(
-        auto_now_add=True, db_index=True, verbose_name=_('Date time')
-    )
-    filename = models.CharField(max_length=128, verbose_name=_('Filename'))
-    file_size = models.PositiveIntegerField(
-        db_index=True, default=0, verbose_name=_('File size')
-    )
-
-    objects = DocumentPageCachedImage()
-
-    class Meta:
-        verbose_name = _('Document page cached image')
-        verbose_name_plural = _('Document page cached images')
-
-    def delete(self, *args, **kwargs):
-        storage_documentimagecache.delete(self.filename)
-        return super(DocumentPageCachedImage, self).delete(*args, **kwargs)
-
-    def natural_key(self):
-        return (self.filename, self.document_page.natural_key())
-    natural_key.dependencies = ['documents.DocumentPage']
-
-    def save(self, *args, **kwargs):
-        self.file_size = storage_documentimagecache.size(self.filename)
-        return super(DocumentPageCachedImage, self).save(*args, **kwargs)
-
-
 class DocumentPageResult(DocumentPage):
     class Meta:
         ordering = ('document_version__document', 'page_number')

mayan/apps/documents/models/document_version_models.py

@@ -7,11 +7,11 @@ import shutil
 import uuid
 
 from django.apps import apps
-from django.core.files.base import ContentFile
 from django.db import models, transaction
 from django.template import Template, Context
 from django.urls import reverse
 from django.utils.encoding import force_text, python_2_unicode_compatible
+from django.utils.functional import cached_property
 from django.utils.translation import ugettext_lazy as _
 
 from mayan.apps.converter.exceptions import InvalidOfficeFormat, PageCountError
@@ -21,10 +21,11 @@ from mayan.apps.converter.utils import get_converter_class
 from mayan.apps.mimetype.api import get_mimetype
 
 from ..events import event_document_new_version, event_document_version_revert
+from ..literals import DOCUMENT_IMAGES_CACHE_NAME
 from ..managers import DocumentVersionManager
 from ..settings import setting_fix_orientation, setting_hash_block_size
 from ..signals import post_document_created, post_version_upload
-from ..storages import storage_documentversion, storage_documentimagecache
+from ..storages import storage_documentversion
 
 from .document_models import Document
 
@@ -61,14 +62,6 @@ class DocumentVersion(models.Model):
     _pre_open_hooks = {}
     _post_save_hooks = {}
 
-    @classmethod
-    def register_pre_open_hook(cls, order, func):
-        cls._pre_open_hooks[order] = func
-
-    @classmethod
-    def register_post_save_hook(cls, order, func):
-        cls._post_save_hooks[order] = func
-
     document = models.ForeignKey(
         on_delete=models.CASCADE, related_name='versions', to=Document,
         verbose_name=_('Document')
@@ -118,18 +111,35 @@ class DocumentVersion(models.Model):
 
     objects = DocumentVersionManager()
 
+    @classmethod
+    def register_pre_open_hook(cls, order, func):
+        cls._pre_open_hooks[order] = func
+
+    @classmethod
+    def register_post_save_hook(cls, order, func):
+        cls._post_save_hooks[order] = func
+
     def __str__(self):
         return self.get_rendered_string()
 
-    @property
-    def cache_filename(self):
-        return 'document-version-{}'.format(self.uuid)
+    @cached_property
+    def cache(self):
+        Cache = apps.get_model(app_label='file_caching', model_name='Cache')
+        return Cache.objects.get(name=DOCUMENT_IMAGES_CACHE_NAME)
+
+    @cached_property
+    def cache_partition(self):
+        partition, created = self.cache.partitions.get_or_create(
+            name='version-{}'.format(self.uuid)
+        )
+        return partition
 
     def delete(self, *args, **kwargs):
         for page in self.pages.all():
             page.delete()
 
         self.file.storage.delete(self.file.name)
+        self.cache_partition.delete()
 
         return super(DocumentVersion, self).delete(*args, **kwargs)
 
@@ -164,43 +174,36 @@ class DocumentVersion(models.Model):
             return first_page.get_api_image_url(*args, **kwargs)
 
     def get_intermediate_file(self):
-        cache_filename = self.cache_filename
-        logger.debug('Intermidiate filename: %s', cache_filename)
-
-        if storage_documentimagecache.exists(cache_filename):
-            logger.debug('Intermidiate file "%s" found.', cache_filename)
-
-            return storage_documentimagecache.open(cache_filename)
+        cache_filename = 'intermediate_file'
+        cache_file = self.cache_partition.get_file(filename=cache_filename)
+        if cache_file:
+            logger.debug('Intermidiate file found.')
+            return cache_file.open()
         else:
-            logger.debug('Intermidiate file "%s" not found.', cache_filename)
+            logger.debug('Intermidiate file not found.')
 
             try:
                 with self.open() as version_file_object:
-                    converter = get_converter_class()(file_object=version_file_object)
-                    with converter.to_pdf() as pdf_file_object:
-
-                        # Since open "wb+" doesn't create files, check if the file
-                        # exists, if not then create it
-                        if not storage_documentimagecache.exists(cache_filename):
-                            storage_documentimagecache.save(
-                                name=cache_filename, content=ContentFile(content='')
+                    converter = get_converter_class()(
+                        file_object=version_file_object
                     )
-
-                        with storage_documentimagecache.open(cache_filename, mode='wb+') as file_object:
+                    with converter.to_pdf() as pdf_file_object:
+                        with self.cache_partition.create_file(filename=cache_filename) as file_object:
                             shutil.copyfileobj(
                                 fsrc=pdf_file_object, fdst=file_object
                             )
 
-                    return storage_documentimagecache.open(cache_filename)
+                        return self.cache_partition.get_file(filename=cache_filename).open()
             except InvalidOfficeFormat:
                 return self.open()
             except Exception as exception:
-                # Cleanup in case of error
                 logger.error(
                     'Error creating intermediate file "%s"; %s.',
                     cache_filename, exception
                 )
-                storage_documentimagecache.delete(cache_filename)
+                cache_file = self.cache_partition.get_file(filename=cache_filename)
+                if cache_file:
+                    cache_file.delete()
                 raise
 
     def get_rendered_string(self, preserve_extension=False):
@@ -223,11 +226,6 @@ class DocumentVersion(models.Model):
         return (self.checksum, self.document.natural_key())
     natural_key.dependencies = ['documents.Document']
 
-    def invalidate_cache(self):
-        storage_documentimagecache.delete(self.cache_filename)
-        for page in self.pages.all():
-            page.invalidate_cache()
-
     @property
     def is_in_trash(self):
         return self.document.is_in_trash
@@ -248,6 +246,17 @@ class DocumentVersion(models.Model):
 
             return result
 
+    @property
+    def pages_all(self):
+        DocumentPage = apps.get_model(
+            app_label='documents', model_name='DocumentPage'
+        )
+        return DocumentPage.passthrough.filter(document_version=self)
+
+    @property
+    def pages(self):
+        return self.version_pages.all()
+
     @property
     def page_count(self):
         """

mayan/apps/documents/queues.py

@@ -61,10 +61,6 @@ queue_documents_periodic.add_task_type(
     schedule=timedelta(seconds=DELETE_STALE_STUBS_INTERVAL),
 )
 
-queue_tools.add_task_type(
-    dotted_path='mayan.apps.documents.tasks.task_clear_image_cache',
-    label=_('Clear image cache')
-)
 queue_tools.add_task_type(
     dotted_path='mayan.apps.documents.tasks.task_scan_duplicates_all',
     label=_('Duplicated document scan')
@@ -82,3 +78,7 @@ queue_uploads.add_task_type(
     dotted_path='mayan.apps.documents.tasks.task_scan_duplicates_for',
     label=_('Scan document duplicates')
 )
+queue_uploads.add_task_type(
+    dotted_path='mayan.apps.documents.tasks.task_upload_new_document',
+    label=_('Upload new document')
+)

mayan/apps/documents/settings.py

@@ -8,11 +8,22 @@ from django.utils.translation import ugettext_lazy as _
 from mayan.apps.smart_settings.classes import Namespace
 
 from .literals import (
-    DEFAULT_DOCUMENTS_HASH_BLOCK_SIZE, DEFAULT_LANGUAGE, DEFAULT_LANGUAGE_CODES
+    DEFAULT_DOCUMENTS_CACHE_MAXIMUM_SIZE, DEFAULT_DOCUMENTS_HASH_BLOCK_SIZE,
+    DEFAULT_LANGUAGE, DEFAULT_LANGUAGE_CODES
 )
+from .utils import callback_update_cache_size
 
 namespace = Namespace(label=_('Documents'), name='documents')
 
+setting_document_cache_maximum_size = namespace.add_setting(
+    global_name='DOCUMENTS_CACHE_MAXIMUM_SIZE',
+    default=DEFAULT_DOCUMENTS_CACHE_MAXIMUM_SIZE,
+    help_text=_(
+        'The threshold at which the DOCUMENT_CACHE_STORAGE_BACKEND will start '
+        'deleting the oldest document image cache files. Specify the size in '
+        'bytes.'
+    ), post_edit_function=callback_update_cache_size
+)
 setting_documentimagecache_storage = namespace.add_setting(
     global_name='DOCUMENTS_CACHE_STORAGE_BACKEND',
     default='django.core.files.storage.FileSystemStorage', help_text=_(

mayan/apps/documents/tasks.py

@@ -9,7 +9,8 @@ from django.db import OperationalError
 from mayan.celery import app
 
 from .literals import (
-    UPDATE_PAGE_COUNT_RETRY_DELAY, UPLOAD_NEW_VERSION_RETRY_DELAY
+    UPDATE_PAGE_COUNT_RETRY_DELAY, UPLOAD_NEW_DOCUMENT_RETRY_DELAY,
+    UPLOAD_NEW_VERSION_RETRY_DELAY
 )
 
 logger = logging.getLogger(__name__)
@@ -41,17 +42,6 @@ def task_check_trash_periods():
     DocumentType.objects.check_trash_periods()
 
 
-@app.task(ignore_result=True)
-def task_clear_image_cache():
-    Document = apps.get_model(
-        app_label='documents', model_name='Document'
-    )
-
-    logger.info('Starting document cache invalidation')
-    Document.objects.invalidate_cache()
-    logger.info('Finished document cache invalidation')
-
-
 @app.task(ignore_result=True)
 def task_delete_document(trashed_document_id):
     DeletedDocument = apps.get_model(
@@ -81,8 +71,7 @@ def task_generate_document_page_image(document_page_id, *args, **kwargs):
         app_label='documents', model_name='DocumentPage'
     )
 
-    document_page = DocumentPage.objects.get(pk=document_page_id)
-
+    document_page = DocumentPage.passthrough.get(pk=document_page_id)
     return document_page.generate_image(*args, **kwargs)
 
 
@@ -127,6 +116,60 @@ def task_update_page_count(self, version_id):
         raise self.retry(exc=exception)
 
 
+@app.task(bind=True, default_retry_delay=UPLOAD_NEW_DOCUMENT_RETRY_DELAY, ignore_result=True)
+def task_upload_new_document(self, document_type_id, shared_uploaded_file_id):
+    DocumentType = apps.get_model(
+        app_label='documents', model_name='DocumentType'
+    )
+
+    SharedUploadedFile = apps.get_model(
+        app_label='common', model_name='SharedUploadedFile'
+    )
+
+    try:
+        document_type = DocumentType.objects.get(pk=document_type_id)
+        shared_file = SharedUploadedFile.objects.get(
+            pk=shared_uploaded_file_id
+        )
+    except OperationalError as exception:
+        logger.warning(
+            'Operational error during attempt to retrieve shared data for '
+            'new document of type: %s; %s. Retrying.', document_type, exception
+        )
+        raise self.retry(exc=exception)
+
+    try:
+        with shared_file.open() as file_object:
+            document_type.new_document(file_object=file_object)
+    except OperationalError as exception:
+        logger.warning(
+            'Operational error during attempt to create new document '
+            'of type: %s; %s. Retrying.', document_type, exception
+        )
+        raise self.retry(exc=exception)
+    except Exception as exception:
+        # This except and else block emulate a finally:
+        logger.error(
+            'Unexpected error during attempt to create new document '
+            'of type: %s; %s', document_type, exception
+        )
+        try:
+            shared_file.delete()
+        except OperationalError as exception:
+            logger.warning(
+                'Operational error during attempt to delete shared '
+                'file: %s; %s.', shared_file, exception
+            )
+    else:
+        try:
+            shared_file.delete()
+        except OperationalError as exception:
+            logger.warning(
+                'Operational error during attempt to delete shared '
+                'file: %s; %s.', shared_file, exception
+            )
+
+
 @app.task(bind=True, default_retry_delay=UPLOAD_NEW_VERSION_RETRY_DELAY, ignore_result=True)
 def task_upload_new_version(self, document_id, shared_uploaded_file_id, user_id, comment=None):
     SharedUploadedFile = apps.get_model(

mayan/apps/documents/tests/test_api.py

@@ -154,11 +154,11 @@ class DocumentAPITestCase(DocumentTestMixin, BaseAPITestCase):
     auto_upload_document = False
 
     def _request_document_upload(self):
-        with open(TEST_DOCUMENT_PATH, mode='rb') as file_descriptor:
+        with open(TEST_DOCUMENT_PATH, mode='rb') as file_object:
             return self.post(
                 viewname='rest_api:document-list', data={
                     'document_type': self.test_document_type.pk,
-                    'file': file_descriptor
+                    'file': file_object
                 }
             )
 
@@ -208,12 +208,12 @@ class DocumentAPITestCase(DocumentTestMixin, BaseAPITestCase):
         # is the latest.
         time.sleep(1)
 
-        with open(TEST_DOCUMENT_PATH, mode='rb') as file_descriptor:
+        with open(TEST_DOCUMENT_PATH, mode='rb') as file_object:
             return self.post(
                 viewname='rest_api:document-version-list', kwargs={
                     'pk': self.test_document.pk,
                 }, data={
-                    'comment': '', 'file': file_descriptor,
+                    'comment': '', 'file': file_object,
                 }
             )
 

mayan/apps/documents/tests/test_document_page_views.py

@@ -2,11 +2,152 @@ from __future__ import unicode_literals
 
 from django.utils.encoding import force_text
 
-from ..permissions import permission_document_view
+from ..permissions import (
+    permission_document_edit, permission_document_view
+)
 
 from .base import GenericDocumentViewTestCase
 
 
+class DocumentPageDisableViewTestCase(GenericDocumentViewTestCase):
+    def setUp(self):
+        super(DocumentPageDisableViewTestCase, self).setUp()
+        self.test_document_page = self.test_document.pages_all.first()
+
+    def _request_test_document_page_disable_view(self):
+        return self.post(
+            viewname='documents:document_page_disable', kwargs={
+                'pk': self.test_document_page.pk
+            }
+        )
+
+    def test_document_page_disable_view_no_permission(self):
+        test_document_page_count = self.test_document.pages.count()
+
+        response = self._request_test_document_page_disable_view()
+        self.assertEqual(response.status_code, 404)
+
+        self.assertEqual(
+            test_document_page_count, self.test_document.pages.count()
+        )
+
+    def test_document_page_disable_view_with_access(self):
+        self.grant_access(
+            obj=self.test_document, permission=permission_document_edit
+        )
+
+        test_document_page_count = self.test_document.pages.count()
+
+        response = self._request_test_document_page_disable_view()
+        self.assertEqual(response.status_code, 302)
+
+        self.assertNotEqual(
+            test_document_page_count, self.test_document.pages.count()
+        )
+
+    def _request_test_document_page_multiple_disable_view(self):
+        return self.post(
+            viewname='documents:document_page_multiple_disable', data={
+                'id_list': self.test_document_page.pk
+            }
+        )
+
+    def test_document_page_multiple_disable_view_no_permission(self):
+        test_document_page_count = self.test_document.pages.count()
+
+        response = self._request_test_document_page_multiple_disable_view()
+        self.assertEqual(response.status_code, 404)
+
+        self.assertEqual(
+            test_document_page_count, self.test_document.pages.count()
+        )
+
+    def test_document_page_multiple_disable_view_with_access(self):
+        self.grant_access(
+            obj=self.test_document, permission=permission_document_edit
+        )
+
+        test_document_page_count = self.test_document.pages.count()
+
+        response = self._request_test_document_page_multiple_disable_view()
+        self.assertEqual(response.status_code, 302)
+
+        self.assertNotEqual(
+            test_document_page_count, self.test_document.pages.count()
+        )
+
+    def _disable_test_document_page(self):
+        self.test_document_page.enabled = False
+        self.test_document_page.save()
+
+    def _request_test_document_page_enable_view(self):
+        return self.post(
+            viewname='documents:document_page_enable', kwargs={
+                'pk': self.test_document_page.pk
+            }
+        )
+
+    def test_document_page_enable_view_no_permission(self):
+        self._disable_test_document_page()
+
+        test_document_page_count = self.test_document.pages.count()
+
+        response = self._request_test_document_page_enable_view()
+        self.assertEqual(response.status_code, 404)
+
+        self.assertEqual(
+            test_document_page_count, self.test_document.pages.count()
+        )
+
+    def test_document_page_enable_view_with_access(self):
+        self._disable_test_document_page()
+        self.grant_access(
+            obj=self.test_document, permission=permission_document_edit
+        )
+
+        test_document_page_count = self.test_document.pages.count()
+
+        response = self._request_test_document_page_enable_view()
+        self.assertEqual(response.status_code, 302)
+
+        self.assertNotEqual(
+            test_document_page_count, self.test_document.pages.count()
+        )
+
+    def _request_test_document_page_multiple_enable_view(self):
+        return self.post(
+            viewname='documents:document_page_multiple_enable', data={
+                'id_list': self.test_document_page.pk
+            }
+        )
+
+    def test_document_page_multiple_enable_view_no_permission(self):
+        self._disable_test_document_page()
+        test_document_page_count = self.test_document.pages.count()
+
+        response = self._request_test_document_page_multiple_enable_view()
+        self.assertEqual(response.status_code, 404)
+
+        self.assertEqual(
+            test_document_page_count, self.test_document.pages.count()
+        )
+
+    def test_document_page_multiple_enable_view_with_access(self):
+        self._disable_test_document_page()
+        self.grant_access(
+            obj=self.test_document, permission=permission_document_edit
+        )
+
+        test_document_page_count = self.test_document.pages.count()
+
+        response = self._request_test_document_page_multiple_enable_view()
+        self.assertEqual(response.status_code, 302)
+
+        self.assertNotEqual(
+            test_document_page_count, self.test_document.pages.count()
+        )
+
+
 class DocumentPageViewTestCase(GenericDocumentViewTestCase):
     def _request_test_document_page_list_view(self):
         return self.get(