Merge branch 'versions/minor' into features/redactions_upstream_merge

Signed-off-by: Roberto Rosario <roberto.rosario@mayan-edms.com>

.gitlab-ci.yml

@@ -63,6 +63,7 @@ job_docker_nightly:
   only:
     - nightly
     - staging
+    - /^clients\/.+$/
 
 job_documentation_build:
   stage: build_documentation
@@ -160,6 +161,7 @@ job_push_python:
     - releases/python
     - staging
     - nightly
+    - /^clients\/.+$/
 
 test-mysql:
   <<: *test_base

.tx/config

@@ -115,6 +115,12 @@ source_lang = en
 source_file = mayan/apps/events/locale/en/LC_MESSAGES/django.po
 type = PO
 
+[mayan-edms.file_caching-3-0]
+file_filter = mayan/apps/file_caching/locale/<lang>/LC_MESSAGES/django.po
+source_lang = en
+source_file = mayan/apps/file_caching/locale/en/LC_MESSAGES/django.po
+type = PO
+
 [mayan-edms.file_metadata-3-0]
 file_filter = mayan/apps/file_metadata/locale/<lang>/LC_MESSAGES/django.po
 source_lang = en
@@ -222,3 +228,10 @@ file_filter = mayan/apps/user_management/locale/<lang>/LC_MESSAGES/django.po
 source_lang = en
 source_file = mayan/apps/user_management/locale/en/LC_MESSAGES/django.po
 type = PO
+
+[mayan-edms.weblink-3-0]
+file_filter = mayan/apps/weblinks/locale/<lang>/LC_MESSAGES/django.po
+source_lang = en
+source_file = mayan/apps/weblinks/locale/en/LC_MESSAGES/django.po
+type = PO
+

CHANGES_BC.rst

@@ -0,0 +1,10 @@
+- Use Select2 widget for the document type selection form.
+- Update source column matching to be additive and not exclusive.
+- Add two columns to show the number of documents per workflow and
+  workflow state.
+- Sort module.
+- Add link to sort individual indexes.
+- Support exclusions from source columns.
+- Improve source column exclusion. Improve for model subclasses in partial querysets.
+- Add sortable index instance label column.
+- Add rectangle drawing transformation.

HISTORY.rst

@@ -36,14 +36,47 @@
 - Remove encapsulate helper.
 - Add support for menu inheritance.
 - Emphasize source column labels.
+- Backport file cache manager app.
+- Convert document image cache to use file cache manager app.
+  Add setting DOCUMENTS_CACHE_MAXIMUM_SIZE defaults to 500 MB.
+- Replace djcelery and replace it with django-celery-beat.
+- Update Celery to version 4.3.0
+  Thanks to Jakob Haufe (@sur5r) and Jesaja Everling (@jeverling)
+  for much of the research and code updates.
+- Support wildcard MIME type associations for the file metadata drivers.
+- Rename MAYAN_GUID to MAYAN_GID
+- Update Gunicorn to use sync workers.
+- Include devpi-server as a development dependency.
+- Update default Docker stack file.
+- Remove Redis from the Docker image.
+- Add Celery flower to the Docker image.
+- Allow PIP proxying to the Docker image during build.
+- Default Celery worker concurrency to 0 (auto).
+- Set DJANGO_SETTINGS_MODULE environment variable to make it
+  available to sub processes.
+- Add entrypoint commands to run single workers, single gunicorn
+  or single celery commands like "flower".
+- Add platform template to return queues for a worker.
+- Update the EXIFTOOL driver to run for all documents
+  regardless of MIME type.
+- Remove task inspection from task manager app.
+- Move pagination navigation inside the toolbar.
+- Remove document image clear link and view.
+  This is now handled by the file caching app.
+- Add web links app.
+- Add support to display column help text
+  as a tooltip.
+- Update numeric dashboard widget to display
+  thousand commas.
+- Add support for disabling document pages.
 
 3.2.6 (2019-07-10)
 ==================
-* Remove the smart settings app * import.
-* Encode settings YAML before hashing.
-* Fix document icon used in the workflow runtime links.
-* Add trashed date time label.
-* Fix thumbnail generation issue. GitLab issue #637.
+- Remove the smart settings app * import.
+- Encode settings YAML before hashing.
+- Fix document icon used in the workflow runtime links.
+- Add trashed date time label.
+- Fix thumbnail generation issue. GitLab issue #637.
   Thanks to Giacomo Cariello (@giacomocariello) for the report
   and the merge request fixing the issue.
 
@@ -95,6 +128,7 @@
 ==================
 - Add support for disabling the random primary key
   test mixin.
+- Add a reusable task to upload documents.
 - Fix mailing profile log columns mappings.
   GitLab issue #626. Thanks to Jesaja Everling (@jeverling)
   for the report.

Makefile

@@ -18,7 +18,7 @@ clean-pyc: ## Remove Python artifacts.
 	find . -name '*.pyc' -exec rm -f {} +
 	find . -name '*.pyo' -exec rm -f {} +
 	find . -name '*~' -exec rm -f {} +
-
+	find . -name '__pycache__' -exec rm -R -f {} + 
 
 # Testing
 
@@ -234,10 +234,10 @@ generate-requirements: ## Generate all requirements files from the project deped
 # Dev server
 
 runserver: ## Run the development server.
-	./manage.py runserver --settings=mayan.settings.development $(ADDRPORT)
+	./manage.py runserver --nothreading --settings=mayan.settings.development $(ADDRPORT)
 
 runserver_plus: ## Run the Django extension's development server.
-	./manage.py runserver_plus --settings=mayan.settings.development $(ADDRPORT)
+	./manage.py runserver_plus --nothreading --settings=mayan.settings.development $(ADDRPORT)
 
 shell_plus: ## Run the shell_plus command.
 	./manage.py shell_plus --settings=mayan.settings.development
@@ -258,7 +258,7 @@ test-with-docker-frontend: ## Launch a front end instance that uses the producti
 	./manage.py runserver --settings=mayan.settings.staging.docker
 
 test-with-docker-worker: ## Launch a worker instance that uses the production-like services.
-	./manage.py celery worker --settings=mayan.settings.staging.docker -B -l INFO -O fair
+	DJANGO_SETTINGS_MODULE=mayan.settings.staging.docker ./manage.py celery worker -A mayan -B -l INFO -O fair
 
 docker-mysql-on: ## Launch and initialize a MySQL Docker container.
 	docker run -d --name mysql -p 3306:3306 -e MYSQL_ALLOW_EMPTY_PASSWORD=True -e MYSQL_DATABASE=mayan_edms mysql

contrib/scripts/process_messages.py

@@ -13,11 +13,12 @@ APP_LIST = (
     'checkouts', 'common', 'converter', 'dashboards', 'dependencies',
     'django_gpg', 'document_comments', 'document_indexing',
     'document_parsing', 'document_signatures', 'document_states',
-    'documents', 'dynamic_search', 'events', 'file_metadata', 'linking',
-    'lock_manager', 'mailer', 'mayan_statistics', 'metadata', 'mirroring',
-    'motd', 'navigation', 'ocr', 'permissions', 'platform', 'rest_api',
-    'smart_settings', 'sources', 'storage', 'tags', 'task_manager',
-    'user_management'
+    'documents', 'dynamic_search', 'events', 'file_caching',
+    'file_metadata', 'linking', 'lock_manager', 'mailer',
+    'mayan_statistics', 'metadata', 'mirroring', 'motd', 'navigation',
+    'ocr', 'permissions', 'platform', 'rest_api', 'smart_settings',
+    'sources', 'storage', 'tags', 'task_manager', 'user_management',
+    'weblinks'
 )
 
 LANGUAGE_LIST = (

docker/Dockerfile

@@ -4,7 +4,7 @@
 # BASE_IMAGE - Bare bones image with the base packages needed to run Mayan EDMS
 ####
 
-FROM debian:9.8-slim as BASE_IMAGE
+FROM debian:10.0-slim as BASE_IMAGE
 
 LABEL maintainer="Roberto Rosario roberto.rosario@mayan-edms.com"
 
@@ -22,6 +22,7 @@ RUN set -x \
 && DEBIAN_FRONTEND=noninteractive \
 apt-get update \
 && apt-get install -y --no-install-recommends \
+    ca-certificates \
     exiftool \
     ghostscript \
     gpgv \
@@ -29,11 +30,11 @@ apt-get update \
     graphviz \
     libfuse2 \
     libmagic1 \
-    libmariadbclient18 \
+    libmariadb3 \
     libreoffice \
     libpq5 \
     poppler-utils \
-    redis-server \
+    python3-distutils \
     sane-utils \
     sudo \
     supervisor \
@@ -52,22 +53,20 @@ apt-get update \
 && if [ "$(uname -m)" = "armv7l" ]; then \
     ln -s /usr/lib/arm-linux-gnueabihf/libz.so /usr/lib/ \
     && ln -s /usr/lib/arm-linux-gnueabihf/libjpeg.so /usr/lib/ \
-; fi \
-# Discard data when Redis runs out of memory
-&& echo "maxmemory-policy allkeys-lru" >> /etc/redis/redis.conf \
-# Disable saving the Redis database
-echo "save \"\"" >> /etc/redis/redis.conf \
-# Only provision 1 database
-&& echo "databases 1" >> /etc/redis/redis.conf
-
+; fi
 
 ####
-# BUILDER_IMAGE - This image buildS the Python package and is discarded afterwards
+# BUILDER_IMAGE - This image builds the Python package and is discarded afterwards
+#                 only the build artifact is carried over to the next image.
 ####
 
 # Reuse image
 FROM BASE_IMAGE as BUILDER_IMAGE
 
+# Python libraries caching
+ARG PIP_INDEX_URL
+ARG PIP_TRUSTED_HOST
+
 WORKDIR /src
 
 # Copy the source files needed to build the Python package
@@ -96,31 +95,32 @@ apt-get install -y --no-install-recommends \
     libssl-dev \
     g++ \
     gcc \
-    python-dev \
-    python-virtualenv \
+    python3-dev \
+    python3-venv \
 && mkdir -p "${PROJECT_INSTALL_DIR}" \
 && chown -R mayan:mayan "${PROJECT_INSTALL_DIR}" \
 && chown -R mayan:mayan /src
 
 USER mayan
-RUN python -m virtualenv "${PROJECT_INSTALL_DIR}" \
+RUN python3 -m venv "${PROJECT_INSTALL_DIR}" \
 && . "${PROJECT_INSTALL_DIR}/bin/activate" \
-&& pip install --no-cache-dir --no-use-pep517 \
-    librabbitmq==1.6.1 \
-    mysql-python==1.2.5 \
-    psycopg2==2.7.3.2 \
-    redis==2.10.6 \
+&& pip install --no-cache-dir \
+    librabbitmq==2.0.0 \
+    mysqlclient==1.4.2.post1 \
+    psycopg2==2.8.3 \
+    redis==3.2.1 \
+    flower==0.9.3 \
 # psutil is needed by ARM builds otherwise gevent and gunicorn fail to start
 && UNAME=`uname -m` && if [ "${UNAME#*arm}" != $UNAME ]; then \
-    pip install --no-cache-dir --no-use-pep517 \
+    pip install --no-cache-dir \
     psutil==5.6.2 \
 ; fi \
 # Install the Python packages needed to build Mayan EDMS
-&& pip install --no-cache-dir --no-use-pep517 -r /src/requirements/build.txt \
+&& pip install --no-cache-dir -r /src/requirements/build.txt \
 # Build Mayan EDMS
-&& python setup.py sdist \
+&& python3 setup.py sdist \
 # Install the built Mayan EDMS package
-&& pip install --no-cache-dir --no-use-pep517 dist/mayan* \
+&& pip install --no-cache-dir dist/mayan* \
 # Install the static content
 && mayan-edms.py installdependencies \
 && MAYAN_STATIC_ROOT=${PROJECT_INSTALL_DIR}/static mayan-edms.py preparestatic --link --noinput
@@ -128,7 +128,7 @@ RUN python -m virtualenv "${PROJECT_INSTALL_DIR}" \
 COPY --chown=mayan:mayan requirements/testing-base.txt "${PROJECT_INSTALL_DIR}"
 
 ####
-# Final image - BASE_IMAGE + Mayan install directory from the builder image
+# Final image - BASE_IMAGE + BUILDER_IMAGE artifact (Mayan install directory)
 ####
 
 FROM BASE_IMAGE
@@ -144,7 +144,7 @@ VOLUME ["/var/lib/mayan"]
 ENTRYPOINT ["entrypoint.sh"]
 
 EXPOSE 8000
-CMD ["mayan"]
+CMD ["run_all"]
 
 RUN ${PROJECT_INSTALL_DIR}/bin/mayan-edms.py platformtemplate supervisord_docker > /etc/supervisor/conf.d/mayan.conf \
 && apt-get clean autoclean \

docker/Makefile

@@ -1,4 +1,9 @@
-APT_PROXY ?= `/sbin/ip route|awk '/docker0/ { print $$9 }'`:3142
+HOST_IP = `/sbin/ip route|awk '/docker0/ { print $$9 }'`
+
+APT_PROXY ?= $(HOST_IP):3142
+PIP_INDEX_URL ?= http://$(HOST_IP):3141/root/pypi/+simple/
+PIP_TRUSTED_HOST ?= $(HOST_IP)
+
 IMAGE_VERSION ?= `cat docker/rootfs/version`
 CONSOLE_COLUMNS ?= `echo $$(tput cols)`
 CONSOLE_LINES ?= `echo $$(tput lines)`
@@ -7,7 +12,7 @@ docker-build: ## Build a new image locally.
 	docker build -t mayanedms/mayanedms:$(IMAGE_VERSION) -f docker/Dockerfile .
 
 docker-build-with-proxy: ## Build a new image locally using an APT proxy as APT_PROXY.
-	docker build -t mayanedms/mayanedms:$(IMAGE_VERSION) -f docker/Dockerfile --build-arg APT_PROXY=$(APT_PROXY) .
+	docker build -t mayanedms/mayanedms:$(IMAGE_VERSION) -f docker/Dockerfile --build-arg APT_PROXY=$(APT_PROXY) --build-arg PIP_INDEX_URL=$(PIP_INDEX_URL) --build-arg PIP_TRUSTED_HOST=$(PIP_TRUSTED_HOST) --build-arg HTTP_PROXY=$(HTTP_PROXY) --build-arg HTTPS_PROXY=$(HTTPS_PROXY) .
 
 docker-shell: ## Launch a bash instance inside a running container. Pass the container name via DOCKER_CONTAINER.
 	docker exec -e TERM=$(TERM) -e "COLUMNS=$(CONSOLE_COLUMNS)" -e "LINES=$(CONSOLE_LINES)" -it $(DOCKER_CONTAINER) /bin/bash
@@ -23,3 +28,13 @@ docker-test-cleanup: ## Delete the test container and the test volume.
 docker-test-all: ## Build and executed the test suite in a test container.
 docker-test-all: docker-build-with-proxy
 	docker run --rm run-tests
+
+docker-compose-build:
+	docker-compose -f docker/docker-compose.yml -p mayan-edms build
+
+docker-compose-build-with-proxy:
+	docker-compose -f docker/docker-compose.yml -p mayan-edms build --build-arg APT_PROXY=$(APT_PROXY) --build-arg PIP_INDEX_URL=$(PIP_INDEX_URL) --build-arg PIP_TRUSTED_HOST=$(PIP_TRUSTED_HOST) --build-arg HTTP_PROXY=$(HTTP_PROXY) --build-arg HTTPS_PROXY=$(HTTPS_PROXY)
+
+docker-compose-up:
+	docker-compose -f docker/docker-compose.yml -p mayan-edms up
+

docker/docker-compose-development.yml

@@ -1,72 +0,0 @@
-version: '2.1'
-
-volumes:
-    broker:
-        driver: local
-    app:
-        driver: local
-    db:
-        driver: local
-    results:
-        driver: local
-
-services:
-  broker:
-    container_name: mayan-edms-broker
-    image: healthcheck/rabbitmq
-    environment:
-      RABBITMQ_DEFAULT_USER: mayan
-      RABBITMQ_DEFAULT_PASS: mayan
-      RABBITMQ_DEFAULT_VHOST: mayan
-    volumes:
-      - broker:/var/lib/rabbitmq
-  results:
-    container_name: mayan-edms-results
-    image: healthcheck/redis
-    volumes:
-      - results:/data
-  #db:
-  #  container_name: mayan-edms-db
-  #  image: healthcheck/mysql
-  #  environment:
-  #    MYSQL_DATABASE: mayan
-  #    MYSQL_PASSWORD: mayan-password
-  #    MYSQL_ROOT_PASSWORD: root-password
-  #    MYSQL_USER: mayan
-  #  volumes:
-  #    - db:/var/lib/mysql
-  db:
-    container_name: mayan-edms-db
-    image: healthcheck/postgres
-    environment:
-      POSTGRES_DB: mayan
-      POSTGRES_PASSWORD: mayan-password
-      POSTGRES_USER: mayan
-    volumes:
-      - db:/var/lib/postgresql/data
-  mayan-edms:
-    container_name: mayan-edms-app
-    image: mayan-edms/next
-    build:
-      context: ./
-      args:
-        - APT_PROXY=172.18.0.1:3142
-    depends_on:
-      broker:
-        condition: service_healthy
-      db:
-        condition: service_healthy
-      results:
-        condition: service_healthy
-    environment:
-      MAYAN_BROKER_URL: amqp://mayan:mayan@broker:5672/mayan
-      MAYAN_CELERY_RESULT_BACKEND: redis://results:6379/0
-      MAYAN_DATABASE_ENGINE: django.db.backends.postgresql
-      MAYAN_DATABASE_HOST: db
-      MAYAN_DATABASE_NAME: mayan
-      MAYAN_DATABASE_PASSWORD: mayan-password
-      MAYAN_DATABASE_USER: mayan
-    ports:
-      - "80:80"
-    volumes:
-      - app:/var/lib/mayan

docker/docker-compose.yml

@@ -1,58 +1,130 @@
-version: '2.1'
+version: '3.7'
 
-volumes:
-    broker:
-        driver: local
-    app:
-        driver: local
-    db:
-        driver: local
-    results:
-        driver: local
+networks:
+  mayan-bridge:
+    driver: bridge
 
 services:
-  broker:
-    container_name: mayan-edms-broker
-    image: healthcheck/rabbitmq
-    environment:
-      RABBITMQ_DEFAULT_USER: mayan
-      RABBITMQ_DEFAULT_PASS: mayan
-      RABBITMQ_DEFAULT_VHOST: mayan
-    volumes:
-      - broker:/var/lib/rabbitmq
-  results:
-    container_name: mayan-edms-results
-    image: healthcheck/redis
-    volumes:
-      - results:/data
-  db:
-    container_name: mayan-edms-db
-    image: healthcheck/postgres
-    environment:
-      POSTGRES_DB: mayan
-      POSTGRES_PASSWORD: mayan-password
-      POSTGRES_USER: mayan
-    volumes:
-      - db:/var/lib/postgresql/data
-  mayan-edms:
-    container_name: mayan-edms-app
-    image: mayanedms/mayanedms:latest
+  app:
+    build:
+      context: ..
+      dockerfile: ./docker/Dockerfile
     depends_on:
-      broker:
-        condition: service_healthy
-      db:
-        condition: service_healthy
-      results:
-        condition: service_healthy
-    environment:
-      MAYAN_BROKER_URL: amqp://mayan:mayan@broker:5672/mayan
-      MAYAN_CELERY_RESULT_BACKEND: redis://results:6379/0
-      MAYAN_DATABASE_ENGINE: django.db.backends.postgresql
-      MAYAN_DATABASE_HOST: db
-      MAYAN_DATABASE_NAME: mayan
-      MAYAN_DATABASE_PASSWORD: mayan-password
-      MAYAN_DATABASE_USER: mayan
+      - postgresql
+      - redis
+      # Enable to use RabbitMQ
+      #- rabbitmq
+    environment: &mayan_env
+      # Enable to use RabbitMQ
+      # MAYAN_CELERY_BROKER_URL: amqp://mayan:mayanrabbitpass@broker:5672/mayan
+      # Disable Redis Broker to use RabbitMQ as Broker
+      MAYAN_CELERY_BROKER_URL: redis://redis:6379/1
+      MAYAN_CELERY_RESULT_BACKEND: redis://redis:6379/0
+      MAYAN_DATABASES: "{'default':{'ENGINE':'django.db.backends.postgresql','NAME':'mayan','PASSWORD':'mayandbpass','USER':'mayan','HOST':'postgresql'}}"
+    image: mayanedms/mayanedms:3.2.6
+    networks:
+      - mayan-bridge
     ports:
       - "80:8000"
+    restart: unless-stopped
     volumes:
-      - app:/var/lib/mayan
+      - /docker-volumes/mayan-edms/media:/var/lib/mayan
+
+  postgresql:
+    environment:
+      POSTGRES_DB: mayan
+      POSTGRES_PASSWORD: mayandbpass
+      POSTGRES_USER: mayan
+    image: postgres:9.6
+    networks:
+      - mayan-bridge
+    restart: unless-stopped
+    volumes:
+      - /docker-volumes/mayan-edms/postgres:/var/lib/postgresql/data
+
+  redis:
+    command:
+      - redis-server
+      - --databases
+      - "2"
+      - --maxmemory-policy
+      - allkeys-lru
+      - --save
+      - ""
+    image: redis:5.0
+    networks:
+      - mayan-bridge
+    restart: unless-stopped
+
+  # Optional services
+
+  # celery_flower:
+  #   command:
+  #     - run_celery
+  #     - flower
+  #   depends_on:
+  #     - postgresql
+  #     - redis
+  #     # Enable to use RabbitMQ
+  #     # - rabbitmq
+  #   environment:
+  #     <<: *mayan_env
+  #   image: mayanedms/mayanedms:3.2.6
+  #   networks:
+  #     - mayan-bridge
+  #   ports:
+  #     - "5555:5555"
+  #   restart: unless-stopped
+
+  # Enable to use RabbitMQ
+  # rabbitmq:
+  #   container_name: mayan-edms-rabbitmq
+  #   image: healthcheck/rabbitmq
+  #   environment:
+  #     RABBITMQ_DEFAULT_USER: mayan
+  #     RABBITMQ_DEFAULT_PASS: mayanrabbitpass
+  #     RABBITMQ_DEFAULT_VHOST: mayan
+  #   networks:
+  #     - mayan-bridge
+  #   restart: unless-stopped
+  #   volumes:
+  #     - /docker-volumes/mayan-edms/rabbitmq:/var/lib/rabbitmq
+
+  # Enable to run stand alone workers
+  # worker_fast:
+  #   command:
+  #     - run_worker
+  #     - fast
+  #   depends_on:
+  #     - postgresql
+  #     - redis
+  #     # Enable to use RabbitMQ
+  #     # - rabbitmq
+  #   environment:
+  #     <<: *mayan_env
+  #   image: mayanedms/mayanedms:3.2.6
+  #   networks:
+  #     - mayan-bridge
+  #   restart: unless-stopped
+  #   volumes:
+  #     - /docker-volumes/mayan-edms/media:/var/lib/mayan
+
+  # Enable to run stand frontend gunicorn
+  # frontend:
+  #   command:
+  #     - run_frontend
+  #   depends_on:
+  #     - postgresql
+  #     - redis
+  #     # Enable to use RabbitMQ
+  #     # - rabbitmq
+  #   environment:
+  #     <<: *mayan_env
+  #   image: mayanedms/mayanedms:3.2.6
+  #   networks:
+  #     - mayan-bridge
+  #   ports:
+  #     - "81:8000"
+  #   restart: unless-stopped
+  #   volumes:
+  #     - /docker-volumes/mayan-edms/media:/var/lib/mayan

docker/rootfs/usr/local/bin/entrypoint.sh

@@ -1,4 +1,7 @@
-#!/bin/sh
+#!/bin/bash
+
+# Use bash and not sh to support argument slicing "${@:2}"
+# sh defaults to dash instead of bash.
 
 set -e
 echo "mayan: starting entrypoint.sh"
@@ -6,19 +9,15 @@ INSTALL_FLAG=/var/lib/mayan/system/SECRET_KEY
 CONCURRENCY_ARGUMENT=--concurrency=
 
 DEFAULT_USER_UID=1000
-DEFAULT_USER_GUID=1000
-
-export MAYAN_DEFAULT_BROKER_URL=redis://127.0.0.1:6379/0
-export MAYAN_DEFAULT_CELERY_RESULT_BACKEND=redis://127.0.0.1:6379/0
+DEFAULT_USER_GID=1000
 
 export MAYAN_ALLOWED_HOSTS='["*"]'
 export MAYAN_BIN=/opt/mayan-edms/bin/mayan-edms.py
-export MAYAN_BROKER_URL=${MAYAN_BROKER_URL:-${MAYAN_DEFAULT_BROKER_URL}}
-export MAYAN_CELERY_RESULT_BACKEND=${MAYAN_CELERY_RESULT_BACKEND:-${MAYAN_DEFAULT_CELERY_RESULT_BACKEND}}
 export MAYAN_INSTALL_DIR=/opt/mayan-edms
 export MAYAN_PYTHON_BIN_DIR=/opt/mayan-edms/bin/
 export MAYAN_MEDIA_ROOT=/var/lib/mayan
 export MAYAN_SETTINGS_MODULE=${MAYAN_SETTINGS_MODULE:-mayan.settings.production}
+export DJANGO_SETTINGS_MODULE=${MAYAN_SETTINGS_MODULE}
 
 export MAYAN_GUNICORN_BIN=${MAYAN_PYTHON_BIN_DIR}gunicorn
 export MAYAN_GUNICORN_WORKERS=${MAYAN_GUNICORN_WORKERS:-2}
@@ -26,13 +25,9 @@ export MAYAN_GUNICORN_TIMEOUT=${MAYAN_GUNICORN_TIMEOUT:-120}
 export MAYAN_PIP_BIN=${MAYAN_PYTHON_BIN_DIR}pip
 export MAYAN_STATIC_ROOT=${MAYAN_INSTALL_DIR}/static
 
-MAYAN_WORKER_FAST_CONCURRENCY=${MAYAN_WORKER_FAST_CONCURRENCY:-1}
-MAYAN_WORKER_MEDIUM_CONCURRENCY=${MAYAN_WORKER_MEDIUM_CONCURRENCY:-1}
-MAYAN_WORKER_SLOW_CONCURRENCY=${MAYAN_WORKER_SLOW_CONCURRENCY:-1}
-
-echo "mayan: changing uid/guid"
-usermod mayan -u ${MAYAN_USER_UID:-${DEFAULT_USER_UID}}
-groupmod mayan -g ${MAYAN_USER_GUID:-${DEFAULT_USER_GUID}}
+MAYAN_WORKER_FAST_CONCURRENCY=${MAYAN_WORKER_FAST_CONCURRENCY:-0}
+MAYAN_WORKER_MEDIUM_CONCURRENCY=${MAYAN_WORKER_MEDIUM_CONCURRENCY:-0}
+MAYAN_WORKER_SLOW_CONCURRENCY=${MAYAN_WORKER_SLOW_CONCURRENCY:-0}
 
 if [ "$MAYAN_WORKER_FAST_CONCURRENCY" -eq 0 ]; then
     MAYAN_WORKER_FAST_CONCURRENCY=
@@ -55,11 +50,9 @@ else
 fi
 export MAYAN_WORKER_SLOW_CONCURRENCY
 
-export CELERY_ALWAYS_EAGER=False
+# Allow importing of user setting modules
 export PYTHONPATH=$PYTHONPATH:$MAYAN_MEDIA_ROOT
 
-chown mayan:mayan /var/lib/mayan -R
-
 apt_get_install() {
     apt-get -q update
     apt-get install -y --force-yes --no-install-recommends --auto-remove "$@"
@@ -67,9 +60,9 @@ apt_get_install() {
     rm -rf /var/lib/apt/lists/*
 }
 
-initialize() {
-    echo "mayan: initialize()"
-    su mayan -c "${MAYAN_BIN} initialsetup --force --no-javascript"
+initialsetup() {
+    echo "mayan: initialsetup()"
+    su mayan -c "${MAYAN_BIN} initialsetup --force --no-dependencies"
 }
 
 os_package_installs() {
@@ -86,43 +79,71 @@ pip_installs() {
     fi
 }
 
-start() {
+run_all() {
     echo "mayan: start()"
     rm -rf /var/run/supervisor.sock
     exec /usr/bin/supervisord -nc /etc/supervisor/supervisord.conf
 }
 
-upgrade() {
-    echo "mayan: upgrade()"
-    su mayan -c "${MAYAN_BIN} performupgrade --no-javascript"
+performupgrade() {
+    echo "mayan: performupgrade()"
+    su mayan -c "${MAYAN_BIN} performupgrade --no-dependencies"
+}
+
+make_ready() {
+    # Check if this is a new install, otherwise try to upgrade the existing
+    # installation on subsequent starts
+    if [ ! -f $INSTALL_FLAG ]; then
+        initialsetup
+    else
+        performupgrade
+    fi
+}
+
+set_uid_guid() {
+    echo "mayan: changing uid/guid"
+    usermod mayan -u ${MAYAN_USER_UID:-${DEFAULT_USER_UID}}
+    groupmod mayan -g ${MAYAN_USER_GID:-${DEFAULT_USER_GID}}
 }
 
 os_package_installs || true
 pip_installs || true
+chown mayan:mayan /var/lib/mayan -R
 
 case "$1" in
 
-mayan) # Check if this is a new install, otherwise try to upgrade the existing
-       # installation on subsequent starts
-       if [ ! -f $INSTALL_FLAG ]; then
-           initialize
-       else
-           upgrade
-       fi
-       start
-       ;;
+run_initialsetup)
+    initialsetup
+    ;;
 
-run-tests) # Check if this is a new install, otherwise try to upgrade the existing
-           # installation on subsequent starts
-           if [ ! -f $INSTALL_FLAG ]; then
-               initialize
-           else
-               upgrade
-           fi
-           run-tests.sh
-           ;;
+run_performupgrade)
+    performupgrade
+    ;;
 
-*) su mayan -c "$@";
-   ;;
+run_all)
+    make_ready
+    run_all
+    ;;
+
+run_celery)
+    run_celery.sh "${@:2}"
+    ;;
+
+run_frontend)
+    run_frontend.sh
+    ;;
+
+run_tests)
+    make_ready
+    run_tests.sh
+    ;;
+
+run_worker)
+    run_worker.sh "${@:2}"
+    ;;
+
+*)
+    su mayan -c "$@"
+    ;;
 
 esac

docker/rootfs/usr/local/bin/run_celery.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+# Use -A and not --app. Both are the same but behave differently
+# -A can be located before the command while --app cannot.
+su mayan -c "${MAYAN_PYTHON_BIN_DIR}celery -A mayan $@"

docker/rootfs/usr/local/bin/run_frontend.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+MAYAN_GUNICORN_MAX_REQUESTS=${MAYAN_GUNICORN_MAX_REQUESTS:-500}
+MAYAN_GUNICORN_MAX_REQUESTS_JITTERS=${MAYAN_GUNICORN_MAX_REQUESTS_JITTERS:-50}
+MAYAN_GUNICORN_WORKER_CLASS=${MAYAN_GUNICORN_WORKER_CLASS:-sync}
+
+su mayan -c "${MAYAN_PYTHON_BIN_DIR}gunicorn -w ${MAYAN_GUNICORN_WORKERS} mayan.wsgi --max-requests ${MAYAN_GUNICORN_MAX_REQUESTS} --max-requests-jitter ${MAYAN_GUNICORN_MAX_REQUESTS_JITTERS} --worker-class ${MAYAN_GUNICORN_WORKER_CLASS} --bind 0.0.0.0:8000 --timeout ${MAYAN_GUNICORN_TIMEOUT}"

docker/rootfs/usr/local/bin/run_worker.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+QUEUE_LIST=`MAYAN_WORKER_NAME=$1 su mayan -c "${MAYAN_PYTHON_BIN_DIR}mayan-edms.py platformtemplate worker_queues"`
+
+# Use -A and not --app. Both are the same but behave differently
+# -A can be located before the command while --app cannot.
+# Pass ${@:2} to allow overriding the defaults arguments
+su mayan -c "${MAYAN_PYTHON_BIN_DIR}celery -A mayan worker -Ofair -l ERROR -Q $QUEUE_LIST ${@:2}"

docker/stack.yml

@@ -9,24 +9,32 @@ volumes:
 services:
 
   db:
-    image: postgres
     environment:
       POSTGRES_DB: mayan
-      POSTGRES_PASSWORD: mayan-password
+      POSTGRES_PASSWORD: mayandbpass
       POSTGRES_USER: mayan
+    image: postgres
     volumes:
       - db:/var/lib/postgresql/data
 
   app:
+    environment:
+      MAYAN_CELERY_BROKER_URL: redis://redis:6379/1
+      MAYAN_CELERY_RESULT_BACKEND: redis://redis:6379/0
+      MAYAN_DATABASES: "{'default':{'ENGINE':'django.db.backends.postgresql','NAME':'mayan','PASSWORD':'mayandbpass','USER':'mayan','HOST':'db'}}"
     image: mayanedms/mayanedms:latest
     ports:
       - 80:8000
-    environment:
-      MAYAN_DATABASE_ENGINE: django.db.backends.postgresql
-      MAYAN_DATABASE_HOST: db
-      MAYAN_DATABASE_NAME: mayan
-      MAYAN_DATABASE_PASSWORD: mayan-password
-      MAYAN_DATABASE_USER: mayan
-      MAYAN_DATABASE_CONN_MAX_AGE: 0
     volumes:
       - app:/var/lib/mayan
+
+  redis:
+    command:
+      - redis-server
+      - --databases
+      - "2"
+      - --maxmemory-policy
+      - allkeys-lru
+      - --save
+      - ""
+    image: redis:5.0

docs/chapters/deploying.rst

@@ -220,11 +220,11 @@ of a restart or power failure. The Gunicorn workers are increased to 3.
 ---------------------------------------------------------------------
    Replace (paying attention to the comma at the end)::
 
-       MAYAN_BROKER_URL="redis://127.0.0.1:6379/0",
+       MAYAN_CELERY_BROKER_URL="redis://127.0.0.1:6379/0",
 
    with::
 
-       MAYAN_BROKER_URL="amqp://mayan:mayanrabbitmqpassword@localhost:5672/mayan",
+       MAYAN_CELERY_BROKER_URL="amqp://mayan:mayanrabbitmqpassword@localhost:5672/mayan",
 
    increase the number of Gunicorn workers to 3 in the line (``-w 2`` section)::
 

docs/chapters/docker.rst

@@ -49,12 +49,7 @@ Finally create and run a Mayan EDMS container::
     --name mayan-edms \
     --restart=always \
     -p 80:8000 \
-    -e MAYAN_DATABASE_ENGINE=django.db.backends.postgresql \
-    -e MAYAN_DATABASE_HOST=172.17.0.1 \
-    -e MAYAN_DATABASE_NAME=mayan \
-    -e MAYAN_DATABASE_PASSWORD=mayanuserpass \
-    -e MAYAN_DATABASE_USER=mayan \
-    -e MAYAN_DATABASE_CONN_MAX_AGE=0 \
+    -e MAYAN_DATABASES="{'default':{'ENGINE':'django.db.backends.postgresql','NAME':'mayan','PASSWORD':'mayanuserpass','USER':'mayan','HOST':'172.17.0.1'}}" \
     -v /docker-volumes/mayan-edms/media:/var/lib/mayan \
     mayanedms/mayanedms:<version>
 
@@ -108,12 +103,7 @@ instead of the IP address of the Docker host (``172.17.0.1``)::
     --network=mayan \
     --restart=always \
     -p 80:8000 \
-    -e MAYAN_DATABASE_ENGINE=django.db.backends.postgresql \
-    -e MAYAN_DATABASE_HOST=mayan-edms-postgres \
-    -e MAYAN_DATABASE_NAME=mayan \
-    -e MAYAN_DATABASE_PASSWORD=mayanuserpass \
-    -e MAYAN_DATABASE_USER=mayan \
-    -e MAYAN_DATABASE_CONN_MAX_AGE=0 \
+    -e MAYAN_DATABASES="{'default':{'ENGINE':'django.db.backends.postgresql','NAME':'mayan','PASSWORD':'mayanuserpass','USER':'mayan','HOST':'mayan-edms-postgres'}}" \
     -v /docker-volumes/mayan-edms/media:/var/lib/mayan \
     mayanedms/mayanedms:<version>
 
@@ -137,102 +127,14 @@ To start the container again::
 Environment Variables
 ---------------------
 
-The Mayan EDMS image can be configure via environment variables.
-
-``MAYAN_DATABASE_ENGINE``
-
-Defaults to ``None``. This environment variable configures the database
-backend to use. If left unset, SQLite will be used. The database backends
-supported by this Docker image are:
-
-- ``'django.db.backends.postgresql'``
-- ``'django.db.backends.mysql'``
-- ``'django.db.backends.sqlite3'``
-
-When using the SQLite backend, the database file will be saved in the Docker
-volume. The SQLite database as used by Mayan EDMS is meant only for development
-or testing, never use it in production.
-
-``MAYAN_DATABASE_NAME``
-
-Defaults to 'mayan'. This optional environment variable can be used to define
-the database name that Mayan EDMS will connect to. For more information read
-the pertinent Django documentation page:
-:django-docs:`Connecting to the database <ref/databases/#connecting-to-the-database>`
-
-``MAYAN_DATABASE_USER``
-
-Defaults to 'mayan'. This optional environment variable is used to set the
-username that will be used to connect to the database. For more information
-read the pertinent Django documentation page:
-:django-docs:`Settings, USER <ref/settings/#user>`
-
-``MAYAN_DATABASE_PASSWORD``
-
-Defaults to ''. This optional environment variable is used to set the
-password that will be used to connect to the database. For more information
-read the pertinent Django documentation page:
-:django-docs:`Settings, PASSWORD <ref/settings/#password>`
-
-``MAYAN_DATABASE_HOST``
-
-Defaults to `None`. This optional environment variable is used to set the
-hostname that will be used to connect to the database. This can be the
-hostname of another container or an IP address. For more information read
-the pertinent Django documentation page:
-:django-docs:`Settings, HOST <ref/settings/#host>`
-
-``MAYAN_DATABASE_PORT``
-
-Defaults to `None`. This optional environment variable is used to set the
-port number to use when connecting to the database. An empty string means
-the default port. Not used with SQLite. For more information read the
-pertinent Django documentation page:
-:django-docs:`Settings, PORT <ref/settings/#port>`
-
-``MAYAN_BROKER_URL``
-
-This optional environment variable determines the broker that Celery will use
-to relay task messages between the frontend code and the background workers.
-For more information read the pertinent Celery Kombu documentation page: `Broker URL`_
-
-.. _Broker URL: http://kombu.readthedocs.io/en/latest/userguide/connections.html#connection-urls
-
-This Docker image supports using Redis and RabbitMQ as brokers.
-
-Caveat: If the `MAYAN_BROKER_URL` and `MAYAN_CELERY_RESULT_BACKEND` environment
-variables are specified, the built-in Redis server inside the container will
-be disabled.
-
-``MAYAN_CELERY_RESULT_BACKEND``
-
-This optional environment variable determines the results backend that Celery
-will use to relay result messages from the background workers to the frontend
-code. For more information read the pertinent Celery Kombu documentation page:
-`Task result backend settings`_
-
-.. _Task result backend settings: http://docs.celeryproject.org/en/3.1/configuration.html#celery-result-backend
-
-This Docker image supports using Redis and RabbitMQ as result backends.
-
-Caveat: If the `MAYAN_BROKER_URL` and `MAYAN_CELERY_RESULT_BACKEND` environment
-variables are specified, the built-in Redis server inside the container will
-be disabled.
+The common set of settings can also be modified via environment variables when
+using the Docker image. In addition to the common set of settings, some Docker
+image specific environment variables are available.
 
 ``MAYAN_SETTINGS_MODULE``
 
 Optional. Allows loading an alternate settings file.
 
-``MAYAN_DATABASE_CONN_MAX_AGE``
-
-Amount in seconds to keep a database connection alive. Allow reuse of database
-connections. For more information read the pertinent Django documentation
-page: :django-docs:`Settings, CONN_MAX_AGE <ref/settings/#conn-max-age>`
-According to new information Gunicorn's microthreads don't share connections
-and will exhaust the available Postgres connections available if a number
-other than 0 is used. Reference: https://serverfault.com/questions/635100/django-conn-max-age-persists-connections-but-doesnt-reuse-them-with-postgresq
-and https://github.com/benoitc/gunicorn/issues/996
-
 ``MAYAN_GUNICORN_WORKERS``
 
 Optional. This environment variable controls the number of frontend workers
@@ -269,12 +171,21 @@ number of CPUs detected).
 Optional. Changes the UID of the ``mayan`` user internal to the Docker
 container. Defaults to 1000.
 
-``MAYAN_USER_GUID``
+``MAYAN_USER_GID``
 
-Optional. Changes the GUID of the ``mayan`` user internal to the Docker
+Optional. Changes the GID of the ``mayan`` user internal to the Docker
 container. Defaults to 1000.
 
 
+Included drivers
+----------------
+
+The Docker image supports using Redis and RabbitMQ as result backends. For
+databases, the image includes support for PostgreSQL and MySQL/MariaDB.
+Support for additional brokers or databases may be added using the
+``MAYAN_APT_INSTALL`` environment variable.
+
+
 .. _docker-accessing-outside-data:
 
 Accessing outside data
@@ -442,6 +353,7 @@ These are:
 
 Nightly images
 ==============
+
 The continuous integration pipeline used for testing development builds also
 produces a resulting Docker image. These are build automatically and their
 stability is not guaranteed. They should never be used in production.

docs/chapters/scaling_up.rst

@@ -94,11 +94,11 @@ For the Docker image, launch a separate RabbitMQ container
 
     docker run -d --name mayan-edms-rabbitmq -e RABBITMQ_DEFAULT_USER=mayan -e RABBITMQ_DEFAULT_PASS=mayanrabbitmqpassword -e RABBITMQ_DEFAULT_VHOST=mayan rabbitmq:3
 
-Pass the MAYAN_BROKER_URL environment variable (https://kombu.readthedocs.io/en/latest/userguide/connections.html#connection-urls)
+Pass the MAYAN_CELERY_BROKER_URL environment variable (https://kombu.readthedocs.io/en/latest/userguide/connections.html#connection-urls)
 to the Mayan EDMS container so that it uses the RabbitMQ container the
 message broker::
 
-    -e MAYAN_BROKER_URL="amqp://mayan:mayanrabbitmqpassword@localhost:5672/mayan",
+    -e MAYAN_CELERY_BROKER_URL="amqp://mayan:mayanrabbitmqpassword@localhost:5672/mayan",
 
 When tasks finish, they leave behind a return status or the result of a
 calculation, these are stored for a while so that whoever requested the

docs/chapters/sources.rst

@@ -15,7 +15,8 @@ The current document sources supported are:
 - IMAP email - Same as the ``POP3`` email source but for email accounts using
   the ``IMAP`` protocol.
 - Watch folder - A filesystem folder that is scanned periodically for files.
-  Any file in the watch folder is automatically uploaded.
+  Any file in the watch folder is automatically uploaded. When the upload for a
+  file is completed, the file is removed from source folder.
 - Staging folder - Folder where networked attached scanned can save image
   files. The files in these staging folders are scanned and a preview is
   generated to help the process of upload. Staging folders and Watch folders

docs/releases/3.2.3.rst

@@ -19,7 +19,6 @@ Changes
   GitLab issue #625. Thanks to Jesaja Everling (@jeverling)
   for the report and the research.
 
-
 Removals
 --------
 

docs/releases/3.3.rst

@@ -49,6 +49,41 @@ Changes
 - Remove encapsulate helper.
 - Add support for menu inheritance.
 - Emphasize source column labels.
+- Backport file cache manager app.
+- Convert document image cache to use file cache manager app.
+  Add setting DOCUMENTS_CACHE_MAXIMUM_SIZE defaults to 500 MB.
+- Update Celery to version 4.3.0. Settings changed:
+  MAYAN_BROKER_URL to MAYAN_CELERY_BROKER_URL,
+  MAYAN_CELERY_ALWAYS_EAGER to MAYAN_CELERY_TASK_ALWAYS_EAGER.
+- Replace djcelery and replace it with django-celery-beat.
+- Update Celery to version 4.3.0 with 55e9b2263cbdb9b449361412fd18d8ee0a442dd3
+  from versions/next, code from GitLab issue #594 and GitLab merge request !55.
+  Thanks to Jakob Haufe (@sur5r) and Jesaja Everling (@jeverling)
+  for much of the research and code updates.
+- Support wildcard MIME type associations for the file metadata drivers.
+- Rename MAYAN_GUID to MAYAN_GID
+- Update Gunicorn to use sync workers.
+- Include devpi-server as a development dependency.
+- Update default Docker stack file.
+- Remove Redis from the Docker image.
+- Add Celery flower to the Docker image.
+- Allow PIP proxying to the Docker image during build.
+- Default Celery worker concurrency to 0 (auto).
+- Set DJANGO_SETTINGS_MODULE environment variable to make it
+  available to sub processes.
+- Add entrypoint commands to run single workers, single gunicorn
+  or single celery commands like "flower".
+- Add platform template to return queues for a worker.
+- Remove task inspection from task manager app.
+- Move pagination navigation inside the toolbar.
+- Remove document image clear link and view.
+  This is now handled by the file caching app.
+- Add web links app.
+- Add support to display column help text
+  as a tooltip.
+- Update numeric dashboard widget to display
+  thousand commas.
+- Add support for disabling document pages.
 
 Removals
 --------
@@ -56,17 +91,17 @@ Removals
 - Database conversion. Reason for removal. The database conversions support
   provided by this feature (SQLite to PostgreSQL) was being confused with
   database migrations and upgrades.
-  
+
   Database upgrades are the responsibility of the app and the framework.
   Database conversions however are not the responsibility of the app (Mayan),
   they are the responsibility of the framework.
-  
+
   Database conversion is outside the scope of what Mayan does but we added
   the code, management command, instructions and testing setup to provide
   this to our users until the framework (Django) decided to add this
   themselves (like they did with migrations).
 
-  Continued confusion about the purpose of the feature and confusion about 
+  Continued confusion about the purpose of the feature and confusion about
   how errors with this feature were a reflexion of the code quality of
   Mayannecessitated the removal of the database conversion feature.
 
@@ -160,7 +195,13 @@ Backward incompatible changes
 Bugs fixed or issues closed
 ---------------------------
 
+- :gitlab-issue:`526` RuntimeWarning: Never call result.get() within a task!
 - :gitlab-issue:`532` Workflow preview isn't updated right after transitions are modified
+- :gitlab-issue:`540` hint-outdated/update documentation
+- :gitlab-issue:`594` 3.2b1: Unable to install/run under Python 3.5/3.6/3.7
 - :gitlab-issue:`634` Failing docker entrypoint when using secret config
+- :gitlab-issue:`635` Build a docker image for Python3
+- :gitlab-issue:`644` Update sane-utils package in docker image.
+
 
 .. _PyPI: https://pypi.python.org/pypi/mayan-edms/

mayan/__init__.py

@@ -3,7 +3,7 @@ from __future__ import unicode_literals
 __title__ = 'Mayan EDMS'
 __version__ = '3.2.6'
 __build__ = 0x030206
-__build_string__ = 'v3.2.6_Wed Jul 10 03:18:15 2019 -0400'
+__build_string__ = 'v3.2.6-68-gab601f9180_Wed Jul 17 04:30:11 2019 -0400'
 __django_version__ = '1.11'
 __author__ = 'Roberto Rosario'
 __author_email__ = 'roberto.rosario@mayan-edms.com'

mayan/apps/acls/classes.py

@@ -12,6 +12,7 @@ logger = logging.getLogger(__name__)
 class ModelPermission(object):
     _functions = {}
     _inheritances = {}
+    _manager_names = {}
     _registry = {}
 
     @classmethod
@@ -97,6 +98,24 @@ class ModelPermission(object):
     def get_inheritance(cls, model):
         return cls._inheritances[model]
 
+    @classmethod
+    def get_manager(cls, model):
+        try:
+            manager_name = cls.get_manager_name(model=model)
+        except KeyError:
+            manager_name = None
+
+        if manager_name:
+            manager = getattr(model, manager_name)
+        else:
+            manager = model._meta.default_manager
+
+        return manager
+
+    @classmethod
+    def get_manager_name(cls, model):
+        return cls._manager_names[model]
+
     @classmethod
     def register_function(cls, model, function):
         cls._functions[model] = function
@@ -104,3 +123,7 @@ class ModelPermission(object):
     @classmethod
     def register_inheritance(cls, model, related):
         cls._inheritances[model] = related
+
+    @classmethod
+    def register_manager(cls, model, manager_name):
+        cls._manager_names[model] = manager_name

mayan/apps/acls/managers.py

@@ -200,28 +200,26 @@ class AccessControlListManager(models.Manager):
 
         return result
 
-    def check_access(self, obj, permissions, user, manager=None):
+    def check_access(self, obj, permissions, user):
         # Allow specific managers for models that have more than one
         # for example the Document model when checking for access for a trashed
         # document.
 
-        if manager:
-            source_queryset = manager.all()
+        meta = getattr(obj, '_meta', None)
+
+        if not meta:
+            logger.debug(
+                ugettext(
+                    'Object "%s" is not a model and cannot be checked for '
+                    'access.'
+                ) % force_text(obj)
+            )
+            return True
         else:
-            meta = getattr(obj, '_meta', None)
+            manager = ModelPermission.get_manager(model=obj._meta.model)
+            source_queryset = manager.all()
 
-            if not meta:
-                logger.debug(
-                    ugettext(
-                        'Object "%s" is not a model and cannot be checked for '
-                        'access.'
-                    ) % force_text(obj)
-                )
-                return True
-            else:
-                source_queryset = obj._meta.default_manager.all()
-
-        restricted_queryset = obj._meta.default_manager.none()
+        restricted_queryset = manager.none()
         for permission in permissions:
             # Default relationship betweens permissions is OR
             # TODO: Add support for AND relationship

mayan/apps/appearance/static/appearance/css/base.css

@@ -98,14 +98,10 @@ hr {
     min-height: 120px;
     padding-bottom: 1px;
     padding-top: 20px;
-    text-shadow: 1px 1px 1px rgba(0, 0, 0, 0.3);
+    text-shadow: 1px 1px 3px rgba(0, 0, 0, 1);
     white-space: normal;
 }
 
-.btn-block .fa {
-    text-shadow: 1px 1px 1px rgba(0, 0, 0, 0.3);
-}
-
 .radio ul li {
     list-style-type:none;
 }
@@ -115,14 +111,10 @@ a i {
 }
 
 .dashboard-widget {
-    box-shadow: 1px 1px 1px rgba(0, 0, 0, 0.3);
+    box-shadow: 1px 1px 3px rgba(0, 0, 0, 0.7);
     border: 1px solid black;
 }
 
-.dashboard-widget .panel-heading i {
-    text-shadow: 1px 1px 1px rgba(0, 0, 0, 0.3);
-}
-
 .dashboard-widget-icon {
     font-size: 200%;
 }
@@ -220,6 +212,18 @@ a i {
     font-weight: bold;
 }
 
+.panel-highlighted {
+    box-shadow: 0px 0px 3px #18bc9c, 10px 10px 20px #000000;
+}
+
+.panel-highlighted:hover {
+    box-shadow: 0px 0px 3px #18bc9c, 10px 10px 20px #000000, 0px 0px 8px #000000;
+}
+
+.panel-item:not(.panel-highlighted):hover  {
+    box-shadow: 0px 0px 8px #000000;
+}
+
 /* Content */
 @media (min-width:1200px) {
     .container-fluid {
@@ -249,14 +253,6 @@ a i {
     margin: auto;
 }
 
-.thin_border {
-    border: 1px solid black;
-    display: block;
-    margin-left: auto;
-    margin-right: auto;
-}
-
-
 .thin_border-thumbnail {
   display: block;
   max-width: 100%;
@@ -266,6 +262,14 @@ a i {
   margin: auto;
 }
 
+/* Must go after .thin_border-thumbnail */
+.thin_border {
+    border: 1px solid black;
+    display: inline;
+    margin-left: 0px;
+    margin-right: 0px;
+}
+
 #ajax-spinner {
     position: fixed;
     top: 16px;
@@ -536,5 +540,20 @@ a i {
 }
 
 .navbar-fixed-top {
-    box-shadow: 0px 3px 5px rgba(0, 0, 0, 0.5);
+    box-shadow: 0px 3px 3px rgba(0, 0, 0, 0.4);
+}
+
+.toolbar {
+    border: 1px solid rgba(0, 0, 0, 0.1);
+    box-shadow: 1px 1px 2px rgba(0, 0, 0, .3);
+    margin-bottom: 10px;
+    padding-bottom: 8px;
+    padding-left: 12px;
+    padding-right: 15px;
+    padding-top: 8px;
+}
+
+#body-plain {
+    padding-top: 0px;
+    margin-top: 10px;
 }

mayan/apps/appearance/static/appearance/js/base.js

@@ -6,7 +6,8 @@ var MayanAppClass = MayanApp;
 
 var partialNavigation = new PartialNavigation({
     initialURL: initialURL,
-    disabledAnchorClasses: ['disabled'],
+    disabledAnchorClasses: [
+        'btn-multi-item-action', 'disabled', 'pagination-disabled'
+    ],
     excludeAnchorClasses: ['fancybox', 'new_window', 'non-ajax'],
-    formBeforeSerializeCallbacks: [MayanApp.MultiObjectFormProcess],
 });

mayan/apps/appearance/static/appearance/js/mayan_app.js

@@ -17,30 +17,36 @@ class MayanApp {
 
     // Class methods and variables
 
-    static MultiObjectFormProcess ($form, options) {
-        /*
-         * ajaxForm callback to add the external item checkboxes to the
-         * submitted form
-         */
+    static countChecked() {
+        var checkCount = $('.check-all-slave:checked').length;
 
-        if ($form.hasClass('form-multi-object-action')) {
-            // Turn form data into an object
-            var formArray = $form.serializeArray().reduce(function (obj, item) {
-                obj[item.name] = item.value;
-                return obj;
-            }, {});
-
-            // Add all checked checkboxes to the form data
-            $('.form-multi-object-action-checkbox:checked').each(function() {
-                var $this = $(this);
-                formArray[$this.attr('name')] = $this.attr('value');
-            });
-
-            // Set the form data as the data to send
-            options.data = formArray;
+        if (checkCount) {
+            $('#multi-item-title').hide();
+            $('#multi-item-actions').show();
+        } else {
+            $('#multi-item-title').show();
+            $('#multi-item-actions').hide();
         }
     }
 
+    static setupMultiItemActions () {
+        $('body').on('change', '.check-all-slave', function () {
+            MayanApp.countChecked();
+        });
+
+        $('body').on('click', '.btn-multi-item-action', function (event) {
+            var id_list = [];
+            $('.check-all-slave:checked').each(function (index, value) {
+                //Split the name (ie:"pk_200") and extract only the ID
+                id_list.push(value.name.split('_')[1]);
+            });
+            event.preventDefault();
+            partialNavigation.setLocation(
+                $(this).attr('href') + '?id_list=' + id_list.join(',')
+            );
+        });
+    }
+
     static setupNavBarState () {
         $('body').on('click', '.a-main-menu-accordion-link', function (event) {
             console.log('ad');
@@ -166,10 +172,10 @@ class MayanApp {
         var self = this;
 
         this.setupAJAXSpinner();
-        this.setupAutoSubmit();
         this.setupFormHotkeys();
         this.setupFullHeightResizing();
         this.setupItemsSelector();
+        MayanApp.setupMultiItemActions();
         this.setupNavbarCollapse();
         MayanApp.setupNavBarState();
         this.setupNewWindowAnchor();
@@ -177,6 +183,7 @@ class MayanApp {
             value.app = self;
             app.doRefreshAJAXMenu(value);
         });
+        this.setupPanelSelection();
         partialNavigation.initialize();
     }
 
@@ -200,14 +207,6 @@ class MayanApp {
         });
     }
 
-    setupAutoSubmit () {
-        $('body').on('change', '.select-auto-submit', function () {
-            if ($(this).val()) {
-                $(this.form).trigger('submit');
-            }
-        });
-    }
-
     setupFormHotkeys () {
         $('body').on('keypress', '.form-hotkey-enter', function (e) {
             if ((e.which && e.which == 13) || (e.keyCode && e.keyCode == 13)) {
@@ -238,9 +237,22 @@ class MayanApp {
         app.lastChecked = null;
 
         $('body').on('click', '.check-all', function (event) {
+            var $this = $(this);
             var checked = $(event.target).prop('checked');
             var $checkBoxes = $('.check-all-slave');
 
+            if (checked === undefined) {
+                checked = $this.data('checked');
+                checked = !checked;
+                $this.data('checked', checked);
+
+                if (checked) {
+                    $this.find('[data-fa-i2svg]').addClass($this.data('icon-checked')).removeClass($this.data('icon-unchecked'));
+                } else {
+                    $this.find('[data-fa-i2svg]').addClass($this.data('icon-unchecked')).removeClass($this.data('icon-checked'));
+                }
+            }
+
             $checkBoxes.prop('checked', checked);
             $checkBoxes.trigger('change');
         });
@@ -286,6 +298,58 @@ class MayanApp {
         });
     }
 
+   setupPanelSelection () {
+        var app = this;
+
+        // Setup panel highlighting on check
+        $('body').on('change', '.check-all-slave', function (event) {
+            var checked = $(event.target).prop('checked');
+            if (checked) {
+                $(this).closest('.panel-item').addClass('panel-highlighted');
+            } else {
+                $(this).closest('.panel-item').removeClass('panel-highlighted');
+            }
+        });
+
+        $('body').on('click', '.panel-item', function (event) {
+            var $this = $(this);
+            var targetSrc = $(event.target).prop('src');
+            var targetHref = $(event.target).prop('href');
+            var targetIsButton = event.target.tagName === 'BUTTON';
+            var lastChecked = null;
+
+            if ((targetSrc === undefined) && (targetHref === undefined) && (targetIsButton === false)) {
+                var $checkbox = $this.find('.check-all-slave');
+                var checked = $checkbox.prop('checked');
+
+                if (checked) {
+                    $checkbox.prop('checked', '');
+                    $checkbox.trigger('change');
+                } else {
+                    $checkbox.prop('checked', 'checked');
+                    $checkbox.trigger('change');
+                }
+
+                if(!app.lastChecked) {
+                    app.lastChecked = $checkbox;
+                }
+
+                if (event.shiftKey) {
+                    var $checkBoxes = $('.check-all-slave');
+
+                    var start = $checkBoxes.index($checkbox);
+                    var end = $checkBoxes.index(app.lastChecked);
+
+                    $checkBoxes.slice(
+                        Math.min(start, end), Math.max(start, end) + 1
+                    ).prop('checked', app.lastChecked.prop('checked')).trigger('change');
+                }
+                app.lastChecked = $checkbox;
+                window.getSelection().removeAllRanges();
+            }
+        });
+    }
+
     setupScrollView () {
         $('.scrollable').scrollview();
     }

mayan/apps/appearance/templates/appearance/base.html

@@ -136,6 +136,9 @@
             },
         {% endfor %}
     ];
+    $(function () {
+      $('[data-toggle="tooltip"]').tooltip();
+    })
 </script>
 {% block javascript %}{% endblock %}
 

mayan/apps/appearance/templates/appearance/base_plain.html

@@ -33,7 +33,7 @@
             }
         </script>
     </head>
-    <body>
+    <body id="body-plain">
         {% block content_plain %}{% endblock %}
 
         <script src="{% static 'appearance/node_modules/jquery/dist/jquery.min.js' %}" type="text/javascript"></script>

mayan/apps/appearance/templates/appearance/generic_list_items_subtemplate.html

@@ -11,41 +11,9 @@
                 {% include 'appearance/no_results.html' %}
             </div>
         {% else %}
-            <h4>
-                {% if page_obj %}
-                    {% if page_obj.paginator.num_pages != 1 %}
-                        {% blocktrans with page_obj.start_index as start and page_obj.end_index as end and page_obj.paginator.object_list|length as total and page_obj.number as page_number and page_obj.paginator.num_pages as total_pages %}Total ({{ start }} - {{ end }} out of {{ total }}) (Page {{ page_number }} of {{ total_pages }}){% endblocktrans %}
-                    {% else %}
-                        {% blocktrans with page_obj.paginator.object_list|length as total %}Total: {{ total }}{% endblocktrans %}
-                    {% endif %}
-                {% else %}
-                    {% blocktrans with object_list|length as total %}Total: {{ total }}{% endblocktrans %}
-                {% endif %}
-            </h4>
-            <hr>
+            {% include "appearance/list_header.html" %}
+            {% navigation_resolve_menu name='multi item' sort_results=True source=object_list.0 as links_multi_menus_results %}
             <div class="well center-block">
-                <div class="clearfix">
-                    <div class="pull-right">
-                        <form action="{% url 'common:multi_object_action_view' %}" class="form-multi-object-action" method="get">
-                            {% if object_list %}
-                                {% if not hide_multi_item_actions %}
-                                    {% get_multi_item_links_form object_list %}
-                                {% endif %}
-                                {% if multi_item_actions %}
-                                    <fieldset style="margin-top: -10px;">
-                                        <input class="check-all" type="checkbox"/>&nbsp;
-                                        {{ multi_item_form }}
-                                    </fieldset>
-                                {% endif %}
-                            {% endif %}
-                        </form>
-                    </div>
-                </div>
-
-                {% if object_list %}
-                    <hr style="border-bottom: 1px solid lightgrey;">
-                {% endif %}
-
                 <div class="row row-items">
                     {% for object in object_list %}
                         <div class="{{ column_class|default:'col-xs-12 col-sm-4 col-md-3 col-lg-2' }}">
@@ -53,9 +21,9 @@
                                 <div class="panel-heading">
                                     <div class="form-group">
                                         <div class="checkbox">
-                                            <label for="id_indexes_0">
-                                                {% if multi_item_actions %}
-                                                    <input class="form-multi-object-action-checkbox check-all-slave checkbox" type="checkbox" name="pk_{{ object.pk }}" />
+                                            <label for="id_indexes_0" style="cursor: auto;">
+                                                {% if links_multi_menus_results %}
+                                                    <input class="form-multi-object-action-checkbox check-all-slave checkbox" name="pk_{{ object.pk }}" style="cursor: pointer;" type="checkbox" />
                                                 {% endif %}
 
                                                 <span style="color: white; word-break: break-all; overflow-wrap: break-word;">
@@ -68,12 +36,7 @@
                                                     {% else %}
                                                         {% navigation_get_source_columns source=object only_identifier=True as source_column %}
                                                         {% navigation_source_column_resolve column=source_column as column_value %}
-
-                                                        {% if source_column.is_attribute_absolute_url or source_column.is_object_absolute_url %}
-                                                            <a href="{% navigation_source_column_get_absolute_url source_column=source_column obj=object %}">{{ column_value }}</a>
-                                                        {% else %}
-                                                            {{ column_value }}
-                                                        {% endif %}
+                                                        {{ column_value }}
                                                     {% endif %}
                                                 </span>
                                             </label>
@@ -82,7 +45,6 @@
 
                                 </div>
                                 <div class="panel-body">
-
                                     {% if not hide_columns %}
                                         {% navigation_get_source_columns source=object exclude_identifier=True as source_columns %}
                                         {% for column in source_columns %}
@@ -136,7 +98,6 @@
                         </div>
                     {% endfor %}
                 </div>
-                {% include 'pagination/pagination.html' %}
             </div>
         {% endif %}
     </div>

mayan/apps/appearance/templates/appearance/generic_list_subtemplate.html

@@ -1,6 +1,7 @@
 {% load i18n %}
 {% load static %}
 
+{% load appearance_tags %}
 {% load common_tags %}
 {% load navigation_tags %}
 
@@ -11,44 +12,16 @@
                 {% include 'appearance/no_results.html' %}
             </div>
         {% else %}
-            <h4>
-                {% if page_obj %}
-                    {% if page_obj.paginator.num_pages != 1 %}
-                        {% blocktrans with page_obj.start_index as start and page_obj.end_index as end and page_obj.paginator.object_list|length as total and page_obj.number as page_number and page_obj.paginator.num_pages as total_pages %}Total ({{ start }} - {{ end }} out of {{ total }}) (Page {{ page_number }} of {{ total_pages }}){% endblocktrans %}
-                    {% else %}
-                        {% blocktrans with page_obj.paginator.object_list|length as total %}Total: {{ total }}{% endblocktrans %}
-                    {% endif %}
-                {% else %}
-                    {% blocktrans with object_list|length as total %}Total: {{ total }}{% endblocktrans %}
-                {% endif %}
-            </h4>
-            <hr>
-
+            {% include "appearance/list_header.html" %}
+            {% navigation_resolve_menu name='multi item' sort_results=True source=object_list.0 as links_multi_menus_results %}
             <div class="well center-block">
-                <div class="clearfix">
-                    <div class="pull-right">
-                        <form action="{% url 'common:multi_object_action_view' %}" class="form-multi-object-action" method="get">
-                            {% if object_list %}
-                                {% if not hide_multi_item_actions %}
-                                    {% get_multi_item_links_form object_list %}
-                                {% endif %}
-                                {% if multi_item_actions %}
-                                    <fieldset style="margin-top: -10px; margin-bottom: 10px;">
-                                        {{ multi_item_form }}
-                                    </fieldset>
-                                {% endif %}
-                            {% endif %}
-                        </form>
-                    </div>
-                </div>
-
                 <div class="table-responsive">
                     <table class="table table-condensed table-striped">
                         <tbody>
                             {% if not hide_header %}
                                 <tr>
-                                    {% if multi_item_actions %}
-                                        <th class="first"><input class="checkbox check-all" type="checkbox" /></th>
+                                    {% if links_multi_menus_results %}
+                                        <th class="first"></th>
                                     {% endif %}
 
                                     {% if not hide_object %}
@@ -58,30 +31,40 @@
                                         {% if source_column %}
                                             <th>
                                                 {% if source_column.is_sortable %}
-                                                    <a href="{% navigation_get_sort_field_querystring column=source_column %}">{{ source_column.label }}
+                                                    <a href="{% navigation_get_sort_field_querystring column=source_column %}">{{ source_column.label }}</a>
                                                     {% if source_column.get_sort_field == sort_field %}
                                                         {% if icon_sort %}{{ icon_sort.render }}{% endif %}
                                                     {% endif %}
-                                                    </a>
                                                 {% else %}
                                                     {{ source_column.label }}
                                                 {% endif %}
+
+                                                {% if source_column.help_text %}
+                                                    <span data-toggle="tooltip" data-placement="bottom" title="{{ source_column.help_text }}">
+                                                    {% get_icon icon_path='mayan.apps.navigation.icons.icon_source_column_help_text' %}
+                                                    </span>
+                                                {% endif %}
                                             </th>
                                         {% endif %}
                                     {% endif %}
 
                                     {% if not hide_columns %}
                                         {% navigation_get_source_columns source=object_list exclude_identifier=True as source_columns %}
-                                        {% for column in source_columns %}
+                                        {% for source_column in source_columns %}
                                             <th>
-                                                {% if column.is_sortable %}
-                                                    <a href="{% navigation_get_sort_field_querystring column=column %}">{{ column.label }}
-                                                    {% if column.get_sort_field == sort_field %}
+                                                {% if source_column.is_sortable %}
+                                                    <a href="{% navigation_get_sort_field_querystring column=source_column %}">{{ source_column.label }}</a>
+                                                    {% if source_column.get_sort_field == sort_field %}
                                                         {% if icon_sort %}{{ icon_sort.render }}{% endif %}
                                                     {% endif %}
-                                                    </a>
                                                 {% else %}
-                                                    {{ column.label }}
+                                                    {{ source_column.label }}
+                                                {% endif %}
+
+                                                {% if source_column.help_text %}
+                                                    <span data-toggle="tooltip" data-placement="bottom" title="{{ source_column.help_text }}">
+                                                    {% get_icon icon_path='mayan.apps.navigation.icons.icon_source_column_help_text' %}
+                                                    </span>
                                                 {% endif %}
                                             </th>
                                         {% endfor %}
@@ -99,9 +82,9 @@
                             {% for object in object_list %}
                                 <tr>
 
-                                    {% if multi_item_actions %}
+                                    {% if links_multi_menus_results %}
                                         <td>
-                                            <input type="checkbox" class="form-multi-object-action-checkbox check-all-slave checkbox" name="pk_{{ object.pk }}" value="" />
+                                            <input class="form-multi-object-action-checkbox check-all-slave checkbox" name="pk_{{ object.pk }}" type="checkbox" value="" />
                                         </td>
                                     {% endif %}
 
@@ -112,11 +95,7 @@
                                         {% navigation_source_column_resolve column=source_column as column_value %}
                                         {% if column_value %}
                                             <td>
-                                                {% if source_column.is_attribute_absolute_url or source_column.is_object_absolute_url %}
-                                                    <a href="{% navigation_source_column_get_absolute_url source_column=source_column obj=object %}">{{ column_value }}</a>
-                                                {% else %}
-                                                    {{ column_value }}
-                                                {% endif %}
+                                                {{ column_value }}
                                             </td>
                                         {% endif %}
                                     {% endif %}
@@ -170,7 +149,6 @@
                         </tbody>
                     </table>
                 </div>
-                {% include 'pagination/pagination.html' %}
             </div>
         {% endif %}
     </div>

mayan/apps/appearance/templates/appearance/list_header.html

@@ -0,0 +1,28 @@
+{% load i18n %}
+{% load static %}
+
+{% load common_tags %}
+{% load navigation_tags %}
+
+{% if object_list %}
+    <h4>
+        {% if page_obj %}
+            {% if page_obj.paginator.num_pages != 1 %}
+                {% blocktrans with page_obj.start_index as start and page_obj.end_index as end and page_obj.paginator.object_list|length as total and page_obj.number as page_number and page_obj.paginator.num_pages as total_pages %}Total ({{ start }} - {{ end }} out of {{ total }}) (Page {{ page_number }} of {{ total_pages }}){% endblocktrans %}
+            {% else %}
+                {% blocktrans with page_obj.paginator.object_list|length as total %}Total: {{ total }}{% endblocktrans %}
+            {% endif %}
+        {% else %}
+            {% blocktrans with object_list|length as total %}Total: {{ total }}{% endblocktrans %}
+        {% endif %}
+    </h4>
+    <hr>
+
+    {% if not hide_multi_item_actions %}
+        {% navigation_resolve_menu name='multi item' sort_results=True source=object_list.0 as links_multi_menus_results %}
+    {% endif %}
+{% endif %}
+
+<div class="clearfix">
+    {% include 'appearance/list_toolbar.html' %}
+</div>

mayan/apps/appearance/templates/appearance/list_toolbar.html

@@ -0,0 +1,90 @@
+{% load i18n %}
+
+{% load common_tags %}
+{% load navigation_tags %}
+
+{% if is_paginated or links_multi_menus_results %}
+    <div class="well center-block toolbar">
+{% endif %}
+
+    {% if links_multi_menus_results %}
+        <div class="pull-left">
+            <div class="btn-toolbar" role="toolbar" style="margin-right: 10px;">
+                <div class="btn-group">
+                        <a class="btn btn-default btn-sm check-all" data-checked=false data-icon-checked="fa fa-check-square" data-icon-unchecked="far fa-square" title="{% trans 'Select/Deselect all' %}">
+                            <i class="far fa-square"></i>
+                        </a>
+                </div>
+            </div>
+        </div>
+    {% endif %}
+
+    {% if is_paginated %}
+        <div class="pull-left">
+            <div class="btn-toolbar" role="toolbar">
+                <div class="btn-group">
+                    {% if page_obj.has_previous %}
+                        <a class="btn btn-default btn-sm" href="?{{ page_obj.previous_page_number.querystring }}">&lsaquo;&lsaquo;</a>
+                    {% else %}
+                        <a class="btn btn-default btn-sm disabled" href="#">&lsaquo;&lsaquo;</a>
+                    {% endif %}
+
+                    {% for page in page_obj.pages %}
+
+                        {% if page %}
+
+                            {% ifequal page page_obj.number %}
+                                <a class="active btn btn-default btn-sm pagination-disabled" href="#">{{ page }}</a>
+                            {% else %}
+                                <a class="btn btn-default btn-sm" href="?{{ page.querystring }}">{{ page }}</a>
+                            {% endifequal %}
+                        {% else %}
+                            <a class="btn btn-default btn-sm disabled" href="#">...</a>
+                        {% endif %}
+
+                    {% endfor %}
+                    {% if page_obj.has_next %}
+                        <a class="btn btn-default btn-sm" href="?{{ page_obj.next_page_number.querystring }}">&rsaquo;&rsaquo;</a>
+                    {% else %}
+                        <a class="btn btn-default btn-sm disabled" href="#">&rsaquo;&rsaquo;</a>
+                    {% endif %}
+                </div>
+            </div>
+        </div>
+    {% endif %}
+
+    {% if links_multi_menus_results %}
+        <p class="pull-right" id="multi-item-title" style="line-height: 16px; padding-top: 8px;">{% trans 'Select items to activate bulk actions. Use Shift + click to select many.' %}</p>
+
+        <div class="pull-right btn-group" id="multi-item-actions" style="display: none;">
+            <button aria-expanded="true" class="btn btn-danger btn-sm dropdown-toggle" data-toggle="dropdown" type="button">
+                {% trans 'Bulk actions' %}
+                <span class="caret"></span>
+                <span class="sr-only">{% trans 'Toggle Dropdown' %}</span>
+            </button>
+            <ul class="dropdown-menu" role="menu">
+                {% for multi_item_menu_results in links_multi_menus_results %}
+                    {% for link_group in multi_item_menu_results.link_groups %}
+                        {% with link_group.links as object_navigation_links %}
+                        {% with 'true' as as_li %}
+                        {% with 'true' as hide_active_anchor %}
+                        {% with 'btn-sm btn-multi-item-action' as link_classes %}
+                            {% include 'navigation/generic_navigation.html' %}
+                        {% endwith %}
+                        {% endwith %}
+                        {% endwith %}
+                        {% endwith %}
+
+                    {% endfor %}
+                    {% if not forloop.last and link_group %}
+                        <li class="divider"></li>
+                    {% endif %}
+                {% endfor %}
+            </ul>
+        </div>
+    {% endif %}
+
+{% if is_paginated or links_multi_menus_results %}
+    <div class="clearfix"></div>
+    </div>
+{% endif %}

mayan/apps/appearance/templates/pagination/pagination.html

@@ -11,7 +11,7 @@
             {% if page %}
 
                 {% ifequal page page_obj.number %}
-                    <li class="active"><a href="#">{{ page }}</a></li>
+                    <li class="active"><a class="pagination-disabled" href="#">{{ page }}</a></li>
                 {% else %}
                     <li><a href="?{{ page.querystring }}">{{ page }}</a></li>
                 {% endifequal %}

mayan/apps/authentication/templates/authentication/login.html

@@ -17,7 +17,7 @@
     {% motd %}
 
     <div class="row">
-        <div class="col-xs-10 col-xs-offset-1 col-sm-8 col-sm-offset-2 col-md-6 col-md-offset-3 col-lg-4 col-lg-offset-4">
+        <div class="col-xs-12 col-sm-10 col-sm-offset-1 col-md-10 col-md-offset-1 col-lg-6 col-lg-offset-3">
             <div class="panel panel-primary">
                 <div class="panel-heading">
                     <h3 class="panel-title">&nbsp;</h3>

mayan/apps/authentication/tests/test_views.py

@@ -14,7 +14,6 @@ from django.utils.http import urlunquote_plus
 from mayan.apps.common.tests import GenericViewTestCase
 from mayan.apps.smart_settings.classes import Namespace
 from mayan.apps.user_management.permissions import permission_user_edit
-from mayan.apps.user_management.tests.mixins import UserTestMixin
 from mayan.apps.user_management.tests.literals import TEST_USER_PASSWORD_EDITED
 
 from ..settings import setting_maximum_session_length
@@ -262,7 +261,7 @@ class UserLoginTestCase(GenericViewTestCase):
         self.assertEqual(response.redirect_chain, [(TEST_REDIRECT_URL, 302)])
 
 
-class UserViewTestCase(UserTestMixin, UserPasswordViewTestMixin, GenericViewTestCase):
+class UserViewTestCase(UserPasswordViewTestMixin, GenericViewTestCase):
     def test_user_set_password_view_no_access(self):
         self._create_test_user()
 

mayan/apps/autoadmin/templates/autoadmin/credentials.html

@@ -4,7 +4,7 @@
 
 {% if autoadmin_properties.account %}
     <div class="row">
-        <div class="col-xs-10 col-xs-offset-1 col-sm-8 col-sm-offset-2 col-md-6 col-md-offset-3 col-lg-4 col-lg-offset-4">
+        <div class="col-xs-12 col-sm-10 col-sm-offset-1 col-md-10 col-md-offset-1 col-lg-6 col-lg-offset-3">
             <br>
             <div class="panel panel-primary">
                 <div class="panel-heading">

mayan/apps/cabinets/migrations/0002_auto_20190729_0236.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.22 on 2019-07-29 02:36
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('cabinets', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='cabinet',
+            name='label',
+            field=models.CharField(help_text='A short text used to identify the cabinet.', max_length=128, verbose_name='Label'),
+        ),
+    ]

mayan/apps/cabinets/models.py

@@ -32,7 +32,10 @@ class Cabinet(MPTTModel):
         blank=True, db_index=True, null=True, on_delete=models.CASCADE,
         related_name='children', to='self'
     )
-    label = models.CharField(max_length=128, verbose_name=_('Label'))
+    label = models.CharField(
+        help_text=_('A short text used to identify the cabinet.'),
+        max_length=128, verbose_name=_('Label')
+    )
     documents = models.ManyToManyField(
         blank=True, related_name='cabinets', to=Document,
         verbose_name=_('Documents')

mayan/apps/cabinets/urls.py

@@ -12,55 +12,62 @@ from .views import (
     CabinetDeleteView, CabinetDetailView, CabinetEditView, CabinetListView,
 )
 
-urlpatterns = [
+urlpatterns_cabinets = [
     url(
-        regex=r'^list/$', view=CabinetListView.as_view(), name='cabinet_list'
+        regex=r'^cabinets/$', view=CabinetListView.as_view(), name='cabinet_list'
     ),
     url(
-        regex=r'^(?P<pk>\d+)/child/add/$', view=CabinetChildAddView.as_view(),
-        name='cabinet_child_add'
-    ),
-    url(
-        regex=r'^create/$', view=CabinetCreateView.as_view(),
+        regex=r'^cabinets/create/$', view=CabinetCreateView.as_view(),
         name='cabinet_create'
     ),
     url(
-        regex=r'^(?P<pk>\d+)/edit/$', view=CabinetEditView.as_view(),
-        name='cabinet_edit'
+        regex=r'^cabinets/(?P<pk>\d+)/children/add/$', view=CabinetChildAddView.as_view(),
+        name='cabinet_child_add'
     ),
     url(
-        regex=r'^(?P<pk>\d+)/delete/$', view=CabinetDeleteView.as_view(),
+        regex=r'^cabinets/(?P<pk>\d+)/delete/$', view=CabinetDeleteView.as_view(),
         name='cabinet_delete'
     ),
     url(
-        regex=r'^(?P<pk>\d+)/$', view=CabinetDetailView.as_view(),
-        name='cabinet_view'
+        regex=r'^cabinets/(?P<pk>\d+)/edit/$', view=CabinetEditView.as_view(),
+        name='cabinet_edit'
     ),
     url(
-        regex=r'^document/(?P<pk>\d+)/cabinet/add/$',
+        regex=r'^cabinets/(?P<pk>\d+)/$', view=CabinetDetailView.as_view(),
+        name='cabinet_view'
+    ),
+]
+
+urlpatterns_documents_cabinets = [
+    url(
+        regex=r'^documents/(?P<pk>\d+)/cabinets/add/$',
         view=DocumentAddToCabinetView.as_view(), name='document_cabinet_add'
     ),
     url(
-        regex=r'^document/multiple/cabinet/add/$',
+        regex=r'^documents/multiple/cabinets/add/$',
         view=DocumentAddToCabinetView.as_view(),
         name='document_multiple_cabinet_add'
     ),
     url(
-        regex=r'^document/(?P<pk>\d+)/cabinet/remove/$',
+        regex=r'^documents/(?P<pk>\d+)/cabinets/remove/$',
         view=DocumentRemoveFromCabinetView.as_view(),
         name='document_cabinet_remove'
     ),
     url(
-        regex=r'^document/multiple/cabinet/remove/$',
+        regex=r'^documents/multiple/cabinets/remove/$',
         view=DocumentRemoveFromCabinetView.as_view(),
         name='multiple_document_cabinet_remove'
     ),
     url(
-        regex=r'^document/(?P<pk>\d+)/cabinet/list/$',
+        regex=r'^documents/(?P<pk>\d+)/cabinets/$',
         view=DocumentCabinetListView.as_view(), name='document_cabinet_list'
     ),
 ]
 
+urlpatterns = []
+urlpatterns.extend(urlpatterns_cabinets)
+urlpatterns.extend(urlpatterns_documents_cabinets)
+
 api_urls = [
     url(
         regex=r'^cabinets/(?P<pk>[0-9]+)/documents/(?P<document_pk>[0-9]+)/$',

mayan/apps/checkouts/migrations/0008_checkedoutdocument.py

@@ -0,0 +1,26 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.20 on 2019-07-25 04:52
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('documents', '0050_auto_20190725_0451'),
+        ('checkouts', '0007_auto_20180310_1715'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='CheckedOutDocument',
+            fields=[
+            ],
+            options={
+                'proxy': True,
+                'indexes': [],
+            },
+            bases=('documents.document',),
+        ),
+    ]

mayan/apps/checkouts/tests/test_views.py

@@ -315,43 +315,6 @@ class DocumentCheckoutViewTestCase(
             response=response, text=self.test_document.label, status_code=200
         )
 
-    def test_document_check_out_new_version(self):
-        """
-        Gitlab issue #231
-        User shown option to upload new version of a document even though it
-        is blocked by checkout - v2.0.0b2
-
-        Expected results:
-            - Link to upload version view should not resolve
-            - Upload version view should reject request
-        """
-        self._create_test_case_superuser()
-        self._check_out_test_document()
-        self.login_superuser()
-
-        response = self.post(
-            viewname='sources:upload_version', kwargs={
-                'document_pk': self.test_document.pk
-            }, follow=True
-        )
-
-        self.assertContains(
-            response=response, text='blocked from uploading',
-            status_code=200
-        )
-
-        response = self.get(
-            viewname='documents:document_version_list', kwargs={
-                'pk': self.test_document.pk
-            }, follow=True
-        )
-
-        # Needed by the url view resolver
-        response.context.current_app = None
-        resolved_link = link_document_version_upload.resolve(context=response.context)
-
-        self.assertEqual(resolved_link, None)
-
     def test_document_check_in_forcefull_view_no_permission(self):
         # Gitlab issue #237
         # Forcefully checking in a document by a user without adequate
@@ -388,3 +351,47 @@ class DocumentCheckoutViewTestCase(
         )
         self.assertEqual(response.status_code, 302)
         self.assertFalse(self.test_document.is_checked_out())
+
+
+class NewVersionBlockViewTestCase(
+    DocumentCheckoutTestMixin, DocumentCheckoutViewTestMixin,
+    GenericDocumentViewTestCase):
+
+    def test_document_check_out_new_version(self):
+        """
+        Gitlab issue #231
+        User shown option to upload new version of a document even though it
+        is blocked by checkout - v2.0.0b2
+
+        Expected results:
+            - Link to upload version view should not resolve
+            - Upload version view should reject request
+        """
+        self._create_test_case_superuser()
+        self._check_out_test_document()
+        self.login_superuser()
+
+        response = self.post(
+            viewname='sources:upload_version', kwargs={
+                'document_pk': self.test_document.pk
+            }, follow=True
+        )
+
+        self.assertContains(
+            response=response, text='blocked from uploading',
+            status_code=200
+        )
+
+        response = self.get(
+            viewname='documents:document_version_list', kwargs={
+                'pk': self.test_document.pk
+            }, follow=True
+        )
+
+        # Needed by the url view resolver
+        response.context.current_app = None
+        resolved_link = link_document_version_upload.resolve(
+            context=response.context
+        )
+
+        self.assertEqual(resolved_link, None)

mayan/apps/common/dependencies.py

@@ -61,102 +61,9 @@ PythonDependency(
         SOFTWARE.
     ''', module=__name__, name='PyYAML', version_string='==5.1.1'
 )
-PythonDependency(
-    copyright_text='''
-        Copyright (c) 2015 Ask Solem & contributors.  All rights reserved.
-        Copyright (c) 2012-2014 GoPivotal, Inc.  All rights reserved.
-        Copyright (c) 2009, 2010, 2011, 2012 Ask Solem, and individual contributors.  All rights reserved.
-
-        Celery is licensed under The BSD License (3 Clause, also known as
-        the new BSD license).  The license is an OSI approved Open Source
-        license and is GPL-compatible(1).
-
-        The license text can also be found here:
-        http://www.opensource.org/licenses/BSD-3-Clause
-
-        License
-        =======
-
-        Redistribution and use in source and binary forms, with or without
-        modification, are permitted provided that the following conditions are met:
-        * Redistributions of source code must retain the above copyright
-        notice, this list of conditions and the following disclaimer.
-        * Redistributions in binary form must reproduce the above copyright
-        notice, this list of conditions and the following disclaimer in the
-        documentation and/or other materials provided with the distribution.
-        * Neither the name of Ask Solem, nor the
-        names of its contributors may be used to endorse or promote products
-        derived from this software without specific prior written permission.
-
-        THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-        AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
-        THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-        PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL Ask Solem OR CONTRIBUTORS
-        BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-        CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-        SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-        INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-        CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-        ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-        POSSIBILITY OF SUCH DAMAGE.
-
-        Documentation License
-        =====================
-
-        The documentation portion of Celery (the rendered contents of the
-        "docs" directory of a software distribution or checkout) is supplied
-        under the Creative Commons Attribution-Noncommercial-Share Alike 3.0
-        United States License as described by
-        http://creativecommons.org/licenses/by-nc-sa/3.0/us/
-
-        Footnotes
-        =========
-        (1) A GPL-compatible license makes it possible to
-        combine Celery with other software that is released
-        under the GPL, it does not mean that we're distributing
-        Celery under the GPL license.  The BSD license, unlike the GPL,
-        let you distribute a modified version without making your
-        changes open source.
-    ''', module=__name__, name='celery', version_string='==3.1.24'
-)
-PythonDependency(
-    copyright_text='''
-        Copyright (c) 2012-2013 GoPivotal, Inc.  All Rights Reserved.
-        Copyright (c) 2009-2012 Ask Solem.  All Rights Reserved.
-        All rights reserved.
-
-        Redistribution and use in source and binary forms, with or without
-        modification, are permitted provided that the following conditions are met:
-
-        * Redistributions of source code must retain the above copyright notice,
-        this list of conditions and the following disclaimer.
-        * Redistributions in binary form must reproduce the above copyright
-        notice, this list of conditions and the following disclaimer in the
-        documentation and/or other materials provided with the distribution.
-
-        Neither the name of Ask Solem nor the names of its contributors may be used
-        to endorse or promote products derived from this software without specific
-        prior written permission.
-
-        THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-        AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
-        THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-        PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
-        BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-        CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-        SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-        INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-        CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-        ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-        POSSIBILITY OF SUCH DAMAGE.
-    ''', module=__name__, name='django-celery', version_string='==3.2.1'
-)
 PythonDependency(
     module=__name__, name='django-downloadview', version_string='==1.9'
 )
-PythonDependency(
-    module=__name__, name='django-environ', version_string='==0.4.5'
-)
 PythonDependency(
     module=__name__, name='django-formtools', version_string='==2.1'
 )
@@ -383,6 +290,10 @@ PythonDependency(
     module=__name__, environment=environment_development, name='Werkzeug',
     version_string='==0.15.4'
 )
+PythonDependency(
+    module=__name__, environment=environment_development, name='devpi-server',
+    version_string='==5.0.0'
+)
 PythonDependency(
     environment=environment_development, module=__name__,
     name='django-debug-toolbar', version_string='==1.11'

mayan/apps/common/http.py

@@ -2,6 +2,7 @@ from __future__ import unicode_literals
 
 from django.http import QueryDict
 from django.utils.encoding import force_bytes
+from django.utils.six import PY3
 
 
 class URL(object):
@@ -20,9 +21,7 @@ class URL(object):
 
     def to_string(self):
         if self._args.keys():
-            query = force_bytes(
-                '?{}'.format(self._args.urlencode())
-            )
+            query = '?{}'.format(self._args.urlencode())
         else:
             query = ''
 
@@ -31,6 +30,9 @@ class URL(object):
         else:
             path = ''
 
-        result = force_bytes('{}{}'.format(path, query))
+        result = '{}{}'.format(path, query)
 
-        return result
+        if PY3:
+            return result
+        else:
+            return force_bytes(result)

mayan/apps/common/management/commands/initialsetup.py

@@ -28,8 +28,8 @@ class Command(management.BaseCommand):
         )
 
         parser.add_argument(
-            '--no-javascript', action='store_true', dest='no_javascript',
-            help='Don\'t download the JavaScript dependencies.',
+            '--no-dependencies', action='store_true', dest='no_dependencies',
+            help='Don\'t download dependencies.',
         )
 
     def initialize_system(self, force=False):
@@ -88,9 +88,9 @@ class Command(management.BaseCommand):
         self.initialize_system(force=options.get('force', False))
         pre_initial_setup.send(sender=self)
 
-        if not options.get('no_javascript', False):
+        if not options.get('no_dependencies', False):
             management.call_command(
-                command_name='installjavascript', interactive=False
+                command_name='installdependencies', interactive=False
             )
 
         management.call_command(

mayan/apps/common/management/commands/literals.py

@@ -1,10 +0,0 @@
-from __future__ import unicode_literals
-
-SETTING_FILE_TEMPLATE = '''
-from __future__ import absolute_import, unicode_literals
-
-from .base import *
-
-SECRET_KEY = '{0}'
-
-'''

mayan/apps/common/management/commands/performupgrade.py

@@ -11,8 +11,8 @@ class Command(management.BaseCommand):
 
     def add_arguments(self, parser):
         parser.add_argument(
-            '--no-javascript', action='store_true', dest='no_javascript',
-            help='Don\'t download the JavaScript dependencies.',
+            '--no-dependencies', action='store_true', dest='no_dependencies',
+            help='Don\'t download dependencies.',
         )
 
     def handle(self, *args, **options):
@@ -25,9 +25,9 @@ class Command(management.BaseCommand):
                 )
             )
 
-        if not options.get('no_javascript', False):
+        if not options.get('no_dependencies', False):
             management.call_command(
-                command_name='installjavascript', interactive=False
+                command_name='installdependencies', interactive=False
             )
 
         try:

mayan/apps/common/management/commands/purgeperiodictasks.py

@@ -2,7 +2,7 @@ from __future__ import unicode_literals
 
 from django.core import management
 
-from djcelery.models import IntervalSchedule, PeriodicTask
+from django_celery_beat.models import IntervalSchedule, PeriodicTask
 
 
 class Command(management.BaseCommand):

mayan/apps/common/mixins.py

@@ -1,6 +1,7 @@
 from __future__ import unicode_literals
 
 from django.contrib import messages
+from django.contrib.contenttypes.models import ContentType
 from django.core.exceptions import ImproperlyConfigured, PermissionDenied
 from django.http import Http404, HttpResponseRedirect
 from django.shortcuts import get_object_or_404
@@ -8,6 +9,7 @@ from django.urls import reverse
 from django.utils.translation import ungettext, ugettext_lazy as _
 from django.views.generic.detail import SingleObjectMixin
 
+from mayan.apps.acls.classes import ModelPermission
 from mayan.apps.acls.models import AccessControlList
 from mayan.apps.permissions import Permission
 
@@ -17,6 +19,28 @@ from .literals import PK_LIST_SEPARATOR
 from .settings import setting_home_view
 
 
+class ContentTypeViewMixin(object):
+    """
+    This mixin makes it easier for views to retrieve a content type from
+    the URL pattern.
+    """
+    content_type_url_kw_args = {
+        'app_label': 'app_label',
+        'model_name': 'model'
+    }
+
+    def get_content_type(self):
+        return get_object_or_404(
+            klass=ContentType,
+            app_label=self.kwargs[
+                self.content_type_url_kw_args['app_label']
+            ],
+            model=self.kwargs[
+                self.content_type_url_kw_args['model_name']
+            ]
+        )
+
+
 class DeleteExtraDataMixin(object):
     """
     Mixin to populate the extra data needed for delete views
@@ -103,7 +127,15 @@ class ExternalObjectMixin(object):
                 'get_external_object_queryset() method.'
             )
 
-        return self.external_object_queryset or self.external_object_class.objects.all()
+        queryset = self.external_object_queryset
+
+        if not queryset:
+            manager = ModelPermission.get_manager(
+                model=self.external_object_class
+            )
+            queryset = manager.all()
+
+        return queryset
 
     def get_external_object_queryset_filtered(self):
         queryset = self.get_external_object_queryset()
@@ -118,6 +150,20 @@ class ExternalObjectMixin(object):
         return queryset
 
 
+class ExternalContentTypeObjectMixin(ContentTypeViewMixin, ExternalObjectMixin):
+    """
+    Mixin to retrieve an external object by content type from the URL pattern.
+    """
+    external_object_pk_url_kwarg = 'object_id'
+
+    def get_external_object_queryset(self):
+        content_type = self.get_content_type()
+        self.external_object_class = content_type.model_class()
+        return super(
+            ExternalContentTypeObjectMixin, self
+        ).get_external_object_queryset()
+
+
 class FormExtraKwargsMixin(object):
     """
     Mixin that allows a view to pass extra keyword arguments to forms
@@ -250,9 +296,9 @@ class ObjectActionMixin(object):
 
     def get_success_message(self, count):
         return ungettext(
-            self.success_message,
-            self.success_message_plural,
-            count
+            singular=self.success_message,
+            plural=self.success_message_plural,
+            number=count
         ) % {
             'count': count,
         }
@@ -271,14 +317,15 @@ class ObjectActionMixin(object):
                 pass
             except ActionError:
                 messages.error(
-                    self.request, self.error_message % {'instance': instance}
+                    message=self.error_message % {'instance': instance},
+                    request=self.request
                 )
             else:
                 self.action_count += 1
 
         messages.success(
-            self.request,
-            self.get_success_message(count=self.action_count)
+            message=self.get_success_message(count=self.action_count),
+            request=self.request
         )
 
 

mayan/apps/common/tests/test_views.py

@@ -4,7 +4,6 @@ from django.contrib.auth import get_user_model
 from django.contrib.contenttypes.models import ContentType
 
 from mayan.apps.acls.classes import ModelPermission
-from mayan.apps.user_management.tests.mixins import UserTestMixin
 
 from ..models import ErrorLogEntry
 from ..permissions_runtime import permission_error_log_view
@@ -13,7 +12,7 @@ from .base import GenericViewTestCase
 from .literals import TEST_ERROR_LOG_ENTRY_RESULT
 
 
-class CommonViewTestCase(UserTestMixin, GenericViewTestCase):
+class CommonViewTestCase(GenericViewTestCase):
     def _request_about_view(self):
         return self.get(viewname='common:about_view')
 

mayan/apps/common/urls.py

@@ -1,7 +1,7 @@
 from __future__ import unicode_literals
 
 from django.conf.urls import url
-from django.views.i18n import JavaScriptCatalog, set_language
+from django.views.i18n import JavaScriptCatalog
 
 from .api_views import (
     APIContentTypeList, APITemplateDetailView, APITemplateListView
@@ -10,30 +10,10 @@ from .views import (
     AboutView, CurrentUserLocaleProfileDetailsView,
     CurrentUserLocaleProfileEditView, FaviconRedirectView, HomeView,
     LicenseView, ObjectErrorLogEntryListClearView, ObjectErrorLogEntryListView,
-    RootView, SetupListView, ToolsListView, multi_object_action_view
+    RootView, SetupListView, ToolsListView
 )
 
-urlpatterns = [
-    url(regex=r'^$', view=RootView.as_view(), name='root'),
-    url(regex=r'^home/$', view=HomeView.as_view(), name='home'),
-    url(regex=r'^about/$', view=AboutView.as_view(), name='about_view'),
-    url(regex=r'^license/$', view=LicenseView.as_view(), name='license_view'),
-    url(
-        regex=r'^object/multiple/action/$', view=multi_object_action_view,
-        name='multi_object_action_view'
-    ),
-    url(regex=r'^setup/$', view=SetupListView.as_view(), name='setup_list'),
-    url(regex=r'^tools/$', view=ToolsListView.as_view(), name='tools_list'),
-    url(
-        regex=r'^user/locale/$',
-        view=CurrentUserLocaleProfileDetailsView.as_view(),
-        name='current_user_locale_profile_details'
-    ),
-    url(
-        regex=r'^user/locale/edit/$',
-        view=CurrentUserLocaleProfileEditView.as_view(),
-        name='current_user_locale_profile_edit'
-    ),
+urlpatterns_error_logs = [
     url(
         regex=r'^object/(?P<app_label>[-\w]+)/(?P<model>[-\w]+)/(?P<object_id>\d+)/errors/$',
         view=ObjectErrorLogEntryListView.as_view(), name='object_error_list'
@@ -45,7 +25,20 @@ urlpatterns = [
     ),
 ]
 
-urlpatterns += [
+urlpatterns_user_locale = [
+    url(
+        regex=r'^user/locale/$',
+        view=CurrentUserLocaleProfileDetailsView.as_view(),
+        name='current_user_locale_profile_details'
+    ),
+    url(
+        regex=r'^user/locale/edit/$',
+        view=CurrentUserLocaleProfileEditView.as_view(),
+        name='current_user_locale_profile_edit'
+    ),
+]
+
+urlpatterns_misc = [
     url(
         regex=r'^favicon\.ico$', view=FaviconRedirectView.as_view()
     ),
@@ -53,11 +46,21 @@ urlpatterns += [
         regex=r'^jsi18n/(?P<packages>\S+?)/$', view=JavaScriptCatalog.as_view(),
         name='javascript_catalog'
     ),
-    url(
-        regex=r'^set_language/$', view=set_language, name='set_language'
-    ),
 ]
 
+urlpatterns = [
+    url(regex=r'^$', view=RootView.as_view(), name='root'),
+    url(regex=r'^home/$', view=HomeView.as_view(), name='home'),
+    url(regex=r'^about/$', view=AboutView.as_view(), name='about_view'),
+    url(regex=r'^license/$', view=LicenseView.as_view(), name='license_view'),
+    url(regex=r'^setup/$', view=SetupListView.as_view(), name='setup_list'),
+    url(regex=r'^tools/$', view=ToolsListView.as_view(), name='tools_list'),
+]
+
+urlpatterns.extend(urlpatterns_error_logs)
+urlpatterns.extend(urlpatterns_misc)
+urlpatterns.extend(urlpatterns_user_locale)
+
 api_urls = [
     url(
         regex=r'^content_types/$', view=APIContentTypeList.as_view(),

mayan/apps/common/views.py

@@ -1,15 +1,11 @@
 from __future__ import absolute_import, unicode_literals
 
-from json import dumps
-
 from django.conf import settings
 from django.contrib import messages
 from django.contrib.contenttypes.models import ContentType
-from django.http import HttpResponseRedirect
 from django.shortcuts import get_object_or_404
-from django.urls import reverse, reverse_lazy
+from django.urls import reverse_lazy
 from django.utils import timezone, translation
-from django.utils.http import urlencode
 from django.utils.translation import ugettext_lazy as _
 from django.views.generic import RedirectView
 
@@ -220,67 +216,3 @@ class ToolsListView(SimpleView):
                 'These modules are used to do system maintenance.'
             )
         }
-
-
-def multi_object_action_view(request):
-    """
-    Proxy view called first when using a multi object action, which
-    then redirects to the appropriate specialized view
-    """
-    next = request.POST.get(
-        'next', request.GET.get(
-            'next', request.META.get(
-                'HTTP_REFERER', reverse(setting_home_view.value)
-            )
-        )
-    )
-
-    action = request.GET.get('action', None)
-    id_list = ','.join(
-        [key[3:] for key in request.GET.keys() if key.startswith('pk_')]
-    )
-    items_property_list = [
-        (key[11:]) for key in request.GET.keys() if key.startswith('properties_')
-    ]
-
-    if not action:
-        messages.error(
-            message=_('No action selected.'), request=request
-        )
-        return HttpResponseRedirect(
-            redirect_to=request.META.get(
-                'HTTP_REFERER', reverse(setting_home_view.value)
-            )
-        )
-
-    if not id_list and not items_property_list:
-        messages.error(
-            message=_('Must select at least one item.'),
-            request=request
-        )
-        return HttpResponseRedirect(
-            redirect_to=request.META.get(
-                'HTTP_REFERER', reverse(setting_home_view.value)
-            )
-        )
-
-    # Separate redirects to keep backwards compatibility with older
-    # functions that don't expect a properties_list parameter
-    if items_property_list:
-        return HttpResponseRedirect(
-            redirect_to='%s?%s' % (
-                action,
-                urlencode(
-                    {
-                        'items_property_list': dumps(items_property_list),
-                        'next': next
-                    }
-                )
-            )
-        )
-    else:
-        return HttpResponseRedirect(
-            redirect_to='%s?%s' % (
-                action, urlencode({'id_list': id_list, 'next': next})
-            )
-        )

mayan/apps/converter/classes.py

@@ -10,7 +10,6 @@ import sh
 
 from django.utils.translation import ugettext_lazy as _
 
-from mayan.apps.common.serialization import yaml_load
 from mayan.apps.mimetype.api import get_mimetype
 from mayan.apps.storage.settings import setting_temporary_directory
 from mayan.apps.storage.utils import (
@@ -147,7 +146,7 @@ class ConverterBase(object):
                 logger.error('Exception launching Libre Office; %s', exception)
                 raise
             finally:
-                fs_cleanup(libreoffice_home_directory)
+                fs_cleanup(filename=libreoffice_home_directory)
 
             # LibreOffice return a PDF file with the same name as the input
             # provided but with the .pdf extension.
@@ -181,7 +180,7 @@ class ConverterBase(object):
             shutil.copyfileobj(
                 fsrc=converted_file_object, fdst=temporary_converted_file_object
             )
-        fs_cleanup(converted_file_path)
+        fs_cleanup(filename=converted_file_path)
         temporary_converted_file_object.seek(0)
         return temporary_converted_file_object
 

mayan/apps/converter/migrations/0014_auto_20190626_1904.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.20 on 2019-06-26 19:04
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('converter', '0013_auto_20180823_2353'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='transformation',
+            name='name',
+            field=models.CharField(choices=[('crop', 'Crop: left, top, right, bottom'), ('draw_rectangle', 'Draw rectangle: left, top, right, bottom, fillcolor, outlinecolor, outlinewidth'), ('draw_rectangle_percent', 'Draw rectangle (percents coordinates): left, top, right, bottom, fillcolor, outlinecolor, outlinewidth'), ('flip', 'Flip'), ('gaussianblur', 'Gaussian blur: radius'), ('lineart', 'Line art'), ('mirror', 'Mirror'), ('resize', 'Resize: width, height'), ('rotate', 'Rotate: degrees, fillcolor'), ('rotate180', 'Rotate 180 degrees'), ('rotate270', 'Rotate 270 degrees'), ('rotate90', 'Rotate 90 degrees'), ('unsharpmask', 'Unsharp masking: radius, percent, threshold'), ('zoom', 'Zoom: percent')], max_length=128, verbose_name='Name'),
+        ),
+    ]

mayan/apps/converter/tests/test_transformations.py

@@ -121,7 +121,7 @@ class TransformationTestCase(GenericDocumentTestCase):
             arguments={'top': '10'}
         )
 
-        self.assertTrue(document_page.generate_image().startswith('page'))
+        self.assertTrue(document_page.generate_image())
 
     def test_crop_transformation_invalid_arguments(self):
         self._silence_logger(name='mayan.apps.converter.managers')
@@ -132,8 +132,7 @@ class TransformationTestCase(GenericDocumentTestCase):
             obj=document_page, transformation=TransformationCrop,
             arguments={'top': 'x', 'left': '-'}
         )
-
-        self.assertTrue(document_page.generate_image().startswith('page'))
+        self.assertTrue(document_page.generate_image())
 
     def test_crop_transformation_non_valid_range_arguments(self):
         self._silence_logger(name='mayan.apps.converter.managers')
@@ -145,7 +144,7 @@ class TransformationTestCase(GenericDocumentTestCase):
             arguments={'top': '-1000', 'bottom': '100000000'}
         )
 
-        self.assertTrue(document_page.generate_image().startswith('page'))
+        self.assertTrue(document_page.generate_image())
 
     def test_crop_transformation_overlapping_ranges_arguments(self):
         self._silence_logger(name='mayan.apps.converter.managers')
@@ -162,7 +161,7 @@ class TransformationTestCase(GenericDocumentTestCase):
             arguments={'left': '1000', 'right': '10000'}
         )
 
-        self.assertTrue(document_page.generate_image().startswith('page'))
+        self.assertTrue(document_page.generate_image())
 
     def test_lineart_transformations(self):
         document_page = self.test_document.pages.first()
@@ -172,7 +171,7 @@ class TransformationTestCase(GenericDocumentTestCase):
             arguments={}
         )
 
-        self.assertTrue(document_page.generate_image().startswith('page'))
+        self.assertTrue(document_page.generate_image())
 
     def test_rotate_transformations(self):
         document_page = self.test_document.pages.first()
@@ -182,18 +181,18 @@ class TransformationTestCase(GenericDocumentTestCase):
             arguments={}
         )
 
-        self.assertTrue(document_page.generate_image().startswith('page'))
+        self.assertTrue(document_page.generate_image())
 
         Transformation.objects.add_to_object(
             obj=document_page, transformation=TransformationRotate180,
             arguments={}
         )
 
-        self.assertTrue(document_page.generate_image().startswith('page'))
+        self.assertTrue(document_page.generate_image())
 
         Transformation.objects.add_to_object(
             obj=document_page, transformation=TransformationRotate270,
             arguments={}
         )
 
-        self.assertTrue(document_page.generate_image().startswith('page'))
+        self.assertTrue(document_page.generate_image())

mayan/apps/converter/transformations.py

@@ -3,7 +3,7 @@ from __future__ import unicode_literals
 import hashlib
 import logging
 
-from PIL import Image, ImageColor, ImageFilter
+from PIL import Image, ImageColor, ImageDraw, ImageFilter
 
 from django.utils.translation import string_concat, ugettext_lazy as _
 from django.utils.encoding import force_bytes
@@ -151,6 +151,208 @@ class TransformationCrop(BaseTransformation):
         return self.image.crop((left, top, right, bottom))
 
 
+class TransformationDrawRectangle(BaseTransformation):
+    arguments = (
+        'left', 'top', 'right', 'bottom', 'fillcolor', 'outlinecolor',
+        'outlinewidth'
+    )
+    label = _('Draw rectangle')
+    name = 'draw_rectangle'
+
+    def execute_on(self, *args, **kwargs):
+        super(TransformationDrawRectangle, self).execute_on(*args, **kwargs)
+
+        try:
+            left = int(self.left or '0')
+        except ValueError:
+            left = 0
+
+        try:
+            top = int(self.top or '0')
+        except ValueError:
+            top = 0
+
+        try:
+            right = int(self.right or '0')
+        except ValueError:
+            right = 0
+
+        try:
+            bottom = int(self.bottom or '0')
+        except ValueError:
+            bottom = 0
+
+        if left < 0:
+            left = 0
+
+        if left > self.image.size[0] - 1:
+            left = self.image.size[0] - 1
+
+        if top < 0:
+            top = 0
+
+        if top > self.image.size[1] - 1:
+            top = self.image.size[1] - 1
+
+        if right < 0:
+            right = 0
+
+        if right > self.image.size[0] - 1:
+            right = self.image.size[0] - 1
+
+        if bottom < 0:
+            bottom = 0
+
+        if bottom > self.image.size[1] - 1:
+            bottom = self.image.size[1] - 1
+
+        # Invert right value
+        # Pillow uses left, top, right, bottom to define a viewport
+        # of real coordinates
+        # We invert the right and bottom to define a viewport
+        # that can crop from the right and bottom borders without
+        # having to know the real dimensions of an image
+        right = self.image.size[0] - right
+        bottom = self.image.size[1] - bottom
+
+        if left > right:
+            left = right - 1
+
+        if top > bottom:
+            top = bottom - 1
+
+        logger.debug(
+            'left: %f, top: %f, right: %f, bottom: %f', left, top, right,
+            bottom
+        )
+
+        fillcolor_value = getattr(self, 'fillcolor', None)
+        if fillcolor_value:
+            fill_color = ImageColor.getrgb(fillcolor_value)
+        else:
+            fill_color = 0
+
+        outlinecolor_value = getattr(self, 'outlinecolor', None)
+        if outlinecolor_value:
+            outline_color = ImageColor.getrgb(outlinecolor_value)
+        else:
+            outline_color = None
+
+        outlinewidth_value = getattr(self, 'outlinewidth', None)
+        if outlinewidth_value:
+            outline_width = int(outlinewidth_value)
+        else:
+            outline_width = 0
+
+        draw = ImageDraw.Draw(self.image)
+        draw.rectangle(
+            (left, top, right, bottom), fill=fill_color, outline=outline_color,
+            width=outline_width
+        )
+
+        return self.image
+
+
+class TransformationDrawRectanglePercent(BaseTransformation):
+    arguments = (
+        'left', 'top', 'right', 'bottom', 'fillcolor', 'outlinecolor',
+        'outlinewidth'
+    )
+    label = _('Draw rectangle (percents coordinates)')
+    name = 'draw_rectangle_percent'
+
+    def execute_on(self, *args, **kwargs):
+        super(TransformationDrawRectanglePercent, self).execute_on(*args, **kwargs)
+
+        try:
+            left = float(self.left or '0')
+        except ValueError:
+            left = 0
+
+        try:
+            top = float(self.top or '0')
+        except ValueError:
+            top = 0
+
+        try:
+            right = float(self.right or '0')
+        except ValueError:
+            right = 0
+
+        try:
+            bottom = float(self.bottom or '0')
+        except ValueError:
+            bottom = 0
+
+        if left < 0:
+            left = 0
+
+        if left > 100:
+            left = 100
+
+        if top < 0:
+            top = 0
+
+        if top > 100:
+            top = 100
+
+        if right < 0:
+            right = 0
+
+        if right > 100:
+            right = 100
+
+        if bottom < 0:
+            bottom = 0
+
+        if bottom > 100:
+            bottom = 100
+
+        logger.debug(
+            'left: %f, top: %f, right: %f, bottom: %f', left, top, right,
+            bottom
+        )
+
+        fillcolor_value = getattr(self, 'fillcolor', None)
+        if fillcolor_value:
+            fill_color = ImageColor.getrgb(fillcolor_value)
+        else:
+            fill_color = 0
+
+        outlinecolor_value = getattr(self, 'outlinecolor', None)
+        if outlinecolor_value:
+            outline_color = ImageColor.getrgb(outlinecolor_value)
+        else:
+            outline_color = None
+
+        outlinewidth_value = getattr(self, 'outlinewidth', None)
+        if outlinewidth_value:
+            outline_width = int(outlinewidth_value)
+        else:
+            outline_width = 0
+
+        left = left / 100.0 * self.image.size[0]
+        top = top / 100.0 * self.image.size[1]
+
+        # Invert right value
+        # Pillow uses left, top, right, bottom to define a viewport
+        # of real coordinates
+        # We invert the right and bottom to define a viewport
+        # that can crop from the right and bottom borders without
+        # having to know the real dimensions of an image
+
+        right = self.image.size[0] - (right / 100.0 * self.image.size[0])
+        bottom = self.image.size[1] - (bottom / 100.0 * self.image.size[1])
+
+        draw = ImageDraw.Draw(self.image)
+        draw.rectangle(
+            (left, top, right, bottom), fill=fill_color, outline=outline_color,
+            width=outline_width
+        )
+
+        return self.image
+
+
 class TransformationFlip(BaseTransformation):
     arguments = ()
     label = _('Flip')
@@ -316,6 +518,10 @@ class TransformationZoom(BaseTransformation):
 
 
 BaseTransformation.register(transformation=TransformationCrop)
+BaseTransformation.register(transformation=TransformationDrawRectangle)
+BaseTransformation.register(
+    transformation=TransformationDrawRectanglePercent
+)
 BaseTransformation.register(transformation=TransformationFlip)
 BaseTransformation.register(transformation=TransformationGaussianBlur)
 BaseTransformation.register(transformation=TransformationLineArt)

mayan/apps/converter/urls.py

@@ -9,19 +9,19 @@ from .views import (
 
 urlpatterns = [
     url(
-        regex=r'^create_for/(?P<app_label>[-\w]+)/(?P<model>[-\w]+)/(?P<object_id>\d+)/$',
-        view=TransformationCreateView.as_view(), name='transformation_create'
-    ),
-    url(
-        regex=r'^list_for/(?P<app_label>[-\w]+)/(?P<model>[-\w]+)/(?P<object_id>\d+)/$',
+        regex=r'^object/(?P<app_label>[-\w]+)/(?P<model>[-\w]+)/(?P<object_id>\d+)/transformations/$',
         view=TransformationListView.as_view(), name='transformation_list'
     ),
     url(
-        regex=r'^delete/(?P<pk>\d+)/$', view=TransformationDeleteView.as_view(),
+        regex=r'^object/(?P<app_label>[-\w]+)/(?P<model>[-\w]+)/(?P<object_id>\d+)/transformations/create/$',
+        view=TransformationCreateView.as_view(), name='transformation_create'
+    ),
+    url(
+        regex=r'^transformations/(?P<pk>\d+)/delete/$', view=TransformationDeleteView.as_view(),
         name='transformation_delete'
     ),
     url(
-        regex=r'^edit/(?P<pk>\d+)/$', view=TransformationEditView.as_view(),
+        regex=r'^transformations/(?P<pk>\d+)/edit/$', view=TransformationEditView.as_view(),
         name='transformation_edit'
     ),
 ]

mayan/apps/dashboards/classes.py

@@ -1,5 +1,6 @@
 from __future__ import unicode_literals
 
+from django.contrib.humanize.templatetags.humanize import intcomma
 from django.template import loader
 
 
@@ -85,7 +86,8 @@ class DashboardWidgetNumeric(BaseDashboardWidget):
 
     def get_context(self):
         return {
-            'count': self.count,
+            'count': intcomma(value=self.count),
+            'count_raw': self.count,
             'icon_class': self.icon_class,
             'label': self.label,
             'link': self.link,

mayan/apps/dashboards/templates/dashboards/numeric_widget.html

@@ -1,5 +1,7 @@
 {% load i18n %}
 
+{% load appearance_tags %}
+
 <div class="col-xs-12 col-sm-6 col-md-4 col-lg-3 match-height">
     <div class="panel panel-secondary dashboard-widget">
         <div class="panel-heading">
@@ -9,7 +11,7 @@
                         <i class="dashboard-widget-icon {{ icon }}"></i>
                     {% elif icon_class %}
                         <div class="dashboard-widget-icon">
-                            {{ icon_class.render }}
+                            {% appearance_icon_render icon_class enable_shadow=True %}
                         </div>
                     {% endif %}
                 </div>

mayan/apps/django_gpg/urls.py

@@ -11,35 +11,35 @@ from .views import (
 
 urlpatterns = [
     url(
-        regex=r'^(?P<pk>\d+)/$', view=KeyDetailView.as_view(),
+        regex=r'^keys/(?P<pk>\d+)/$', view=KeyDetailView.as_view(),
         name='key_detail'
     ),
     url(
-        regex=r'^(?P<pk>\d+)/delete/$', view=KeyDeleteView.as_view(),
+        regex=r'^keys/(?P<pk>\d+)/delete/$', view=KeyDeleteView.as_view(),
         name='key_delete'
     ),
     url(
-        regex=r'^(?P<pk>\d+)/download/$', view=KeyDownloadView.as_view(),
+        regex=r'^keys/(?P<pk>\d+)/download/$', view=KeyDownloadView.as_view(),
         name='key_download'
     ),
     url(
-        regex=r'^list/private/$', view=PrivateKeyListView.as_view(),
+        regex=r'^keys/private/$', view=PrivateKeyListView.as_view(),
         name='key_private_list'
     ),
     url(
-        regex=r'^list/public/$', view=PublicKeyListView.as_view(),
+        regex=r'^keys/public/$', view=PublicKeyListView.as_view(),
         name='key_public_list'
     ),
     url(
-        regex=r'^upload/$', view=KeyUploadView.as_view(), name='key_upload'
+        regex=r'^keys/upload/$', view=KeyUploadView.as_view(), name='key_upload'
     ),
-    url(regex=r'^query/$', view=KeyQueryView.as_view(), name='key_query'),
+    url(regex=r'^keys/query/$', view=KeyQueryView.as_view(), name='key_query'),
     url(
-        regex=r'^query/results/$', view=KeyQueryResultView.as_view(),
+        regex=r'^keys/query/results/$', view=KeyQueryResultView.as_view(),
         name='key_query_results'
     ),
     url(
-        regex=r'^receive/(?P<key_id>.+)/$', view=KeyReceive.as_view(),
+        regex=r'^keys/receive/(?P<key_id>.+)/$', view=KeyReceive.as_view(),
         name='key_receive'
     ),
 ]

mayan/apps/document_comments/urls.py

@@ -11,25 +11,25 @@ from .views import (
 
 urlpatterns = [
     url(
-        regex=r'^(?P<pk>\d+)/comment/add/$',
+        regex=r'^documents/(?P<pk>\d+)/comments/$',
+        view=DocumentCommentListView.as_view(), name='comments_for_document'
+    ),
+    url(
+        regex=r'^documents/(?P<pk>\d+)/comments/add/$',
         view=DocumentCommentCreateView.as_view(), name='comment_add'
     ),
     url(
-        regex=r'^comment/(?P<pk>\d+)/delete/$',
+        regex=r'^comments/(?P<pk>\d+)/delete/$',
         view=DocumentCommentDeleteView.as_view(), name='comment_delete'
     ),
     url(
-        regex=r'^comment/(?P<pk>\d+)/$',
+        regex=r'^comments/(?P<pk>\d+)/$',
         view=DocumentCommentDetailView.as_view(), name='comment_details'
     ),
     url(
-        regex=r'^comment/(?P<pk>\d+)/edit/$',
+        regex=r'^comments/(?P<pk>\d+)/edit/$',
         view=DocumentCommentEditView.as_view(), name='comment_edit'
     ),
-    url(
-        regex=r'^(?P<pk>\d+)/comment/list/$',
-        view=DocumentCommentListView.as_view(), name='comments_for_document'
-    ),
 ]
 
 api_urls = [

mayan/apps/document_indexing/tests/test_views.py

@@ -196,3 +196,36 @@ class IndexToolsViewTestCase(
 
         # An instance root exists
         self.assertTrue(self.test_index.instance_root.pk)
+
+    def test_index_rebuild_view_no_permission(self):
+        self._create_test_index()
+
+        self.test_index.node_templates.create(
+            parent=self.test_index.template_root,
+            expression=TEST_INDEX_TEMPLATE_DOCUMENT_LABEL_EXPRESSION,
+            link_documents=True
+        )
+
+        response = self._request_test_index_rebuild_view()
+        self.assertEqual(response.status_code, 404)
+
+        self.assertEqual(IndexInstanceNode.objects.count(), 0)
+
+    def test_index_rebuild_view_with_access(self):
+        self._create_test_index()
+
+        self.test_index.node_templates.create(
+            parent=self.test_index.template_root,
+            expression=TEST_INDEX_TEMPLATE_DOCUMENT_LABEL_EXPRESSION,
+            link_documents=True
+        )
+
+        self.grant_access(
+            obj=self.test_index,
+            permission=permission_document_indexing_rebuild
+        )
+
+        response = self._request_test_index_rebuild_view()
+        self.assertEqual(response.status_code, 302)
+
+        self.assertNotEqual(IndexInstanceNode.objects.count(), 0)

mayan/apps/document_indexing/urls.py

@@ -15,72 +15,81 @@ from .views import (
     TemplateNodeCreateView, TemplateNodeDeleteView, TemplateNodeEditView
 )
 
-urlpatterns = [
+urlpatterns_indexes = [
     url(
-        regex=r'^setup/document_types/(?P<pk>\d+)/index_templates/$',
+        regex=r'^document_types/(?P<pk>\d+)/index_templates/$',
         view=DocumentTypeIndexesView.as_view(),
         name='document_type_index_templates'
     ),
     url(
-        regex=r'^setup/index/list/$', view=SetupIndexListView.as_view(),
+        regex=r'^indexes/$', view=SetupIndexListView.as_view(),
         name='index_setup_list'
     ),
     url(
-        regex=r'^setup/index/create/$', view=SetupIndexCreateView.as_view(),
+        regex=r'^indexes/create/$', view=SetupIndexCreateView.as_view(),
         name='index_setup_create'
     ),
     url(
-        regex=r'^setup/index/(?P<pk>\d+)/edit/$',
-        view=SetupIndexEditView.as_view(), name='index_setup_edit'
-    ),
-    url(
-        regex=r'^setup/index/(?P<pk>\d+)/delete/$',
+        regex=r'^indexes/(?P<pk>\d+)/delete/$',
         view=SetupIndexDeleteView.as_view(), name='index_setup_delete'
     ),
     url(
-        regex=r'^setup/index/(?P<pk>\d+)/template/$',
-        view=SetupIndexTreeTemplateListView.as_view(), name='index_setup_view'
+        regex=r'^indexes/(?P<pk>\d+)/edit/$',
+        view=SetupIndexEditView.as_view(), name='index_setup_edit'
     ),
     url(
-        regex=r'^setup/index/(?P<pk>\d+)/document_types/$',
+        regex=r'^indexes/(?P<pk>\d+)/document_types/$',
         view=SetupIndexDocumentTypesView.as_view(),
         name='index_setup_document_types'
     ),
     url(
-        regex=r'^setup/index/(?P<pk>\d+)/rebuild/$',
+        regex=r'^indexes/(?P<pk>\d+)/rebuild/$',
         view=SetupIndexRebuildView.as_view(), name='index_setup_rebuild'
     ),
     url(
-        regex=r'^setup/template/node/(?P<pk>\d+)/create/child/$',
+        regex=r'^indexes/(?P<pk>\d+)/nodes/$',
+        view=SetupIndexTreeTemplateListView.as_view(), name='index_setup_view'
+    ),
+    url(
+        regex=r'^indexes/nodes/(?P<pk>\d+)/children/create/$',
         view=TemplateNodeCreateView.as_view(), name='template_node_create'
     ),
     url(
-        regex=r'^setup/template/node/(?P<pk>\d+)/edit/$',
-        view=TemplateNodeEditView.as_view(), name='template_node_edit'
-    ),
-    url(
-        regex=r'^setup/template/node/(?P<pk>\d+)/delete/$',
+        regex=r'^indexes/nodes/(?P<pk>\d+)/delete/$',
         view=TemplateNodeDeleteView.as_view(), name='template_node_delete'
     ),
-
     url(
-        regex=r'^index/list/$', view=IndexListView.as_view(), name='index_list'
+        regex=r'^indexes/nodes/(?P<pk>\d+)/edit/$',
+        view=TemplateNodeEditView.as_view(), name='template_node_edit'
+    ),
+]
+
+urlpatterns_index_instances = [
+    url(
+        regex=r'^index_instances/$', view=IndexListView.as_view(), name='index_list'
     ),
     url(
-        regex=r'^instance/node/(?P<pk>\d+)/$',
+        regex=r'^index_instances/nodes/(?P<pk>\d+)/$',
         view=IndexInstanceNodeView.as_view(), name='index_instance_node_view'
     ),
+    url(
+        regex=r'^documents/(?P<pk>\d+)/index_instances/$',
+        view=DocumentIndexNodeListView.as_view(), name='document_index_list'
+    ),
+]
 
+urlpatterns_tools = [
     url(
         regex=r'^indexes/rebuild/$', view=IndexesRebuildView.as_view(),
         name='rebuild_index_instances'
     ),
-    url(
-        regex=r'^list/for/document/(?P<pk>\d+)/$',
-        view=DocumentIndexNodeListView.as_view(), name='document_index_list'
-    ),
 ]
 
+urlpatterns = []
+urlpatterns.extend(urlpatterns_indexes)
+urlpatterns.extend(urlpatterns_index_instances)
+urlpatterns.extend(urlpatterns_tools)
+
 api_urls = [
     url(
         regex=r'^indexes/node/(?P<pk>[0-9]+)/documents/$',

mayan/apps/document_parsing/apps.py

@@ -86,7 +86,7 @@ class DocumentParsingApp(MayanAppConfig):
         )
 
         ModelField(
-            model=Document, name='versions__pages__content__content'
+            model=Document, name='versions__version_pages__content__content'
         )
 
         ModelPermission.register(
@@ -118,7 +118,7 @@ class DocumentParsingApp(MayanAppConfig):
         )
 
         document_search.add_model_field(
-            field='versions__pages__content__content', label=_('Content')
+            field='versions__version_pages__content__content', label=_('Content')
         )
 
         document_page_search.add_model_field(

mayan/apps/document_parsing/links.py

@@ -10,6 +10,11 @@ from .permissions import (
     permission_parse_document
 )
 
+
+def is_document_page_disabled(context):
+    return not context['resolved_object'].enabled
+
+
 link_document_content = Link(
     args='resolved_object.id',
     icon_class_path='mayan.apps.document_parsing.icons.icon_document_content',
@@ -17,7 +22,7 @@ link_document_content = Link(
     view='document_parsing:document_content'
 )
 link_document_page_content = Link(
-    args='resolved_object.id',
+    args='resolved_object.id', conditional_disable=is_document_page_disabled,
     icon_class_path='mayan.apps.document_parsing.icons.icon_document_content',
     permissions=(permission_content_view,), text=_('Content'),
     view='document_parsing:document_page_content'

mayan/apps/document_signatures/urls.py

@@ -12,37 +12,37 @@ from .views import (
 
 urlpatterns = [
     url(
-        regex=r'^(?P<pk>\d+)/details/$',
+        regex=r'^signatures/(?P<pk>\d+)/details/$',
         view=DocumentVersionSignatureDetailView.as_view(),
         name='document_version_signature_details'
     ),
     url(
-        regex=r'^signature/(?P<pk>\d+)/download/$',
+        regex=r'^signatures/(?P<pk>\d+)/download/$',
         view=DocumentVersionSignatureDownloadView.as_view(),
         name='document_version_signature_download'
     ),
     url(
-        regex=r'^document/version/(?P<pk>\d+)/signatures/list/$',
+        regex=r'^documents/versions/(?P<pk>\d+)/signatures/$',
         view=DocumentVersionSignatureListView.as_view(),
         name='document_version_signature_list'
     ),
     url(
-        regex=r'^documents/version/(?P<pk>\d+)/signature/detached/upload/$',
+        regex=r'^documents/versions/(?P<pk>\d+)/signatures/detached/upload/$',
         view=DocumentVersionSignatureUploadView.as_view(),
         name='document_version_signature_upload'
     ),
     url(
-        regex=r'^documents/version/(?P<pk>\d+)/signature/detached/create/$',
+        regex=r'^documents/versions/(?P<pk>\d+)/signatures/detached/create/$',
         view=DocumentVersionDetachedSignatureCreateView.as_view(),
         name='document_version_signature_detached_create'
     ),
     url(
-        regex=r'^documents/version/(?P<pk>\d+)/signature/embedded/create/$',
+        regex=r'^documents/versions/(?P<pk>\d+)/signatures/embedded/create/$',
         view=DocumentVersionEmbeddedSignatureCreateView.as_view(),
         name='document_version_signature_embedded_create'
     ),
     url(
-        regex=r'^signature/(?P<pk>\d+)/delete/$',
+        regex=r'^signatures/(?P<pk>\d+)/delete/$',
         view=DocumentVersionSignatureDeleteView.as_view(),
         name='document_version_signature_delete'
     ),

mayan/apps/document_states/api_views.py

@@ -27,7 +27,6 @@ from .serializers import (
 )
 
 from .settings import settings_workflow_image_cache_time
-from .storages import storage_workflowimagecache
 from .tasks import task_generate_workflow_image
 
 
@@ -204,7 +203,8 @@ class APIWorkflowImageView(generics.RetrieveAPIView):
         )
 
         cache_filename = task.get(timeout=WORKFLOW_IMAGE_TASK_TIMEOUT)
-        with storage_workflowimagecache.open(cache_filename) as file_object:
+        cache_file = self.get_object().cache_partition.get_file(filename=cache_filename)
+        with cache_file.open() as file_object:
             response = HttpResponse(file_object.read(), content_type='image')
             if '_hash' in request.GET:
                 patch_cache_control(

mayan/apps/document_states/apps.py

@@ -1,7 +1,7 @@
 from __future__ import unicode_literals
 
 from django.apps import apps
-from django.db.models.signals import post_save
+from django.db.models.signals import post_migrate, post_save
 from django.utils.translation import ugettext_lazy as _
 
 from mayan.apps.acls.classes import ModelPermission
@@ -25,7 +25,8 @@ from .classes import DocumentStateHelper, WorkflowAction
 from .events import event_workflow_created, event_workflow_edited
 from .dependencies import *  # NOQA
 from .handlers import (
-    handler_index_document, handler_launch_workflow, handler_trigger_transition
+    handler_create_workflow_image_cache, handler_index_document,
+    handler_launch_workflow, handler_trigger_transition
 )
 from .html_widgets import WorkflowLogExtraDataWidget, widget_transition_events
 from .links import (
@@ -328,6 +329,17 @@ class DocumentStatesApp(MayanAppConfig):
                 link_workflow_template_preview
             ), sources=(Workflow,)
         )
+
+        menu_list_facet.unbind_links(
+            links=(
+                link_acl_list, link_events_for_object,
+                link_object_event_types_user_subcriptions_list,
+                link_workflow_template_document_types,
+                link_workflow_template_state_list, link_workflow_template_transition_list,
+                link_workflow_template_preview
+            ), sources=(WorkflowRuntimeProxy,)
+        )
+
         menu_list_facet.bind_links(
             links=(
                 link_document_type_workflow_templates,
@@ -441,6 +453,10 @@ class DocumentStatesApp(MayanAppConfig):
 
         # Index updating
 
+        post_migrate.connect(
+            dispatch_uid='workflows_handler_create_workflow_image_cache',
+            receiver=handler_create_workflow_image_cache,
+        )
         post_save.connect(
             dispatch_uid='workflows_handler_index_document_save',
             receiver=handler_index_document,

mayan/apps/document_states/handlers.py

@@ -6,6 +6,22 @@ from django.utils.translation import ugettext_lazy as _
 from mayan.apps.document_indexing.tasks import task_index_document
 from mayan.apps.events.classes import EventType
 
+from .literals import (
+    WORKFLOW_IMAGE_CACHE_STORAGE_INSTANCE_PATH, WORKFLOW_IMAGE_CACHE_NAME
+)
+from .settings import setting_workflow_image_cache_maximum_size
+
+
+def handler_create_workflow_image_cache(sender, **kwargs):
+    Cache = apps.get_model(app_label='file_caching', model_name='Cache')
+    Cache.objects.update_or_create(
+        defaults={
+            'label': _('Workflow images'),
+            'storage_instance_path': WORKFLOW_IMAGE_CACHE_STORAGE_INSTANCE_PATH,
+            'maximum_size': setting_workflow_image_cache_maximum_size.value,
+        }, name=WORKFLOW_IMAGE_CACHE_NAME,
+    )
+
 
 def handler_index_document(sender, **kwargs):
     task_index_document.apply_async(

mayan/apps/document_states/literals.py

@@ -2,6 +2,8 @@ from __future__ import unicode_literals
 
 from django.utils.translation import ugettext_lazy as _
 
+DEFAULT_WORKFLOW_IMAGE_CACHE_MAXIMUM_SIZE = 50 * 2 ** 20  # 50 Megabytes
+
 FIELD_TYPE_CHOICE_CHAR = 1
 FIELD_TYPE_CHOICE_INTEGER = 2
 FIELD_TYPE_CHOICES = (
@@ -30,4 +32,6 @@ WORKFLOW_ACTION_WHEN_CHOICES = (
     (WORKFLOW_ACTION_ON_ENTRY, _('On entry')),
     (WORKFLOW_ACTION_ON_EXIT, _('On exit')),
 )
+WORKFLOW_IMAGE_CACHE_NAME = 'workflow_images'
+WORKFLOW_IMAGE_CACHE_STORAGE_INSTANCE_PATH = 'mayan.apps.document_states.storages.storage_workflowimagecache'
 WORKFLOW_IMAGE_TASK_TIMEOUT = 60

mayan/apps/document_states/models.py

@@ -6,24 +6,23 @@ import logging
 
 from furl import furl
 from graphviz import Digraph
-import yaml
-try:
-    from yaml import CSafeLoader as SafeLoader
-except ImportError:
-    from yaml import SafeLoader
 
+from django.apps import apps
 from django.conf import settings
 from django.core import serializers
 from django.core.exceptions import PermissionDenied, ValidationError
-from django.core.files.base import ContentFile
 from django.db import IntegrityError, models, transaction
 from django.db.models import F, Max, Q
 from django.urls import reverse
-from django.utils.encoding import force_text, python_2_unicode_compatible
+from django.utils.encoding import (
+    force_bytes, force_text, python_2_unicode_compatible
+)
+from django.utils.functional import cached_property
 from django.utils.module_loading import import_string
 from django.utils.translation import ugettext_lazy as _
 
 from mayan.apps.acls.models import AccessControlList
+from mayan.apps.common.serialization import yaml_load
 from mayan.apps.common.validators import YAMLValidator, validate_internal_name
 from mayan.apps.documents.models import Document, DocumentType
 from mayan.apps.documents.permissions import permission_document_view
@@ -33,11 +32,11 @@ from .error_logs import error_log_state_actions
 from .events import event_workflow_created, event_workflow_edited
 from .literals import (
     FIELD_TYPE_CHOICES, WIDGET_CLASS_CHOICES, WORKFLOW_ACTION_WHEN_CHOICES,
-    WORKFLOW_ACTION_ON_ENTRY, WORKFLOW_ACTION_ON_EXIT
+    WORKFLOW_ACTION_ON_ENTRY, WORKFLOW_ACTION_ON_EXIT,
+    WORKFLOW_IMAGE_CACHE_NAME
 )
 from .managers import WorkflowManager
 from .permissions import permission_workflow_transition
-from .storages import storage_workflowimagecache
 
 logger = logging.getLogger(__name__)
 
@@ -74,19 +73,37 @@ class Workflow(models.Model):
     def __str__(self):
         return self.label
 
-    def generate_image(self):
-        cache_filename = '{}-{}'.format(self.id, self.get_hash())
-        image = self.render()
+    @cached_property
+    def cache(self):
+        Cache = apps.get_model(app_label='file_caching', model_name='Cache')
+        return Cache.objects.get(name=WORKFLOW_IMAGE_CACHE_NAME)
 
-        # Since open "wb+" doesn't create files, check if the file
-        # exists, if not then create it
-        if not storage_workflowimagecache.exists(cache_filename):
-            storage_workflowimagecache.save(
-                name=cache_filename, content=ContentFile(content='')
+    @cached_property
+    def cache_partition(self):
+        partition, created = self.cache.partitions.get_or_create(
+            name='{}'.format(self.pk)
+        )
+        return partition
+
+    def delete(self, *args, **kwargs):
+        self.cache_partition.delete()
+        return super(Workflow, self).delete(*args, **kwargs)
+
+    def generate_image(self):
+        cache_filename = '{}'.format(self.get_hash())
+
+        if self.cache_partition.get_file(filename=cache_filename):
+            logger.debug(
+                'workflow cache file "%s" found', cache_filename
+            )
+        else:
+            logger.debug(
+                'workflow cache file "%s" not found', cache_filename
             )
 
-        with storage_workflowimagecache.open(cache_filename, mode='wb+') as file_object:
-            file_object.write(image)
+            image = self.render()
+            with self.cache_partition.create_file(filename=cache_filename) as file_object:
+                file_object.write(image)
 
         return cache_filename
 
@@ -109,12 +126,16 @@ class Workflow(models.Model):
             Workflow.objects.filter(pk=self.pk)
         ) + list(
             WorkflowState.objects.filter(workflow__pk=self.pk)
+        ) + list(
+            WorkflowStateAction.objects.filter(state__workflow__pk=self.pk)
         ) + list(
             WorkflowTransition.objects.filter(workflow__pk=self.pk)
         )
 
         return hashlib.sha256(
-            serializers.serialize('json', objects_lists)
+            force_bytes(
+                serializers.serialize('json', objects_lists)
+            )
         ).hexdigest()
 
     def get_initial_state(self):
@@ -465,7 +486,7 @@ class WorkflowTransitionField(models.Model):
         return self.label
 
     def get_widget_kwargs(self):
-        return yaml.load(stream=self.widget_kwargs, Loader=SafeLoader)
+        return yaml_load(stream=self.widget_kwargs)
 
 
 @python_2_unicode_compatible

mayan/apps/document_states/settings.py

@@ -7,8 +7,20 @@ from django.utils.translation import ugettext_lazy as _
 
 from mayan.apps.smart_settings.classes import Namespace
 
+from .literals import DEFAULT_WORKFLOW_IMAGE_CACHE_MAXIMUM_SIZE
+from .utils import callback_update_workflow_image_cache_size
+
 namespace = Namespace(label=_('Workflows'), name='document_states')
 
+setting_workflow_image_cache_maximum_size = namespace.add_setting(
+    global_name='WORKFLOW_IMAGE_CACHE_MAXIMUM_SIZE',
+    default=DEFAULT_WORKFLOW_IMAGE_CACHE_MAXIMUM_SIZE,
+    help_text=_(
+        'The threshold at which the WORKFLOW_IMAGE_CACHE_STORAGE_BACKEND will '
+        'start deleting the oldest workflow image cache files. Specify the '
+        'size in bytes.'
+    ), post_edit_function=callback_update_workflow_image_cache_size
+)
 settings_workflow_image_cache_time = namespace.add_setting(
     global_name='WORKFLOWS_IMAGE_CACHE_TIME', default='31556926',
     help_text=_(

mayan/apps/document_states/tests/test_models.py

@@ -0,0 +1,11 @@
+from __future__ import unicode_literals
+
+from mayan.apps.common.tests import BaseTestCase
+
+from .mixins import WorkflowTestMixin
+
+
+class WorkflowModelTestCase(WorkflowTestMixin, BaseTestCase):
+    def test_workflow_template_preview(self):
+        self._create_test_workflow()
+        self.assertTrue(self.test_workflow.get_api_image_url())

mayan/apps/document_states/urls.py

@@ -225,7 +225,7 @@ urlpatterns_workflow_transition_fields = [
     ),
 ]
 
-urlpatterns = [
+urlpatterns_tools = [
     url(
         regex=r'^tools/workflows/launch/$',
         view=ToolLaunchWorkflows.as_view(),
@@ -233,6 +233,8 @@ urlpatterns = [
     ),
 ]
 
+urlpatterns = []
+urlpatterns.extend(urlpatterns_tools)
 urlpatterns.extend(urlpatterns_workflow_instances)
 urlpatterns.extend(urlpatterns_workflow_runtime_proxies)
 urlpatterns.extend(urlpatterns_workflow_states)

mayan/apps/document_states/utils.py

@@ -0,0 +1,12 @@
+from __future__ import unicode_literals
+
+from django.apps import apps
+
+from .literals import WORKFLOW_IMAGE_CACHE_NAME
+
+
+def callback_update_workflow_image_cache_size(setting):
+    Cache = apps.get_model(app_label='file_caching', model_name='Cache')
+    cache = Cache.objects.get(name=WORKFLOW_IMAGE_CACHE_NAME)
+    cache.maximum_size = setting.value
+    cache.save()

mayan/apps/document_states/views/__init__.py

@@ -1 +0,0 @@
-

mayan/apps/document_states/views/workflow_template_state_views.py

@@ -1,53 +1,30 @@
 from __future__ import absolute_import, unicode_literals
 
-from django.contrib import messages
-from django.db import transaction
 from django.http import Http404, HttpResponseRedirect
 from django.shortcuts import get_object_or_404
 from django.template import RequestContext
-from django.urls import reverse, reverse_lazy
+from django.urls import reverse
 from django.utils.translation import ugettext_lazy as _
 
 from mayan.apps.common.generics import (
-    AddRemoveView, ConfirmView, FormView, SingleObjectCreateView,
-    SingleObjectDeleteView, SingleObjectDetailView,
+    FormView, SingleObjectCreateView, SingleObjectDeleteView,
     SingleObjectDynamicFormCreateView, SingleObjectDynamicFormEditView,
     SingleObjectEditView, SingleObjectListView
 )
 from mayan.apps.common.mixins import ExternalObjectMixin
-from mayan.apps.documents.events import event_document_type_edited
-from mayan.apps.documents.models import DocumentType
-from mayan.apps.documents.permissions import permission_document_type_edit
-from mayan.apps.events.classes import EventType
-from mayan.apps.events.models import StoredEventType
 
 from ..classes import WorkflowAction
-from ..events import event_workflow_edited
 from ..forms import (
-    WorkflowActionSelectionForm, WorkflowForm, WorkflowPreviewForm,
-    WorkflowStateActionDynamicForm, WorkflowStateForm, WorkflowTransitionForm,
-    WorkflowTransitionTriggerEventRelationshipFormSet
-)
-from ..icons import (
-    icon_workflow_template_list, icon_workflow_state, icon_workflow_state_action,
-    icon_workflow_transition, icon_workflow_transition_field
+    WorkflowActionSelectionForm, WorkflowStateActionDynamicForm,
+    WorkflowStateForm
 )
+from ..icons import icon_workflow_state, icon_workflow_state_action
 from ..links import (
-    link_workflow_template_create, link_workflow_template_state_create,
+    link_workflow_template_state_create,
     link_workflow_template_state_action_selection,
-    link_workflow_template_transition_create,
-    link_workflow_template_transition_field_create,
 )
-from ..models import (
-    Workflow, WorkflowState, WorkflowStateAction, WorkflowTransition,
-    WorkflowTransitionField
-)
-from ..permissions import (
-    permission_workflow_create, permission_workflow_delete,
-    permission_workflow_edit, permission_workflow_tools,
-    permission_workflow_view,
-)
-from ..tasks import task_launch_all_workflows
+from ..models import Workflow, WorkflowState, WorkflowStateAction
+from ..permissions import permission_workflow_edit, permission_workflow_view
 
 
 class WorkflowTemplateStateActionCreateView(SingleObjectDynamicFormCreateView):

mayan/apps/document_states/views/workflow_template_transition_views.py

@@ -1,53 +1,28 @@
 from __future__ import absolute_import, unicode_literals
 
 from django.contrib import messages
-from django.db import transaction
-from django.http import Http404, HttpResponseRedirect
-from django.shortcuts import get_object_or_404
 from django.template import RequestContext
-from django.urls import reverse, reverse_lazy
+from django.urls import reverse
 from django.utils.translation import ugettext_lazy as _
 
 from mayan.apps.common.generics import (
-    AddRemoveView, ConfirmView, FormView, SingleObjectCreateView,
-    SingleObjectDeleteView, SingleObjectDetailView,
-    SingleObjectDynamicFormCreateView, SingleObjectDynamicFormEditView,
+    FormView, SingleObjectCreateView, SingleObjectDeleteView,
     SingleObjectEditView, SingleObjectListView
 )
 from mayan.apps.common.mixins import ExternalObjectMixin
-from mayan.apps.documents.events import event_document_type_edited
-from mayan.apps.documents.models import DocumentType
-from mayan.apps.documents.permissions import permission_document_type_edit
 from mayan.apps.events.classes import EventType
 from mayan.apps.events.models import StoredEventType
 
-from ..classes import WorkflowAction
-from ..events import event_workflow_edited
 from ..forms import (
-    WorkflowActionSelectionForm, WorkflowForm, WorkflowPreviewForm,
-    WorkflowStateActionDynamicForm, WorkflowStateForm, WorkflowTransitionForm,
-    WorkflowTransitionTriggerEventRelationshipFormSet
-)
-from ..icons import (
-    icon_workflow_template_list, icon_workflow_state, icon_workflow_state_action,
-    icon_workflow_transition, icon_workflow_transition_field
+    WorkflowTransitionForm, WorkflowTransitionTriggerEventRelationshipFormSet
 )
+from ..icons import icon_workflow_transition, icon_workflow_transition_field
 from ..links import (
-    link_workflow_template_create, link_workflow_template_state_create,
-    link_workflow_template_state_action_selection,
     link_workflow_template_transition_create,
     link_workflow_template_transition_field_create,
 )
-from ..models import (
-    Workflow, WorkflowState, WorkflowStateAction, WorkflowTransition,
-    WorkflowTransitionField
-)
-from ..permissions import (
-    permission_workflow_create, permission_workflow_delete,
-    permission_workflow_edit, permission_workflow_tools,
-    permission_workflow_view,
-)
-from ..tasks import task_launch_all_workflows
+from ..models import Workflow, WorkflowTransition, WorkflowTransitionField
+from ..permissions import permission_workflow_edit, permission_workflow_view
 
 
 class WorkflowTemplateTransitionCreateView(ExternalObjectMixin, SingleObjectCreateView):

mayan/apps/document_states/views/workflow_template_views.py

@@ -2,46 +2,23 @@ from __future__ import absolute_import, unicode_literals
 
 from django.contrib import messages
 from django.db import transaction
-from django.http import Http404, HttpResponseRedirect
-from django.shortcuts import get_object_or_404
 from django.template import RequestContext
-from django.urls import reverse, reverse_lazy
+from django.urls import reverse_lazy
 from django.utils.translation import ugettext_lazy as _
 
 from mayan.apps.common.generics import (
-    AddRemoveView, ConfirmView, FormView, SingleObjectCreateView,
-    SingleObjectDeleteView, SingleObjectDetailView,
-    SingleObjectDynamicFormCreateView, SingleObjectDynamicFormEditView,
-    SingleObjectEditView, SingleObjectListView
+    AddRemoveView, ConfirmView, SingleObjectCreateView, SingleObjectDeleteView,
+    SingleObjectDetailView, SingleObjectEditView, SingleObjectListView
 )
-from mayan.apps.common.mixins import ExternalObjectMixin
 from mayan.apps.documents.events import event_document_type_edited
 from mayan.apps.documents.models import DocumentType
 from mayan.apps.documents.permissions import permission_document_type_edit
-from mayan.apps.events.classes import EventType
-from mayan.apps.events.models import StoredEventType
 
-from ..classes import WorkflowAction
 from ..events import event_workflow_edited
-from ..forms import (
-    WorkflowActionSelectionForm, WorkflowForm, WorkflowPreviewForm,
-    WorkflowStateActionDynamicForm, WorkflowStateForm, WorkflowTransitionForm,
-    WorkflowTransitionTriggerEventRelationshipFormSet
-)
-from ..icons import (
-    icon_workflow_template_list, icon_workflow_state, icon_workflow_state_action,
-    icon_workflow_transition, icon_workflow_transition_field
-)
-from ..links import (
-    link_workflow_template_create, link_workflow_template_state_create,
-    link_workflow_template_state_action_selection,
-    link_workflow_template_transition_create,
-    link_workflow_template_transition_field_create,
-)
-from ..models import (
-    Workflow, WorkflowState, WorkflowStateAction, WorkflowTransition,
-    WorkflowTransitionField
-)
+from ..forms import WorkflowForm, WorkflowPreviewForm
+from ..icons import icon_workflow_template_list
+from ..links import link_workflow_template_create
+from ..models import Workflow
 from ..permissions import (
     permission_workflow_create, permission_workflow_delete,
     permission_workflow_edit, permission_workflow_tools,

mayan/apps/documents/api_views.py

@@ -36,7 +36,6 @@ from .serializers import (
     WritableDocumentTypeSerializer, WritableDocumentVersionSerializer
 )
 from .settings import settings_document_page_image_cache_time
-from .storages import storage_documentimagecache
 from .tasks import task_generate_document_page_image
 
 logger = logging.getLogger(__name__)
@@ -165,7 +164,7 @@ class APIDocumentPageImageView(generics.RetrieveAPIView):
 
         AccessControlList.objects.check_access(
             obj=document, permissions=(permission_required,),
-            user=self.request.user, manager=Document.passthrough
+            user=self.request.user
         )
         return document
 
@@ -175,7 +174,7 @@ class APIDocumentPageImageView(generics.RetrieveAPIView):
         )
 
     def get_queryset(self):
-        return self.get_document_version().pages.all()
+        return self.get_document_version().pages_all.all()
 
     def get_serializer(self, *args, **kwargs):
         return None
@@ -205,11 +204,13 @@ class APIDocumentPageImageView(generics.RetrieveAPIView):
         )
 
         cache_filename = task.get(timeout=DOCUMENT_IMAGE_TASK_TIMEOUT)
-        with storage_documentimagecache.open(cache_filename) as file_object:
+        cache_file = self.get_object().cache_partition.get_file(filename=cache_filename)
+        with cache_file.open() as file_object:
             response = HttpResponse(file_object.read(), content_type='image')
             if '_hash' in request.GET:
                 patch_cache_control(
-                    response, max_age=settings_document_page_image_cache_time.value
+                    response=response,
+                    max_age=settings_document_page_image_cache_time.value
                 )
             return response
 

mayan/apps/documents/apps.py

@@ -1,6 +1,6 @@
 from __future__ import absolute_import, unicode_literals
 
-from django.db.models.signals import post_delete
+from django.db.models.signals import post_delete, post_migrate
 from django.utils.translation import ugettext_lazy as _
 
 from mayan.apps.acls.classes import ModelPermission
@@ -43,11 +43,11 @@ from .events import (
     event_document_view
 )
 from .handlers import (
-    handler_create_default_document_type, handler_remove_empty_duplicates_lists,
-    handler_scan_duplicates_for,
+    handler_create_default_document_type, handler_create_document_cache,
+    handler_remove_empty_duplicates_lists, handler_scan_duplicates_for
 )
 from .links import (
-    link_clear_image_cache, link_document_clear_transformations,
+    link_document_clear_transformations,
     link_document_clone_transformations, link_document_delete,
     link_document_document_type_edit, link_document_download,
     link_document_duplicates_list, link_document_edit,
@@ -60,6 +60,8 @@ from .links import (
     link_document_multiple_download, link_document_multiple_favorites_add,
     link_document_multiple_favorites_remove, link_document_multiple_restore,
     link_document_multiple_trash, link_document_multiple_update_page_count,
+    link_document_page_disable, link_document_page_multiple_disable,
+    link_document_page_enable, link_document_page_multiple_enable,
     link_document_page_navigation_first, link_document_page_navigation_last,
     link_document_page_navigation_next, link_document_page_navigation_previous,
     link_document_page_return, link_document_page_rotate_left,
@@ -100,6 +102,11 @@ from .widgets import (
 )
 
 
+def is_document_page_enabled(context):
+    return context['object'].enabled
+
+
+
 class DocumentsApp(MayanAppConfig):
     app_namespace = 'documents'
     app_url = 'documents'
@@ -214,12 +221,21 @@ class DocumentsApp(MayanAppConfig):
         ModelPermission.register_inheritance(
             model=Document, related='document_type',
         )
+        ModelPermission.register_manager(
+            model=Document, manager_name='passthrough'
+        )
         ModelPermission.register_inheritance(
             model=DocumentPage, related='document_version__document',
         )
+        ModelPermission.register_manager(
+            model=DocumentPage, manager_name='passthrough'
+        )
         ModelPermission.register_inheritance(
             model=DocumentPageResult, related='document_version__document',
         )
+        ModelPermission.register_manager(
+            model=DocumentPageResult, manager_name='passthrough'
+        )
         ModelPermission.register_inheritance(
             model=DocumentTypeFilename, related='document_type',
         )
@@ -262,13 +278,21 @@ class DocumentsApp(MayanAppConfig):
         # DocumentPage
         SourceColumn(
             attribute='get_label', is_identifier=True,
-            is_object_absolute_url=True, source=DocumentPage
+            is_object_absolute_url=True, source=DocumentPage,
+            widget_condition=is_document_page_enabled
         )
         SourceColumn(
             func=lambda context: document_page_thumbnail_widget.render(
                 instance=context['object']
             ), label=_('Thumbnail'), source=DocumentPage
         )
+        SourceColumn(
+            attribute='enabled', include_label=True, source=DocumentPage,
+            widget=TwoStateWidget
+        )
+        SourceColumn(
+            attribute='page_number', include_label=True, source=DocumentPage
+        )
 
         SourceColumn(
             attribute='get_label', is_identifier=True,
@@ -377,7 +401,7 @@ class DocumentsApp(MayanAppConfig):
 
         menu_setup.bind_links(links=(link_document_type_setup,))
         menu_tools.bind_links(
-            links=(link_clear_image_cache, link_duplicated_document_scan)
+            links=(link_duplicated_document_scan,)
         )
 
         # Document type links
@@ -503,6 +527,16 @@ class DocumentsApp(MayanAppConfig):
                 link_document_page_navigation_last
             ), sources=(DocumentPage,)
         )
+        menu_multi_item.bind_links(
+            links=(
+                link_document_page_multiple_disable,
+                link_document_page_multiple_enable
+            ), sources=(DocumentPage,)
+        )
+        menu_object.bind_links(
+            links=(link_document_page_disable, link_document_page_enable),
+            sources=(DocumentPage,)
+        )
         menu_list_facet.bind_links(
             links=(link_transformation_list,), sources=(DocumentPage,)
         )
@@ -527,6 +561,10 @@ class DocumentsApp(MayanAppConfig):
             dispatch_uid='handler_create_default_document_type',
             receiver=handler_create_default_document_type
         )
+        post_migrate.connect(
+            dispatch_uid='documents_handler_create_document_cache',
+            receiver=handler_create_document_cache,
+        )
         post_version_upload.connect(
             dispatch_uid='handler_scan_duplicates_for',
             receiver=handler_scan_duplicates_for

mayan/apps/documents/handlers.py

@@ -1,8 +1,13 @@
 from __future__ import unicode_literals
 
 from django.apps import apps
+from django.utils.translation import ugettext_lazy as _
 
-from .literals import DEFAULT_DOCUMENT_TYPE_LABEL
+from .literals import (
+    DEFAULT_DOCUMENT_TYPE_LABEL, DOCUMENT_CACHE_STORAGE_INSTANCE_PATH,
+    DOCUMENT_IMAGES_CACHE_NAME
+)
+from .settings import setting_document_cache_maximum_size
 from .signals import post_initial_document_type
 from .tasks import task_clean_empty_duplicate_lists, task_scan_duplicates_for
 
@@ -21,6 +26,17 @@ def handler_create_default_document_type(sender, **kwargs):
         )
 
 
+def handler_create_document_cache(sender, **kwargs):
+    Cache = apps.get_model(app_label='file_caching', model_name='Cache')
+    Cache.objects.update_or_create(
+        defaults={
+            'label': _('Document images'),
+            'storage_instance_path': DOCUMENT_CACHE_STORAGE_INSTANCE_PATH,
+            'maximum_size': setting_document_cache_maximum_size.value,
+        }, name=DOCUMENT_IMAGES_CACHE_NAME,
+    )
+
+
 def handler_scan_duplicates_for(sender, instance, **kwargs):
     task_scan_duplicates_for.apply_async(
         kwargs={'document_id': instance.document.pk}

mayan/apps/documents/icons.py

@@ -12,8 +12,6 @@ icon_document_type = Icon(
 
 icon_menu_documents = Icon(driver_name='fontawesome', symbol='book')
 
-icon_clear_image_cache = Icon(driver_name='fontawesome', symbol='file-image')
-
 icon_dashboard_document_types = icon_document_type
 icon_dashboard_documents_in_trash = Icon(
     driver_name='fontawesome', symbol='trash-alt'
@@ -27,8 +25,6 @@ icon_dashboard_new_documents_this_month = Icon(
 icon_dashboard_total_document = Icon(
     driver_name='fontawesome', symbol='book'
 )
-
-
 icon_document_quick_download = Icon(
     driver_name='fontawesome', symbol='download'
 )
@@ -106,6 +102,14 @@ icon_favorite_document_remove = Icon(
     secondary_symbol='minus'
 )
 
+# Document pages
+
+icon_document_page_disable = Icon(
+    driver_name='fontawesomecss', css_classes='far fa-eye-slash'
+)
+icon_document_page_enable = Icon(
+    driver_name='fontawesomecss', css_classes='far fa-eye'
+)
 icon_document_page_navigation_first = Icon(
     driver_name='fontawesome', symbol='step-backward'
 )

mayan/apps/documents/links.py

@@ -8,7 +8,7 @@ from mayan.apps.converter.permissions import (
 from mayan.apps.navigation.classes import Link
 
 from .icons import (
-    icon_clear_image_cache, icon_document_list_recent_access,
+    icon_document_list_recent_access,
     icon_recent_added_document_list, icon_document_page_navigation_first,
     icon_document_page_navigation_last, icon_document_page_navigation_next,
     icon_document_page_navigation_previous, icon_document_page_return,
@@ -19,14 +19,14 @@ from .icons import (
     icon_duplicated_document_list, icon_duplicated_document_scan
 )
 from .permissions import (
-    permission_document_delete, permission_document_download,
-    permission_document_properties_edit, permission_document_print,
-    permission_document_restore, permission_document_tools,
-    permission_document_version_revert, permission_document_view,
-    permission_document_trash, permission_document_type_create,
-    permission_document_type_delete, permission_document_type_edit,
-    permission_document_type_view, permission_empty_trash,
-    permission_document_version_view
+    permission_document_delete, permission_document_edit,
+    permission_document_download, permission_document_properties_edit,
+    permission_document_print, permission_document_restore,
+    permission_document_tools, permission_document_version_revert,
+    permission_document_view, permission_document_trash,
+    permission_document_type_create, permission_document_type_delete,
+    permission_document_type_edit, permission_document_type_view,
+    permission_empty_trash, permission_document_version_view
 )
 from .settings import setting_zoom_max_level, setting_zoom_min_level
 
@@ -43,11 +43,11 @@ def is_not_current_version(context):
 
 
 def is_first_page(context):
-    return context['resolved_object'].page_number <= 1
+    return context['resolved_object'].siblings.first() == context['resolved_object']
 
 
 def is_last_page(context):
-    return context['resolved_object'].page_number >= context['resolved_object'].document_version.pages.count()
+    return context['resolved_object'].siblings.last() == context['resolved_object']
 
 
 def is_max_zoom(context):
@@ -58,6 +58,14 @@ def is_min_zoom(context):
     return context['zoom'] <= setting_zoom_min_level.value
 
 
+def is_document_page_enabled(context):
+    return context['resolved_object'].enabled
+
+
+def is_document_page_disabled(context):
+    return not context['resolved_object'].enabled
+
+
 # Facet
 link_document_preview = Link(
     args='resolved_object.id',
@@ -264,22 +272,37 @@ link_document_list_deleted = Link(
     text=_('Trash can'), view='documents:document_list_deleted'
 )
 
-# Tools
-link_clear_image_cache = Link(
-    icon_class=icon_clear_image_cache,
-    description=_(
-        'Clear the graphics representations used to speed up the documents\' '
-        'display and interactive transformations results.'
-    ), permissions=(permission_document_tools,),
-    text=_('Clear document image cache'),
-    view='documents:document_clear_image_cache'
-)
 link_trash_can_empty = Link(
     permissions=(permission_empty_trash,), text=_('Empty trash'),
     view='documents:trash_can_empty'
 )
 
 # Document pages
+
+link_document_page_disable = Link(
+    condition=is_document_page_enabled,
+    icon_class_path='mayan.apps.documents.icons.icon_document_page_disable',
+    kwargs={'pk': 'resolved_object.id'},
+    permissions=(permission_document_edit,), text=_('Disable page'),
+    view='documents:document_page_disable'
+)
+link_document_page_multiple_disable = Link(
+    icon_class_path='mayan.apps.documents.icons.icon_document_page_disable',
+    text=_('Disable pages'),
+    view='documents:document_page_multiple_disable'
+)
+link_document_page_enable = Link(
+    condition=is_document_page_disabled,
+    icon_class_path='mayan.apps.documents.icons.icon_document_page_enable',
+    kwargs={'pk': 'resolved_object.id'},
+    permissions=(permission_document_edit,), text=_('Enable page'),
+    view='documents:document_page_enable'
+)
+link_document_page_multiple_enable = Link(
+    icon_class_path='mayan.apps.documents.icons.icon_document_page_enable',
+    text=_('Enable pages'),
+    view='documents:document_page_multiple_enable'
+)
 link_document_page_navigation_first = Link(
     args='resolved_object.pk', conditional_disable=is_first_page,
     icon_class=icon_document_page_navigation_first,
@@ -323,6 +346,7 @@ link_document_page_rotate_right = Link(
     text=_('Rotate right'), view='documents:document_page_rotate_right',
 )
 link_document_page_view = Link(
+    conditional_disable=is_document_page_disabled,
     icon_class_path='mayan.apps.documents.icons.icon_document_page_view',
     permissions=(permission_document_view,), text=_('Page image'),
     view='documents:document_page_view', args='resolved_object.pk'

mayan/apps/documents/literals.py

@@ -9,6 +9,7 @@ CHECK_TRASH_PERIOD_INTERVAL = 60
 DELETE_STALE_STUBS_INTERVAL = 60 * 10  # 10 minutes
 DEFAULT_DELETE_PERIOD = 30
 DEFAULT_DELETE_TIME_UNIT = TIME_DELTA_UNIT_DAYS
+DEFAULT_DOCUMENTS_CACHE_MAXIMUM_SIZE = 500 * 2 ** 20  # 500 Megabytes
 DEFAULT_DOCUMENTS_HASH_BLOCK_SIZE = 65535
 DEFAULT_LANGUAGE = 'eng'
 DEFAULT_LANGUAGE_CODES = (
@@ -30,8 +31,11 @@ DEFAULT_LANGUAGE_CODES = (
 DEFAULT_ZIP_FILENAME = 'document_bundle.zip'
 DEFAULT_DOCUMENT_TYPE_LABEL = _('Default')
 DOCUMENT_IMAGE_TASK_TIMEOUT = 120
+DOCUMENT_IMAGES_CACHE_NAME = 'document_images'
+DOCUMENT_CACHE_STORAGE_INSTANCE_PATH = 'mayan.apps.documents.storages.storage_documentimagecache'
 STUB_EXPIRATION_INTERVAL = 60 * 60 * 24  # 24 hours
 UPDATE_PAGE_COUNT_RETRY_DELAY = 10
+UPLOAD_NEW_DOCUMENT_RETRY_DELAY = 10
 UPLOAD_NEW_VERSION_RETRY_DELAY = 10
 
 PAGE_RANGE_ALL = 'all'

mayan/apps/documents/managers.py

@@ -22,28 +22,9 @@ class DocumentManager(models.Manager):
 
     def get_queryset(self):
         return TrashCanQuerySet(
-            self.model, using=self._db
+            model=self.model, using=self._db
         ).filter(in_trash=False).filter(is_stub=False)
 
-    def invalidate_cache(self):
-        for document in self.model.objects.all():
-            document.invalidate_cache()
-
-
-class DocumentPageCachedImage(models.Manager):
-    def get_by_natural_key(self, filename, document_page_natural_key):
-        DocumentPage = apps.get_model(
-            app_label='documents', model_name='DocumentPage'
-        )
-        try:
-            document_page = DocumentPage.objects.get_by_natural_key(
-                *document_page_natural_key
-            )
-        except DocumentPage.DoesNotExist:
-            raise self.model.DoesNotExist
-
-        return self.get(document_page__pk=document_page.pk, filename=filename)
-
 
 class DocumentPageManager(models.Manager):
     def get_by_natural_key(self, page_number, document_version_natural_key):
@@ -57,6 +38,11 @@ class DocumentPageManager(models.Manager):
 
         return self.get(document_version__pk=document_version.pk, page_number=page_number)
 
+    def get_queryset(self):
+        return models.QuerySet(
+            model=self.model, using=self._db
+        ).filter(enabled=True)
+
 
 class DocumentTypeManager(models.Manager):
     def check_delete_periods(self):

mayan/apps/documents/migrations/0049_auto_20190715_0454.py

@@ -0,0 +1,37 @@
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+from ..storages import storage_documentimagecache
+
+
+def operation_clear_old_cache(apps, schema_editor):
+    DocumentPageCachedImage = apps.get_model(
+        'documents', 'DocumentPageCachedImage'
+    )
+
+    for cached_image in DocumentPageCachedImage.objects.using(schema_editor.connection.alias).all():
+        # Delete each cached image directly since the model doesn't exists and
+        # will not trigger the physical deletion of the stored file
+        storage_documentimagecache.delete(cached_image.filename)
+        cached_image.delete()
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('documents', '0048_auto_20190711_0544'),
+    ]
+
+    operations = [
+        migrations.RunPython(
+            code=operation_clear_old_cache,
+            reverse_code=migrations.RunPython.noop
+        ),
+        migrations.RemoveField(
+            model_name='documentpagecachedimage',
+            name='document_page',
+        ),
+        migrations.DeleteModel(
+            name='DocumentPageCachedImage',
+        ),
+    ]

mayan/apps/documents/migrations/0050_auto_20190725_0451.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.20 on 2019-07-25 04:51
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('documents', '0049_auto_20190715_0454'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='document',
+            name='language',
+            field=models.CharField(blank=True, default='eng', help_text='The dominant language in the document.', max_length=8, verbose_name='Language'),
+        ),
+    ]

mayan/apps/documents/migrations/0051_documentpage_enabled.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.22 on 2019-07-29 07:03
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('documents', '0050_auto_20190725_0451'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='documentpage',
+            name='enabled',
+            field=models.BooleanField(default=True, verbose_name='Enabled'),
+        ),
+    ]

mayan/apps/documents/models/document_models.py

@@ -136,10 +136,6 @@ class Document(models.Model):
         if latest_version:
             return latest_version.get_api_image_url(*args, **kwargs)
 
-    def invalidate_cache(self):
-        for document_version in self.versions.all():
-            document_version.invalidate_cache()
-
     @property
     def is_in_trash(self):
         return self.in_trash
@@ -240,6 +236,18 @@ class Document(models.Model):
     def page_count(self):
         return self.latest_version.page_count
 
+    @property
+    def pages_all(self):
+        try:
+            return self.latest_version.pages_all
+        except AttributeError:
+            # Document has no version yet
+            DocumentPage = apps.get_model(
+                app_label='documents', model_name='DocumentPage'
+            )
+
+            return DocumentPage.objects.none()
+
     @property
     def pages(self):
         try:

mayan/apps/documents/models/document_page_models.py

@@ -4,13 +4,14 @@ import logging
 
 from furl import furl
 
-from django.core.files.base import ContentFile
 from django.db import models
 from django.urls import reverse
 from django.utils.encoding import force_text, python_2_unicode_compatible
+from django.utils.functional import cached_property
 from django.utils.translation import ugettext_lazy as _
 
 from mayan.apps.converter.literals import DEFAULT_ZOOM_LEVEL, DEFAULT_ROTATION
+
 from mayan.apps.converter.models import Transformation
 from mayan.apps.converter.transformations import (
     BaseTransformation, TransformationResize, TransformationRotate,
@@ -18,17 +19,16 @@ from mayan.apps.converter.transformations import (
 )
 from mayan.apps.converter.utils import get_converter_class
 
-from ..managers import DocumentPageCachedImage, DocumentPageManager
+from ..managers import DocumentPageManager
 from ..settings import (
     setting_disable_base_image_cache, setting_disable_transformed_image_cache,
     setting_display_width, setting_display_height, setting_zoom_max_level,
     setting_zoom_min_level
 )
-from ..storages import storage_documentimagecache
 
 from .document_version_models import DocumentVersion
 
-__all__ = ('DocumentPage', 'DocumentPageCachedImage', 'DocumentPageResult')
+__all__ = ('DocumentPage', 'DocumentPageResult')
 logger = logging.getLogger(__name__)
 
 
@@ -38,15 +38,17 @@ class DocumentPage(models.Model):
     Model that describes a document version page
     """
     document_version = models.ForeignKey(
-        on_delete=models.CASCADE, related_name='pages', to=DocumentVersion,
+        on_delete=models.CASCADE, related_name='version_pages', to=DocumentVersion,
         verbose_name=_('Document version')
     )
+    enabled = models.BooleanField(default=True, verbose_name=_('Enabled'))
     page_number = models.PositiveIntegerField(
         db_index=True, default=1, editable=False,
         verbose_name=_('Page number')
     )
 
     objects = DocumentPageManager()
+    passthrough = models.Manager()
 
     class Meta:
         ordering = ('page_number',)
@@ -56,12 +58,15 @@ class DocumentPage(models.Model):
     def __str__(self):
         return self.get_label()
 
-    @property
-    def cache_filename(self):
-        return 'page-cache-{}'.format(self.uuid)
+    @cached_property
+    def cache_partition(self):
+        partition, created = self.document_version.cache.partitions.get_or_create(
+            name=self.uuid
+        )
+        return partition
 
     def delete(self, *args, **kwargs):
-        self.invalidate_cache()
+        self.cache_partition.delete()
         super(DocumentPage, self).delete(*args, **kwargs)
 
     def detect_orientation(self):
@@ -80,29 +85,24 @@ class DocumentPage(models.Model):
 
     def generate_image(self, *args, **kwargs):
         transformation_list = self.get_combined_transformation_list(*args, **kwargs)
-
-        cache_filename = '{}-{}'.format(
-            self.cache_filename, BaseTransformation.combine(transformation_list)
-        )
+        combined_cache_filename = BaseTransformation.combine(transformation_list)
 
         # Check is transformed image is available
-        logger.debug('transformations cache filename: %s', cache_filename)
+        logger.debug('transformations cache filename: %s', combined_cache_filename)
 
-        if not setting_disable_transformed_image_cache.value and storage_documentimagecache.exists(cache_filename):
+        if not setting_disable_transformed_image_cache.value and self.cache_partition.get_file(filename=combined_cache_filename):
             logger.debug(
-                'transformations cache file "%s" found', cache_filename
+                'transformations cache file "%s" found', combined_cache_filename
             )
         else:
             logger.debug(
-                'transformations cache file "%s" not found', cache_filename
+                'transformations cache file "%s" not found', combined_cache_filename
             )
             image = self.get_image(transformations=transformation_list)
-            with storage_documentimagecache.open(cache_filename, 'wb+') as file_object:
+            with self.cache_partition.create_file(filename=combined_cache_filename) as file_object:
                 file_object.write(image.getvalue())
 
-            self.cached_images.create(filename=cache_filename)
-
-        return cache_filename
+        return combined_cache_filename
 
     def get_absolute_url(self):
         return reverse(
@@ -159,7 +159,6 @@ class DocumentPage(models.Model):
             zoom_level = setting_zoom_max_level.value
 
         # Generate transformation hash
-
         transformation_list = []
 
         # Stored transformations first
@@ -186,13 +185,15 @@ class DocumentPage(models.Model):
         return transformation_list
 
     def get_image(self, transformations=None):
-        cache_filename = self.cache_filename
+        cache_filename = 'base_image'
         logger.debug('Page cache filename: %s', cache_filename)
 
-        if not setting_disable_base_image_cache.value and storage_documentimagecache.exists(cache_filename):
+        cache_file = self.cache_partition.get_file(filename=cache_filename)
+
+        if not setting_disable_base_image_cache.value and cache_file:
             logger.debug('Page cache file "%s" found', cache_filename)
 
-            with storage_documentimagecache.open(cache_filename) as file_object:
+            with cache_file.open() as file_object:
                 converter = get_converter_class()(
                     file_object=file_object
                 )
@@ -200,8 +201,8 @@ class DocumentPage(models.Model):
                 converter.seek_page(page_number=0)
 
                 # This code is also repeated below to allow using a context
-                # manager with storage_documentimagecache.open and close it
-                # automatically.
+                # manager with cache_file.open and close it automatically.
+                # Apply runtime transformations
                 for transformation in transformations:
                     converter.transform(transformation=transformation)
 
@@ -218,14 +219,11 @@ class DocumentPage(models.Model):
 
                     page_image = converter.get_page()
 
-                    # Since open "wb+" doesn't create files, check if the file
-                    # exists, if not then create it
-                    if not storage_documentimagecache.exists(cache_filename):
-                        storage_documentimagecache.save(name=cache_filename, content=ContentFile(content=''))
-
-                    with storage_documentimagecache.open(cache_filename, 'wb+') as file_object:
+                    # Since open "wb+" doesn't create files, create it explicitly
+                    with self.cache_partition.create_file(filename=cache_filename) as file_object:
                         file_object.write(page_image.getvalue())
 
+                    # Apply runtime transformations
                     for transformation in transformations:
                         converter.transform(transformation=transformation)
 
@@ -236,14 +234,8 @@ class DocumentPage(models.Model):
                     'Error creating page cache file "%s"; %s',
                     cache_filename, exception
                 )
-                storage_documentimagecache.delete(cache_filename)
                 raise
 
-    def invalidate_cache(self):
-        storage_documentimagecache.delete(self.cache_filename)
-        for cached_image in self.cached_images.all():
-            cached_image.delete()
-
     @property
     def is_in_trash(self):
         return self.document.is_in_trash
@@ -254,7 +246,7 @@ class DocumentPage(models.Model):
         ) % {
             'document': force_text(self.document),
             'page_num': self.page_number,
-            'total_pages': self.document_version.pages.count()
+            'total_pages': self.document_version.pages_all.count()
         }
     get_label.short_description = _('Label')
 
@@ -277,38 +269,6 @@ class DocumentPage(models.Model):
         return '{}-{}'.format(self.document_version.uuid, self.pk)
 
 
-class DocumentPageCachedImage(models.Model):
-    document_page = models.ForeignKey(
-        on_delete=models.CASCADE, related_name='cached_images',
-        to=DocumentPage, verbose_name=_('Document page')
-    )
-    datetime = models.DateTimeField(
-        auto_now_add=True, db_index=True, verbose_name=_('Date time')
-    )
-    filename = models.CharField(max_length=128, verbose_name=_('Filename'))
-    file_size = models.PositiveIntegerField(
-        db_index=True, default=0, verbose_name=_('File size')
-    )
-
-    objects = DocumentPageCachedImage()
-
-    class Meta:
-        verbose_name = _('Document page cached image')
-        verbose_name_plural = _('Document page cached images')
-
-    def delete(self, *args, **kwargs):
-        storage_documentimagecache.delete(self.filename)
-        return super(DocumentPageCachedImage, self).delete(*args, **kwargs)
-
-    def natural_key(self):
-        return (self.filename, self.document_page.natural_key())
-    natural_key.dependencies = ['documents.DocumentPage']
-
-    def save(self, *args, **kwargs):
-        self.file_size = storage_documentimagecache.size(self.filename)
-        return super(DocumentPageCachedImage, self).save(*args, **kwargs)
-
-
 class DocumentPageResult(DocumentPage):
     class Meta:
         ordering = ('document_version__document', 'page_number')

mayan/apps/documents/models/document_version_models.py

@@ -7,11 +7,11 @@ import shutil
 import uuid
 
 from django.apps import apps
-from django.core.files.base import ContentFile
 from django.db import models, transaction
 from django.template import Template, Context
 from django.urls import reverse
 from django.utils.encoding import force_text, python_2_unicode_compatible
+from django.utils.functional import cached_property
 from django.utils.translation import ugettext_lazy as _
 
 from mayan.apps.converter.exceptions import InvalidOfficeFormat, PageCountError
@@ -21,10 +21,11 @@ from mayan.apps.converter.utils import get_converter_class
 from mayan.apps.mimetype.api import get_mimetype
 
 from ..events import event_document_new_version, event_document_version_revert
+from ..literals import DOCUMENT_IMAGES_CACHE_NAME
 from ..managers import DocumentVersionManager
 from ..settings import setting_fix_orientation, setting_hash_block_size
 from ..signals import post_document_created, post_version_upload
-from ..storages import storage_documentversion, storage_documentimagecache
+from ..storages import storage_documentversion
 
 from .document_models import Document
 
@@ -61,14 +62,6 @@ class DocumentVersion(models.Model):
     _pre_open_hooks = {}
     _post_save_hooks = {}
 
-    @classmethod
-    def register_pre_open_hook(cls, order, func):
-        cls._pre_open_hooks[order] = func
-
-    @classmethod
-    def register_post_save_hook(cls, order, func):
-        cls._post_save_hooks[order] = func
-
     document = models.ForeignKey(
         on_delete=models.CASCADE, related_name='versions', to=Document,
         verbose_name=_('Document')
@@ -118,18 +111,35 @@ class DocumentVersion(models.Model):
 
     objects = DocumentVersionManager()
 
+    @classmethod
+    def register_pre_open_hook(cls, order, func):
+        cls._pre_open_hooks[order] = func
+
+    @classmethod
+    def register_post_save_hook(cls, order, func):
+        cls._post_save_hooks[order] = func
+
     def __str__(self):
         return self.get_rendered_string()
 
-    @property
-    def cache_filename(self):
-        return 'document-version-{}'.format(self.uuid)
+    @cached_property
+    def cache(self):
+        Cache = apps.get_model(app_label='file_caching', model_name='Cache')
+        return Cache.objects.get(name=DOCUMENT_IMAGES_CACHE_NAME)
+
+    @cached_property
+    def cache_partition(self):
+        partition, created = self.cache.partitions.get_or_create(
+            name='version-{}'.format(self.uuid)
+        )
+        return partition
 
     def delete(self, *args, **kwargs):
         for page in self.pages.all():
             page.delete()
 
         self.file.storage.delete(self.file.name)
+        self.cache_partition.delete()
 
         return super(DocumentVersion, self).delete(*args, **kwargs)
 
@@ -164,43 +174,36 @@ class DocumentVersion(models.Model):
             return first_page.get_api_image_url(*args, **kwargs)
 
     def get_intermediate_file(self):
-        cache_filename = self.cache_filename
-        logger.debug('Intermidiate filename: %s', cache_filename)
-
-        if storage_documentimagecache.exists(cache_filename):
-            logger.debug('Intermidiate file "%s" found.', cache_filename)
-
-            return storage_documentimagecache.open(cache_filename)
+        cache_filename = 'intermediate_file'
+        cache_file = self.cache_partition.get_file(filename=cache_filename)
+        if cache_file:
+            logger.debug('Intermidiate file found.')
+            return cache_file.open()
         else:
-            logger.debug('Intermidiate file "%s" not found.', cache_filename)
+            logger.debug('Intermidiate file not found.')
 
             try:
                 with self.open() as version_file_object:
-                    converter = get_converter_class()(file_object=version_file_object)
+                    converter = get_converter_class()(
+                        file_object=version_file_object
+                    )
                     with converter.to_pdf() as pdf_file_object:
-
-                        # Since open "wb+" doesn't create files, check if the file
-                        # exists, if not then create it
-                        if not storage_documentimagecache.exists(cache_filename):
-                            storage_documentimagecache.save(
-                                name=cache_filename, content=ContentFile(content='')
-                            )
-
-                        with storage_documentimagecache.open(cache_filename, mode='wb+') as file_object:
+                        with self.cache_partition.create_file(filename=cache_filename) as file_object:
                             shutil.copyfileobj(
                                 fsrc=pdf_file_object, fdst=file_object
                             )
 
-                    return storage_documentimagecache.open(cache_filename)
+                        return self.cache_partition.get_file(filename=cache_filename).open()
             except InvalidOfficeFormat:
                 return self.open()
             except Exception as exception:
-                # Cleanup in case of error
                 logger.error(
                     'Error creating intermediate file "%s"; %s.',
                     cache_filename, exception
                 )
-                storage_documentimagecache.delete(cache_filename)
+                cache_file = self.cache_partition.get_file(filename=cache_filename)
+                if cache_file:
+                    cache_file.delete()
                 raise
 
     def get_rendered_string(self, preserve_extension=False):
@@ -223,11 +226,6 @@ class DocumentVersion(models.Model):
         return (self.checksum, self.document.natural_key())
     natural_key.dependencies = ['documents.Document']
 
-    def invalidate_cache(self):
-        storage_documentimagecache.delete(self.cache_filename)
-        for page in self.pages.all():
-            page.invalidate_cache()
-
     @property
     def is_in_trash(self):
         return self.document.is_in_trash
@@ -248,6 +246,17 @@ class DocumentVersion(models.Model):
 
             return result
 
+    @property
+    def pages_all(self):
+        DocumentPage = apps.get_model(
+            app_label='documents', model_name='DocumentPage'
+        )
+        return DocumentPage.passthrough.filter(document_version=self)
+
+    @property
+    def pages(self):
+        return self.version_pages.all()
+
     @property
     def page_count(self):
         """

mayan/apps/documents/queues.py

@@ -61,10 +61,6 @@ queue_documents_periodic.add_task_type(
     schedule=timedelta(seconds=DELETE_STALE_STUBS_INTERVAL),
 )
 
-queue_tools.add_task_type(
-    dotted_path='mayan.apps.documents.tasks.task_clear_image_cache',
-    label=_('Clear image cache')
-)
 queue_tools.add_task_type(
     dotted_path='mayan.apps.documents.tasks.task_scan_duplicates_all',
     label=_('Duplicated document scan')
@@ -82,3 +78,7 @@ queue_uploads.add_task_type(
     dotted_path='mayan.apps.documents.tasks.task_scan_duplicates_for',
     label=_('Scan document duplicates')
 )
+queue_uploads.add_task_type(
+    dotted_path='mayan.apps.documents.tasks.task_upload_new_document',
+    label=_('Upload new document')
+)

mayan/apps/documents/settings.py

@@ -8,11 +8,22 @@ from django.utils.translation import ugettext_lazy as _
 from mayan.apps.smart_settings.classes import Namespace
 
 from .literals import (
-    DEFAULT_DOCUMENTS_HASH_BLOCK_SIZE, DEFAULT_LANGUAGE, DEFAULT_LANGUAGE_CODES
+    DEFAULT_DOCUMENTS_CACHE_MAXIMUM_SIZE, DEFAULT_DOCUMENTS_HASH_BLOCK_SIZE,
+    DEFAULT_LANGUAGE, DEFAULT_LANGUAGE_CODES
 )
+from .utils import callback_update_cache_size
 
 namespace = Namespace(label=_('Documents'), name='documents')
 
+setting_document_cache_maximum_size = namespace.add_setting(
+    global_name='DOCUMENTS_CACHE_MAXIMUM_SIZE',
+    default=DEFAULT_DOCUMENTS_CACHE_MAXIMUM_SIZE,
+    help_text=_(
+        'The threshold at which the DOCUMENT_CACHE_STORAGE_BACKEND will start '
+        'deleting the oldest document image cache files. Specify the size in '
+        'bytes.'
+    ), post_edit_function=callback_update_cache_size
+)
 setting_documentimagecache_storage = namespace.add_setting(
     global_name='DOCUMENTS_CACHE_STORAGE_BACKEND',
     default='django.core.files.storage.FileSystemStorage', help_text=_(

mayan/apps/documents/tasks.py

@@ -9,7 +9,8 @@ from django.db import OperationalError
 from mayan.celery import app
 
 from .literals import (
-    UPDATE_PAGE_COUNT_RETRY_DELAY, UPLOAD_NEW_VERSION_RETRY_DELAY
+    UPDATE_PAGE_COUNT_RETRY_DELAY, UPLOAD_NEW_DOCUMENT_RETRY_DELAY,
+    UPLOAD_NEW_VERSION_RETRY_DELAY
 )
 
 logger = logging.getLogger(__name__)
@@ -41,17 +42,6 @@ def task_check_trash_periods():
     DocumentType.objects.check_trash_periods()
 
 
-@app.task(ignore_result=True)
-def task_clear_image_cache():
-    Document = apps.get_model(
-        app_label='documents', model_name='Document'
-    )
-
-    logger.info('Starting document cache invalidation')
-    Document.objects.invalidate_cache()
-    logger.info('Finished document cache invalidation')
-
-
 @app.task(ignore_result=True)
 def task_delete_document(trashed_document_id):
     DeletedDocument = apps.get_model(
@@ -81,8 +71,7 @@ def task_generate_document_page_image(document_page_id, *args, **kwargs):
         app_label='documents', model_name='DocumentPage'
     )
 
-    document_page = DocumentPage.objects.get(pk=document_page_id)
-
+    document_page = DocumentPage.passthrough.get(pk=document_page_id)
     return document_page.generate_image(*args, **kwargs)
 
 
@@ -127,6 +116,60 @@ def task_update_page_count(self, version_id):
         raise self.retry(exc=exception)
 
 
+@app.task(bind=True, default_retry_delay=UPLOAD_NEW_DOCUMENT_RETRY_DELAY, ignore_result=True)
+def task_upload_new_document(self, document_type_id, shared_uploaded_file_id):
+    DocumentType = apps.get_model(
+        app_label='documents', model_name='DocumentType'
+    )
+
+    SharedUploadedFile = apps.get_model(
+        app_label='common', model_name='SharedUploadedFile'
+    )
+
+    try:
+        document_type = DocumentType.objects.get(pk=document_type_id)
+        shared_file = SharedUploadedFile.objects.get(
+            pk=shared_uploaded_file_id
+        )
+    except OperationalError as exception:
+        logger.warning(
+            'Operational error during attempt to retrieve shared data for '
+            'new document of type: %s; %s. Retrying.', document_type, exception
+        )
+        raise self.retry(exc=exception)
+
+    try:
+        with shared_file.open() as file_object:
+            document_type.new_document(file_object=file_object)
+    except OperationalError as exception:
+        logger.warning(
+            'Operational error during attempt to create new document '
+            'of type: %s; %s. Retrying.', document_type, exception
+        )
+        raise self.retry(exc=exception)
+    except Exception as exception:
+        # This except and else block emulate a finally:
+        logger.error(
+            'Unexpected error during attempt to create new document '
+            'of type: %s; %s', document_type, exception
+        )
+        try:
+            shared_file.delete()
+        except OperationalError as exception:
+            logger.warning(
+                'Operational error during attempt to delete shared '
+                'file: %s; %s.', shared_file, exception
+            )
+    else:
+        try:
+            shared_file.delete()
+        except OperationalError as exception:
+            logger.warning(
+                'Operational error during attempt to delete shared '
+                'file: %s; %s.', shared_file, exception
+            )
+
+
 @app.task(bind=True, default_retry_delay=UPLOAD_NEW_VERSION_RETRY_DELAY, ignore_result=True)
 def task_upload_new_version(self, document_id, shared_uploaded_file_id, user_id, comment=None):
     SharedUploadedFile = apps.get_model(

mayan/apps/documents/tests/test_api.py

@@ -154,11 +154,11 @@ class DocumentAPITestCase(DocumentTestMixin, BaseAPITestCase):
     auto_upload_document = False
 
     def _request_document_upload(self):
-        with open(TEST_DOCUMENT_PATH, mode='rb') as file_descriptor:
+        with open(TEST_DOCUMENT_PATH, mode='rb') as file_object:
             return self.post(
                 viewname='rest_api:document-list', data={
                     'document_type': self.test_document_type.pk,
-                    'file': file_descriptor
+                    'file': file_object
                 }
             )
 
@@ -208,12 +208,12 @@ class DocumentAPITestCase(DocumentTestMixin, BaseAPITestCase):
         # is the latest.
         time.sleep(1)
 
-        with open(TEST_DOCUMENT_PATH, mode='rb') as file_descriptor:
+        with open(TEST_DOCUMENT_PATH, mode='rb') as file_object:
             return self.post(
                 viewname='rest_api:document-version-list', kwargs={
                     'pk': self.test_document.pk,
                 }, data={
-                    'comment': '', 'file': file_descriptor,
+                    'comment': '', 'file': file_object,
                 }
             )
 

mayan/apps/documents/tests/test_document_page_views.py

@@ -2,11 +2,152 @@ from __future__ import unicode_literals
 
 from django.utils.encoding import force_text
 
-from ..permissions import permission_document_view
+from ..permissions import (
+    permission_document_edit, permission_document_view
+)
 
 from .base import GenericDocumentViewTestCase
 
 
+class DocumentPageDisableViewTestCase(GenericDocumentViewTestCase):
+    def setUp(self):
+        super(DocumentPageDisableViewTestCase, self).setUp()
+        self.test_document_page = self.test_document.pages_all.first()
+
+    def _request_test_document_page_disable_view(self):
+        return self.post(
+            viewname='documents:document_page_disable', kwargs={
+                'pk': self.test_document_page.pk
+            }
+        )
+
+    def test_document_page_disable_view_no_permission(self):
+        test_document_page_count = self.test_document.pages.count()
+
+        response = self._request_test_document_page_disable_view()
+        self.assertEqual(response.status_code, 404)
+
+        self.assertEqual(
+            test_document_page_count, self.test_document.pages.count()
+        )
+
+    def test_document_page_disable_view_with_access(self):
+        self.grant_access(
+            obj=self.test_document, permission=permission_document_edit
+        )
+
+        test_document_page_count = self.test_document.pages.count()
+
+        response = self._request_test_document_page_disable_view()
+        self.assertEqual(response.status_code, 302)
+
+        self.assertNotEqual(
+            test_document_page_count, self.test_document.pages.count()
+        )
+
+    def _request_test_document_page_multiple_disable_view(self):
+        return self.post(
+            viewname='documents:document_page_multiple_disable', data={
+                'id_list': self.test_document_page.pk
+            }
+        )
+
+    def test_document_page_multiple_disable_view_no_permission(self):
+        test_document_page_count = self.test_document.pages.count()
+
+        response = self._request_test_document_page_multiple_disable_view()
+        self.assertEqual(response.status_code, 404)
+
+        self.assertEqual(
+            test_document_page_count, self.test_document.pages.count()
+        )
+
+    def test_document_page_multiple_disable_view_with_access(self):
+        self.grant_access(
+            obj=self.test_document, permission=permission_document_edit
+        )
+
+        test_document_page_count = self.test_document.pages.count()
+
+        response = self._request_test_document_page_multiple_disable_view()
+        self.assertEqual(response.status_code, 302)
+
+        self.assertNotEqual(
+            test_document_page_count, self.test_document.pages.count()
+        )
+
+    def _disable_test_document_page(self):
+        self.test_document_page.enabled = False
+        self.test_document_page.save()
+
+    def _request_test_document_page_enable_view(self):
+        return self.post(
+            viewname='documents:document_page_enable', kwargs={
+                'pk': self.test_document_page.pk
+            }
+        )
+
+    def test_document_page_enable_view_no_permission(self):
+        self._disable_test_document_page()
+
+        test_document_page_count = self.test_document.pages.count()
+
+        response = self._request_test_document_page_enable_view()
+        self.assertEqual(response.status_code, 404)
+
+        self.assertEqual(
+            test_document_page_count, self.test_document.pages.count()
+        )
+
+    def test_document_page_enable_view_with_access(self):
+        self._disable_test_document_page()
+        self.grant_access(
+            obj=self.test_document, permission=permission_document_edit
+        )
+
+        test_document_page_count = self.test_document.pages.count()
+
+        response = self._request_test_document_page_enable_view()
+        self.assertEqual(response.status_code, 302)
+
+        self.assertNotEqual(
+            test_document_page_count, self.test_document.pages.count()
+        )
+
+    def _request_test_document_page_multiple_enable_view(self):
+        return self.post(
+            viewname='documents:document_page_multiple_enable', data={
+                'id_list': self.test_document_page.pk
+            }
+        )
+
+    def test_document_page_multiple_enable_view_no_permission(self):
+        self._disable_test_document_page()
+        test_document_page_count = self.test_document.pages.count()
+
+        response = self._request_test_document_page_multiple_enable_view()
+        self.assertEqual(response.status_code, 404)
+
+        self.assertEqual(
+            test_document_page_count, self.test_document.pages.count()
+        )
+
+    def test_document_page_multiple_enable_view_with_access(self):
+        self._disable_test_document_page()
+        self.grant_access(
+            obj=self.test_document, permission=permission_document_edit
+        )
+
+        test_document_page_count = self.test_document.pages.count()
+
+        response = self._request_test_document_page_multiple_enable_view()
+        self.assertEqual(response.status_code, 302)
+
+        self.assertNotEqual(
+            test_document_page_count, self.test_document.pages.count()
+        )
+
+
 class DocumentPageViewTestCase(GenericDocumentViewTestCase):
     def _request_test_document_page_list_view(self):
         return self.get(