From 5b304ea7424f258f5ace0a08a2ad6380fb358dcc Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Sat, 5 Oct 2019 15:08:53 -0400 Subject: [PATCH 01/26] Bump version to 3.3 beta 1 Signed-off-by: Roberto Rosario --- docker/rootfs/version | 2 +- mayan/__init__.py | 6 +++--- requirements/base.txt | 3 --- setup.py | 3 --- 4 files changed, 4 insertions(+), 10 deletions(-) diff --git a/docker/rootfs/version b/docker/rootfs/version index f092941a75..d9b6a7830f 100755 --- a/docker/rootfs/version +++ b/docker/rootfs/version @@ -1 +1 @@ -3.2.8 +3.3beta1 diff --git a/mayan/__init__.py b/mayan/__init__.py index 4dffd359e1..924c608969 100644 --- a/mayan/__init__.py +++ b/mayan/__init__.py @@ -1,9 +1,9 @@ from __future__ import unicode_literals __title__ = 'Mayan EDMS' -__version__ = '3.2.8' -__build__ = 0x030208 -__build_string__ = 'v3.2.8_Tue Oct 1 13:31:40 2019 -0400' +__version__ = '3.3beta1' +__build__ = 0x030300 +__build_string__ = 'v3.2.8-168-gce4413d539_Sat Oct 5 03:31:51 2019 -0400' __django_version__ = '1.11' __author__ = 'Roberto Rosario' __author_email__ = 'roberto.rosario@mayan-edms.com' diff --git a/requirements/base.txt b/requirements/base.txt index e74cc93d45..87dd80811c 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,13 +1,10 @@ Pillow==6.0.0 PyPDF2==1.26.0 PyYAML==5.1.1 -celery==4.3.0 django-activity-stream==0.7.0 -django-celery-beat==1.5.0 django-colorful==1.3 django-cors-headers==2.5.2 django-downloadview==1.9 -django-environ==0.4.5 django-formtools==2.1 django-mathfilters==0.4.0 django-model-utils==3.1.2 diff --git a/setup.py b/setup.py index 67898584f0..a8ab7ec567 100644 --- a/setup.py +++ b/setup.py @@ -60,13 +60,10 @@ django==1.11.24 Pillow==6.0.0 PyPDF2==1.26.0 PyYAML==5.1.1 -celery==4.3.0 django-activity-stream==0.7.0 -django-celery-beat==1.5.0 django-colorful==1.3 django-cors-headers==2.5.2 django-downloadview==1.9 -django-environ==0.4.5 django-formtools==2.1 django-mathfilters==0.4.0 django-model-utils==3.1.2 From 042727aaa957ffd7f4d223730c42bd46e91534fc Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Sat, 5 Oct 2019 15:09:46 -0400 Subject: [PATCH 02/26] Update build string Signed-off-by: Roberto Rosario --- mayan/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mayan/__init__.py b/mayan/__init__.py index 924c608969..9401b78d20 100644 --- a/mayan/__init__.py +++ b/mayan/__init__.py @@ -3,7 +3,7 @@ from __future__ import unicode_literals __title__ = 'Mayan EDMS' __version__ = '3.3beta1' __build__ = 0x030300 -__build_string__ = 'v3.2.8-168-gce4413d539_Sat Oct 5 03:31:51 2019 -0400' +__build_string__ = 'v3.3beta1_Sat Oct 5 15:08:53 2019 -0400' __django_version__ = '1.11' __author__ = 'Roberto Rosario' __author_email__ = 'roberto.rosario@mayan-edms.com' From cb6cb4121fb7aabc63c8e752ff7ad3cc43b60355 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Sun, 6 Oct 2019 03:04:34 -0400 Subject: [PATCH 03/26] Fix typos Signed-off-by: Roberto Rosario --- docs/releases/3.3.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/releases/3.3.rst b/docs/releases/3.3.rst index 17ff89a513..47878877f9 100644 --- a/docs/releases/3.3.rst +++ b/docs/releases/3.3.rst @@ -93,7 +93,7 @@ Changes Removals -------- -- Database conversion. Reason for removal. The database conversions support +- Database conversion. Reason for removal: The database conversions support provided by this feature (SQLite to PostgreSQL) was being confused with database migrations and upgrades. @@ -108,7 +108,7 @@ Removals Continued confusion about the purpose of the feature and confusion about how errors with this feature were a reflexion of the code quality of - Mayannecessitated the removal of the database conversion feature. + Mayan necessitated the removal of the database conversion feature. - Django environ From 949c0ab2855cde22853a05e9ea9b0b6d0e2f6901 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Mon, 7 Oct 2019 10:43:52 -0400 Subject: [PATCH 04/26] Remove empty ine Signed-off-by: Roberto Rosario --- .../document_states/templates/document_states/extra_data.html | 1 - 1 file changed, 1 deletion(-) diff --git a/mayan/apps/document_states/templates/document_states/extra_data.html b/mayan/apps/document_states/templates/document_states/extra_data.html index b996601769..94ed6b96c2 100644 --- a/mayan/apps/document_states/templates/document_states/extra_data.html +++ b/mayan/apps/document_states/templates/document_states/extra_data.html @@ -5,4 +5,3 @@ {% endfor %} {% endif %} - From 339b7dd836bcec28223cd72befe8f52a390fe469 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Mon, 7 Oct 2019 16:43:00 -0400 Subject: [PATCH 05/26] Add missing dependencies import Signed-off-by: Roberto Rosario --- mayan/apps/task_manager/apps.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mayan/apps/task_manager/apps.py b/mayan/apps/task_manager/apps.py index 51044d3c52..4ae48a9e48 100644 --- a/mayan/apps/task_manager/apps.py +++ b/mayan/apps/task_manager/apps.py @@ -8,6 +8,7 @@ from mayan.apps.common.menus import menu_tools from mayan.apps.navigation.classes import SourceColumn from .classes import CeleryQueue, Task +from .dependencies import * # NOQA from .links import link_task_manager from .settings import * # NOQA From 9cf1d44ee7baeaa47fecd031331fbcfa373555e1 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 8 Oct 2019 09:33:39 -0400 Subject: [PATCH 06/26] Move Makefile versions to variables Signed-off-by: Roberto Rosario --- Makefile | 35 +++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index e5de22ab31..489c40d584 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,15 @@ .PHONY: clean-pyc clean-build +DOCKER_MYSQL_IMAGE = mysql:8.0 +DOCKER_ORACLE_IMAGE = wnameless/oracle-xe-11g +DOCKER_POSTGRES_IMAGE = postgres:9.6-alpine +DOCKER_REDIS_IMAGE = redis:5.0-alpine + +PYTHON_MYSQL_VERSION = 1.4.4 +PYTHON_PSYCOPG2_VERSION = 2.8.3 +PYTHON_RABBITMQ_VERSION = 2.0.0 +PYTHON_REDIS_VERSION = 3.2.1 + help: @echo "Usage: make \n" @awk 'BEGIN {FS = ":.*##"} /^[a-zA-Z_-]+:.*?## / { printf " * %-40s -%s\n", $$1, $$2 }' $(MAKEFILE_LIST)|sort @@ -18,7 +28,7 @@ clean-pyc: ## Remove Python artifacts. find . -name '*.pyc' -exec rm -f {} + find . -name '*.pyo' -exec rm -f {} + find . -name '*~' -exec rm -f {} + - find . -name '__pycache__' -exec rm -R -f {} + + find . -name '__pycache__' -exec rm -R -f {} + # Testing @@ -33,10 +43,10 @@ test-all: clean-pyc test-launch-postgres: @docker rm -f test-postgres || true @docker volume rm test-postgres || true - docker run -d --name test-postgres -p 5432:5432 -v test-postgres:/var/lib/postgresql/data healthcheck/postgres + docker run -d --name test-postgres -p 5432:5432 -v test-postgres:/var/lib/postgresql/data $(DOCKER_POSTGRES_IMAGE) sudo apt-get install -q libpq-dev - pip install psycopg2 - while ! docker inspect --format='{{json .State.Health}}' test-postgres|grep 'Status":"healthy"'; do sleep 1; done + pip install psycopg2==$(PYTHON_PSYCOPG2_VERSION) + while ! nc -z 127.0.0.1 5432; do sleep 1; done test-with-postgres: ## MODULE= - Run tests for a single app, module or test class against a Postgres database container. test-with-postgres: test-launch-postgres @@ -53,10 +63,10 @@ test-with-postgres-all: test-launch-postgres test-launch-mysql: @docker rm -f test-mysql || true @docker volume rm test-mysql || true - docker run -d --name test-mysql -p 3306:3306 -e MYSQL_ALLOW_EMPTY_PASSWORD=True -e MYSQL_DATABASE=mayan -v test-mysql:/var/lib/mysql healthcheck/mysql + docker run -d --name test-mysql -p 3306:3306 -e MYSQL_ALLOW_EMPTY_PASSWORD=True -e MYSQL_DATABASE=mayan -v test-mysql:/var/lib/mysql $(DOCKER_MYSQL_IMAGE) sudo apt-get install -q libmysqlclient-dev mysql-client - pip install mysqlclient - while ! docker inspect --format='{{json .State.Health}}' test-mysql|grep 'Status":"healthy"'; do sleep 1; done + pip install mysqlclient==$(PYTHON_MYSQL_VERSION) + while ! nc -z 127.0.0.1 3306; do sleep 1; done mysql -h 127.0.0.1 -P 3306 -uroot -e "set global character_set_server=utf8mb4;" test-with-mysql: ## MODULE= - Run tests for a single app, module or test class against a MySQL database container. @@ -75,7 +85,7 @@ test-with-mysql-all: test-launch-mysql test-launch-oracle: @docker rm -f test-oracle || true @docker volume rm test-oracle || true - docker run -d --name test-oracle -p 49160:22 -p 49161:1521 -e ORACLE_ALLOW_REMOTE=true -v test-oracle:/u01/app/oracle wnameless/oracle-xe-11g + docker run -d --name test-oracle -p 49160:22 -p 49161:1521 -e ORACLE_ALLOW_REMOTE=true -v test-oracle:/u01/app/oracle $(DOCKER_ORACLE_IMAGE) # https://gist.github.com/kimus/10012910 pip install cx_Oracle while ! nc -z 127.0.0.1 49161; do sleep 1; done @@ -243,11 +253,12 @@ shell_plus: ## Run the shell_plus command. ./manage.py shell_plus --settings=mayan.settings.development test-with-docker-services-on: ## Launch and initialize production-like services using Docker (Postgres and Redis). - docker run -d --name redis -p 6379:6379 redis - docker run -d --name postgres -p 5432:5432 postgres + docker run -d --name redis -p 6379:6379 $(DOCKER_REDIS_IMAGE) + docker run -d --name postgres -p 5432:5432 $(DOCKER_POSTGRES_IMAGE) while ! nc -z 127.0.0.1 6379; do sleep 1; done while ! nc -z 127.0.0.1 5432; do sleep 1; done sleep 4 + pip install psycopg2==$(PYTHON_PSYCOPG2_VERSION) redis==$(PYTHON_REDIS_VERSION) ./manage.py initialsetup --settings=mayan.settings.staging.docker test-with-docker-services-off: ## Stop and delete the Docker production-like services. @@ -261,7 +272,7 @@ test-with-docker-worker: ## Launch a worker instance that uses the production-li DJANGO_SETTINGS_MODULE=mayan.settings.staging.docker ./manage.py celery worker -A mayan -B -l INFO -O fair docker-mysql-on: ## Launch and initialize a MySQL Docker container. - docker run -d --name mysql -p 3306:3306 -e MYSQL_ALLOW_EMPTY_PASSWORD=True -e MYSQL_DATABASE=mayan_edms mysql + docker run -d --name mysql -p 3306:3306 -e MYSQL_ALLOW_EMPTY_PASSWORD=True -e MYSQL_DATABASE=mayan_edms $(DOCKER_MYSQL_IMAGE) while ! nc -z 127.0.0.1 3306; do sleep 1; done docker-mysql-off: ## Stop and delete the MySQL Docker container. @@ -269,7 +280,7 @@ docker-mysql-off: ## Stop and delete the MySQL Docker container. docker rm mysql docker-postgres-on: ## Launch and initialize a PostgreSQL Docker container. - docker run -d --name postgres -p 5432:5432 postgres + docker run -d --name postgres -p 5432:5432 $(DOCKER_POSTGRES_IMAGE) while ! nc -z 127.0.0.1 5432; do sleep 1; done docker-postgres-off: ## Stop and delete the PostgreSQL Docker container. From 653f55f84a67fa0f8765934da8f8e323eb5e0a3f Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 8 Oct 2019 09:43:10 -0400 Subject: [PATCH 07/26] Invalidate the layer cache in tests Signed-off-by: Roberto Rosario --- mayan/apps/common/tests/base.py | 3 ++- mayan/apps/converter/tests/mixins.py | 8 ++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/mayan/apps/common/tests/base.py b/mayan/apps/common/tests/base.py index 3c352df9df..81802e63e7 100644 --- a/mayan/apps/common/tests/base.py +++ b/mayan/apps/common/tests/base.py @@ -5,6 +5,7 @@ from django.test import TestCase from django_downloadview import assert_download_response from mayan.apps.acls.tests.mixins import ACLTestCaseMixin +from mayan.apps.converter.tests.mixins import LayerTestCaseMixin from mayan.apps.permissions.classes import Permission from mayan.apps.smart_settings.classes import Namespace from mayan.apps.user_management.tests.mixins import UserTestMixin @@ -19,7 +20,7 @@ from .mixins import ( class BaseTestCase( - SilenceLoggerTestCaseMixin, ConnectionsCheckTestCaseMixin, + LayerTestCaseMixin, SilenceLoggerTestCaseMixin, ConnectionsCheckTestCaseMixin, RandomPrimaryKeyModelMonkeyPatchMixin, ACLTestCaseMixin, ModelTestCaseMixin, OpenFileCheckTestCaseMixin, TempfileCheckTestCasekMixin, UserTestMixin, TestCase diff --git a/mayan/apps/converter/tests/mixins.py b/mayan/apps/converter/tests/mixins.py index b5a914a232..7fffc10f4d 100644 --- a/mayan/apps/converter/tests/mixins.py +++ b/mayan/apps/converter/tests/mixins.py @@ -14,6 +14,12 @@ from .literals import ( ) +class LayerTestCaseMixin(object): + def setUp(self): + super(LayerTestCaseMixin, self).setUp() + Layer.invalidate_cache() + + class LayerTestMixin(PermissionTestMixin): test_layer = Layer( label='Test layer', name='test_layer', order=1000, @@ -38,8 +44,6 @@ class LayerTestMixin(PermissionTestMixin): 'select': self.test_layer_permission, 'view': self.test_layer_permission, } - Layer.invalidate_cache() - Layer.update() class TransformationTestMixin(LayerTestMixin): From 517bb4e9a203c04679db7db0fc605b8f639f57e8 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 8 Oct 2019 09:33:39 -0400 Subject: [PATCH 08/26] Move Makefile versions to variables Signed-off-by: Roberto Rosario --- Makefile | 35 +++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index e5de22ab31..489c40d584 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,15 @@ .PHONY: clean-pyc clean-build +DOCKER_MYSQL_IMAGE = mysql:8.0 +DOCKER_ORACLE_IMAGE = wnameless/oracle-xe-11g +DOCKER_POSTGRES_IMAGE = postgres:9.6-alpine +DOCKER_REDIS_IMAGE = redis:5.0-alpine + +PYTHON_MYSQL_VERSION = 1.4.4 +PYTHON_PSYCOPG2_VERSION = 2.8.3 +PYTHON_RABBITMQ_VERSION = 2.0.0 +PYTHON_REDIS_VERSION = 3.2.1 + help: @echo "Usage: make \n" @awk 'BEGIN {FS = ":.*##"} /^[a-zA-Z_-]+:.*?## / { printf " * %-40s -%s\n", $$1, $$2 }' $(MAKEFILE_LIST)|sort @@ -18,7 +28,7 @@ clean-pyc: ## Remove Python artifacts. find . -name '*.pyc' -exec rm -f {} + find . -name '*.pyo' -exec rm -f {} + find . -name '*~' -exec rm -f {} + - find . -name '__pycache__' -exec rm -R -f {} + + find . -name '__pycache__' -exec rm -R -f {} + # Testing @@ -33,10 +43,10 @@ test-all: clean-pyc test-launch-postgres: @docker rm -f test-postgres || true @docker volume rm test-postgres || true - docker run -d --name test-postgres -p 5432:5432 -v test-postgres:/var/lib/postgresql/data healthcheck/postgres + docker run -d --name test-postgres -p 5432:5432 -v test-postgres:/var/lib/postgresql/data $(DOCKER_POSTGRES_IMAGE) sudo apt-get install -q libpq-dev - pip install psycopg2 - while ! docker inspect --format='{{json .State.Health}}' test-postgres|grep 'Status":"healthy"'; do sleep 1; done + pip install psycopg2==$(PYTHON_PSYCOPG2_VERSION) + while ! nc -z 127.0.0.1 5432; do sleep 1; done test-with-postgres: ## MODULE= - Run tests for a single app, module or test class against a Postgres database container. test-with-postgres: test-launch-postgres @@ -53,10 +63,10 @@ test-with-postgres-all: test-launch-postgres test-launch-mysql: @docker rm -f test-mysql || true @docker volume rm test-mysql || true - docker run -d --name test-mysql -p 3306:3306 -e MYSQL_ALLOW_EMPTY_PASSWORD=True -e MYSQL_DATABASE=mayan -v test-mysql:/var/lib/mysql healthcheck/mysql + docker run -d --name test-mysql -p 3306:3306 -e MYSQL_ALLOW_EMPTY_PASSWORD=True -e MYSQL_DATABASE=mayan -v test-mysql:/var/lib/mysql $(DOCKER_MYSQL_IMAGE) sudo apt-get install -q libmysqlclient-dev mysql-client - pip install mysqlclient - while ! docker inspect --format='{{json .State.Health}}' test-mysql|grep 'Status":"healthy"'; do sleep 1; done + pip install mysqlclient==$(PYTHON_MYSQL_VERSION) + while ! nc -z 127.0.0.1 3306; do sleep 1; done mysql -h 127.0.0.1 -P 3306 -uroot -e "set global character_set_server=utf8mb4;" test-with-mysql: ## MODULE= - Run tests for a single app, module or test class against a MySQL database container. @@ -75,7 +85,7 @@ test-with-mysql-all: test-launch-mysql test-launch-oracle: @docker rm -f test-oracle || true @docker volume rm test-oracle || true - docker run -d --name test-oracle -p 49160:22 -p 49161:1521 -e ORACLE_ALLOW_REMOTE=true -v test-oracle:/u01/app/oracle wnameless/oracle-xe-11g + docker run -d --name test-oracle -p 49160:22 -p 49161:1521 -e ORACLE_ALLOW_REMOTE=true -v test-oracle:/u01/app/oracle $(DOCKER_ORACLE_IMAGE) # https://gist.github.com/kimus/10012910 pip install cx_Oracle while ! nc -z 127.0.0.1 49161; do sleep 1; done @@ -243,11 +253,12 @@ shell_plus: ## Run the shell_plus command. ./manage.py shell_plus --settings=mayan.settings.development test-with-docker-services-on: ## Launch and initialize production-like services using Docker (Postgres and Redis). - docker run -d --name redis -p 6379:6379 redis - docker run -d --name postgres -p 5432:5432 postgres + docker run -d --name redis -p 6379:6379 $(DOCKER_REDIS_IMAGE) + docker run -d --name postgres -p 5432:5432 $(DOCKER_POSTGRES_IMAGE) while ! nc -z 127.0.0.1 6379; do sleep 1; done while ! nc -z 127.0.0.1 5432; do sleep 1; done sleep 4 + pip install psycopg2==$(PYTHON_PSYCOPG2_VERSION) redis==$(PYTHON_REDIS_VERSION) ./manage.py initialsetup --settings=mayan.settings.staging.docker test-with-docker-services-off: ## Stop and delete the Docker production-like services. @@ -261,7 +272,7 @@ test-with-docker-worker: ## Launch a worker instance that uses the production-li DJANGO_SETTINGS_MODULE=mayan.settings.staging.docker ./manage.py celery worker -A mayan -B -l INFO -O fair docker-mysql-on: ## Launch and initialize a MySQL Docker container. - docker run -d --name mysql -p 3306:3306 -e MYSQL_ALLOW_EMPTY_PASSWORD=True -e MYSQL_DATABASE=mayan_edms mysql + docker run -d --name mysql -p 3306:3306 -e MYSQL_ALLOW_EMPTY_PASSWORD=True -e MYSQL_DATABASE=mayan_edms $(DOCKER_MYSQL_IMAGE) while ! nc -z 127.0.0.1 3306; do sleep 1; done docker-mysql-off: ## Stop and delete the MySQL Docker container. @@ -269,7 +280,7 @@ docker-mysql-off: ## Stop and delete the MySQL Docker container. docker rm mysql docker-postgres-on: ## Launch and initialize a PostgreSQL Docker container. - docker run -d --name postgres -p 5432:5432 postgres + docker run -d --name postgres -p 5432:5432 $(DOCKER_POSTGRES_IMAGE) while ! nc -z 127.0.0.1 5432; do sleep 1; done docker-postgres-off: ## Stop and delete the PostgreSQL Docker container. From 4659269349981533fe4024d4fd688e824f871cc9 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 8 Oct 2019 09:43:10 -0400 Subject: [PATCH 09/26] Invalidate the layer cache in tests Signed-off-by: Roberto Rosario --- mayan/apps/common/tests/base.py | 3 ++- mayan/apps/converter/tests/mixins.py | 8 ++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/mayan/apps/common/tests/base.py b/mayan/apps/common/tests/base.py index 3c352df9df..81802e63e7 100644 --- a/mayan/apps/common/tests/base.py +++ b/mayan/apps/common/tests/base.py @@ -5,6 +5,7 @@ from django.test import TestCase from django_downloadview import assert_download_response from mayan.apps.acls.tests.mixins import ACLTestCaseMixin +from mayan.apps.converter.tests.mixins import LayerTestCaseMixin from mayan.apps.permissions.classes import Permission from mayan.apps.smart_settings.classes import Namespace from mayan.apps.user_management.tests.mixins import UserTestMixin @@ -19,7 +20,7 @@ from .mixins import ( class BaseTestCase( - SilenceLoggerTestCaseMixin, ConnectionsCheckTestCaseMixin, + LayerTestCaseMixin, SilenceLoggerTestCaseMixin, ConnectionsCheckTestCaseMixin, RandomPrimaryKeyModelMonkeyPatchMixin, ACLTestCaseMixin, ModelTestCaseMixin, OpenFileCheckTestCaseMixin, TempfileCheckTestCasekMixin, UserTestMixin, TestCase diff --git a/mayan/apps/converter/tests/mixins.py b/mayan/apps/converter/tests/mixins.py index b5a914a232..7fffc10f4d 100644 --- a/mayan/apps/converter/tests/mixins.py +++ b/mayan/apps/converter/tests/mixins.py @@ -14,6 +14,12 @@ from .literals import ( ) +class LayerTestCaseMixin(object): + def setUp(self): + super(LayerTestCaseMixin, self).setUp() + Layer.invalidate_cache() + + class LayerTestMixin(PermissionTestMixin): test_layer = Layer( label='Test layer', name='test_layer', order=1000, @@ -38,8 +44,6 @@ class LayerTestMixin(PermissionTestMixin): 'select': self.test_layer_permission, 'view': self.test_layer_permission, } - Layer.invalidate_cache() - Layer.update() class TransformationTestMixin(LayerTestMixin): From 7b3a83ee39c700ca54ab505a2daed72faebe6f59 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 8 Oct 2019 09:48:54 -0400 Subject: [PATCH 10/26] Update GitLab CI to use Python 3 and virtualenv Signed-off-by: Roberto Rosario --- .gitlab-ci.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a5b56f4358..b620d70cb2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -152,7 +152,9 @@ job_push_python: - locale-gen en_US.UTF-8 - update-locale LANG=en_US.UTF-8 - export LC_ALL=en_US.UTF-8 - - apt-get install -qq curl exiftool gcc ghostscript gnupg1 graphviz libfuse2 libjpeg-dev libmagic1 libpng-dev libtiff-dev poppler-utils libreoffice poppler-utils python-dev python-pip tesseract-ocr tesseract-ocr-deu + - apt-get install -qq curl exiftool gcc ghostscript gnupg1 graphviz libfuse2 libjpeg-dev libmagic1 libpng-dev libtiff-dev poppler-utils libreoffice poppler-utils python-dev python-virtualenv python3-dev tesseract-ocr tesseract-ocr-deu + - virtualenv venv -p /usr/bin/python3 + - . venv/bin/activate - pip install -r requirements.txt -r requirements/testing-base.txt only: - releases/all @@ -170,6 +172,7 @@ test-mysql: - mysql:8.0.3 script: - apt-get install -qq libmysqlclient-dev mysql-client + - . venv/bin/activate - pip install mysqlclient - mysql -h"$MYSQL_PORT_3306_TCP_ADDR" -P"$MYSQL_PORT_3306_TCP_PORT" -uroot -p"$MYSQL_ENV_MYSQL_ROOT_PASSWORD" -e "set global character_set_server=utf8mb4;" - python manage.py test --mayan-apps --settings=mayan.settings.testing.gitlab-ci.db_mysql --nomigrations @@ -185,6 +188,7 @@ test-postgres: - postgres script: - apt-get install -qq libpq-dev + - . venv/bin/activate - pip install psycopg2 - python manage.py test --mayan-apps --settings=mayan.settings.testing.gitlab-ci.db_postgres --nomigrations tags: @@ -193,6 +197,7 @@ test-postgres: test-sqlite: <<: *test_base script: + - . venv/bin/activate - python manage.py test --mayan-apps --settings=mayan.settings.testing.gitlab-ci --nomigrations deploy_demo: From 1b327b99f01814af6984e97f5f37ae68b1e29b93 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 8 Oct 2019 15:14:16 -0400 Subject: [PATCH 11/26] Update run_test Docker command name Signed-off-by: Roberto Rosario --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b620d70cb2..772bd315d3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -19,7 +19,7 @@ job_docker_build: - docker build --pull -t "$CI_REGISTRY_IMAGE" -f docker/Dockerfile . - VERSION=`cat docker/rootfs/version` - docker tag "$CI_REGISTRY_IMAGE" "$CI_REGISTRY_IMAGE:$VERSION" - - docker run --rm "$CI_REGISTRY_IMAGE:$VERSION" run-tests + - docker run --rm "$CI_REGISTRY_IMAGE:$VERSION" run_tests - docker push "$CI_REGISTRY_IMAGE:$VERSION" - docker push "$CI_REGISTRY_IMAGE:latest" - docker tag "$CI_REGISTRY_IMAGE:$VERSION" registry-1.docker.io/mayanedms/mayanedms:"$VERSION" @@ -58,7 +58,7 @@ job_docker_nightly: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY script: - docker build --pull -t "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG" -f docker/Dockerfile . - - docker run --rm "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG" run-tests + - docker run --rm "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG" run_tests - docker push "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG" only: - nightly From cc8147d002546bd2db06a20ddbd38f54172eb238 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 8 Oct 2019 15:15:50 -0400 Subject: [PATCH 12/26] Update requirements and setup Signed-off-by: Roberto Rosario --- mayan/__init__.py | 2 +- requirements/base.txt | 2 ++ setup.py | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/mayan/__init__.py b/mayan/__init__.py index 9401b78d20..c1392d304f 100644 --- a/mayan/__init__.py +++ b/mayan/__init__.py @@ -3,7 +3,7 @@ from __future__ import unicode_literals __title__ = 'Mayan EDMS' __version__ = '3.3beta1' __build__ = 0x030300 -__build_string__ = 'v3.3beta1_Sat Oct 5 15:08:53 2019 -0400' +__build_string__ = 'v3.3beta1-9-g1b327b99f0_Tue Oct 8 15:15:08 2019 -0400' __django_version__ = '1.11' __author__ = 'Roberto Rosario' __author_email__ = 'roberto.rosario@mayan-edms.com' diff --git a/requirements/base.txt b/requirements/base.txt index 87dd80811c..337001f564 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,7 +1,9 @@ Pillow==6.0.0 PyPDF2==1.26.0 PyYAML==5.1.1 +celery==4.3.0 django-activity-stream==0.7.0 +django-celery-beat==1.5.0 django-colorful==1.3 django-cors-headers==2.5.2 django-downloadview==1.9 diff --git a/setup.py b/setup.py index a8ab7ec567..5b4c273ef0 100644 --- a/setup.py +++ b/setup.py @@ -60,7 +60,9 @@ django==1.11.24 Pillow==6.0.0 PyPDF2==1.26.0 PyYAML==5.1.1 +celery==4.3.0 django-activity-stream==0.7.0 +django-celery-beat==1.5.0 django-colorful==1.3 django-cors-headers==2.5.2 django-downloadview==1.9 From 8cf807899a0ebedaa15f492875aec286a95a083e Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 8 Oct 2019 18:45:53 -0400 Subject: [PATCH 13/26] Initial commit to support page mapping Signed-off-by: Roberto Rosario --- mayan/apps/document_parsing/apps.py | 10 +- mayan/apps/document_parsing/models.py | 4 +- mayan/apps/document_parsing/views.py | 6 +- mayan/apps/documents/admin.py | 19 +- mayan/apps/documents/api_views.py | 137 ++++++++- mayan/apps/documents/apps.py | 4 +- mayan/apps/documents/dashboard_widgets.py | 6 +- mayan/apps/documents/literals.py | 2 + mayan/apps/documents/managers.py | 28 +- .../migrations/0052_auto_20191007_1921.py | 27 ++ .../migrations/0053_auto_20191007_1922.py | 27 ++ .../migrations/0054_auto_20191008_1522.py | 31 ++ .../migrations/0055_auto_20191008_2116.py | 46 +++ mayan/apps/documents/models/__init__.py | 1 + .../apps/documents/models/document_models.py | 69 +++-- .../documents/models/document_page_models.py | 121 +++++--- .../models/document_version_models.py | 20 +- .../models/document_version_page_models.py | 284 ++++++++++++++++++ mayan/apps/documents/queues.py | 8 + mayan/apps/documents/search.py | 4 +- mayan/apps/documents/serializers.py | 53 +++- mayan/apps/documents/statistics.py | 6 +- mayan/apps/documents/tasks.py | 38 ++- mayan/apps/documents/urls.py | 23 +- .../documents/views/document_page_views.py | 12 +- .../migrations/0003_auto_20191008_1510.py | 20 ++ mayan/apps/metadata/apps.py | 2 +- mayan/apps/ocr/apps.py | 8 +- mayan/apps/ocr/models.py | 4 +- mayan/apps/ocr/tasks.py | 6 +- mayan/apps/ocr/tests/test_api.py | 6 +- mayan/apps/ocr/tests/test_events.py | 4 +- mayan/apps/ocr/tests/test_views.py | 48 +-- mayan/apps/ocr/views.py | 6 +- mayan/apps/redactions/apps.py | 2 +- mayan/apps/tags/apps.py | 2 +- 36 files changed, 914 insertions(+), 180 deletions(-) create mode 100644 mayan/apps/documents/migrations/0052_auto_20191007_1921.py create mode 100644 mayan/apps/documents/migrations/0053_auto_20191007_1922.py create mode 100644 mayan/apps/documents/migrations/0054_auto_20191008_1522.py create mode 100644 mayan/apps/documents/migrations/0055_auto_20191008_2116.py create mode 100644 mayan/apps/documents/models/document_version_page_models.py create mode 100644 mayan/apps/file_caching/migrations/0003_auto_20191008_1510.py diff --git a/mayan/apps/document_parsing/apps.py b/mayan/apps/document_parsing/apps.py index ea4355c5a8..71cd9930ac 100644 --- a/mayan/apps/document_parsing/apps.py +++ b/mayan/apps/document_parsing/apps.py @@ -63,7 +63,7 @@ class DocumentParsingApp(MayanAppConfig): app_label='documents', model_name='Document' ) DocumentPage = apps.get_model( - app_label='documents', model_name='DocumentPage' + app_label='documents', model_name='DocumentVersionPage' ) DocumentType = apps.get_model( app_label='documents', model_name='DocumentType' @@ -100,9 +100,9 @@ class DocumentParsingApp(MayanAppConfig): ) ) - ModelField( - model=Document, name='versions__version_pages__content__content' - ) + #ModelField( + # model=Document, name='versions__pages__content__content' + #) ModelPermission.register( model=Document, permissions=( @@ -133,7 +133,7 @@ class DocumentParsingApp(MayanAppConfig): ) document_search.add_model_field( - field='versions__version_pages__content__content', label=_('Content') + field='versions__pages__content__content', label=_('Content') ) document_page_search.add_model_field( diff --git a/mayan/apps/document_parsing/models.py b/mayan/apps/document_parsing/models.py index 5e9b52320e..8d70eacd5b 100644 --- a/mayan/apps/document_parsing/models.py +++ b/mayan/apps/document_parsing/models.py @@ -17,8 +17,8 @@ class DocumentPageContent(models.Model): This model store's the parsed content of a document page. """ document_page = models.OneToOneField( - on_delete=models.CASCADE, related_name='content', to=DocumentPage, - verbose_name=_('Document page') + on_delete=models.CASCADE, related_name='content', + to=DocumentPage, verbose_name=_('Document page') ) content = models.TextField( blank=True, help_text=_( diff --git a/mayan/apps/document_parsing/views.py b/mayan/apps/document_parsing/views.py index de9cb3ef0b..2fde8527d0 100644 --- a/mayan/apps/document_parsing/views.py +++ b/mayan/apps/document_parsing/views.py @@ -12,7 +12,9 @@ from mayan.apps.common.generics import ( ) from mayan.apps.common.mixins import ExternalObjectMixin from mayan.apps.documents.forms import DocumentTypeFilteredSelectForm -from mayan.apps.documents.models import Document, DocumentPage, DocumentType +from mayan.apps.documents.models import ( + Document, DocumentType, DocumentVersionPage +) from .forms import DocumentContentForm, DocumentPageContentForm from .models import DocumentPageContent, DocumentVersionParseError @@ -87,7 +89,7 @@ class DocumentContentDownloadView(SingleObjectDownloadView): class DocumentPageContentView(SingleObjectDetailView): form_class = DocumentPageContentForm - model = DocumentPage + model = DocumentVersionPage object_permission = permission_content_view def dispatch(self, request, *args, **kwargs): diff --git a/mayan/apps/documents/admin.py b/mayan/apps/documents/admin.py index 62d30c0e1f..59b6004f31 100644 --- a/mayan/apps/documents/admin.py +++ b/mayan/apps/documents/admin.py @@ -3,18 +3,12 @@ from __future__ import unicode_literals from django.contrib import admin from .models import ( - DeletedDocument, Document, DocumentPage, DocumentType, - DocumentTypeFilename, DocumentVersion, DuplicatedDocument, RecentDocument + DeletedDocument, Document, DocumentType, DocumentTypeFilename, + DocumentVersion, DocumentVersionPage, DuplicatedDocument, + RecentDocument ) -class DocumentPageInline(admin.StackedInline): - model = DocumentPage - extra = 1 - classes = ('collapse-open',) - allow_add = True - - class DocumentTypeFilenameInline(admin.StackedInline): model = DocumentTypeFilename extra = 1 @@ -29,6 +23,13 @@ class DocumentVersionInline(admin.StackedInline): allow_add = True +class DocumentVersionPageInline(admin.StackedInline): + model = DocumentVersionPage + extra = 1 + classes = ('collapse-open',) + allow_add = True + + @admin.register(DeletedDocument) class DeletedDocumentAdmin(admin.ModelAdmin): date_hierarchy = 'deleted_date_time' diff --git a/mayan/apps/documents/api_views.py b/mayan/apps/documents/api_views.py index 4d17038722..7f0d6d0d46 100644 --- a/mayan/apps/documents/api_views.py +++ b/mayan/apps/documents/api_views.py @@ -33,7 +33,8 @@ from .serializers import ( DocumentTypeSerializer, DocumentVersionSerializer, NewDocumentSerializer, NewDocumentVersionSerializer, RecentDocumentSerializer, WritableDocumentSerializer, - WritableDocumentTypeSerializer, WritableDocumentVersionSerializer + WritableDocumentTypeSerializer, WritableDocumentVersionSerializer, + DocumentVersionPageSerializer ) from .settings import settings_document_page_image_cache_time from .tasks import task_generate_document_page_image @@ -168,13 +169,13 @@ class APIDocumentPageImageView(generics.RetrieveAPIView): ) return document - def get_document_version(self): - return get_object_or_404( - self.get_document().versions.all(), pk=self.kwargs['version_pk'] - ) + #def get_document_version(self): + # return get_object_or_404( + # self.get_document().versions.all(), pk=self.kwargs['version_pk'] + # ) def get_queryset(self): - return self.get_document_version().pages_all.all() + return self.get_document().pages_all.all() def get_serializer(self, *args, **kwargs): return None @@ -221,6 +222,95 @@ class APIDocumentPageImageView(generics.RetrieveAPIView): return response +class APIDocumentVersionPageImageView(generics.RetrieveAPIView): + """ + get: Returns an image representation of the selected document version page. + """ + lookup_url_kwarg = 'page_pk' + + def get_document(self): + if self.request.method == 'GET': + permission_required = permission_document_view + else: + permission_required = permission_document_edit + + document = get_object_or_404(Document.passthrough, pk=self.kwargs['pk']) + + AccessControlList.objects.check_access( + obj=document, permissions=(permission_required,), + user=self.request.user + ) + return document + + def get_document_version(self): + return get_object_or_404( + self.get_document().versions.all(), pk=self.kwargs['version_pk'] + ) + + def get_queryset(self): + return self.get_document_version().pages_all.all() + + def get_serializer(self, *args, **kwargs): + return None + + def get_serializer_class(self): + return None + + @cache_control(private=True) + def retrieve(self, request, *args, **kwargs): + width = request.GET.get('width') + height = request.GET.get('height') + zoom = request.GET.get('zoom') + + if zoom: + zoom = int(zoom) + + rotation = request.GET.get('rotation') + + if rotation: + rotation = int(rotation) + + maximum_layer_order = request.GET.get('maximum_layer_order') + if maximum_layer_order: + maximum_layer_order = int(maximum_layer_order) + + task = task_generate_document_version_page_image.apply_async( + kwargs=dict( + document_version_page_id=self.get_object().pk, width=width, + height=height, zoom=zoom, rotation=rotation, + maximum_layer_order=maximum_layer_order, + user_id=request.user.pk + ) + ) + + cache_filename = task.get(timeout=DOCUMENT_IMAGE_TASK_TIMEOUT) + cache_file = self.get_object().cache_partition.get_file(filename=cache_filename) + with cache_file.open() as file_object: + response = HttpResponse(file_object.read(), content_type='image') + if '_hash' in request.GET: + patch_cache_control( + response=response, + max_age=settings_document_page_image_cache_time.value + ) + return response + + +class APIDocumentPageListView(generics.ListAPIView): + serializer_class = DocumentPageSerializer + + def get_document(self): + document = get_object_or_404(Document, pk=self.kwargs['pk']) + + AccessControlList.objects.check_access( + obj=document, permissions=(permission_document_view,), + user=self.request.user + ) + return document + + def get_queryset(self): + return self.get_document().pages.all() + + class APIDocumentPageView(generics.RetrieveUpdateAPIView): """ get: Returns the selected document page details. @@ -230,6 +320,38 @@ class APIDocumentPageView(generics.RetrieveUpdateAPIView): lookup_url_kwarg = 'page_pk' serializer_class = DocumentPageSerializer + def get_document(self): + if self.request.method == 'GET': + permission_required = permission_document_view + else: + permission_required = permission_document_edit + + document = get_object_or_404(Document, pk=self.kwargs['pk']) + + AccessControlList.objects.check_access( + obj=document, permissions=(permission_required,), + user=self.request.user + ) + return document + + #def get_document_version(self): + # return get_object_or_404( + # self.get_document().versions.all(), pk=self.kwargs['version_pk'] + # ) + + def get_queryset(self): + return self.get_document().pages.all() + + +class APIDocumentVersionPageView(generics.RetrieveUpdateAPIView): + """ + get: Returns the selected document verion page details. + patch: Edit the selected document version page. + put: Edit the selected document version page. + """ + lookup_url_kwarg = 'page_pk' + serializer_class = DocumentVersionPageSerializer + def get_document(self): if self.request.method == 'GET': permission_required = permission_document_view @@ -289,8 +411,7 @@ class APIDocumentTypeView(generics.RetrieveUpdateDestroyAPIView): 'GET': (permission_document_type_view,), 'PUT': (permission_document_type_edit,), 'PATCH': (permission_document_type_edit,), - 'DELETE': (permission_document_type_delete,) - } + 'DELETE': (permission_document_type_delete,) } permission_classes = (MayanPermission,) queryset = DocumentType.objects.all() diff --git a/mayan/apps/documents/apps.py b/mayan/apps/documents/apps.py index 09e85928c7..738395c805 100644 --- a/mayan/apps/documents/apps.py +++ b/mayan/apps/documents/apps.py @@ -120,8 +120,8 @@ class DocumentsApp(MayanAppConfig): DeletedDocument = self.get_model(model_name='DeletedDocument') Document = self.get_model(model_name='Document') - DocumentPage = self.get_model(model_name='DocumentPage') - DocumentPageResult = self.get_model(model_name='DocumentPageResult') + DocumentPage = self.get_model(model_name='DocumentVersionPage') + DocumentPageResult = self.get_model(model_name='DocumentVersionPageResult') DocumentType = self.get_model(model_name='DocumentType') DocumentTypeFilename = self.get_model(model_name='DocumentTypeFilename') DocumentVersion = self.get_model(model_name='DocumentVersion') diff --git a/mayan/apps/documents/dashboard_widgets.py b/mayan/apps/documents/dashboard_widgets.py index 2ca9dd1b89..221e2cfbbd 100644 --- a/mayan/apps/documents/dashboard_widgets.py +++ b/mayan/apps/documents/dashboard_widgets.py @@ -32,12 +32,12 @@ class DashboardWidgetDocumentPagesTotal(DashboardWidgetNumeric): AccessControlList = apps.get_model( app_label='acls', model_name='AccessControlList' ) - DocumentPage = apps.get_model( - app_label='documents', model_name='DocumentPage' + DocumentVersionPage = apps.get_model( + app_label='documents', model_name='DocumentVersionPage' ) self.count = AccessControlList.objects.restrict_queryset( permission=permission_document_view, user=request.user, - queryset=DocumentPage.objects.all() + queryset=DocumentVersionPage.objects.all() ).count() return super(DashboardWidgetDocumentPagesTotal, self).render(request) diff --git a/mayan/apps/documents/literals.py b/mayan/apps/documents/literals.py index e29ed17e8d..f5b4587b07 100644 --- a/mayan/apps/documents/literals.py +++ b/mayan/apps/documents/literals.py @@ -42,3 +42,5 @@ PAGE_RANGE_RANGE = 'range' PAGE_RANGE_CHOICES = ( (PAGE_RANGE_ALL, _('All pages')), (PAGE_RANGE_RANGE, _('Page range')) ) + +RETRY_DELAY_DOCUMENT_RESET_PAGES = 30 diff --git a/mayan/apps/documents/managers.py b/mayan/apps/documents/managers.py index 6c9dd92640..7ec57862ef 100644 --- a/mayan/apps/documents/managers.py +++ b/mayan/apps/documents/managers.py @@ -28,15 +28,15 @@ class DocumentManager(models.Manager): class DocumentPageManager(models.Manager): def get_by_natural_key(self, page_number, document_version_natural_key): - DocumentVersion = apps.get_model( - app_label='documents', model_name='DocumentVersion' + Document = apps.get_model( + app_label='documents', model_name='Document' ) try: - document_version = DocumentVersion.objects.get_by_natural_key(*document_version_natural_key) - except DocumentVersion.DoesNotExist: + document = Document.objects.get_by_natural_key(*document_version_natural_key) + except Document.DoesNotExist: raise self.model.DoesNotExist - return self.get(document_version__pk=document_version.pk, page_number=page_number) + return self.get(document__pk=document.pk, page_number=page_number) def get_queryset(self): return models.QuerySet( @@ -124,6 +124,24 @@ class DocumentVersionManager(models.Manager): return self.get(document__pk=document.pk, checksum=checksum) +class DocumentVersionPageManager(models.Manager): + def get_by_natural_key(self, page_number, document_version_natural_key): + DocumentVersion = apps.get_model( + app_label='documents', model_name='DocumentVersion' + ) + try: + document_version = DocumentVersion.objects.get_by_natural_key(*document_version_natural_key) + except DocumentVersion.DoesNotExist: + raise self.model.DoesNotExist + + return self.get(document_version__pk=document_version.pk, page_number=page_number) + + def get_queryset(self): + return models.QuerySet( + model=self.model, using=self._db + ).filter(enabled=True) + + class DuplicatedDocumentManager(models.Manager): def clean_empty_duplicate_lists(self): self.filter(documents=None).delete() diff --git a/mayan/apps/documents/migrations/0052_auto_20191007_1921.py b/mayan/apps/documents/migrations/0052_auto_20191007_1921.py new file mode 100644 index 0000000000..5cfcc3040d --- /dev/null +++ b/mayan/apps/documents/migrations/0052_auto_20191007_1921.py @@ -0,0 +1,27 @@ +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('documents', '0051_documentpage_enabled'), + ] + + operations = [ + migrations.DeleteModel( + name='DocumentPageResult', + ), + migrations.RenameModel('DocumentPage', 'DocumentVersionPage'), + migrations.AlterField( + model_name='documentversionpage', + name='document_version', + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name='version_pages', to='documents.DocumentVersion', + verbose_name='Document version' + ), + ), + ] diff --git a/mayan/apps/documents/migrations/0053_auto_20191007_1922.py b/mayan/apps/documents/migrations/0053_auto_20191007_1922.py new file mode 100644 index 0000000000..ee4175bd28 --- /dev/null +++ b/mayan/apps/documents/migrations/0053_auto_20191007_1922.py @@ -0,0 +1,27 @@ +from __future__ import unicode_literals + +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ('documents', '0052_auto_20191007_1921'), + ('ocr', '0008_auto_20180917_0646'), + ('document_parsing', '0004_auto_20180917_0645'), + ] + + operations = [ + migrations.CreateModel( + name='DocumentPageResult', + fields=[ + ], + options={ + 'verbose_name': 'Document version page', + 'verbose_name_plural': 'Document version pages', + 'ordering': ('document_version__document', 'page_number'), + 'proxy': True, + 'indexes': [], + }, + bases=('documents.documentversionpage',), + ), + ] diff --git a/mayan/apps/documents/migrations/0054_auto_20191008_1522.py b/mayan/apps/documents/migrations/0054_auto_20191008_1522.py new file mode 100644 index 0000000000..4bfca889db --- /dev/null +++ b/mayan/apps/documents/migrations/0054_auto_20191008_1522.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.22 on 2019-10-08 15:22 +from __future__ import unicode_literals + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('documents', '0053_auto_20191007_1922'), + ] + + operations = [ + migrations.DeleteModel( + name='DocumentPageResult', + ), + migrations.CreateModel( + name='DocumentVersionPageResult', + fields=[ + ], + options={ + 'verbose_name': 'Document version page', + 'verbose_name_plural': 'Document version pages', + 'ordering': ('document_version__document', 'page_number'), + 'proxy': True, + 'indexes': [], + }, + bases=('documents.documentversionpage',), + ), + ] diff --git a/mayan/apps/documents/migrations/0055_auto_20191008_2116.py b/mayan/apps/documents/migrations/0055_auto_20191008_2116.py new file mode 100644 index 0000000000..5f6ad39fa8 --- /dev/null +++ b/mayan/apps/documents/migrations/0055_auto_20191008_2116.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.22 on 2019-10-08 21:16 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('contenttypes', '0002_remove_content_type_name'), + ('documents', '0054_auto_20191008_1522'), + ] + + operations = [ + migrations.CreateModel( + name='DocumentPage', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('enabled', models.BooleanField(default=True, verbose_name='Enabled')), + ('page_number', models.PositiveIntegerField(blank=True, db_index=True, null=True, verbose_name='Page number')), + ('object_id', models.PositiveIntegerField()), + ('content_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType')), + ('document', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='pages', to='documents.Document', verbose_name='Document')), + ], + options={ + 'verbose_name': 'Document page', + 'verbose_name_plural': 'Document pages', + 'ordering': ('page_number',), + }, + ), + migrations.AlterModelOptions( + name='documentversionpage', + options={'ordering': ('page_number',), 'verbose_name': 'Document version page', 'verbose_name_plural': 'Document version pages'}, + ), + migrations.AlterField( + model_name='documentversionpage', + name='document_version', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='pages', to='documents.DocumentVersion', verbose_name='Document version'), + ), + migrations.AlterUniqueTogether( + name='documentpage', + unique_together=set([('document', 'page_number')]), + ), + ] diff --git a/mayan/apps/documents/models/__init__.py b/mayan/apps/documents/models/__init__.py index bb4b55b82a..69063382c8 100644 --- a/mayan/apps/documents/models/__init__.py +++ b/mayan/apps/documents/models/__init__.py @@ -2,4 +2,5 @@ from .document_models import * # NOQA from .document_page_models import * # NOQA from .document_type_models import * # NOQA from .document_version_models import * # NOQA +from .document_version_page_models import * # NOQA from .misc_models import * # NOQA diff --git a/mayan/apps/documents/models/document_models.py b/mayan/apps/documents/models/document_models.py index 8b6fef2400..651254166c 100644 --- a/mayan/apps/documents/models/document_models.py +++ b/mayan/apps/documents/models/document_models.py @@ -5,9 +5,10 @@ import uuid from django.apps import apps from django.core.files import File -from django.db import models +from django.db import models, transaction from django.urls import reverse from django.utils.encoding import python_2_unicode_compatible +from django.utils.functional import cached_property from django.utils.timezone import now from django.utils.translation import ugettext, ugettext_lazy as _ @@ -15,6 +16,7 @@ from ..events import ( event_document_create, event_document_properties_edit, event_document_type_change, ) +from ..literals import DOCUMENT_IMAGES_CACHE_NAME from ..managers import DocumentManager, PassthroughManager, TrashCanManager from ..settings import setting_language from ..signals import post_document_type_change @@ -102,6 +104,18 @@ class Document(models.Model): ) return RecentDocument.objects.add_document_for_user(user, self) + @cached_property + def cache(self): + Cache = apps.get_model(app_label='file_caching', model_name='Cache') + return Cache.objects.get(name=DOCUMENT_IMAGES_CACHE_NAME) + + @cached_property + def cache_partition(self): + partition, created = self.cache.partitions.get_or_create( + name='document-{}'.format(self.uuid) + ) + return partition + def delete(self, *args, **kwargs): to_trash = kwargs.pop('to_trash', True) @@ -165,6 +179,22 @@ class Document(models.Model): """ return self.latest_version.open(*args, **kwargs) + def reset_pages(self): + with transaction.atomic(): + for page in self.pages.all(): + page.delete() + + self.latest_version.update_page_count() + + for version_page in self.latest_version.pages.all(): + document_page = self.pages.create( + #content_type = models.ForeignKey( + # on_delete=models.CASCADE, to=ContentType + #) + #object_id = models.PositiveIntegerField() + content_object = version_page + ) + def restore(self): self.in_trash = False self.save() @@ -234,28 +264,31 @@ class Document(models.Model): @property def page_count(self): - return self.latest_version.page_count + return self.pages.count() + #return self.latest_version.page_count @property def pages_all(self): - try: - return self.latest_version.pages_all - except AttributeError: - # Document has no version yet - DocumentPage = apps.get_model( - app_label='documents', model_name='DocumentPage' - ) + return self.pages.all() + #try: + # return self.latest_version.pages_all + #except AttributeError: + # # Document has no version yet + # DocumentPage = apps.get_model( + # app_label='documents', model_name='DocumentPage' + # ) - return DocumentPage.objects.none() + # return DocumentPage.objects.none() @property def pages(self): - try: - return self.latest_version.pages - except AttributeError: - # Document has no version yet - DocumentPage = apps.get_model( - app_label='documents', model_name='DocumentPage' - ) + return self.pages.all() + #try: + # return self.latest_version.pages + #except AttributeError: + # # Document has no version yet + # DocumentPage = apps.get_model( + # app_label='documents', model_name='DocumentVersionPage' + # ) - return DocumentPage.objects.none() + # return DocumentPage.objects.none() diff --git a/mayan/apps/documents/models/document_page_models.py b/mayan/apps/documents/models/document_page_models.py index 00be66ab4b..87cac77f7a 100644 --- a/mayan/apps/documents/models/document_page_models.py +++ b/mayan/apps/documents/models/document_page_models.py @@ -4,14 +4,16 @@ import logging from furl import furl +from django.contrib.contenttypes.fields import GenericForeignKey +from django.contrib.contenttypes.models import ContentType from django.db import models +from django.db.models import Max from django.urls import reverse from django.utils.encoding import force_text, python_2_unicode_compatible from django.utils.functional import cached_property from django.utils.translation import ugettext_lazy as _ from mayan.apps.converter.literals import DEFAULT_ZOOM_LEVEL, DEFAULT_ROTATION - from mayan.apps.converter.models import LayerTransformation from mayan.apps.converter.transformations import ( BaseTransformation, TransformationResize, TransformationRotate, @@ -26,25 +28,32 @@ from ..settings import ( setting_zoom_min_level ) -from .document_version_models import DocumentVersion +from .document_models import Document +#from .document_version_page_models import DocumentVersionPage -__all__ = ('DocumentPage', 'DocumentPageResult') +__all__ = ('DocumentPage',)# 'DocumentPageResult') logger = logging.getLogger(__name__) @python_2_unicode_compatible class DocumentPage(models.Model): """ - Model that describes a document version page + Model that describes a document page """ - document_version = models.ForeignKey( - on_delete=models.CASCADE, related_name='version_pages', to=DocumentVersion, - verbose_name=_('Document version') + document = models.ForeignKey( + on_delete=models.CASCADE, related_name='pages', to=Document, + verbose_name=_('Document') ) enabled = models.BooleanField(default=True, verbose_name=_('Enabled')) page_number = models.PositiveIntegerField( - db_index=True, default=1, editable=False, - verbose_name=_('Page number') + db_index=True, blank=True, null=True, verbose_name=_('Page number') + ) + content_type = models.ForeignKey( + on_delete=models.CASCADE, to=ContentType + ) + object_id = models.PositiveIntegerField() + content_object = GenericForeignKey( + ct_field='content_type', fk_field='object_id' ) objects = DocumentPageManager() @@ -52,6 +61,7 @@ class DocumentPage(models.Model): class Meta: ordering = ('page_number',) + unique_together = ('document', 'page_number') verbose_name = _('Document page') verbose_name_plural = _('Document pages') @@ -60,7 +70,7 @@ class DocumentPage(models.Model): @cached_property def cache_partition(self): - partition, created = self.document_version.cache.partitions.get_or_create( + partition, created = self.document.cache.partitions.get_or_create( name=self.uuid ) return partition @@ -69,19 +79,19 @@ class DocumentPage(models.Model): self.cache_partition.delete() super(DocumentPage, self).delete(*args, **kwargs) - def detect_orientation(self): - with self.document_version.open() as file_object: - converter = get_converter_class()( - file_object=file_object, - mime_type=self.document_version.mimetype - ) - return converter.detect_orientation( - page_number=self.page_number - ) + #def detect_orientation(self): + # with self.document_version.open() as file_object: + # converter = get_converter_class()( + # file_object=file_object, + # mime_type=self.document_version.mimetype + # ) + # return converter.detect_orientation( + # page_number=self.page_number + # ) - @property - def document(self): - return self.document_version.document + #@property + #def document(self): + # return self.document_version.document def generate_image(self, user=None, **kwargs): transformation_list = self.get_combined_transformation_list(user=user, **kwargs) @@ -90,7 +100,7 @@ class DocumentPage(models.Model): # Check is transformed image is available logger.debug('transformations cache filename: %s', combined_cache_filename) - if not setting_disable_transformed_image_cache.value and self.cache_partition.get_file(filename=combined_cache_filename): + if self.cache_partition.get_file(filename=combined_cache_filename): logger.debug( 'transformations cache file "%s" found', combined_cache_filename ) @@ -128,7 +138,8 @@ class DocumentPage(models.Model): final_url.args = kwargs final_url.path = reverse( viewname='rest_api:documentpage-image', kwargs={ - 'pk': self.document.pk, 'version_pk': self.document_version.pk, + 'pk': self.document.pk, + #'version_pk': self.document_version.pk, 'page_pk': self.pk } ) @@ -190,12 +201,12 @@ class DocumentPage(models.Model): return transformation_list def get_image(self, transformations=None): - cache_filename = 'base_image' + cache_filename = 'document_page' logger.debug('Page cache filename: %s', cache_filename) cache_file = self.cache_partition.get_file(filename=cache_filename) - if not setting_disable_base_image_cache.value and cache_file: + if cache_file: logger.debug('Page cache file "%s" found', cache_filename) with cache_file.open() as file_object: @@ -216,7 +227,12 @@ class DocumentPage(models.Model): logger.debug('Page cache file "%s" not found', cache_filename) try: - with self.document_version.get_intermediate_file() as file_object: + #with self.document_version.get_intermediate_file() as file_object: + #Render or get cached document version page + self.content_object.get_image() + cache_filename = 'base_image' + cache_file = self.content_object.cache_partition.get_file(filename=cache_filename) + with cache_file.open() as file_object: converter = get_converter_class()( file_object=file_object ) @@ -241,28 +257,39 @@ class DocumentPage(models.Model): ) raise + def get_label(self): + return _( + 'Page %(page_number)d out of %(total_pages)d of %(document)s' + ) % { + 'document': force_text(self.document), + 'page_number': self.page_number, + 'total_pages': self.document.pages_all.count() + } + get_label.short_description = _('Label') + @property def is_in_trash(self): return self.document.is_in_trash - def get_label(self): - return _( - 'Page %(page_num)d out of %(total_pages)d of %(document)s' - ) % { - 'document': force_text(self.document), - 'page_num': self.page_number, - 'total_pages': self.document_version.pages_all.count() - } - get_label.short_description = _('Label') - def natural_key(self): - return (self.page_number, self.document_version.natural_key()) - natural_key.dependencies = ['documents.DocumentVersion'] + return (self.page_number, self.document.natural_key()) + natural_key.dependencies = ['documents.Document'] + + def save(self, *args, **kwargs): + if not self.page_number: + last_page_number = DocumentPage.objects.filter( + document=self.document + ).aggregate(Max('page_number'))['page_number__max'] + if last_page_number is not None: + self.page_number = last_page_number + 1 + else: + self.page_number = 1 + super(DocumentPage, self).save(*args, **kwargs) @property def siblings(self): return DocumentPage.objects.filter( - document_version=self.document_version + document=self.document ) @property @@ -271,12 +298,12 @@ class DocumentPage(models.Model): Make cache UUID a mix of version ID and page ID to avoid using stale images """ - return '{}-{}'.format(self.document_version.uuid, self.pk) + return '{}-{}'.format(self.document.uuid, self.pk) -class DocumentPageResult(DocumentPage): - class Meta: - ordering = ('document_version__document', 'page_number') - proxy = True - verbose_name = _('Document page') - verbose_name_plural = _('Document pages') +#class DocumentVersionPageResult(DocumentVersionPage): +# class Meta: +# ordering = ('document_version__document', 'page_number') +# proxy = True +# verbose_name = _('Document version page') +# verbose_name_plural = _('Document version pages') diff --git a/mayan/apps/documents/models/document_version_models.py b/mayan/apps/documents/models/document_version_models.py index 1c24fbfeda..7a398fecb1 100644 --- a/mayan/apps/documents/models/document_version_models.py +++ b/mayan/apps/documents/models/document_version_models.py @@ -246,16 +246,16 @@ class DocumentVersion(models.Model): return result - @property - def pages_all(self): - DocumentPage = apps.get_model( - app_label='documents', model_name='DocumentPage' - ) - return DocumentPage.passthrough.filter(document_version=self) + #@property + #def pages_all(self): + # DocumentPage = apps.get_model( + # app_label='documents', model_name='DocumentVersionPage' + # ) + # return DocumentPage.passthrough.filter(document_version=self) - @property - def pages(self): - return self.version_pages.all() + #@property + #def pages(self): + # return self.pages.all() @property def page_count(self): @@ -410,7 +410,7 @@ class DocumentVersion(models.Model): pass else: DocumentPage = apps.get_model( - app_label='documents', model_name='DocumentPage' + app_label='documents', model_name='DocumentVersionPage' ) with transaction.atomic(): diff --git a/mayan/apps/documents/models/document_version_page_models.py b/mayan/apps/documents/models/document_version_page_models.py new file mode 100644 index 0000000000..724de125b7 --- /dev/null +++ b/mayan/apps/documents/models/document_version_page_models.py @@ -0,0 +1,284 @@ +from __future__ import absolute_import, unicode_literals + +import logging + +from furl import furl + +from django.db import models +from django.urls import reverse +from django.utils.encoding import force_text, python_2_unicode_compatible +from django.utils.functional import cached_property +from django.utils.translation import ugettext_lazy as _ + +from mayan.apps.converter.literals import DEFAULT_ZOOM_LEVEL, DEFAULT_ROTATION + +from mayan.apps.converter.models import LayerTransformation +from mayan.apps.converter.transformations import ( + BaseTransformation, TransformationResize, TransformationRotate, + TransformationZoom +) +from mayan.apps.converter.utils import get_converter_class + +from ..managers import DocumentVersionPageManager +from ..settings import ( + setting_disable_base_image_cache, setting_disable_transformed_image_cache, + setting_display_width, setting_display_height, setting_zoom_max_level, + setting_zoom_min_level +) + +from .document_version_models import DocumentVersion + +__all__ = ('DocumentVersionPage', 'DocumentVersionPageResult') +logger = logging.getLogger(__name__) + + +@python_2_unicode_compatible +class DocumentVersionPage(models.Model): + """ + Model that describes a document version page + """ + document_version = models.ForeignKey( + on_delete=models.CASCADE, related_name='pages', to=DocumentVersion, + verbose_name=_('Document version') + ) + enabled = models.BooleanField(default=True, verbose_name=_('Enabled')) + page_number = models.PositiveIntegerField( + db_index=True, default=1, editable=False, + verbose_name=_('Page number') + ) + + objects = DocumentVersionPageManager() + #passthrough = models.Manager() + + class Meta: + ordering = ('page_number',) + verbose_name = _('Document version page') + verbose_name_plural = _('Document version pages') + + def __str__(self): + return self.get_label() + + @cached_property + def cache_partition(self): + partition, created = self.document_version.cache.partitions.get_or_create( + name=self.uuid + ) + return partition + + def delete(self, *args, **kwargs): + self.cache_partition.delete() + super(DocumentVersionPage, self).delete(*args, **kwargs) + + #def detect_orientation(self): + # with self.document_version.open() as file_object: + # converter = get_converter_class()( + # file_object=file_object, + # mime_type=self.document_version.mimetype + # ) + # return converter.detect_orientation( + # page_number=self.page_number + # ) + + @property + def document(self): + return self.document_version.document + + def generate_image(self, user=None, **kwargs): + transformation_list = self.get_combined_transformation_list(user=user, **kwargs) + combined_cache_filename = BaseTransformation.combine(transformation_list) + + # Check is transformed image is available + logger.debug('transformations cache filename: %s', combined_cache_filename) + + if self.cache_partition.get_file(filename=combined_cache_filename): + logger.debug( + 'transformations cache file "%s" found', combined_cache_filename + ) + else: + logger.debug( + 'transformations cache file "%s" not found', combined_cache_filename + ) + image = self.get_image(transformations=transformation_list) + with self.cache_partition.create_file(filename=combined_cache_filename) as file_object: + file_object.write(image.getvalue()) + + return combined_cache_filename + + #def get_absolute_url(self): + # return reverse( + # viewname='documents:document_version_page_view', kwargs={ + # 'pk': self.pk + # } + # ) + + def get_api_image_url(self, *args, **kwargs): + """ + Create an unique URL combining: + - the page's image URL + - the interactive argument + - a hash from the server side and interactive transformations + The purpose of this unique URL is to allow client side caching + if document page images. + """ + transformations_hash = BaseTransformation.combine( + self.get_combined_transformation_list(*args, **kwargs) + ) + + kwargs.pop('transformations', None) + + final_url = furl() + final_url.args = kwargs + final_url.path = reverse( + viewname='rest_api:documentversionpage-image', kwargs={ + 'pk': self.document.pk, 'version_pk': self.document_version.pk, + 'page_pk': self.pk + } + ) + final_url.args['_hash'] = transformations_hash + + return final_url.tostr() + + def get_combined_transformation_list(self, user=None, *args, **kwargs): + """ + Return a list of transformation containing the server side + document page transformation as well as tranformations created + from the arguments as transient interactive transformation. + """ + # Convert arguments into transformations + transformations = kwargs.get('transformations', []) + + # Set sensible defaults if the argument is not specified or if the + # argument is None + width = kwargs.get('width', setting_display_width.value) or setting_display_width.value + height = kwargs.get('height', setting_display_height.value) or setting_display_height.value + rotation = kwargs.get('rotation', DEFAULT_ROTATION) or DEFAULT_ROTATION + zoom_level = kwargs.get('zoom', DEFAULT_ZOOM_LEVEL) or DEFAULT_ZOOM_LEVEL + + if zoom_level < setting_zoom_min_level.value: + zoom_level = setting_zoom_min_level.value + + if zoom_level > setting_zoom_max_level.value: + zoom_level = setting_zoom_max_level.value + + # Generate transformation hash + transformation_list = [] + + maximum_layer_order = kwargs.get('maximum_layer_order', None) + + # Stored transformations first + for stored_transformation in LayerTransformation.objects.get_for_object( + self, maximum_layer_order=maximum_layer_order, as_classes=True, + user=user + ): + transformation_list.append(stored_transformation) + + # Interactive transformations second + for transformation in transformations: + transformation_list.append(transformation) + + if rotation: + transformation_list.append( + TransformationRotate(degrees=rotation) + ) + + if width: + transformation_list.append( + TransformationResize(width=width, height=height) + ) + + if zoom_level: + transformation_list.append(TransformationZoom(percent=zoom_level)) + + return transformation_list + + def get_image(self, transformations=None): + cache_filename = 'base_image' + logger.debug('Page cache filename: %s', cache_filename) + + cache_file = self.cache_partition.get_file(filename=cache_filename) + + if cache_file: + logger.debug('Page cache file "%s" found', cache_filename) + + with cache_file.open() as file_object: + converter = get_converter_class()( + file_object=file_object + ) + + converter.seek_page(page_number=0) + + # This code is also repeated below to allow using a context + # manager with cache_file.open and close it automatically. + # Apply runtime transformations + for transformation in transformations or []: + converter.transform(transformation=transformation) + + return converter.get_page() + else: + logger.debug('Page cache file "%s" not found', cache_filename) + + try: + with self.document_version.get_intermediate_file() as file_object: + converter = get_converter_class()( + file_object=file_object + ) + converter.seek_page(page_number=self.page_number - 1) + + page_image = converter.get_page() + + # Since open "wb+" doesn't create files, create it explicitly + with self.cache_partition.create_file(filename=cache_filename) as file_object: + file_object.write(page_image.getvalue()) + + # Apply runtime transformations + for transformation in transformations or []: + converter.transform(transformation=transformation) + + return converter.get_page() + except Exception as exception: + # Cleanup in case of error + logger.error( + 'Error creating page cache file "%s"; %s', + cache_filename, exception + ) + raise + + #@property + #def is_in_trash(self): + # return self.document.is_in_trash + + def get_label(self): + return _( + 'Version page %(page_number)d out of %(total_pages)d of %(document)s' + ) % { + 'document': force_text(self.document), + 'page_number': self.page_number, + 'total_pages': self.document_version.pages.count() + } + get_label.short_description = _('Label') + + def natural_key(self): + return (self.page_number, self.document_version.natural_key()) + natural_key.dependencies = ['documents.DocumentVersion'] + + #@property + #def siblings(self): + # return DocumentVersionPage.objects.filter( + # document_version=self.document_version + # ) + + @property + def uuid(self): + """ + Make cache UUID a mix of version ID and page ID to avoid using stale + images + """ + return '{}-{}'.format(self.document_version.uuid, self.pk) + + +class DocumentVersionPageResult(DocumentVersionPage): + class Meta: + ordering = ('document_version__document', 'page_number') + proxy = True + verbose_name = _('Document version page') + verbose_name_plural = _('Document version pages') diff --git a/mayan/apps/documents/queues.py b/mayan/apps/documents/queues.py index 5ac0fa5f01..a98b0e757f 100644 --- a/mayan/apps/documents/queues.py +++ b/mayan/apps/documents/queues.py @@ -30,6 +30,10 @@ queue_converter.add_task_type( dotted_path='mayan.apps.documents.tasks.task_generate_document_page_image', label=_('Generate document page image') ) +queue_converter.add_task_type( + dotted_path='mayan.apps.documents.tasks.task_generate_document_version_page_image', + label=_('Generate document version page image') +) queue_documents.add_task_type( dotted_path='mayan.apps.documents.tasks.task_delete_document', @@ -66,6 +70,10 @@ queue_tools.add_task_type( label=_('Duplicated document scan') ) +queue_uploads.add_task_type( + dotted_path='mayan.apps.documents.tasks.task_document_reset_pages', + label=_('Reset document pages') +) queue_uploads.add_task_type( dotted_path='mayan.apps.documents.tasks.task_update_page_count', label=_('Update document page count') diff --git a/mayan/apps/documents/search.py b/mayan/apps/documents/search.py index de03a36dd0..20044afadd 100644 --- a/mayan/apps/documents/search.py +++ b/mayan/apps/documents/search.py @@ -20,7 +20,7 @@ def transformation_format_uuid(term_string): def get_queryset_page_search_queryset(): # Ignore documents in trash can DocumentPage = apps.get_model( - app_label='documents', model_name='DocumentPage' + app_label='documents', model_name='DocumentVersionPage' ) return DocumentPage.objects.filter(document_version__document__in_trash=False) @@ -49,7 +49,7 @@ document_search.add_model_field( document_page_search = SearchModel( app_label='documents', list_mode=LIST_MODE_CHOICE_ITEM, - model_name='DocumentPage', permission=permission_document_view, + model_name='DocumentVersionPage', permission=permission_document_view, queryset=get_queryset_page_search_queryset, serializer_path='mayan.apps.documents.serializers.DocumentPageSerializer' ) diff --git a/mayan/apps/documents/serializers.py b/mayan/apps/documents/serializers.py index 9632a546cb..5a6c172f7a 100644 --- a/mayan/apps/documents/serializers.py +++ b/mayan/apps/documents/serializers.py @@ -8,42 +8,40 @@ from rest_framework.reverse import reverse from mayan.apps.common.models import SharedUploadedFile from .models import ( - Document, DocumentVersion, DocumentPage, DocumentType, - DocumentTypeFilename, RecentDocument + Document, DocumentPage, DocumentType, DocumentTypeFilename, + DocumentVersion, DocumentVersionPage, RecentDocument ) from .settings import setting_language from .tasks import task_upload_new_version class DocumentPageSerializer(serializers.HyperlinkedModelSerializer): - document_version_url = serializers.SerializerMethodField() + document_url = serializers.SerializerMethodField() image_url = serializers.SerializerMethodField() url = serializers.SerializerMethodField() class Meta: - fields = ('document_version_url', 'image_url', 'page_number', 'url') + fields = ('document_url', 'image_url', 'page_number', 'url') model = DocumentPage - def get_document_version_url(self, instance): + def get_document_url(self, instance): return reverse( - viewname='rest_api:documentversion-detail', args=( - instance.document.pk, instance.document_version.pk, + viewname='rest_api:document-detail', args=( + instance.document.pk, ), request=self.context['request'], format=self.context['format'] ) def get_image_url(self, instance): return reverse( viewname='rest_api:documentpage-image', args=( - instance.document.pk, instance.document_version.pk, - instance.pk, + instance.document.pk, instance.pk, ), request=self.context['request'], format=self.context['format'] ) def get_url(self, instance): return reverse( viewname='rest_api:documentpage-detail', args=( - instance.document.pk, instance.document_version.pk, - instance.pk, + instance.document.pk, instance.pk, ), request=self.context['request'], format=self.context['format'] ) @@ -97,6 +95,39 @@ class WritableDocumentTypeSerializer(serializers.ModelSerializer): return obj.documents.count() +class DocumentVersionPageSerializer(serializers.HyperlinkedModelSerializer): + document_version_url = serializers.SerializerMethodField() + image_url = serializers.SerializerMethodField() + url = serializers.SerializerMethodField() + + class Meta: + fields = ('document_version_url', 'image_url', 'page_number', 'url') + model = DocumentVersionPage + + def get_document_version_url(self, instance): + return reverse( + viewname='rest_api:documentversion-detail', args=( + instance.document.pk, instance.document_version.pk, + ), request=self.context['request'], format=self.context['format'] + ) + + def get_image_url(self, instance): + return reverse( + viewname='rest_api:documentversionpage-image', args=( + instance.document.pk, instance.document_version.pk, + instance.pk, + ), request=self.context['request'], format=self.context['format'] + ) + + def get_url(self, instance): + return reverse( + viewname='rest_api:documentversionpage-detail', args=( + instance.document.pk, instance.document_version.pk, + instance.pk, + ), request=self.context['request'], format=self.context['format'] + ) + + class DocumentVersionSerializer(serializers.HyperlinkedModelSerializer): document_url = serializers.SerializerMethodField() download_url = serializers.SerializerMethodField() diff --git a/mayan/apps/documents/statistics.py b/mayan/apps/documents/statistics.py index 2f1059e7d1..94d91be994 100644 --- a/mayan/apps/documents/statistics.py +++ b/mayan/apps/documents/statistics.py @@ -41,7 +41,7 @@ def new_documents_per_month(): def new_document_pages_per_month(): DocumentPage = apps.get_model( - app_label='documents', model_name='DocumentPage' + app_label='documents', model_name='DocumentVersionPage' ) qss = qsstats.QuerySetStats( @@ -106,7 +106,7 @@ def new_document_pages_this_month(user=None): app_label='acls', model_name='AccessControlList' ) DocumentPage = apps.get_model( - app_label='documents', model_name='DocumentPage' + app_label='documents', model_name='DocumentVersionPage' ) queryset = DocumentPage.objects.all() @@ -195,7 +195,7 @@ def total_document_version_per_month(): def total_document_page_per_month(): DocumentPage = apps.get_model( - app_label='documents', model_name='DocumentPage' + app_label='documents', model_name='DocumentVersionPage' ) qss = qsstats.QuerySetStats( diff --git a/mayan/apps/documents/tasks.py b/mayan/apps/documents/tasks.py index 34bfad3616..f62a015538 100644 --- a/mayan/apps/documents/tasks.py +++ b/mayan/apps/documents/tasks.py @@ -9,7 +9,8 @@ from django.db import OperationalError from mayan.celery import app from .literals import ( - UPDATE_PAGE_COUNT_RETRY_DELAY, UPLOAD_NEW_VERSION_RETRY_DELAY + RETRY_DELAY_DOCUMENT_RESET_PAGES, UPDATE_PAGE_COUNT_RETRY_DELAY, + UPLOAD_NEW_VERSION_RETRY_DELAY ) logger = logging.getLogger(__name__) @@ -64,6 +65,25 @@ def task_delete_stubs(): logger.info(msg='Finshed') +@app.task(bind=True, default_retry_delay=RETRY_DELAY_DOCUMENT_RESET_PAGES, ignore_result=True) +def task_document_reset_pages(self, document_id): + Document = apps.get_model( + app_label='documents', model_name='Document' + ) + + document = Document.objects.get(pk=document_id) + + try: + document.reset_pages() + except OperationalError as exception: + logger.warning( + 'Operational error during attempt to reset pages for ' + 'document: %s; %s. Retrying.', document, + exception + ) + raise self.retry(exc=exception) + + @app.task() def task_generate_document_page_image(document_page_id, user_id=None, **kwargs): DocumentPage = apps.get_model( @@ -80,6 +100,22 @@ def task_generate_document_page_image(document_page_id, user_id=None, **kwargs): return document_page.generate_image(user=user, **kwargs) +@app.task() +def task_generate_document_version_page_image(document_version_page_id, user_id=None, **kwargs): + DocumentVersionPage = apps.get_model( + app_label='documents', model_name='DocumentVersionPage' + ) + User = get_user_model() + + if user_id: + user = User.objects.get(pk=user_id) + else: + user = None + + document_version_page = DocumentVersionPage.objects.get(pk=document_version_page_id) + return document_version_page.generate_image(user=user, **kwargs) + + @app.task(ignore_result=True) def task_scan_duplicates_all(): DuplicatedDocument = apps.get_model( diff --git a/mayan/apps/documents/urls.py b/mayan/apps/documents/urls.py index 84aec25094..ed5219f230 100644 --- a/mayan/apps/documents/urls.py +++ b/mayan/apps/documents/urls.py @@ -4,13 +4,15 @@ from django.conf.urls import url from .api_views import ( APITrashedDocumentListView, APIDeletedDocumentRestoreView, - APIDeletedDocumentView, APIDocumentDownloadView, APIDocumentView, - APIDocumentListView, APIDocumentVersionDownloadView, + APIDeletedDocumentView, APIDocumentDownloadView, APIDocumentPageListView, + APIDocumentView, APIDocumentListView, APIDocumentVersionDownloadView, APIDocumentPageImageView, APIDocumentPageView, APIDocumentTypeDocumentListView, APIDocumentTypeListView, APIDocumentTypeView, APIDocumentVersionsListView, APIDocumentVersionPageListView, APIDocumentVersionView, - APIRecentDocumentListView + APIRecentDocumentListView, + APIDocumentVersionPageView, + APIDocumentVersionPageImageView ) from .views.document_views import ( DocumentDocumentTypeEditView, DocumentDownloadFormView, @@ -405,6 +407,11 @@ api_urls = [ view=APIDocumentVersionPageListView.as_view(), name='documentversion-page-list' ), + url( + regex=r'^documents/(?P[0-9]+)/pages/$', + view=APIDocumentPageListView.as_view(), + name='document-page-list' + ), url( regex=r'^documents/(?P[0-9]+)/versions/(?P[0-9]+)/download/$', view=APIDocumentVersionDownloadView.as_view(), @@ -416,12 +423,20 @@ api_urls = [ ), url( regex=r'^documents/(?P[0-9]+)/versions/(?P[0-9]+)/pages/(?P[0-9]+)$', + view=APIDocumentVersionPageView.as_view(), name='documentversionpage-detail' + ), + url( + regex=r'^documents/(?P[0-9]+)/pages/(?P[0-9]+)$', view=APIDocumentPageView.as_view(), name='documentpage-detail' ), url( - regex=r'^documents/(?P[0-9]+)/versions/(?P[0-9]+)/pages/(?P[0-9]+)/image/$', + regex=r'^documents/(?P[0-9]+)/pages/(?P[0-9]+)/image/$', view=APIDocumentPageImageView.as_view(), name='documentpage-image' ), + url( + regex=r'^documents/(?P[0-9]+)/versions/(?P[0-9]+)/pages/(?P[0-9]+)/image/$', + view=APIDocumentVersionPageImageView.as_view(), name='documentversionpage-image' + ), url( regex=r'^trashed_documents/$', view=APITrashedDocumentListView.as_view(), name='trasheddocument-list' diff --git a/mayan/apps/documents/views/document_page_views.py b/mayan/apps/documents/views/document_page_views.py index f05767d627..614cba4585 100644 --- a/mayan/apps/documents/views/document_page_views.py +++ b/mayan/apps/documents/views/document_page_views.py @@ -21,7 +21,7 @@ from mayan.apps.converter.literals import DEFAULT_ROTATION, DEFAULT_ZOOM_LEVEL from ..forms import DocumentPageForm from ..icons import icon_document_pages from ..links import link_document_update_page_count -from ..models import Document, DocumentPage +from ..models import Document, DocumentVersionPage from ..permissions import permission_document_edit, permission_document_view from ..settings import ( setting_rotation_step, setting_zoom_percent_step, setting_zoom_max_level, @@ -69,7 +69,7 @@ class DocumentPageListView(ExternalObjectMixin, SingleObjectListView): class DocumentPageNavigationBase(ExternalObjectMixin, RedirectView): - external_object_class = DocumentPage + external_object_class = DocumentVersionPage external_object_permission = permission_document_view external_object_pk_url_kwarg = 'pk' @@ -164,7 +164,7 @@ class DocumentPageNavigationPrevious(DocumentPageNavigationBase): class DocumentPageView(ExternalObjectMixin, SimpleView): - external_object_class = DocumentPage + external_object_class = DocumentVersionPage external_object_permission = permission_document_view external_object_pk_url_kwarg = 'pk' template_name = 'appearance/generic_form.html' @@ -204,7 +204,7 @@ class DocumentPageViewResetView(RedirectView): class DocumentPageInteractiveTransformation(ExternalObjectMixin, RedirectView): - external_object_class = DocumentPage + external_object_class = DocumentVersionPage external_object_permission = permission_document_view external_object_pk_url_kwarg = 'pk' @@ -289,7 +289,7 @@ class DocumentPageDisable(MultipleObjectConfirmActionView): return result def get_source_queryset(self): - return DocumentPage.passthrough.all() + return DocumentVersionPage.passthrough.all() def object_action(self, form, instance): instance.enabled = False @@ -319,7 +319,7 @@ class DocumentPageEnable(MultipleObjectConfirmActionView): return result def get_source_queryset(self): - return DocumentPage.passthrough.all() + return DocumentVersionPage.passthrough.all() def object_action(self, form, instance): instance.enabled = True diff --git a/mayan/apps/file_caching/migrations/0003_auto_20191008_1510.py b/mayan/apps/file_caching/migrations/0003_auto_20191008_1510.py new file mode 100644 index 0000000000..14382e0479 --- /dev/null +++ b/mayan/apps/file_caching/migrations/0003_auto_20191008_1510.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.22 on 2019-10-08 15:10 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('file_caching', '0002_auto_20190729_0236'), + ] + + operations = [ + migrations.AlterField( + model_name='cache', + name='name', + field=models.CharField(db_index=True, help_text='Internal name of the cache.', max_length=128, unique=True, verbose_name='Name'), + ), + ] diff --git a/mayan/apps/metadata/apps.py b/mayan/apps/metadata/apps.py index 44fd6bb546..5856484193 100644 --- a/mayan/apps/metadata/apps.py +++ b/mayan/apps/metadata/apps.py @@ -76,7 +76,7 @@ class MetadataApp(MayanAppConfig): app_label='documents', model_name='Document' ) DocumentPageResult = apps.get_model( - app_label='documents', model_name='DocumentPageResult' + app_label='documents', model_name='DocumentVersionPageResult' ) DocumentType = apps.get_model( diff --git a/mayan/apps/ocr/apps.py b/mayan/apps/ocr/apps.py index 33b4f1893b..8ec9be9800 100644 --- a/mayan/apps/ocr/apps.py +++ b/mayan/apps/ocr/apps.py @@ -62,7 +62,7 @@ class OCRApp(MayanAppConfig): app_label='documents', model_name='Document' ) DocumentPage = apps.get_model( - app_label='documents', model_name='DocumentPage' + app_label='documents', model_name='DocumentVersionPage' ) DocumentType = apps.get_model( app_label='documents', model_name='DocumentType' @@ -96,9 +96,9 @@ class OCRApp(MayanAppConfig): ) ) - ModelField( - model=Document, name='versions__version_pages__ocr_content__content' - ) + #ModelField( + # model=Document, name='versions__pages__ocr_content__content' + #) ModelPermission.register( model=Document, permissions=( diff --git a/mayan/apps/ocr/models.py b/mayan/apps/ocr/models.py index 9f42843a7e..8402d2079c 100644 --- a/mayan/apps/ocr/models.py +++ b/mayan/apps/ocr/models.py @@ -4,7 +4,9 @@ from django.db import models from django.utils.encoding import force_text, python_2_unicode_compatible from django.utils.translation import ugettext_lazy as _ -from mayan.apps.documents.models import DocumentPage, DocumentType, DocumentVersion +from mayan.apps.documents.models import ( + DocumentPage, DocumentType, DocumentVersion +) from .managers import ( DocumentPageOCRContentManager, DocumentTypeSettingsManager diff --git a/mayan/apps/ocr/tasks.py b/mayan/apps/ocr/tasks.py index a6d9c82efb..4ba1f5268f 100644 --- a/mayan/apps/ocr/tasks.py +++ b/mayan/apps/ocr/tasks.py @@ -19,8 +19,8 @@ def task_do_ocr(self, document_version_pk): DocumentVersion = apps.get_model( app_label='documents', model_name='DocumentVersion' ) - DocumentPageOCRContent = apps.get_model( - app_label='ocr', model_name='DocumentPageOCRContent' + DocumentVersionPageOCRContent = apps.get_model( + app_label='ocr', model_name='DocumentVersionPageOCRContent' ) lock_id = 'task_do_ocr_doc_version-%d' % document_version_pk @@ -39,7 +39,7 @@ def task_do_ocr(self, document_version_pk): 'Starting document OCR for document version: %s', document_version ) - DocumentPageOCRContent.objects.process_document_version( + DocumentVersionPageOCRContent.objects.process_document_version( document_version=document_version ) except OperationalError as exception: diff --git a/mayan/apps/ocr/tests/test_api.py b/mayan/apps/ocr/tests/test_api.py index e847550141..95c03e288a 100644 --- a/mayan/apps/ocr/tests/test_api.py +++ b/mayan/apps/ocr/tests/test_api.py @@ -65,7 +65,7 @@ class OCRAPITestCase(DocumentTestMixin, BaseAPITestCase): hasattr(self.test_document.pages.first(), 'ocr_content') ) - def _request_document_page_content_view(self): + def _request_document_version_page_content_view(self): return self.get( viewname='rest_api:document-page-ocr-content-view', kwargs={ 'document_pk': self.test_document.pk, @@ -75,7 +75,7 @@ class OCRAPITestCase(DocumentTestMixin, BaseAPITestCase): ) def test_get_document_version_page_content_no_access(self): - response = self._request_document_page_content_view() + response = self._request_document_version_page_content_view() self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) def test_get_document_version_page_content_with_access(self): @@ -83,7 +83,7 @@ class OCRAPITestCase(DocumentTestMixin, BaseAPITestCase): self.grant_access( permission=permission_ocr_content_view, obj=self.test_document ) - response = self._request_document_page_content_view() + response = self._request_document_version_page_content_view() self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertTrue( TEST_DOCUMENT_CONTENT in response.data['content'] diff --git a/mayan/apps/ocr/tests/test_events.py b/mayan/apps/ocr/tests/test_events.py index 4a4318ec64..c51cee8fad 100644 --- a/mayan/apps/ocr/tests/test_events.py +++ b/mayan/apps/ocr/tests/test_events.py @@ -8,13 +8,13 @@ from ..events import ( event_ocr_document_content_deleted, event_ocr_document_version_submit, event_ocr_document_version_finish ) -from ..models import DocumentPageOCRContent +from ..models import DocumentVersionPageOCRContent class OCREventsTestCase(GenericDocumentTestCase): def test_document_content_deleted_event(self): Action.objects.all().delete() - DocumentPageOCRContent.objects.delete_content_for( + DocumentVersionPageOCRContent.objects.delete_content_for( document=self.test_document ) diff --git a/mayan/apps/ocr/tests/test_views.py b/mayan/apps/ocr/tests/test_views.py index 6ed278c54a..d7759fbfc1 100644 --- a/mayan/apps/ocr/tests/test_views.py +++ b/mayan/apps/ocr/tests/test_views.py @@ -2,7 +2,7 @@ from __future__ import unicode_literals from mayan.apps.documents.tests.base import GenericDocumentViewTestCase -from ..models import DocumentPageOCRContent +from ..models import DocumentVersionPageOCRContent from ..permissions import ( permission_ocr_content_view, permission_ocr_document, permission_document_type_ocr_setup @@ -27,9 +27,9 @@ class OCRViewTestMixin(object): } ) - def _request_document_page_content_view(self): + def _request_document_version_page_content_view(self): return self.get( - viewname='ocr:document_page_ocr_content', kwargs={ + viewname='ocr:document_version_page_ocr_content', kwargs={ 'pk': self.test_document.pages.first().pk } ) @@ -86,8 +86,8 @@ class OCRViewsTestCase(OCRViewTestMixin, GenericDocumentViewTestCase): self.assertEqual(response.status_code, 404) self.assertTrue( - DocumentPageOCRContent.objects.filter( - document_page=self.test_document.pages.first() + DocumentVersionPageOCRContent.objects.filter( + document_version_page=self.test_document.pages.first() ).exists() ) @@ -101,28 +101,11 @@ class OCRViewsTestCase(OCRViewTestMixin, GenericDocumentViewTestCase): self.assertEqual(response.status_code, 302) self.assertFalse( - DocumentPageOCRContent.objects.filter( - document_page=self.test_document.pages.first() + DocumentVersionPageOCRContent.objects.filter( + document_version_page=self.test_document.pages.first() ).exists() ) - def test_document_page_content_view_no_permissions(self): - self.test_document.submit_for_ocr() - - response = self._request_document_page_content_view() - self.assertEqual(response.status_code, 404) - - def test_document_page_content_view_with_access(self): - self.test_document.submit_for_ocr() - self.grant_access( - obj=self.test_document, permission=permission_ocr_content_view - ) - - response = self._request_document_page_content_view() - self.assertContains( - response=response, text=TEST_DOCUMENT_CONTENT, status_code=200 - ) - def test_document_submit_view_no_permission(self): response = self._request_document_submit_view() self.assertEqual(response.status_code, 404) @@ -188,6 +171,23 @@ class OCRViewsTestCase(OCRViewTestMixin, GenericDocumentViewTestCase): ), ) + def test_document_version_page_content_view_no_permissions(self): + self.test_document.submit_for_ocr() + + response = self._request_document_version_page_content_view() + self.assertEqual(response.status_code, 404) + + def test_document_version_page_content_view_with_access(self): + self.test_document.submit_for_ocr() + self.grant_access( + obj=self.test_document, permission=permission_ocr_content_view + ) + + response = self._request_document_version_page_content_view() + self.assertContains( + response=response, text=TEST_DOCUMENT_CONTENT, status_code=200 + ) + class DocumentTypeOCRViewTestMixin(object): def _request_document_type_ocr_settings_view(self): diff --git a/mayan/apps/ocr/views.py b/mayan/apps/ocr/views.py index 41b978388e..1524b3d006 100644 --- a/mayan/apps/ocr/views.py +++ b/mayan/apps/ocr/views.py @@ -12,7 +12,9 @@ from mayan.apps.common.generics import ( ) from mayan.apps.common.mixins import ExternalObjectMixin from mayan.apps.documents.forms import DocumentTypeFilteredSelectForm -from mayan.apps.documents.models import Document, DocumentPage, DocumentType +from mayan.apps.documents.models import ( + Document, DocumentType, DocumentVersionPage +) from .forms import DocumentPageOCRContentForm, DocumentOCRContentForm from .models import DocumentPageOCRContent, DocumentVersionOCRError @@ -74,7 +76,7 @@ class DocumentOCRContentView(SingleObjectDetailView): class DocumentPageOCRContentView(SingleObjectDetailView): form_class = DocumentPageOCRContentForm - model = DocumentPage + model = DocumentVersionPage object_permission = permission_ocr_content_view def dispatch(self, request, *args, **kwargs): diff --git a/mayan/apps/redactions/apps.py b/mayan/apps/redactions/apps.py index a7460b817d..732262545f 100644 --- a/mayan/apps/redactions/apps.py +++ b/mayan/apps/redactions/apps.py @@ -28,7 +28,7 @@ class RedactionsApp(MayanAppConfig): super(RedactionsApp, self).ready() DocumentPage = apps.get_model( - app_label='documents', model_name='DocumentPage' + app_label='documents', model_name='DocumentVersionPage' ) link_redaction_list = link_transformation_list.copy( diff --git a/mayan/apps/tags/apps.py b/mayan/apps/tags/apps.py index c64811558a..f3d36c1611 100644 --- a/mayan/apps/tags/apps.py +++ b/mayan/apps/tags/apps.py @@ -62,7 +62,7 @@ class TagsApp(MayanAppConfig): ) DocumentPageResult = apps.get_model( - app_label='documents', model_name='DocumentPageResult' + app_label='documents', model_name='DocumentVersionPageResult' ) DocumentTag = self.get_model(model_name='DocumentTag') From 5b37c7715d319f74d3b4f7c9e076b1038d646b6f Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 9 Oct 2019 00:38:08 -0400 Subject: [PATCH 14/26] Fix document page render Solve page_number > 1 error. Add page_all to Document model. Enable redactions. Remove unused methods. Signed-off-by: Roberto Rosario --- mayan/apps/documents/api_views.py | 19 ++---- mayan/apps/documents/apps.py | 2 +- mayan/apps/documents/managers.py | 5 -- .../apps/documents/models/document_models.py | 61 ++++++++----------- .../documents/models/document_page_models.py | 16 ++--- .../models/document_version_models.py | 11 ---- .../documents/views/document_page_views.py | 12 ++-- mayan/apps/redactions/apps.py | 2 +- 8 files changed, 49 insertions(+), 79 deletions(-) diff --git a/mayan/apps/documents/api_views.py b/mayan/apps/documents/api_views.py index 7f0d6d0d46..87c4c9bd3d 100644 --- a/mayan/apps/documents/api_views.py +++ b/mayan/apps/documents/api_views.py @@ -37,7 +37,10 @@ from .serializers import ( DocumentVersionPageSerializer ) from .settings import settings_document_page_image_cache_time -from .tasks import task_generate_document_page_image +from .tasks import ( + task_generate_document_page_image, + task_generate_document_version_page_image +) logger = logging.getLogger(__name__) @@ -169,11 +172,6 @@ class APIDocumentPageImageView(generics.RetrieveAPIView): ) return document - #def get_document_version(self): - # return get_object_or_404( - # self.get_document().versions.all(), pk=self.kwargs['version_pk'] - # ) - def get_queryset(self): return self.get_document().pages_all.all() @@ -248,7 +246,7 @@ class APIDocumentVersionPageImageView(generics.RetrieveAPIView): ) def get_queryset(self): - return self.get_document_version().pages_all.all() + return self.get_document_version().pages.all() def get_serializer(self, *args, **kwargs): return None @@ -334,11 +332,6 @@ class APIDocumentPageView(generics.RetrieveUpdateAPIView): ) return document - #def get_document_version(self): - # return get_object_or_404( - # self.get_document().versions.all(), pk=self.kwargs['version_pk'] - # ) - def get_queryset(self): return self.get_document().pages.all() @@ -544,7 +537,7 @@ class APIRecentDocumentListView(generics.ListAPIView): class APIDocumentVersionPageListView(generics.ListAPIView): - serializer_class = DocumentPageSerializer + serializer_class = DocumentVersionPageSerializer def get_document(self): document = get_object_or_404(Document, pk=self.kwargs['pk']) diff --git a/mayan/apps/documents/apps.py b/mayan/apps/documents/apps.py index 738395c805..95136a2a00 100644 --- a/mayan/apps/documents/apps.py +++ b/mayan/apps/documents/apps.py @@ -120,7 +120,7 @@ class DocumentsApp(MayanAppConfig): DeletedDocument = self.get_model(model_name='DeletedDocument') Document = self.get_model(model_name='Document') - DocumentPage = self.get_model(model_name='DocumentVersionPage') + DocumentPage = self.get_model(model_name='DocumentPage') DocumentPageResult = self.get_model(model_name='DocumentVersionPageResult') DocumentType = self.get_model(model_name='DocumentType') DocumentTypeFilename = self.get_model(model_name='DocumentTypeFilename') diff --git a/mayan/apps/documents/managers.py b/mayan/apps/documents/managers.py index 7ec57862ef..28236f109c 100644 --- a/mayan/apps/documents/managers.py +++ b/mayan/apps/documents/managers.py @@ -136,11 +136,6 @@ class DocumentVersionPageManager(models.Manager): return self.get(document_version__pk=document_version.pk, page_number=page_number) - def get_queryset(self): - return models.QuerySet( - model=self.model, using=self._db - ).filter(enabled=True) - class DuplicatedDocumentManager(models.Manager): def clean_empty_duplicate_lists(self): diff --git a/mayan/apps/documents/models/document_models.py b/mayan/apps/documents/models/document_models.py index 651254166c..0843ea0056 100644 --- a/mayan/apps/documents/models/document_models.py +++ b/mayan/apps/documents/models/document_models.py @@ -146,9 +146,9 @@ class Document(models.Model): ) def get_api_image_url(self, *args, **kwargs): - latest_version = self.latest_version - if latest_version: - return latest_version.get_api_image_url(*args, **kwargs) + first_page = self.pages.first() + if first_page: + return first_page.get_api_image_url(*args, **kwargs) @property def is_in_trash(self): @@ -179,6 +179,30 @@ class Document(models.Model): """ return self.latest_version.open(*args, **kwargs) + @property + def page_count(self): + return self.pages.count() + + @property + def pages(self): + return self.pages.all() + #try: + # return self.latest_version.pages + #except AttributeError: + # # Document has no version yet + # DocumentPage = apps.get_model( + # app_label='documents', model_name='DocumentVersionPage' + # ) + + # return DocumentPage.objects.none() + + @property + def pages_all(self): + DocumentPage = apps.get_model( + app_label='documents', model_name='DocumentPage' + ) + return DocumentPage.passthrough.filter(document=self) + def reset_pages(self): with transaction.atomic(): for page in self.pages.all(): @@ -261,34 +285,3 @@ class Document(models.Model): @property def latest_version(self): return self.versions.order_by('timestamp').last() - - @property - def page_count(self): - return self.pages.count() - #return self.latest_version.page_count - - @property - def pages_all(self): - return self.pages.all() - #try: - # return self.latest_version.pages_all - #except AttributeError: - # # Document has no version yet - # DocumentPage = apps.get_model( - # app_label='documents', model_name='DocumentPage' - # ) - - # return DocumentPage.objects.none() - - @property - def pages(self): - return self.pages.all() - #try: - # return self.latest_version.pages - #except AttributeError: - # # Document has no version yet - # DocumentPage = apps.get_model( - # app_label='documents', model_name='DocumentVersionPage' - # ) - - # return DocumentPage.objects.none() diff --git a/mayan/apps/documents/models/document_page_models.py b/mayan/apps/documents/models/document_page_models.py index 87cac77f7a..1a7747f35f 100644 --- a/mayan/apps/documents/models/document_page_models.py +++ b/mayan/apps/documents/models/document_page_models.py @@ -89,10 +89,6 @@ class DocumentPage(models.Model): # page_number=self.page_number # ) - #@property - #def document(self): - # return self.document_version.document - def generate_image(self, user=None, **kwargs): transformation_list = self.get_combined_transformation_list(user=user, **kwargs) combined_cache_filename = BaseTransformation.combine(transformation_list) @@ -138,9 +134,7 @@ class DocumentPage(models.Model): final_url.args = kwargs final_url.path = reverse( viewname='rest_api:documentpage-image', kwargs={ - 'pk': self.document.pk, - #'version_pk': self.document_version.pk, - 'page_pk': self.pk + 'pk': self.document.pk, 'page_pk': self.pk } ) final_url.args['_hash'] = transformations_hash @@ -229,17 +223,23 @@ class DocumentPage(models.Model): try: #with self.document_version.get_intermediate_file() as file_object: #Render or get cached document version page + + #self.content_object.generate_image() self.content_object.get_image() cache_filename = 'base_image' cache_file = self.content_object.cache_partition.get_file(filename=cache_filename) + with cache_file.open() as file_object: converter = get_converter_class()( file_object=file_object ) - converter.seek_page(page_number=self.page_number - 1) + converter.seek_page(page_number=0) + #self.page_number - 1) page_image = converter.get_page() + cache_filename = 'document_page' + # Since open "wb+" doesn't create files, create it explicitly with self.cache_partition.create_file(filename=cache_filename) as file_object: file_object.write(page_image.getvalue()) diff --git a/mayan/apps/documents/models/document_version_models.py b/mayan/apps/documents/models/document_version_models.py index 7a398fecb1..b8cb7bd533 100644 --- a/mayan/apps/documents/models/document_version_models.py +++ b/mayan/apps/documents/models/document_version_models.py @@ -246,17 +246,6 @@ class DocumentVersion(models.Model): return result - #@property - #def pages_all(self): - # DocumentPage = apps.get_model( - # app_label='documents', model_name='DocumentVersionPage' - # ) - # return DocumentPage.passthrough.filter(document_version=self) - - #@property - #def pages(self): - # return self.pages.all() - @property def page_count(self): """ diff --git a/mayan/apps/documents/views/document_page_views.py b/mayan/apps/documents/views/document_page_views.py index 614cba4585..ba425f2252 100644 --- a/mayan/apps/documents/views/document_page_views.py +++ b/mayan/apps/documents/views/document_page_views.py @@ -21,7 +21,7 @@ from mayan.apps.converter.literals import DEFAULT_ROTATION, DEFAULT_ZOOM_LEVEL from ..forms import DocumentPageForm from ..icons import icon_document_pages from ..links import link_document_update_page_count -from ..models import Document, DocumentVersionPage +from ..models import Document, DocumentPage, DocumentVersionPage from ..permissions import permission_document_edit, permission_document_view from ..settings import ( setting_rotation_step, setting_zoom_percent_step, setting_zoom_max_level, @@ -69,7 +69,7 @@ class DocumentPageListView(ExternalObjectMixin, SingleObjectListView): class DocumentPageNavigationBase(ExternalObjectMixin, RedirectView): - external_object_class = DocumentVersionPage + external_object_class = DocumentPage external_object_permission = permission_document_view external_object_pk_url_kwarg = 'pk' @@ -164,7 +164,7 @@ class DocumentPageNavigationPrevious(DocumentPageNavigationBase): class DocumentPageView(ExternalObjectMixin, SimpleView): - external_object_class = DocumentVersionPage + external_object_class = DocumentPage external_object_permission = permission_document_view external_object_pk_url_kwarg = 'pk' template_name = 'appearance/generic_form.html' @@ -204,7 +204,7 @@ class DocumentPageViewResetView(RedirectView): class DocumentPageInteractiveTransformation(ExternalObjectMixin, RedirectView): - external_object_class = DocumentVersionPage + external_object_class = DocumentPage external_object_permission = permission_document_view external_object_pk_url_kwarg = 'pk' @@ -289,7 +289,7 @@ class DocumentPageDisable(MultipleObjectConfirmActionView): return result def get_source_queryset(self): - return DocumentVersionPage.passthrough.all() + return DocumentPage.passthrough.all() def object_action(self, form, instance): instance.enabled = False @@ -319,7 +319,7 @@ class DocumentPageEnable(MultipleObjectConfirmActionView): return result def get_source_queryset(self): - return DocumentVersionPage.passthrough.all() + return DocumentPage.passthrough.all() def object_action(self, form, instance): instance.enabled = True diff --git a/mayan/apps/redactions/apps.py b/mayan/apps/redactions/apps.py index 732262545f..a7460b817d 100644 --- a/mayan/apps/redactions/apps.py +++ b/mayan/apps/redactions/apps.py @@ -28,7 +28,7 @@ class RedactionsApp(MayanAppConfig): super(RedactionsApp, self).ready() DocumentPage = apps.get_model( - app_label='documents', model_name='DocumentVersionPage' + app_label='documents', model_name='DocumentPage' ) link_redaction_list = link_transformation_list.copy( From d0ee8aba168a99561047a75aad15ace69934f5c3 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 9 Oct 2019 11:53:29 -0400 Subject: [PATCH 15/26] Add document pages reset view Add document version page count update view. Add tests. Register permission_document_tools to the Document model. Signed-off-by: Roberto Rosario --- mayan/apps/documents/apps.py | 39 ++++--- mayan/apps/documents/icons.py | 5 +- mayan/apps/documents/links.py | 28 +++-- .../apps/documents/models/document_models.py | 21 ++-- mayan/apps/documents/tests/mixins.py | 50 ++++++--- .../tests/test_document_version_views.py | 44 +++++--- .../documents/tests/test_document_views.py | 30 +++--- mayan/apps/documents/urls.py | 32 ++++-- .../documents/views/document_page_views.py | 8 +- .../documents/views/document_version_views.py | 49 ++++++++- mayan/apps/documents/views/document_views.py | 101 +++++++++--------- 11 files changed, 255 insertions(+), 152 deletions(-) diff --git a/mayan/apps/documents/apps.py b/mayan/apps/documents/apps.py index 95136a2a00..c527fcf519 100644 --- a/mayan/apps/documents/apps.py +++ b/mayan/apps/documents/apps.py @@ -59,7 +59,7 @@ from .links import ( link_document_multiple_delete, link_document_multiple_document_type_edit, link_document_multiple_download, link_document_multiple_favorites_add, link_document_multiple_favorites_remove, link_document_multiple_restore, - link_document_multiple_trash, link_document_multiple_update_page_count, + link_document_multiple_trash, link_document_multiple_pages_reset, link_document_page_disable, link_document_page_multiple_disable, link_document_page_enable, link_document_page_multiple_enable, link_document_page_navigation_first, link_document_page_navigation_last, @@ -74,8 +74,10 @@ from .links import ( link_document_type_filename_create, link_document_type_filename_delete, link_document_type_filename_edit, link_document_type_filename_list, link_document_type_list, link_document_type_policies, - link_document_type_setup, link_document_update_page_count, + link_document_type_setup, link_document_pages_reset, link_document_version_download, link_document_version_list, + link_document_version_multiple_page_count_update, + link_document_version_page_count_update, link_document_version_return_document, link_document_version_return_list, link_document_version_revert, link_document_version_view, link_duplicated_document_list, link_duplicated_document_scan, @@ -87,10 +89,10 @@ from .permissions import ( permission_document_download, permission_document_edit, permission_document_new_version, permission_document_print, permission_document_properties_edit, permission_document_restore, - permission_document_trash, permission_document_type_delete, - permission_document_type_edit, permission_document_type_view, - permission_document_version_revert, permission_document_version_view, - permission_document_view + permission_document_tools, permission_document_trash, + permission_document_type_delete, permission_document_type_edit, + permission_document_type_view, permission_document_version_revert, + permission_document_version_view, permission_document_view, ) # Just import to initialize the search models from .search import document_search, document_page_search # NOQA @@ -190,13 +192,15 @@ class DocumentsApp(MayanAppConfig): permission_acl_edit, permission_acl_view, permission_document_delete, permission_document_download, permission_document_edit, permission_document_new_version, - permission_document_print, permission_document_properties_edit, - permission_document_restore, permission_document_trash, - permission_document_version_revert, + permission_document_print, + permission_document_properties_edit, + permission_document_restore, permission_document_tools, + permission_document_trash, permission_document_version_revert, permission_document_version_view, permission_document_view, permission_events_view, permission_transformation_create, permission_transformation_delete, - permission_transformation_edit, permission_transformation_view, + permission_transformation_edit, + permission_transformation_view, ) ) @@ -454,7 +458,7 @@ class DocumentsApp(MayanAppConfig): link_document_quick_download, link_document_download, link_document_clear_transformations, link_document_clone_transformations, - link_document_update_page_count, + link_document_pages_reset, ), sources=(Document,) ) menu_object.bind_links( @@ -495,7 +499,7 @@ class DocumentsApp(MayanAppConfig): link_document_multiple_favorites_remove, link_document_multiple_clear_transformations, link_document_multiple_trash, link_document_multiple_download, - link_document_multiple_update_page_count, + link_document_multiple_pages_reset, link_document_multiple_document_type_edit, ), sources=(Document,) ) @@ -547,6 +551,17 @@ class DocumentsApp(MayanAppConfig): link_document_version_return_list ), sources=(DocumentVersion,) ) + menu_multi_item.bind_links( + links=( + link_document_version_multiple_page_count_update, + ), sources=(DocumentVersion,) + ) + menu_object.bind_links( + links=( + link_document_version_page_count_update, + ), sources=(DocumentVersion,) + ) + menu_list_facet.bind_links( links=(link_document_version_view,), sources=(DocumentVersion,) ) diff --git a/mayan/apps/documents/icons.py b/mayan/apps/documents/icons.py index fb891fd585..ce365c8678 100644 --- a/mayan/apps/documents/icons.py +++ b/mayan/apps/documents/icons.py @@ -36,7 +36,10 @@ icon_document_edit = Icon( ) icon_document = Icon(driver_name='fontawesome', symbol='book') icon_document_list = icon_document -icon_document_page_count_update = Icon( +icon_document_pages_reset = Icon( + driver_name='fontawesome', symbol='copy' +) +icon_document_version_page_count_update = Icon( driver_name='fontawesome', symbol='copy' ) icon_document_preview = Icon(driver_name='fontawesome', symbol='eye') diff --git a/mayan/apps/documents/links.py b/mayan/apps/documents/links.py index 271aa16579..89cbd31782 100644 --- a/mayan/apps/documents/links.py +++ b/mayan/apps/documents/links.py @@ -168,12 +168,12 @@ link_document_quick_download = Link( permissions=(permission_document_download,), text=_('Quick download'), view='documents:document_download', ) -link_document_update_page_count = Link( +link_document_pages_reset = Link( args='resolved_object.pk', - icon_class_path='mayan.apps.documents.icons.icon_document_page_count_update', + icon_class_path='mayan.apps.documents.icons.icon_document_pages_reset', permissions=(permission_document_tools,), - text=_('Recalculate page count'), - view='documents:document_update_page_count' + text=_('Reset pages'), + view='documents:document_pages_reset' ) link_document_restore = Link( permissions=(permission_document_restore,), @@ -217,10 +217,10 @@ link_document_multiple_download = Link( text=_('Advanced download'), view='documents:document_multiple_download_form' ) -link_document_multiple_update_page_count = Link( - icon_class_path='mayan.apps.documents.icons.icon_document_page_count_update', - text=_('Recalculate page count'), - view='documents:document_multiple_update_page_count' +link_document_multiple_pages_reset = Link( + icon_class_path='mayan.apps.documents.icons.icon_document_pages_reset', + text=_('Reset pages'), + view='documents:document_multiple_pages_reset' ) link_document_multiple_restore = Link( icon_class_path='mayan.apps.documents.icons.icon_trashed_document_restore', @@ -246,6 +246,18 @@ link_document_version_return_list = Link( permissions=(permission_document_version_view,), text=_('Versions'), view='documents:document_version_list', ) +link_document_version_page_count_update = Link( + args='resolved_object.pk', + icon_class_path='mayan.apps.documents.icons.icon_document_version_page_count_update', + permissions=(permission_document_tools,), + text=_('Update page count'), + view='documents:document_version_page_count_update' +) +link_document_version_multiple_page_count_update = Link( + icon_class_path='mayan.apps.documents.icons.icon_document_version_page_count_update', + text=_('Update page count'), + view='documents:document_version_multiple_page_count_update' +) link_document_version_view = Link( args='resolved_object.pk', icon_class_path='mayan.apps.documents.icons.icon_document_version_view', diff --git a/mayan/apps/documents/models/document_models.py b/mayan/apps/documents/models/document_models.py index 0843ea0056..7ba6984be9 100644 --- a/mayan/apps/documents/models/document_models.py +++ b/mayan/apps/documents/models/document_models.py @@ -170,6 +170,9 @@ class Document(models.Model): document_version.save(_user=_user) logger.info('New document version queued for document: %s', self) + + self.reset_pages(update_page_count=False) + return document_version def open(self, *args, **kwargs): @@ -186,15 +189,6 @@ class Document(models.Model): @property def pages(self): return self.pages.all() - #try: - # return self.latest_version.pages - #except AttributeError: - # # Document has no version yet - # DocumentPage = apps.get_model( - # app_label='documents', model_name='DocumentVersionPage' - # ) - - # return DocumentPage.objects.none() @property def pages_all(self): @@ -203,19 +197,16 @@ class Document(models.Model): ) return DocumentPage.passthrough.filter(document=self) - def reset_pages(self): + def reset_pages(self, update_page_count=True): with transaction.atomic(): for page in self.pages.all(): page.delete() - self.latest_version.update_page_count() + if update_page_count: + self.latest_version.update_page_count() for version_page in self.latest_version.pages.all(): document_page = self.pages.create( - #content_type = models.ForeignKey( - # on_delete=models.CASCADE, to=ContentType - #) - #object_id = models.PositiveIntegerField() content_object = version_page ) diff --git a/mayan/apps/documents/tests/mixins.py b/mayan/apps/documents/tests/mixins.py index bb996c6cde..08266eedbe 100644 --- a/mayan/apps/documents/tests/mixins.py +++ b/mayan/apps/documents/tests/mixins.py @@ -69,6 +69,7 @@ class DocumentTestMixin(object): self.test_document = document self.test_documents.append(document) + self.test_document_version = document.latest_version class DocumentTypeViewTestMixin(object): @@ -148,6 +149,26 @@ class DocumentVersionTestMixin(object): ) +class DocumentVersionViewTestMixin(object): + def _request_document_version_list_view(self): + return self.get( + viewname='documents:document_version_list', + kwargs={'pk': self.test_document.pk} + ) + + def _request_document_version_revert_view(self, document_version): + return self.post( + viewname='documents:document_version_revert', + kwargs={'pk': document_version.pk} + ) + + def _request_test_document_version_page_count_update_view(self): + return self.post( + viewname='documents:document_version_page_count_update', + kwargs={'pk': self.test_document_version.pk} + ) + + class DocumentViewTestMixin(object): def _request_document_properties_view(self): return self.get( @@ -200,6 +221,12 @@ class DocumentViewTestMixin(object): data={'id_list': self.test_document.pk} ) + def _request_document_pages_reset_view(self): + return self.post( + viewname='documents:document_pages_reset', + kwargs={'pk': self.test_document.pk} + ) + def _request_document_version_download(self, data=None): data = data or {} return self.get( @@ -208,18 +235,6 @@ class DocumentViewTestMixin(object): }, data=data ) - def _request_document_update_page_count_view(self): - return self.post( - viewname='documents:document_update_page_count', - kwargs={'pk': self.test_document.pk} - ) - - def _request_document_multiple_update_page_count_view(self): - return self.post( - viewname='documents:document_multiple_update_page_count', - data={'id_list': self.test_document.pk} - ) - def _request_document_clear_transformations_view(self): return self.post( viewname='documents:document_clear_transformations', @@ -232,8 +247,11 @@ class DocumentViewTestMixin(object): data={'id_list': self.test_document.pk} ) - def _request_empty_trash_view(self): - return self.post(viewname='documents:trash_can_empty') + def _request_document_multiple_pages_reset_view(self): + return self.post( + viewname='documents:document_multiple_pages_reset', + data={'id_list': self.test_document.pk} + ) def _request_document_print_view(self): return self.get( @@ -243,3 +261,7 @@ class DocumentViewTestMixin(object): 'page_group': PAGE_RANGE_ALL } ) + + def _request_empty_trash_view(self): + return self.post(viewname='documents:trash_can_empty') + diff --git a/mayan/apps/documents/tests/test_document_version_views.py b/mayan/apps/documents/tests/test_document_version_views.py index 4f712a638b..d2ab419378 100644 --- a/mayan/apps/documents/tests/test_document_version_views.py +++ b/mayan/apps/documents/tests/test_document_version_views.py @@ -1,21 +1,19 @@ from __future__ import unicode_literals from ..permissions import ( - permission_document_version_revert, permission_document_version_view, + permission_document_tools, permission_document_version_revert, + permission_document_version_view, ) from .base import GenericDocumentViewTestCase from .literals import TEST_VERSION_COMMENT -from .mixins import DocumentVersionTestMixin +from .mixins import DocumentVersionTestMixin, DocumentVersionViewTestMixin -class DocumentVersionTestCase(DocumentVersionTestMixin, GenericDocumentViewTestCase): - def _request_document_version_list_view(self): - return self.get( - viewname='documents:document_version_list', - kwargs={'pk': self.test_document.pk} - ) - +class DocumentVersionTestCase( + DocumentVersionViewTestMixin, DocumentVersionTestMixin, + GenericDocumentViewTestCase +): def test_document_version_list_no_permission(self): self._upload_new_version() @@ -33,12 +31,6 @@ class DocumentVersionTestCase(DocumentVersionTestMixin, GenericDocumentViewTestC response=response, text=TEST_VERSION_COMMENT, status_code=200 ) - def _request_document_version_revert_view(self, document_version): - return self.post( - viewname='documents:document_version_revert', - kwargs={'pk': document_version.pk} - ) - def test_document_version_revert_no_permission(self): first_version = self.test_document.latest_version self._upload_new_version() @@ -64,3 +56,25 @@ class DocumentVersionTestCase(DocumentVersionTestMixin, GenericDocumentViewTestC self.assertEqual(response.status_code, 302) self.assertEqual(self.test_document.versions.count(), 1) + + def test_document_version_page_count_update_view_no_permission(self): + self.test_document_version.pages.all().delete() + + response = self._request_test_document_version_page_count_update_view() + self.assertEqual(response.status_code, 404) + + self.assertEqual(self.test_document_version.pages.count(), 0) + + def test_document_version_page_count_update_view_with_access(self): + page_count = self.test_document_version.pages.count() + + self.test_document_version.pages.all().delete() + + self.grant_access( + obj=self.test_document, permission=permission_document_tools + ) + + response = self._request_test_document_version_page_count_update_view() + self.assertEqual(response.status_code, 302) + + self.assertEqual(self.test_document_version.pages.count(), page_count) diff --git a/mayan/apps/documents/tests/test_document_views.py b/mayan/apps/documents/tests/test_document_views.py index 381a71211d..45c96f6786 100644 --- a/mayan/apps/documents/tests/test_document_views.py +++ b/mayan/apps/documents/tests/test_document_views.py @@ -292,46 +292,44 @@ class DocumentsViewsTestCase( ) ) - def test_document_update_page_count_view_no_permission(self): + def test_document_pages_reset_view_no_permission(self): self.test_document.pages.all().delete() - self.assertEqual(self.test_document.pages.count(), 0) - response = self._request_document_update_page_count_view() + response = self._request_document_pages_reset_view() self.assertEqual(response.status_code, 404) self.assertEqual(self.test_document.pages.count(), 0) - def test_document_update_page_count_view_with_permission(self): - # TODO: Revise permission association - + def test_document_pages_reset_view_with_access(self): page_count = self.test_document.pages.count() self.test_document.pages.all().delete() - self.assertEqual(self.test_document.pages.count(), 0) - self.grant_permission(permission=permission_document_tools) + self.grant_access( + obj=self.test_document, permission=permission_document_tools + ) - response = self._request_document_update_page_count_view() + response = self._request_document_pages_reset_view() self.assertEqual(response.status_code, 302) self.assertEqual(self.test_document.pages.count(), page_count) - def test_document_multiple_update_page_count_view_no_permission(self): + def test_document_multiple_pages_reset_view_no_permission(self): self.test_document.pages.all().delete() - self.assertEqual(self.test_document.pages.count(), 0) - response = self._request_document_multiple_update_page_count_view() + response = self._request_document_multiple_pages_reset_view() self.assertEqual(response.status_code, 404) self.assertEqual(self.test_document.pages.count(), 0) - def test_document_multiple_update_page_count_view_with_permission(self): + def test_document_multiple_pages_reset_view_with_access(self): page_count = self.test_document.pages.count() self.test_document.pages.all().delete() - self.assertEqual(self.test_document.pages.count(), 0) - self.grant_permission(permission=permission_document_tools) + self.grant_access( + obj=self.test_document, permission=permission_document_tools + ) - response = self._request_document_multiple_update_page_count_view() + response = self._request_document_multiple_pages_reset_view() self.assertEqual(response.status_code, 302) self.assertEqual(self.test_document.pages.count(), page_count) diff --git a/mayan/apps/documents/urls.py b/mayan/apps/documents/urls.py index ed5219f230..a0e25cc670 100644 --- a/mayan/apps/documents/urls.py +++ b/mayan/apps/documents/urls.py @@ -18,9 +18,10 @@ from .views.document_views import ( DocumentDocumentTypeEditView, DocumentDownloadFormView, DocumentDownloadView, DocumentDuplicatesListView, DocumentEditView, DocumentListView, DocumentPreviewView, DocumentPrint, - DocumentTransformationsClearView, DocumentTransformationsCloneView, - DocumentUpdatePageCountView, DocumentView, DuplicatedDocumentListView, - RecentAccessDocumentListView, RecentAddedDocumentListView + DocumentPagesResetView, DocumentTransformationsClearView, + DocumentTransformationsCloneView, DocumentView, + DuplicatedDocumentListView, RecentAccessDocumentListView, + RecentAddedDocumentListView ) from .views.document_page_views import ( DocumentPageDisable, DocumentPageEnable, DocumentPageListView, @@ -32,7 +33,8 @@ from .views.document_page_views import ( ) from .views.document_version_views import ( DocumentVersionDownloadFormView, DocumentVersionDownloadView, - DocumentVersionListView, DocumentVersionRevertView, DocumentVersionView, + DocumentVersionListView, DocumentVersionRevertView, + DocumentVersionUpdatePageCountView, DocumentVersionView, ) from .views.document_type_views import ( DocumentTypeCreateView, DocumentTypeDeleteView, @@ -174,14 +176,14 @@ urlpatterns_documents = [ name='document_print' ), url( - regex=r'^documents/(?P\d+)/reset_page_count/$', - view=DocumentUpdatePageCountView.as_view(), - name='document_update_page_count' + regex=r'^documents/(?P\d+)/pages/reset/$', + view=DocumentPagesResetView.as_view(), + name='document_pages_reset' ), url( - regex=r'^documents/multiple/reset_page_count/$', - view=DocumentUpdatePageCountView.as_view(), - name='document_multiple_update_page_count' + regex=r'^documents/multiple/pages/reset/$', + view=DocumentPagesResetView.as_view(), + name='document_multiple_pages_reset' ), url( regex=r'^documents/(?P\d+)/download/form/$', @@ -307,6 +309,16 @@ urlpatterns_document_versions = [ view=DocumentVersionDownloadView.as_view(), name='document_version_download' ), + url( + regex=r'^documents/versions/(?P\d+)/pages/update/$', + view=DocumentVersionUpdatePageCountView.as_view(), + name='document_version_page_count_update' + ), + url( + regex=r'^documents/versions/multiple/pages/update/$', + view=DocumentVersionUpdatePageCountView.as_view(), + name='document_version_multiple_page_count_update' + ), url( regex=r'^documents/versions/(?P\d+)/revert/$', view=DocumentVersionRevertView.as_view(), diff --git a/mayan/apps/documents/views/document_page_views.py b/mayan/apps/documents/views/document_page_views.py index ba425f2252..46fcacd9e7 100644 --- a/mayan/apps/documents/views/document_page_views.py +++ b/mayan/apps/documents/views/document_page_views.py @@ -20,7 +20,7 @@ from mayan.apps.converter.literals import DEFAULT_ROTATION, DEFAULT_ZOOM_LEVEL from ..forms import DocumentPageForm from ..icons import icon_document_pages -from ..links import link_document_update_page_count +from ..links import link_document_pages_reset from ..models import Document, DocumentPage, DocumentVersionPage from ..permissions import permission_document_edit, permission_document_view from ..settings import ( @@ -50,13 +50,13 @@ class DocumentPageListView(ExternalObjectMixin, SingleObjectListView): 'hide_object': True, 'list_as_items': True, 'no_results_icon': icon_document_pages, - 'no_results_main_link': link_document_update_page_count.resolve( + 'no_results_main_link': link_document_pages_reset.resolve( request=self.request, resolved_object=self.external_object ), 'no_results_text': _( 'This could mean that the document is of a format that is ' - 'not supported, that it is corrupted or that the upload ' - 'process was interrupted. Use the document page recalculation ' + 'not supported, that it is corrupted, or that the upload ' + 'process was interrupted. Use the document page reset ' 'action to attempt to introspect the page count again.' ), 'no_results_title': _('No document pages available'), diff --git a/mayan/apps/documents/views/document_version_views.py b/mayan/apps/documents/views/document_version_views.py index 4c53bbff36..8a2c3cd1ed 100644 --- a/mayan/apps/documents/views/document_version_views.py +++ b/mayan/apps/documents/views/document_version_views.py @@ -3,10 +3,11 @@ from __future__ import absolute_import, unicode_literals import logging from django.contrib import messages -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import ugettext_lazy as _, ungettext from mayan.apps.common.generics import ( - ConfirmView, SingleObjectDetailView, SingleObjectListView + ConfirmView, MultipleObjectConfirmActionView, SingleObjectDetailView, + SingleObjectListView ) from mayan.apps.common.mixins import ExternalObjectMixin @@ -14,9 +15,10 @@ from ..events import event_document_view from ..forms import DocumentVersionDownloadForm, DocumentVersionPreviewForm from ..models import Document, DocumentVersion from ..permissions import ( - permission_document_download, permission_document_version_revert, - permission_document_version_view + permission_document_download, permission_document_tools, + permission_document_version_revert, permission_document_version_view ) +from ..tasks import task_update_page_count from .document_views import DocumentDownloadFormView, DocumentDownloadView @@ -142,6 +144,45 @@ class DocumentVersionRevertView(ExternalObjectMixin, ConfirmView): ) +class DocumentVersionUpdatePageCountView(MultipleObjectConfirmActionView): + model = DocumentVersion + object_permission = permission_document_tools + success_message = _( + '%(count)d document version queued for page count recalculation' + ) + success_message_plural = _( + '%(count)d documents version queued for page count recalculation' + ) + + def get_extra_context(self): + queryset = self.object_list + + result = { + 'title': ungettext( + singular='Recalculate the page count of the selected document version?', + plural='Recalculate the page count of the selected document versions?', + number=queryset.count() + ) + } + + if queryset.count() == 1: + result.update( + { + 'object': queryset.first(), + 'title': _( + 'Recalculate the page count of the document version: %s?' + ) % queryset.first() + } + ) + + return result + + def object_action(self, form, instance): + task_update_page_count.apply_async( + kwargs={'version_id': instance.pk} + ) + + class DocumentVersionView(SingleObjectDetailView): form_class = DocumentVersionPreviewForm model = DocumentVersion diff --git a/mayan/apps/documents/views/document_views.py b/mayan/apps/documents/views/document_views.py index 566077c99c..aebc6d6324 100644 --- a/mayan/apps/documents/views/document_views.py +++ b/mayan/apps/documents/views/document_views.py @@ -44,14 +44,14 @@ from ..permissions import ( from ..settings import ( setting_print_width, setting_print_height, setting_recent_added_count ) -from ..tasks import task_update_page_count +from ..tasks import task_document_reset_pages from ..utils import parse_range __all__ = ( 'DocumentListView', 'DocumentDocumentTypeEditView', 'DocumentDuplicatesListView', 'DocumentEditView', 'DocumentPreviewView', 'DocumentView', 'DocumentDownloadFormView', 'DocumentDownloadView', - 'DocumentUpdatePageCountView', 'DocumentTransformationsClearView', + 'DocumentPagesResetView', 'DocumentTransformationsClearView', 'DocumentTransformationsCloneView', 'DocumentPrint', 'DuplicatedDocumentListView', 'RecentAccessDocumentListView', 'RecentAddedDocumentListView' @@ -418,6 +418,52 @@ class DocumentPreviewView(SingleObjectDetailView): } +class DocumentPagesResetView(MultipleObjectConfirmActionView): + model = Document + object_permission = permission_document_tools + success_message = _('%(count)d document queued for pages reset') + success_message_plural = _('%(count)d documents queued for pages reset') + + def get_extra_context(self): + queryset = self.object_list + + result = { + 'title': ungettext( + singular='Reset the pages of the selected document?', + plural='Reset the pages of the selected documents?', + number=queryset.count() + ) + } + + if queryset.count() == 1: + result.update( + { + 'object': queryset.first(), + 'title': _( + 'Reset the pages of the document: %s?' + ) % queryset.first() + } + ) + + return result + + def object_action(self, form, instance): + latest_version = instance.latest_version + if latest_version: + task_document_reset_pages.apply_async( + kwargs={'document_id': instance.pk} + ) + else: + messages.error( + self.request, _( + 'Document "%(document)s" is empty. Upload at least one ' + 'document version before attempting to reset the pages. ' + ) % { + 'document': instance, + } + ) + + class DocumentView(SingleObjectDetailView): form_class = DocumentPropertiesForm model = Document @@ -436,57 +482,6 @@ class DocumentView(SingleObjectDetailView): } -class DocumentUpdatePageCountView(MultipleObjectConfirmActionView): - model = Document - object_permission = permission_document_tools - success_message = _( - '%(count)d document queued for page count recalculation' - ) - success_message_plural = _( - '%(count)d documents queued for page count recalculation' - ) - - def get_extra_context(self): - queryset = self.object_list - - result = { - 'title': ungettext( - singular='Recalculate the page count of the selected document?', - plural='Recalculate the page count of the selected documents?', - number=queryset.count() - ) - } - - if queryset.count() == 1: - result.update( - { - 'object': queryset.first(), - 'title': _( - 'Recalculate the page count of the document: %s?' - ) % queryset.first() - } - ) - - return result - - def object_action(self, form, instance): - latest_version = instance.latest_version - if latest_version: - task_update_page_count.apply_async( - kwargs={'version_id': latest_version.pk} - ) - else: - messages.error( - self.request, _( - 'Document "%(document)s" is empty. Upload at least one ' - 'document version before attempting to detect the ' - 'page count.' - ) % { - 'document': instance, - } - ) - - class DocumentTransformationsClearView(MultipleObjectConfirmActionView): model = Document object_permission = permission_transformation_delete From 7fbb94a8ae1118d7b767863723250c9b58462a76 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 9 Oct 2019 16:38:00 -0400 Subject: [PATCH 16/26] Migration updates Squash version page migrations. Add manual OCR and parsing migrations. Fix tests. Page search updates. Signed-off-by: Roberto Rosario --- mayan/apps/document_parsing/admin.py | 8 +- mayan/apps/document_parsing/api_views.py | 6 +- mayan/apps/document_parsing/apps.py | 2 +- mayan/apps/document_parsing/forms.py | 6 +- .../migrations/0005_rename_page_content.py | 28 ++++++ mayan/apps/document_parsing/models.py | 12 +-- mayan/apps/document_parsing/serializers.py | 4 +- mayan/apps/document_parsing/urls.py | 9 +- mayan/apps/document_parsing/views.py | 32 ++++++- mayan/apps/documents/apps.py | 6 +- ...7_1921.py => 0052_rename_document_page.py} | 15 ++- .../migrations/0053_auto_20191007_1922.py | 27 ------ ...create_document_page_and_result_models.py} | 41 ++++++--- .../migrations/0054_auto_20191008_1522.py | 31 ------- .../migrations/0054_reset_document_pages.py | 62 +++++++++++++ .../apps/documents/models/document_models.py | 46 +++++----- .../documents/models/document_page_models.py | 15 ++- .../models/document_version_models.py | 14 +-- .../models/document_version_page_models.py | 12 +-- mayan/apps/documents/queues.py | 2 +- mayan/apps/documents/search.py | 49 ++++++++-- mayan/apps/documents/tasks.py | 4 +- mayan/apps/documents/tests/mixins.py | 2 +- mayan/apps/documents/tests/test_api.py | 31 ++++++- .../tests/test_document_page_views.py | 79 ++++++++-------- mayan/apps/documents/tests/test_search.py | 8 +- .../tests/test_trashed_document_views.py | 92 ++++++++++--------- mayan/apps/documents/views/document_views.py | 4 +- mayan/apps/ocr/admin.py | 9 +- mayan/apps/ocr/api_views.py | 6 +- mayan/apps/ocr/apps.py | 2 +- mayan/apps/ocr/forms.py | 6 +- .../migrations/0009_rename_page_content.py | 28 ++++++ mayan/apps/ocr/models.py | 12 +-- mayan/apps/ocr/serializers.py | 4 +- mayan/apps/ocr/views.py | 4 +- 36 files changed, 449 insertions(+), 269 deletions(-) create mode 100644 mayan/apps/document_parsing/migrations/0005_rename_page_content.py rename mayan/apps/documents/migrations/{0052_auto_20191007_1921.py => 0052_rename_document_page.py} (59%) delete mode 100644 mayan/apps/documents/migrations/0053_auto_20191007_1922.py rename mayan/apps/documents/migrations/{0055_auto_20191008_2116.py => 0053_create_document_page_and_result_models.py} (55%) delete mode 100644 mayan/apps/documents/migrations/0054_auto_20191008_1522.py create mode 100644 mayan/apps/documents/migrations/0054_reset_document_pages.py create mode 100644 mayan/apps/ocr/migrations/0009_rename_page_content.py diff --git a/mayan/apps/document_parsing/admin.py b/mayan/apps/document_parsing/admin.py index 258da5ec3d..5fb8ca6a37 100644 --- a/mayan/apps/document_parsing/admin.py +++ b/mayan/apps/document_parsing/admin.py @@ -3,13 +3,13 @@ from __future__ import unicode_literals from django.contrib import admin from .models import ( - DocumentPageContent, DocumentVersionParseError + DocumentVersionPageContent, DocumentVersionParseError ) -@admin.register(DocumentPageContent) -class DocumentPageContentAdmin(admin.ModelAdmin): - list_display = ('document_page',) +@admin.register(DocumentVersionPageContent) +class DocumentVersionPageContentAdmin(admin.ModelAdmin): + list_display = ('document_version_page',) @admin.register(DocumentVersionParseError) diff --git a/mayan/apps/document_parsing/api_views.py b/mayan/apps/document_parsing/api_views.py index 7d29b61d94..ce024f34be 100644 --- a/mayan/apps/document_parsing/api_views.py +++ b/mayan/apps/document_parsing/api_views.py @@ -8,7 +8,7 @@ from rest_framework.response import Response from mayan.apps.documents.models import Document from mayan.apps.rest_api.permissions import MayanPermission -from .models import DocumentPageContent +from .models import DocumentVersionPageContent from .permissions import permission_content_view from .serializers import DocumentPageContentSerializer @@ -41,8 +41,8 @@ class APIDocumentPageContentView(generics.RetrieveAPIView): try: content = instance.content - except DocumentPageContent.DoesNotExist: - content = DocumentPageContent.objects.none() + except DocumentVersionPageContent.DoesNotExist: + content = DocumentVersionPageContent.objects.none() serializer = self.get_serializer(content) return Response(serializer.data) diff --git a/mayan/apps/document_parsing/apps.py b/mayan/apps/document_parsing/apps.py index 71cd9930ac..45eb734d08 100644 --- a/mayan/apps/document_parsing/apps.py +++ b/mayan/apps/document_parsing/apps.py @@ -133,7 +133,7 @@ class DocumentParsingApp(MayanAppConfig): ) document_search.add_model_field( - field='versions__pages__content__content', label=_('Content') + field='pages__content__content', label=_('Content') ) document_page_search.add_model_field( diff --git a/mayan/apps/document_parsing/forms.py b/mayan/apps/document_parsing/forms.py index 5c4d6af26d..12778c6acb 100644 --- a/mayan/apps/document_parsing/forms.py +++ b/mayan/apps/document_parsing/forms.py @@ -8,7 +8,7 @@ from django.utils.translation import ugettext_lazy as _, ugettext from mayan.apps.common.widgets import TextAreaDiv -from .models import DocumentPageContent +from .models import DocumentVersionPageContent class DocumentContentForm(forms.Form): @@ -29,7 +29,7 @@ class DocumentContentForm(forms.Form): for page in document_pages: try: page_content = page.content.content - except DocumentPageContent.DoesNotExist: + except DocumentVersionPageContent.DoesNotExist: pass else: content.append(conditional_escape(force_text(page_content))) @@ -73,7 +73,7 @@ class DocumentPageContentForm(forms.Form): try: page_content = document_page.content.content - except DocumentPageContent.DoesNotExist: + except DocumentVersionPageContent.DoesNotExist: pass else: content = conditional_escape(force_text(page_content)) diff --git a/mayan/apps/document_parsing/migrations/0005_rename_page_content.py b/mayan/apps/document_parsing/migrations/0005_rename_page_content.py new file mode 100644 index 0000000000..2a78b99f73 --- /dev/null +++ b/mayan/apps/document_parsing/migrations/0005_rename_page_content.py @@ -0,0 +1,28 @@ +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + dependencies = [ + ('document_parsing', '0004_auto_20180917_0645'), + ('documents', '0052_rename_document_page'), + ] + + operations = [ + migrations.RenameModel( + 'DocumentPageContent', 'DocumentVersionPageContent' + ), + migrations.AlterField( + model_name='documentversionpagecontent', + name='document_page', + field=models.OneToOneField( + on_delete=django.db.models.deletion.CASCADE, + name='document_version_page', + related_name='content', + to='documents.DocumentVersionPage', + verbose_name='Document version page' + ), + ), + ] diff --git a/mayan/apps/document_parsing/models.py b/mayan/apps/document_parsing/models.py index 8d70eacd5b..9c7483f448 100644 --- a/mayan/apps/document_parsing/models.py +++ b/mayan/apps/document_parsing/models.py @@ -5,20 +5,20 @@ from django.utils.encoding import force_text, python_2_unicode_compatible from django.utils.translation import ugettext_lazy as _ from mayan.apps.documents.models import ( - DocumentPage, DocumentType, DocumentVersion + DocumentPage, DocumentType, DocumentVersion, DocumentVersionPage ) from .managers import DocumentPageContentManager, DocumentTypeSettingsManager @python_2_unicode_compatible -class DocumentPageContent(models.Model): +class DocumentVersionPageContent(models.Model): """ This model store's the parsed content of a document page. """ - document_page = models.OneToOneField( + document_version_page = models.OneToOneField( on_delete=models.CASCADE, related_name='content', - to=DocumentPage, verbose_name=_('Document page') + to=DocumentVersionPage, verbose_name=_('Document version page') ) content = models.TextField( blank=True, help_text=_( @@ -30,8 +30,8 @@ class DocumentPageContent(models.Model): objects = DocumentPageContentManager() class Meta: - verbose_name = _('Document page content') - verbose_name_plural = _('Document pages contents') + verbose_name = _('Document version page content') + verbose_name_plural = _('Document version pages contents') def __str__(self): return force_text(self.document_page) diff --git a/mayan/apps/document_parsing/serializers.py b/mayan/apps/document_parsing/serializers.py index 7161d2fc40..53d90d354e 100644 --- a/mayan/apps/document_parsing/serializers.py +++ b/mayan/apps/document_parsing/serializers.py @@ -2,10 +2,10 @@ from __future__ import unicode_literals from rest_framework import serializers -from .models import DocumentPageContent +from .models import DocumentVersionPageContent class DocumentPageContentSerializer(serializers.ModelSerializer): class Meta: fields = ('content',) - model = DocumentPageContent + model = DocumentVersionPageContent diff --git a/mayan/apps/document_parsing/urls.py b/mayan/apps/document_parsing/urls.py index f590236176..877ae67a4e 100644 --- a/mayan/apps/document_parsing/urls.py +++ b/mayan/apps/document_parsing/urls.py @@ -7,7 +7,9 @@ from .views import ( DocumentContentView, DocumentContentDeleteView, DocumentContentDownloadView, DocumentPageContentView, DocumentParsingErrorsListView, DocumentSubmitView, - DocumentTypeSettingsEditView, DocumentTypeSubmitView, ParseErrorListView + DocumentTypeSettingsEditView, DocumentTypeSubmitView, + DocumentVersionPageContentView, + ParseErrorListView ) urlpatterns = [ @@ -34,6 +36,11 @@ urlpatterns = [ regex=r'^documents/pages/(?P\d+)/content/$', view=DocumentPageContentView.as_view(), name='document_page_content' ), + url( + regex=r'^documents/versions/pages/(?P\d+)/content/$', + view=DocumentVersionPageContentView.as_view(), + name='document_version_page_content' + ), url( regex=r'^documents/(?P\d+)/submit/$', view=DocumentSubmitView.as_view(), name='document_submit' diff --git a/mayan/apps/document_parsing/views.py b/mayan/apps/document_parsing/views.py index 2fde8527d0..7201253c18 100644 --- a/mayan/apps/document_parsing/views.py +++ b/mayan/apps/document_parsing/views.py @@ -13,11 +13,11 @@ from mayan.apps.common.generics import ( from mayan.apps.common.mixins import ExternalObjectMixin from mayan.apps.documents.forms import DocumentTypeFilteredSelectForm from mayan.apps.documents.models import ( - Document, DocumentType, DocumentVersionPage + Document, DocumentPage, DocumentType, DocumentVersionPage ) from .forms import DocumentContentForm, DocumentPageContentForm -from .models import DocumentPageContent, DocumentVersionParseError +from .models import DocumentVersionPageContent, DocumentVersionParseError from .permissions import ( permission_content_view, permission_document_type_parsing_setup, permission_parse_document @@ -48,7 +48,7 @@ class DocumentContentDeleteView(MultipleObjectConfirmActionView): return result def object_action(self, form, instance): - DocumentPageContent.objects.delete_content_for( + DocumentVersionPageContent.objects.delete_content_for( document=instance, user=self.request.user ) @@ -89,7 +89,7 @@ class DocumentContentDownloadView(SingleObjectDownloadView): class DocumentPageContentView(SingleObjectDetailView): form_class = DocumentPageContentForm - model = DocumentVersionPage + model = DocumentPage object_permission = permission_content_view def dispatch(self, request, *args, **kwargs): @@ -109,6 +109,30 @@ class DocumentPageContentView(SingleObjectDetailView): } +class DocumentVersionPageContentView(SingleObjectDetailView): + form_class = DocumentPageContentForm + model = DocumentVersionPage + object_permission = permission_content_view + + def dispatch(self, request, *args, **kwargs): + result = super(DocumentPageContentView, self).dispatch( + request, *args, **kwargs + ) + self.get_object().document.add_as_recent_document_for_user( + request.user + ) + return result + + def get_extra_context(self): + return { + 'hide_labels': True, + 'object': self.get_object(), + 'title': _( + 'Content for document version page: %s' + ) % self.get_object(), + } + + class DocumentParsingErrorsListView(SingleObjectListView): view_permission = permission_content_view diff --git a/mayan/apps/documents/apps.py b/mayan/apps/documents/apps.py index c527fcf519..4faae2f8bd 100644 --- a/mayan/apps/documents/apps.py +++ b/mayan/apps/documents/apps.py @@ -127,6 +127,7 @@ class DocumentsApp(MayanAppConfig): DocumentType = self.get_model(model_name='DocumentType') DocumentTypeFilename = self.get_model(model_name='DocumentTypeFilename') DocumentVersion = self.get_model(model_name='DocumentVersion') + DocumentVersionPage = self.get_model(model_name='DocumentVersionPage') DuplicatedDocument = self.get_model(model_name='DuplicatedDocument') DynamicSerializerField.add_serializer( @@ -228,7 +229,7 @@ class DocumentsApp(MayanAppConfig): model=Document, manager_name='passthrough' ) ModelPermission.register_inheritance( - model=DocumentPage, related='document_version__document', + model=DocumentPage, related='document', ) ModelPermission.register_manager( model=DocumentPage, manager_name='passthrough' @@ -245,6 +246,9 @@ class DocumentsApp(MayanAppConfig): ModelPermission.register_inheritance( model=DocumentVersion, related='document', ) + ModelPermission.register_inheritance( + model=DocumentVersionPage, related='document_version', + ) # Document and document page thumbnail widget document_page_thumbnail_widget = DocumentPageThumbnailWidget() diff --git a/mayan/apps/documents/migrations/0052_auto_20191007_1921.py b/mayan/apps/documents/migrations/0052_rename_document_page.py similarity index 59% rename from mayan/apps/documents/migrations/0052_auto_20191007_1921.py rename to mayan/apps/documents/migrations/0052_rename_document_page.py index 5cfcc3040d..d6751c3810 100644 --- a/mayan/apps/documents/migrations/0052_auto_20191007_1921.py +++ b/mayan/apps/documents/migrations/0052_rename_document_page.py @@ -5,7 +5,6 @@ import django.db.models.deletion class Migration(migrations.Migration): - dependencies = [ ('documents', '0051_documentpage_enabled'), ] @@ -20,8 +19,20 @@ class Migration(migrations.Migration): name='document_version', field=models.ForeignKey( on_delete=django.db.models.deletion.CASCADE, - related_name='version_pages', to='documents.DocumentVersion', + related_name='pages', to='documents.DocumentVersion', verbose_name='Document version' ), ), + migrations.RemoveField( + model_name='documentversionpage', + name='enabled', + ), + migrations.AlterModelOptions( + name='documentversionpage', + options={ + 'ordering': ('page_number',), + 'verbose_name': 'Document version page', + 'verbose_name_plural': 'Document version pages' + }, + ), ] diff --git a/mayan/apps/documents/migrations/0053_auto_20191007_1922.py b/mayan/apps/documents/migrations/0053_auto_20191007_1922.py deleted file mode 100644 index ee4175bd28..0000000000 --- a/mayan/apps/documents/migrations/0053_auto_20191007_1922.py +++ /dev/null @@ -1,27 +0,0 @@ -from __future__ import unicode_literals - -from django.db import migrations - - -class Migration(migrations.Migration): - dependencies = [ - ('documents', '0052_auto_20191007_1921'), - ('ocr', '0008_auto_20180917_0646'), - ('document_parsing', '0004_auto_20180917_0645'), - ] - - operations = [ - migrations.CreateModel( - name='DocumentPageResult', - fields=[ - ], - options={ - 'verbose_name': 'Document version page', - 'verbose_name_plural': 'Document version pages', - 'ordering': ('document_version__document', 'page_number'), - 'proxy': True, - 'indexes': [], - }, - bases=('documents.documentversionpage',), - ), - ] diff --git a/mayan/apps/documents/migrations/0055_auto_20191008_2116.py b/mayan/apps/documents/migrations/0053_create_document_page_and_result_models.py similarity index 55% rename from mayan/apps/documents/migrations/0055_auto_20191008_2116.py rename to mayan/apps/documents/migrations/0053_create_document_page_and_result_models.py index 5f6ad39fa8..f7f4081c6e 100644 --- a/mayan/apps/documents/migrations/0055_auto_20191008_2116.py +++ b/mayan/apps/documents/migrations/0053_create_document_page_and_result_models.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.22 on 2019-10-08 21:16 from __future__ import unicode_literals from django.db import migrations, models @@ -7,10 +5,9 @@ import django.db.models.deletion class Migration(migrations.Migration): - dependencies = [ ('contenttypes', '0002_remove_content_type_name'), - ('documents', '0054_auto_20191008_1522'), + ('documents', '0052_rename_document_page'), ] operations = [ @@ -25,22 +22,36 @@ class Migration(migrations.Migration): ('document', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='pages', to='documents.Document', verbose_name='Document')), ], options={ + 'unique_together': set([('document', 'page_number')]), 'verbose_name': 'Document page', 'verbose_name_plural': 'Document pages', 'ordering': ('page_number',), }, ), - migrations.AlterModelOptions( - name='documentversionpage', - options={'ordering': ('page_number',), 'verbose_name': 'Document version page', 'verbose_name_plural': 'Document version pages'}, + migrations.CreateModel( + name='DocumentPageResult', + fields=[ + ], + options={ + 'verbose_name': 'Document page result', + 'verbose_name_plural': 'Document pages result', + 'ordering': ('document', 'page_number'), + 'proxy': True, + 'indexes': [], + }, + bases=('documents.documentpage',), ), - migrations.AlterField( - model_name='documentversionpage', - name='document_version', - field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='pages', to='documents.DocumentVersion', verbose_name='Document version'), - ), - migrations.AlterUniqueTogether( - name='documentpage', - unique_together=set([('document', 'page_number')]), + migrations.CreateModel( + name='DocumentVersionPageResult', + fields=[ + ], + options={ + 'verbose_name': 'Document version page', + 'verbose_name_plural': 'Document version pages', + 'ordering': ('document_version__document', 'page_number'), + 'proxy': True, + 'indexes': [], + }, + bases=('documents.documentversionpage',), ), ] diff --git a/mayan/apps/documents/migrations/0054_auto_20191008_1522.py b/mayan/apps/documents/migrations/0054_auto_20191008_1522.py deleted file mode 100644 index 4bfca889db..0000000000 --- a/mayan/apps/documents/migrations/0054_auto_20191008_1522.py +++ /dev/null @@ -1,31 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.22 on 2019-10-08 15:22 -from __future__ import unicode_literals - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('documents', '0053_auto_20191007_1922'), - ] - - operations = [ - migrations.DeleteModel( - name='DocumentPageResult', - ), - migrations.CreateModel( - name='DocumentVersionPageResult', - fields=[ - ], - options={ - 'verbose_name': 'Document version page', - 'verbose_name_plural': 'Document version pages', - 'ordering': ('document_version__document', 'page_number'), - 'proxy': True, - 'indexes': [], - }, - bases=('documents.documentversionpage',), - ), - ] diff --git a/mayan/apps/documents/migrations/0054_reset_document_pages.py b/mayan/apps/documents/migrations/0054_reset_document_pages.py new file mode 100644 index 0000000000..413f423e3d --- /dev/null +++ b/mayan/apps/documents/migrations/0054_reset_document_pages.py @@ -0,0 +1,62 @@ +from __future__ import unicode_literals + +from django.db import migrations + + +def get_latest_version(document): + return document.versions.order_by('timestamp').last() + + +def operation_reset_document_pages(apps, schema_editor): + Document = apps.get_model(app_label='documents', model_name='Document') + DocumentPage = apps.get_model( + app_label='documents', model_name='DocumentPage' + ) + + # Define inside the function to use the migration's apps instance + def pages_reset(document): + ContentType = apps.get_model('contenttypes', 'ContentType') + DocumentVersionPage = apps.get_model( + app_label='documents', model_name='DocumentVersionPage' + ) + + content_type = ContentType.objects.get_for_model( + model=DocumentVersionPage + ) + + for document_page in document.pages.all(): + document_page.delete() + + for version_page in get_latest_version(document=document).pages.all(): + document_page = document.pages.create( + content_type=content_type, + page_number=version_page.page_number, + object_id=version_page.pk, + ) + + for document in Document.objects.using(schema_editor.connection.alias).all(): + pages_reset(document=document) + + +def operation_reset_document_pages_reverse(apps, schema_editor): + Document = apps.get_model(app_label='documents', model_name='Document') + DocumentPage = apps.get_model( + app_label='documents', model_name='DocumentPage' + ) + + for document in Document.objects.using(schema_editor.connection.alias).all(): + for document_page in document.pages.all(): + document_page.delete() + + +class Migration(migrations.Migration): + dependencies = [ + ('documents', '0053_create_document_page_and_result_models'), + ] + + operations = [ + migrations.RunPython( + code=operation_reset_document_pages, + reverse_code=operation_reset_document_pages_reverse + ), + ] diff --git a/mayan/apps/documents/models/document_models.py b/mayan/apps/documents/models/document_models.py index 7ba6984be9..2177119e14 100644 --- a/mayan/apps/documents/models/document_models.py +++ b/mayan/apps/documents/models/document_models.py @@ -116,6 +116,14 @@ class Document(models.Model): ) return partition + @property + def checksum(self): + return self.latest_version.checksum + + @property + def date_updated(self): + return self.latest_version.timestamp + def delete(self, *args, **kwargs): to_trash = kwargs.pop('to_trash', True) @@ -140,6 +148,14 @@ class Document(models.Model): else: return False + @property + def file_mime_encoding(self): + return self.latest_version.encoding + + @property + def file_mimetype(self): + return self.latest_version.mimetype + def get_absolute_url(self): return reverse( viewname='documents:document_preview', kwargs={'pk': self.pk} @@ -154,6 +170,10 @@ class Document(models.Model): def is_in_trash(self): return self.in_trash + @property + def latest_version(self): + return self.versions.order_by('timestamp').last() + def natural_key(self): return (self.uuid,) natural_key.dependencies = ['documents.DocumentType'] @@ -171,8 +191,6 @@ class Document(models.Model): logger.info('New document version queued for document: %s', self) - self.reset_pages(update_page_count=False) - return document_version def open(self, *args, **kwargs): @@ -197,7 +215,7 @@ class Document(models.Model): ) return DocumentPage.passthrough.filter(document=self) - def reset_pages(self, update_page_count=True): + def pages_reset(self, update_page_count=True): with transaction.atomic(): for page in self.pages.all(): page.delete() @@ -254,25 +272,3 @@ class Document(models.Model): @property def size(self): return self.latest_version.size - - # Compatibility methods - - @property - def checksum(self): - return self.latest_version.checksum - - @property - def date_updated(self): - return self.latest_version.timestamp - - @property - def file_mime_encoding(self): - return self.latest_version.encoding - - @property - def file_mimetype(self): - return self.latest_version.mimetype - - @property - def latest_version(self): - return self.versions.order_by('timestamp').last() diff --git a/mayan/apps/documents/models/document_page_models.py b/mayan/apps/documents/models/document_page_models.py index 1a7747f35f..b6784a1ff8 100644 --- a/mayan/apps/documents/models/document_page_models.py +++ b/mayan/apps/documents/models/document_page_models.py @@ -29,9 +29,8 @@ from ..settings import ( ) from .document_models import Document -#from .document_version_page_models import DocumentVersionPage -__all__ = ('DocumentPage',)# 'DocumentPageResult') +__all__ = ('DocumentPage', 'DocumentPageResult') logger = logging.getLogger(__name__) @@ -301,9 +300,9 @@ class DocumentPage(models.Model): return '{}-{}'.format(self.document.uuid, self.pk) -#class DocumentVersionPageResult(DocumentVersionPage): -# class Meta: -# ordering = ('document_version__document', 'page_number') -# proxy = True -# verbose_name = _('Document version page') -# verbose_name_plural = _('Document version pages') +class DocumentPageResult(DocumentPage): + class Meta: + ordering = ('document', 'page_number') + proxy = True + verbose_name = _('Document page result') + verbose_name_plural = _('Document pages result') diff --git a/mayan/apps/documents/models/document_version_models.py b/mayan/apps/documents/models/document_version_models.py index b8cb7bd533..7d8cca828a 100644 --- a/mayan/apps/documents/models/document_version_models.py +++ b/mayan/apps/documents/models/document_version_models.py @@ -246,12 +246,12 @@ class DocumentVersion(models.Model): return result - @property - def page_count(self): - """ - The number of pages that the document posses. - """ - return self.pages.count() + #@property + #def page_count(self): + # """ + # The number of pages that the document posses. + # """ + # return self.pages.count() def revert(self, _user=None): """ @@ -326,6 +326,8 @@ class DocumentVersion(models.Model): sender=Document, instance=self.document ) + self.document.pages_reset(update_page_count=False) + def save_to_file(self, file_object): """ Save a copy of the document from the document storage backend diff --git a/mayan/apps/documents/models/document_version_page_models.py b/mayan/apps/documents/models/document_version_page_models.py index 724de125b7..c0c5b7f8f3 100644 --- a/mayan/apps/documents/models/document_version_page_models.py +++ b/mayan/apps/documents/models/document_version_page_models.py @@ -41,14 +41,12 @@ class DocumentVersionPage(models.Model): on_delete=models.CASCADE, related_name='pages', to=DocumentVersion, verbose_name=_('Document version') ) - enabled = models.BooleanField(default=True, verbose_name=_('Enabled')) page_number = models.PositiveIntegerField( db_index=True, default=1, editable=False, verbose_name=_('Page number') ) objects = DocumentVersionPageManager() - #passthrough = models.Manager() class Meta: ordering = ('page_number',) @@ -261,11 +259,11 @@ class DocumentVersionPage(models.Model): return (self.page_number, self.document_version.natural_key()) natural_key.dependencies = ['documents.DocumentVersion'] - #@property - #def siblings(self): - # return DocumentVersionPage.objects.filter( - # document_version=self.document_version - # ) + @property + def siblings(self): + return DocumentVersionPage.objects.filter( + document_version=self.document_version + ) @property def uuid(self): diff --git a/mayan/apps/documents/queues.py b/mayan/apps/documents/queues.py index a98b0e757f..820115a93e 100644 --- a/mayan/apps/documents/queues.py +++ b/mayan/apps/documents/queues.py @@ -71,7 +71,7 @@ queue_tools.add_task_type( ) queue_uploads.add_task_type( - dotted_path='mayan.apps.documents.tasks.task_document_reset_pages', + dotted_path='mayan.apps.documents.tasks.task_document_pages_reset', label=_('Reset document pages') ) queue_uploads.add_task_type( diff --git a/mayan/apps/documents/search.py b/mayan/apps/documents/search.py index 20044afadd..82517002f9 100644 --- a/mayan/apps/documents/search.py +++ b/mayan/apps/documents/search.py @@ -17,12 +17,20 @@ def transformation_format_uuid(term_string): return term_string -def get_queryset_page_search_queryset(): +def get_queryset_document_page_search_queryset(): # Ignore documents in trash can DocumentPage = apps.get_model( + app_label='documents', model_name='DocumentPage' + ) + return DocumentPage.objects.filter(document__in_trash=False) + + +def get_queryset_document_version_page_search_queryset(): + # Ignore documents in trash can + DocumentVersionPage = apps.get_model( app_label='documents', model_name='DocumentVersionPage' ) - return DocumentPage.objects.filter(document_version__document__in_trash=False) + return DocumentVersionPage.objects.filter(document_version__document__in_trash=False) document_search = SearchModel( @@ -49,25 +57,50 @@ document_search.add_model_field( document_page_search = SearchModel( app_label='documents', list_mode=LIST_MODE_CHOICE_ITEM, - model_name='DocumentVersionPage', permission=permission_document_view, - queryset=get_queryset_page_search_queryset, + model_name='DocumentPage', permission=permission_document_view, + queryset=get_queryset_document_page_search_queryset, serializer_path='mayan.apps.documents.serializers.DocumentPageSerializer' ) +document_version_page_search = SearchModel( + app_label='documents', list_mode=LIST_MODE_CHOICE_ITEM, + model_name='DocumentVersionPage', permission=permission_document_view, + queryset=get_queryset_document_version_page_search_queryset, + serializer_path='mayan.apps.documents.serializers.DocumentPageVersionSerializer' +) + document_page_search.add_model_field( - field='document_version__document__document_type__label', + field='document__document_type__label', label=_('Document type') ) document_page_search.add_model_field( - field='document_version__document__versions__mimetype', + field='document__versions__mimetype', label=_('MIME type') ) document_page_search.add_model_field( + field='document__label', label=_('Label') +) +document_page_search.add_model_field( + field='document__description', label=_('Description') +) +document_page_search.add_model_field( + field='document__document_version__checksum', label=_('Checksum') +) + +document_version_page_search.add_model_field( + field='document_version__document__document_type__label', + label=_('Document type') +) +document_version_page_search.add_model_field( + field='document_version__document__versions__mimetype', + label=_('MIME type') +) +document_version_page_search.add_model_field( field='document_version__document__label', label=_('Label') ) -document_page_search.add_model_field( +document_version_page_search.add_model_field( field='document_version__document__description', label=_('Description') ) -document_page_search.add_model_field( +document_version_page_search.add_model_field( field='document_version__checksum', label=_('Checksum') ) diff --git a/mayan/apps/documents/tasks.py b/mayan/apps/documents/tasks.py index f62a015538..a2215ae69f 100644 --- a/mayan/apps/documents/tasks.py +++ b/mayan/apps/documents/tasks.py @@ -66,7 +66,7 @@ def task_delete_stubs(): @app.task(bind=True, default_retry_delay=RETRY_DELAY_DOCUMENT_RESET_PAGES, ignore_result=True) -def task_document_reset_pages(self, document_id): +def task_document_pages_reset(self, document_id): Document = apps.get_model( app_label='documents', model_name='Document' ) @@ -74,7 +74,7 @@ def task_document_reset_pages(self, document_id): document = Document.objects.get(pk=document_id) try: - document.reset_pages() + document.pages_reset() except OperationalError as exception: logger.warning( 'Operational error during attempt to reset pages for ' diff --git a/mayan/apps/documents/tests/mixins.py b/mayan/apps/documents/tests/mixins.py index 08266eedbe..d11afaa921 100644 --- a/mayan/apps/documents/tests/mixins.py +++ b/mayan/apps/documents/tests/mixins.py @@ -70,6 +70,7 @@ class DocumentTestMixin(object): self.test_document = document self.test_documents.append(document) self.test_document_version = document.latest_version + self.test_document_page = document.pages_all.first() class DocumentTypeViewTestMixin(object): @@ -264,4 +265,3 @@ class DocumentViewTestMixin(object): def _request_empty_trash_view(self): return self.post(viewname='documents:trash_can_empty') - diff --git a/mayan/apps/documents/tests/test_api.py b/mayan/apps/documents/tests/test_api.py index dea50b2618..388b2f2de8 100644 --- a/mayan/apps/documents/tests/test_api.py +++ b/mayan/apps/documents/tests/test_api.py @@ -530,8 +530,7 @@ class DocumentPageAPIViewTestMixin(object): page = self.test_document.pages.first() return self.get( viewname='rest_api:documentpage-image', kwargs={ - 'pk': page.document.pk, 'version_pk': page.document_version.pk, - 'page_pk': page.pk + 'pk': page.document.pk, 'page_pk': page.pk } ) @@ -552,6 +551,33 @@ class DocumentPageAPIViewTestCase( self.assertEqual(response.status_code, status.HTTP_200_OK) +class DocumentVersionPageAPIViewTestMixin(object): + def _request_document_version_page_image(self): + page = self.test_document_version.pages.first() + return self.get( + viewname='rest_api:documentversionpage-image', kwargs={ + 'pk': page.document.pk, 'version_pk': page.document_version.pk, + 'page_pk': page.pk + } + ) + + +class DocumentVersionPageAPIViewTestCase( + DocumentVersionPageAPIViewTestMixin, DocumentTestMixin, BaseAPITestCase +): + def test_document_version_page_api_image_view_no_access(self): + response = self._request_document_version_page_image() + self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) + + def test_document_version_page_api_image_view_with_access(self): + self.grant_access( + obj=self.test_document, permission=permission_document_view + ) + + response = self._request_document_version_page_image() + self.assertEqual(response.status_code, status.HTTP_200_OK) + + class TrashedDocumentAPIViewTestMixin(object): def _request_test_document_api_trash_view(self): return self.delete( @@ -580,7 +606,6 @@ class TrashedDocumentAPIViewTestMixin(object): return self.get( viewname='rest_api:documentpage-image', kwargs={ 'pk': latest_version.document.pk, - 'version_pk': latest_version.pk, 'page_pk': latest_version.pages.first().pk } ) diff --git a/mayan/apps/documents/tests/test_document_page_views.py b/mayan/apps/documents/tests/test_document_page_views.py index 29ee28cec3..96b2702968 100644 --- a/mayan/apps/documents/tests/test_document_page_views.py +++ b/mayan/apps/documents/tests/test_document_page_views.py @@ -9,10 +9,10 @@ from ..permissions import ( from .base import GenericDocumentViewTestCase -class DocumentPageDisableViewTestCase(GenericDocumentViewTestCase): - def setUp(self): - super(DocumentPageDisableViewTestCase, self).setUp() - self.test_document_page = self.test_document.pages_all.first() +class DocumentPageDisableViewTestMixin(object): + def _disable_test_document_page(self): + self.test_document_page.enabled = False + self.test_document_page.save() def _request_test_document_page_disable_view(self): return self.post( @@ -21,6 +21,31 @@ class DocumentPageDisableViewTestCase(GenericDocumentViewTestCase): } ) + def _request_test_document_page_enable_view(self): + return self.post( + viewname='documents:document_page_enable', kwargs={ + 'pk': self.test_document_page.pk + } + ) + + def _request_test_document_page_multiple_disable_view(self): + return self.post( + viewname='documents:document_page_multiple_disable', data={ + 'id_list': self.test_document_page.pk + } + ) + + def _request_test_document_page_multiple_enable_view(self): + return self.post( + viewname='documents:document_page_multiple_enable', data={ + 'id_list': self.test_document_page.pk + } + ) + + +class DocumentPageDisableViewTestCase( + DocumentPageDisableViewTestMixin, GenericDocumentViewTestCase +): def test_document_page_disable_view_no_permission(self): test_document_page_count = self.test_document.pages.count() @@ -45,13 +70,6 @@ class DocumentPageDisableViewTestCase(GenericDocumentViewTestCase): test_document_page_count, self.test_document.pages.count() ) - def _request_test_document_page_multiple_disable_view(self): - return self.post( - viewname='documents:document_page_multiple_disable', data={ - 'id_list': self.test_document_page.pk - } - ) - def test_document_page_multiple_disable_view_no_permission(self): test_document_page_count = self.test_document.pages.count() @@ -76,17 +94,6 @@ class DocumentPageDisableViewTestCase(GenericDocumentViewTestCase): test_document_page_count, self.test_document.pages.count() ) - def _disable_test_document_page(self): - self.test_document_page.enabled = False - self.test_document_page.save() - - def _request_test_document_page_enable_view(self): - return self.post( - viewname='documents:document_page_enable', kwargs={ - 'pk': self.test_document_page.pk - } - ) - def test_document_page_enable_view_no_permission(self): self._disable_test_document_page() @@ -114,13 +121,6 @@ class DocumentPageDisableViewTestCase(GenericDocumentViewTestCase): test_document_page_count, self.test_document.pages.count() ) - def _request_test_document_page_multiple_enable_view(self): - return self.post( - viewname='documents:document_page_multiple_enable', data={ - 'id_list': self.test_document_page.pk - } - ) - def test_document_page_multiple_enable_view_no_permission(self): self._disable_test_document_page() test_document_page_count = self.test_document.pages.count() @@ -148,7 +148,7 @@ class DocumentPageDisableViewTestCase(GenericDocumentViewTestCase): ) -class DocumentPageViewTestCase(GenericDocumentViewTestCase): +class DocumentPageViewTestMixin(object): def _request_test_document_page_list_view(self): return self.get( viewname='documents:document_pages', kwargs={ @@ -156,6 +156,18 @@ class DocumentPageViewTestCase(GenericDocumentViewTestCase): } ) + def _request_test_document_page_view(self, document_page): + return self.get( + viewname='documents:document_page_view', kwargs={ + 'pk': document_page.pk, + } + ) + + +class DocumentPageViewTestCase( + DocumentPageViewTestMixin, GenericDocumentViewTestCase +): + def test_document_page_list_view_no_permission(self): response = self._request_test_document_page_list_view() self.assertEqual(response.status_code, 404) @@ -170,13 +182,6 @@ class DocumentPageViewTestCase(GenericDocumentViewTestCase): response=response, text=self.test_document.label, status_code=200 ) - def _request_test_document_page_view(self, document_page): - return self.get( - viewname='documents:document_page_view', kwargs={ - 'pk': document_page.pk, - } - ) - def test_document_page_view_no_permissions(self): response = self._request_test_document_page_view( document_page=self.test_document.pages.first() diff --git a/mayan/apps/documents/tests/test_search.py b/mayan/apps/documents/tests/test_search.py index 531f6e8d2f..07f85fdb00 100644 --- a/mayan/apps/documents/tests/test_search.py +++ b/mayan/apps/documents/tests/test_search.py @@ -1,9 +1,11 @@ from __future__ import unicode_literals from mayan.apps.common.tests.base import BaseTestCase -from mayan.apps.documents.permissions import permission_document_view -from mayan.apps.documents.search import document_search, document_page_search -from mayan.apps.documents.tests.mixins import DocumentTestMixin + +from ..permissions import permission_document_view +from ..search import document_search, document_page_search + +from .mixins import DocumentTestMixin class DocumentSearchTestCase(DocumentTestMixin, BaseTestCase): diff --git a/mayan/apps/documents/tests/test_trashed_document_views.py b/mayan/apps/documents/tests/test_trashed_document_views.py index 8be806f36e..332ee9dc55 100644 --- a/mayan/apps/documents/tests/test_trashed_document_views.py +++ b/mayan/apps/documents/tests/test_trashed_document_views.py @@ -9,7 +9,7 @@ from ..permissions import ( from .base import GenericDocumentViewTestCase -class TrashedDocumentTestCase(GenericDocumentViewTestCase): +class TrashedDocumentTestMixin(object): def _request_document_restore_get_view(self): return self.get( viewname='documents:document_restore', kwargs={ @@ -17,6 +17,46 @@ class TrashedDocumentTestCase(GenericDocumentViewTestCase): } ) + def _request_document_restore_post_view(self): + return self.post( + viewname='documents:document_restore', kwargs={ + 'pk': self.test_document.pk + } + ) + + def _request_document_trash_get_view(self): + return self.get( + viewname='documents:document_trash', kwargs={ + 'pk': self.test_document.pk + } + ) + + def _request_document_trash_post_view(self): + return self.post( + viewname='documents:document_trash', kwargs={ + 'pk': self.test_document.pk + } + ) + + def _request_trashed_document_delete_get_view(self): + return self.get( + viewname='documents:document_delete', kwargs={ + 'pk': self.test_document.pk + } + ) + + def _request_trashed_document_delete_post_view(self): + return self.post( + viewname='documents:document_delete', kwargs={ + 'pk': self.test_document.pk + } + ) + + def _request_trashed_document_list_view(self): + return self.get(viewname='documents:document_list_deleted') + + +class TrashedDocumentTestCase(GenericDocumentViewTestCase): def test_document_restore_get_view_no_permission(self): self.test_document.delete() self.assertEqual(Document.objects.count(), 0) @@ -43,13 +83,6 @@ class TrashedDocumentTestCase(GenericDocumentViewTestCase): self.assertEqual(Document.objects.count(), document_count) - def _request_document_restore_post_view(self): - return self.post( - viewname='documents:document_restore', kwargs={ - 'pk': self.test_document.pk - } - ) - def test_document_restore_post_view_no_permission(self): self.test_document.delete() self.assertEqual(Document.objects.count(), 0) @@ -74,13 +107,6 @@ class TrashedDocumentTestCase(GenericDocumentViewTestCase): self.assertEqual(DeletedDocument.objects.count(), 0) self.assertEqual(Document.objects.count(), 1) - def _request_document_trash_get_view(self): - return self.get( - viewname='documents:document_trash', kwargs={ - 'pk': self.test_document.pk - } - ) - def test_document_trash_get_view_no_permissions(self): document_count = Document.objects.count() @@ -101,13 +127,6 @@ class TrashedDocumentTestCase(GenericDocumentViewTestCase): self.assertEqual(Document.objects.count(), document_count) - def _request_document_trash_post_view(self): - return self.post( - viewname='documents:document_trash', kwargs={ - 'pk': self.test_document.pk - } - ) - def test_document_trash_post_view_no_permissions(self): response = self._request_document_trash_post_view() self.assertEqual(response.status_code, 404) @@ -126,13 +145,6 @@ class TrashedDocumentTestCase(GenericDocumentViewTestCase): self.assertEqual(DeletedDocument.objects.count(), 1) self.assertEqual(Document.objects.count(), 0) - def _request_document_delete_get_view(self): - return self.get( - viewname='documents:document_delete', kwargs={ - 'pk': self.test_document.pk - } - ) - def test_document_delete_get_view_no_permissions(self): self.test_document.delete() self.assertEqual(Document.objects.count(), 0) @@ -140,7 +152,7 @@ class TrashedDocumentTestCase(GenericDocumentViewTestCase): trashed_document_count = DeletedDocument.objects.count() - response = self._request_document_delete_get_view() + response = self._request_trashed_document_delete_get_view() self.assertEqual(response.status_code, 404) self.assertEqual( @@ -158,26 +170,19 @@ class TrashedDocumentTestCase(GenericDocumentViewTestCase): trashed_document_count = DeletedDocument.objects.count() - response = self._request_document_delete_get_view() + response = self._request_trashed_document_delete_get_view() self.assertEqual(response.status_code, 200) self.assertEqual( DeletedDocument.objects.count(), trashed_document_count ) - def _request_document_delete_post_view(self): - return self.post( - viewname='documents:document_delete', kwargs={ - 'pk': self.test_document.pk - } - ) - def test_document_delete_post_view_no_permissions(self): self.test_document.delete() self.assertEqual(Document.objects.count(), 0) self.assertEqual(DeletedDocument.objects.count(), 1) - response = self._request_document_delete_post_view() + response = self._request_trashed_document_delete_post_view() self.assertEqual(response.status_code, 404) self.assertEqual(Document.objects.count(), 0) @@ -192,19 +197,16 @@ class TrashedDocumentTestCase(GenericDocumentViewTestCase): obj=self.test_document, permission=permission_document_delete ) - response = self._request_document_delete_post_view() + response = self._request_trashed_document_delete_post_view() self.assertEqual(response.status_code, 302) self.assertEqual(DeletedDocument.objects.count(), 0) self.assertEqual(Document.objects.count(), 0) - def _request_document_list_deleted_view(self): - return self.get(viewname='documents:document_list_deleted') - def test_deleted_document_list_view_no_permissions(self): self.test_document.delete() - response = self._request_document_list_deleted_view() + response = self._request_trashed_document_list_view() self.assertNotContains( response=response, text=self.test_document.label, status_code=200 ) @@ -216,7 +218,7 @@ class TrashedDocumentTestCase(GenericDocumentViewTestCase): obj=self.test_document, permission=permission_document_view ) - response = self._request_document_list_deleted_view() + response = self._request_trashed_document_list_view() self.assertContains( response=response, text=self.test_document.label, status_code=200 ) diff --git a/mayan/apps/documents/views/document_views.py b/mayan/apps/documents/views/document_views.py index aebc6d6324..dab008a75c 100644 --- a/mayan/apps/documents/views/document_views.py +++ b/mayan/apps/documents/views/document_views.py @@ -44,7 +44,7 @@ from ..permissions import ( from ..settings import ( setting_print_width, setting_print_height, setting_recent_added_count ) -from ..tasks import task_document_reset_pages +from ..tasks import task_document_pages_reset from ..utils import parse_range __all__ = ( @@ -450,7 +450,7 @@ class DocumentPagesResetView(MultipleObjectConfirmActionView): def object_action(self, form, instance): latest_version = instance.latest_version if latest_version: - task_document_reset_pages.apply_async( + task_document_pages_reset.apply_async( kwargs={'document_id': instance.pk} ) else: diff --git a/mayan/apps/ocr/admin.py b/mayan/apps/ocr/admin.py index 865481d602..ded8af09ad 100644 --- a/mayan/apps/ocr/admin.py +++ b/mayan/apps/ocr/admin.py @@ -3,13 +3,14 @@ from __future__ import unicode_literals from django.contrib import admin from .models import ( - DocumentPageOCRContent, DocumentTypeSettings, DocumentVersionOCRError + DocumentTypeSettings, DocumentVersionPageOCRContent, + DocumentVersionOCRError ) -@admin.register(DocumentPageOCRContent) -class DocumentPageOCRContentAdmin(admin.ModelAdmin): - list_display = ('document_page',) +@admin.register(DocumentVersionPageOCRContent) +class DocumentVersionPageOCRContentAdmin(admin.ModelAdmin): + list_display = ('document_version_page',) @admin.register(DocumentTypeSettings) diff --git a/mayan/apps/ocr/api_views.py b/mayan/apps/ocr/api_views.py index 0dea74beb3..20a4fe833e 100644 --- a/mayan/apps/ocr/api_views.py +++ b/mayan/apps/ocr/api_views.py @@ -8,7 +8,7 @@ from rest_framework.response import Response from mayan.apps.documents.models import Document, DocumentVersion from mayan.apps.rest_api.permissions import MayanPermission -from .models import DocumentPageOCRContent +from .models import DocumentVersionPageOCRContent from .permissions import permission_ocr_content_view, permission_ocr_document from .serializers import DocumentPageOCRContentSerializer @@ -90,8 +90,8 @@ class APIDocumentPageOCRContentView(generics.RetrieveAPIView): try: ocr_content = instance.ocr_content - except DocumentPageOCRContent.DoesNotExist: - ocr_content = DocumentPageOCRContent.objects.none() + except DocumentVersionPageOCRContent.DoesNotExist: + ocr_content = DocumentVersionPageOCRContent.objects.none() serializer = self.get_serializer(ocr_content) return Response(serializer.data) diff --git a/mayan/apps/ocr/apps.py b/mayan/apps/ocr/apps.py index 8ec9be9800..ab33bc3bee 100644 --- a/mayan/apps/ocr/apps.py +++ b/mayan/apps/ocr/apps.py @@ -128,7 +128,7 @@ class OCRApp(MayanAppConfig): ) document_search.add_model_field( - field='versions__version_pages__ocr_content__content', label=_('OCR') + field='pages__ocr_content__content', label=_('OCR') ) document_page_search.add_model_field( diff --git a/mayan/apps/ocr/forms.py b/mayan/apps/ocr/forms.py index 6951861ecf..bd0b346b60 100644 --- a/mayan/apps/ocr/forms.py +++ b/mayan/apps/ocr/forms.py @@ -8,7 +8,7 @@ from django.utils.translation import ugettext_lazy as _, ugettext from mayan.apps.common.widgets import TextAreaDiv -from .models import DocumentPageOCRContent +from .models import DocumentVersionPageOCRContent class DocumentPageOCRContentForm(forms.Form): @@ -30,7 +30,7 @@ class DocumentPageOCRContentForm(forms.Form): try: page_content = page.ocr_content.content - except DocumentPageOCRContent.DoesNotExist: + except DocumentVersionPageOCRContent.DoesNotExist: pass else: content = conditional_escape(force_text(page_content)) @@ -66,7 +66,7 @@ class DocumentOCRContentForm(forms.Form): for page in document_pages: try: page_content = page.ocr_content.content - except DocumentPageOCRContent.DoesNotExist: + except DocumentVersionPageOCRContent.DoesNotExist: pass else: content.append(conditional_escape(force_text(page_content))) diff --git a/mayan/apps/ocr/migrations/0009_rename_page_content.py b/mayan/apps/ocr/migrations/0009_rename_page_content.py new file mode 100644 index 0000000000..20f6e3babb --- /dev/null +++ b/mayan/apps/ocr/migrations/0009_rename_page_content.py @@ -0,0 +1,28 @@ +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + dependencies = [ + ('ocr', '0008_auto_20180917_0646'), + ('documents', '0052_rename_document_page'), + ] + + operations = [ + migrations.RenameModel( + 'DocumentPageOCRContent', 'DocumentVersionPageOCRContent' + ), + migrations.AlterField( + model_name='documentversionpageocrcontent', + name='document_page', + field=models.OneToOneField( + on_delete=django.db.models.deletion.CASCADE, + name='document_version_page', + related_name='ocr_content', + to='documents.DocumentVersionPage', + verbose_name='Document version page' + ), + ), + ] diff --git a/mayan/apps/ocr/models.py b/mayan/apps/ocr/models.py index 8402d2079c..c54a2e12a7 100644 --- a/mayan/apps/ocr/models.py +++ b/mayan/apps/ocr/models.py @@ -5,7 +5,7 @@ from django.utils.encoding import force_text, python_2_unicode_compatible from django.utils.translation import ugettext_lazy as _ from mayan.apps.documents.models import ( - DocumentPage, DocumentType, DocumentVersion + DocumentPage, DocumentType, DocumentVersion, DocumentVersionPage ) from .managers import ( @@ -38,13 +38,13 @@ class DocumentTypeSettings(models.Model): @python_2_unicode_compatible -class DocumentPageOCRContent(models.Model): +class DocumentVersionPageOCRContent(models.Model): """ This model stores the OCR results for a document page. """ - document_page = models.OneToOneField( + document_version_page = models.OneToOneField( on_delete=models.CASCADE, related_name='ocr_content', - to=DocumentPage, verbose_name=_('Document page') + to=DocumentVersionPage, verbose_name=_('Document version page') ) content = models.TextField( blank=True, help_text=_( @@ -55,8 +55,8 @@ class DocumentPageOCRContent(models.Model): objects = DocumentPageOCRContentManager() class Meta: - verbose_name = _('Document page OCR content') - verbose_name_plural = _('Document pages OCR contents') + verbose_name = _('Document version page OCR content') + verbose_name_plural = _('Document version pages OCR contents') def __str__(self): return force_text(self.document_page) diff --git a/mayan/apps/ocr/serializers.py b/mayan/apps/ocr/serializers.py index 3d9c06c18d..780e9a48a6 100644 --- a/mayan/apps/ocr/serializers.py +++ b/mayan/apps/ocr/serializers.py @@ -2,10 +2,10 @@ from __future__ import unicode_literals from rest_framework import serializers -from .models import DocumentPageOCRContent +from .models import DocumentVersionPageOCRContent class DocumentPageOCRContentSerializer(serializers.ModelSerializer): class Meta: fields = ('content',) - model = DocumentPageOCRContent + model = DocumentVersionPageOCRContent diff --git a/mayan/apps/ocr/views.py b/mayan/apps/ocr/views.py index 1524b3d006..7541784886 100644 --- a/mayan/apps/ocr/views.py +++ b/mayan/apps/ocr/views.py @@ -17,7 +17,7 @@ from mayan.apps.documents.models import ( ) from .forms import DocumentPageOCRContentForm, DocumentOCRContentForm -from .models import DocumentPageOCRContent, DocumentVersionOCRError +from .models import DocumentVersionPageOCRContent, DocumentVersionOCRError from .permissions import ( permission_ocr_content_view, permission_ocr_document, permission_document_type_ocr_setup @@ -48,7 +48,7 @@ class DocumentOCRContentDeleteView(MultipleObjectConfirmActionView): return result def object_action(self, form, instance): - DocumentPageOCRContent.objects.delete_content_for( + DocumentVersionPageOCRContent.objects.delete_content_for( document=instance, user=self.request.user ) From 64abf66f2265db6130036dd276901fdac810252f Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 9 Oct 2019 16:55:02 -0400 Subject: [PATCH 17/26] Fix tests Signed-off-by: Roberto Rosario --- mayan/apps/document_parsing/parsers.py | 7 ++++--- mayan/apps/documents/tests/test_trashed_document_views.py | 4 +++- mayan/apps/ocr/managers.py | 4 +++- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/mayan/apps/document_parsing/parsers.py b/mayan/apps/document_parsing/parsers.py index 0e4d0cff70..8ad24b4115 100644 --- a/mayan/apps/document_parsing/parsers.py +++ b/mayan/apps/document_parsing/parsers.py @@ -68,8 +68,9 @@ class Parser(object): self.process_document_page(document_page=document_page) def process_document_page(self, document_page): - DocumentPageContent = apps.get_model( - app_label='document_parsing', model_name='DocumentPageContent' + DocumentVersionPageContent = apps.get_model( + app_label='document_parsing', + model_name='DocumentVersionPageContent' ) logger.info( @@ -80,7 +81,7 @@ class Parser(object): file_object = document_page.document_version.get_intermediate_file() try: - document_page_content, created = DocumentPageContent.objects.get_or_create( + document_page_content, created = DocumentVersionPageContent.objects.get_or_create( document_page=document_page ) document_page_content.content = self.execute( diff --git a/mayan/apps/documents/tests/test_trashed_document_views.py b/mayan/apps/documents/tests/test_trashed_document_views.py index 332ee9dc55..775bef0f38 100644 --- a/mayan/apps/documents/tests/test_trashed_document_views.py +++ b/mayan/apps/documents/tests/test_trashed_document_views.py @@ -56,7 +56,9 @@ class TrashedDocumentTestMixin(object): return self.get(viewname='documents:document_list_deleted') -class TrashedDocumentTestCase(GenericDocumentViewTestCase): +class TrashedDocumentTestCase( + TrashedDocumentTestMixin, GenericDocumentViewTestCase +): def test_document_restore_get_view_no_permission(self): self.test_document.delete() self.assertEqual(Document.objects.count(), 0) diff --git a/mayan/apps/ocr/managers.py b/mayan/apps/ocr/managers.py index ee8e1bd95a..f4631275d5 100644 --- a/mayan/apps/ocr/managers.py +++ b/mayan/apps/ocr/managers.py @@ -46,7 +46,9 @@ class DocumentPageOCRContentManager(models.Manager): ) ) - cache_filename = task.get(timeout=DOCUMENT_IMAGE_TASK_TIMEOUT, disable_sync_subtasks=False) + cache_filename = task.get( + timeout=DOCUMENT_IMAGE_TASK_TIMEOUT, disable_sync_subtasks=False + ) with document_page.cache_partition.get_file(filename=cache_filename).open() as file_object: document_page_content, created = DocumentPageOCRContent.objects.get_or_create( From f163dc78d4757b47d5f0860e63e1a15010941311 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 9 Oct 2019 17:15:59 -0400 Subject: [PATCH 18/26] Fix search setup Signed-off-by: Roberto Rosario --- mayan/apps/cabinets/apps.py | 10 ++++++++-- mayan/apps/document_comments/apps.py | 10 ++++++++-- mayan/apps/document_parsing/apps.py | 8 +++++--- mayan/apps/documents/search.py | 23 +++++++++++++---------- mayan/apps/documents/tests/test_api.py | 6 ++---- mayan/apps/dynamic_search/classes.py | 9 ++++++++- mayan/apps/file_metadata/apps.py | 16 +++++++++++++--- mayan/apps/metadata/apps.py | 14 ++++++++++++-- mayan/apps/ocr/apps.py | 12 ++++++++---- mayan/apps/tags/apps.py | 9 +++++++-- 10 files changed, 84 insertions(+), 33 deletions(-) diff --git a/mayan/apps/cabinets/apps.py b/mayan/apps/cabinets/apps.py index 6011bd4874..a2a0dc24bd 100644 --- a/mayan/apps/cabinets/apps.py +++ b/mayan/apps/cabinets/apps.py @@ -10,12 +10,14 @@ from mayan.apps.common.menus import ( menu_facet, menu_list_facet, menu_main, menu_multi_item, menu_object, menu_secondary ) +from mayan.apps.documents.search import ( + document_page_search, document_search, document_version_page_search +) from mayan.apps.events.classes import ModelEventType from mayan.apps.events.links import ( link_events_for_object, link_object_event_types_user_subcriptions_list, ) from mayan.apps.events.permissions import permission_events_view -from mayan.apps.documents.search import document_page_search, document_search from mayan.apps.navigation.classes import SourceColumn from .dependencies import * # NOQA @@ -115,12 +117,16 @@ class CabinetsApp(MayanAppConfig): ) document_page_search.add_model_field( - field='document_version__document__cabinets__label', + field='document__cabinets__label', label=_('Cabinets') ) document_search.add_model_field( field='cabinets__label', label=_('Cabinets') ) + document_version_page_search.add_model_field( + field='document_version__document__cabinets__label', + label=_('Cabinets') + ) menu_facet.bind_links( links=(link_document_cabinet_list,), sources=(Document,) diff --git a/mayan/apps/document_comments/apps.py b/mayan/apps/document_comments/apps.py index 371d7d9be4..ee34bb8038 100644 --- a/mayan/apps/document_comments/apps.py +++ b/mayan/apps/document_comments/apps.py @@ -8,7 +8,9 @@ from mayan.apps.common.apps import MayanAppConfig from mayan.apps.common.menus import ( menu_facet, menu_list_facet, menu_object, menu_secondary ) -from mayan.apps.documents.search import document_page_search, document_search +from mayan.apps.documents.search import ( + document_page_search, document_search, document_version_page_search +) from mayan.apps.events.classes import ModelEventType from mayan.apps.events.links import ( link_events_for_object, link_object_event_types_user_subcriptions_list @@ -80,13 +82,17 @@ class DocumentCommentsApp(MayanAppConfig): SourceColumn(attribute='comment', source=Comment) document_page_search.add_model_field( - field='document_version__document__comments__comment', + field='document__comments__comment', label=_('Comments') ) document_search.add_model_field( field='comments__comment', label=_('Comments') ) + document_version_page_search.add_model_field( + field='document_version__document__comments__comment', + label=_('Comments') + ) menu_facet.bind_links( links=(link_comments_for_document,), sources=(Document,) diff --git a/mayan/apps/document_parsing/apps.py b/mayan/apps/document_parsing/apps.py index 45eb734d08..7def216213 100644 --- a/mayan/apps/document_parsing/apps.py +++ b/mayan/apps/document_parsing/apps.py @@ -12,7 +12,9 @@ from mayan.apps.common.classes import ModelField from mayan.apps.common.menus import ( menu_facet, menu_list_facet, menu_multi_item, menu_secondary, menu_tools ) -from mayan.apps.documents.search import document_search, document_page_search +from mayan.apps.documents.search import ( + document_search, document_page_search, document_version_page_search +) from mayan.apps.documents.signals import post_version_upload from mayan.apps.events.classes import ModelEventType from mayan.apps.navigation.classes import SourceColumn @@ -133,10 +135,10 @@ class DocumentParsingApp(MayanAppConfig): ) document_search.add_model_field( - field='pages__content__content', label=_('Content') + field='versions__pages__content__content', label=_('Content') ) - document_page_search.add_model_field( + document_version_page_search.add_model_field( field='content__content', label=_('Content') ) diff --git a/mayan/apps/documents/search.py b/mayan/apps/documents/search.py index 82517002f9..b42228a277 100644 --- a/mayan/apps/documents/search.py +++ b/mayan/apps/documents/search.py @@ -38,7 +38,7 @@ document_search = SearchModel( model_name='Document', permission=permission_document_view, serializer_path='mayan.apps.documents.serializers.DocumentSerializer' ) - +''' document_search.add_model_field( field='document_type__label', label=_('Document type') ) @@ -51,10 +51,10 @@ document_search.add_model_field( field='uuid', label=_('UUID'), transformation_function=transformation_format_uuid ) -document_search.add_model_field( - field='versions__checksum', label=_('Checksum') -) - +#document_search.add_model_field( +# field='versions__checksum', label=_('Checksum') +#) +''' document_page_search = SearchModel( app_label='documents', list_mode=LIST_MODE_CHOICE_ITEM, model_name='DocumentPage', permission=permission_document_view, @@ -65,10 +65,10 @@ document_page_search = SearchModel( document_version_page_search = SearchModel( app_label='documents', list_mode=LIST_MODE_CHOICE_ITEM, model_name='DocumentVersionPage', permission=permission_document_view, - queryset=get_queryset_document_version_page_search_queryset, + #queryset=get_queryset_document_version_page_search_queryset, serializer_path='mayan.apps.documents.serializers.DocumentPageVersionSerializer' ) - +''' document_page_search.add_model_field( field='document__document_type__label', label=_('Document type') @@ -83,10 +83,12 @@ document_page_search.add_model_field( document_page_search.add_model_field( field='document__description', label=_('Description') ) -document_page_search.add_model_field( - field='document__document_version__checksum', label=_('Checksum') -) +''' +#document_page_search.add_model_field( +# field='document__document_version__checksum', label=_('Checksum') +#) +''' document_version_page_search.add_model_field( field='document_version__document__document_type__label', label=_('Document type') @@ -104,3 +106,4 @@ document_version_page_search.add_model_field( document_version_page_search.add_model_field( field='document_version__checksum', label=_('Checksum') ) +''' diff --git a/mayan/apps/documents/tests/test_api.py b/mayan/apps/documents/tests/test_api.py index 388b2f2de8..60694de79e 100644 --- a/mayan/apps/documents/tests/test_api.py +++ b/mayan/apps/documents/tests/test_api.py @@ -601,12 +601,10 @@ class TrashedDocumentAPIViewTestMixin(object): ) def _request_test_trashed_document_api_image_view(self): - latest_version = self.test_document.latest_version - return self.get( viewname='rest_api:documentpage-image', kwargs={ - 'pk': latest_version.document.pk, - 'page_pk': latest_version.pages.first().pk + 'pk': self.test_document.pk, + 'page_pk': self.test_document.pages.first().pk } ) diff --git a/mayan/apps/dynamic_search/classes.py b/mayan/apps/dynamic_search/classes.py index 2476f07a15..1630bcc7fb 100644 --- a/mayan/apps/dynamic_search/classes.py +++ b/mayan/apps/dynamic_search/classes.py @@ -184,7 +184,14 @@ class SearchModel(object): query_string=query_string, global_and_search=global_and_search ) - queryset = self.get_queryset().filter(search_query.query).distinct() + try: + queryset = self.get_queryset().filter(search_query.query).distinct() + except Exception: + logger.error( + 'Error filtering model %s with queryset: %s', self.model, + search_query.query + ) + raise if self.permission: queryset = AccessControlList.objects.restrict_queryset( diff --git a/mayan/apps/file_metadata/apps.py b/mayan/apps/file_metadata/apps.py index a9830d467d..675ebe3365 100644 --- a/mayan/apps/file_metadata/apps.py +++ b/mayan/apps/file_metadata/apps.py @@ -12,7 +12,9 @@ from mayan.apps.common.menus import ( menu_tools ) from mayan.apps.document_indexing.handlers import handler_index_document -from mayan.apps.documents.search import document_page_search, document_search +from mayan.apps.documents.search import ( + document_page_search, document_search, document_version_page_search +) from mayan.apps.documents.signals import post_version_upload from mayan.apps.events.classes import ModelEventType from mayan.apps.navigation.classes import SourceColumn @@ -156,11 +158,19 @@ class FileMetadataApp(MayanAppConfig): label=_('File metadata value') ) - document_page_search.add_model_field( + #document_page_search.add_model_field( + # field='document__document_version__file_metadata_drivers__entries__key', + # label=_('File metadata key') + #) + #document_page_search.add_model_field( + # field='document__document_version__file_metadata_drivers__entries__value', + # label=_('File metadata value') + #) + document_version_page_search.add_model_field( field='document_version__file_metadata_drivers__entries__key', label=_('File metadata key') ) - document_page_search.add_model_field( + document_version_page_search.add_model_field( field='document_version__file_metadata_drivers__entries__value', label=_('File metadata value') ) diff --git a/mayan/apps/metadata/apps.py b/mayan/apps/metadata/apps.py index 5856484193..b0224e9ceb 100644 --- a/mayan/apps/metadata/apps.py +++ b/mayan/apps/metadata/apps.py @@ -16,7 +16,9 @@ from mayan.apps.common.menus import ( menu_facet, menu_list_facet, menu_multi_item, menu_object, menu_secondary, menu_setup ) -from mayan.apps.documents.search import document_page_search, document_search +from mayan.apps.documents.search import ( + document_page_search, document_search, document_version_page_search +) from mayan.apps.documents.signals import post_document_type_change from mayan.apps.events.classes import ModelEventType from mayan.apps.events.links import ( @@ -188,10 +190,18 @@ class MetadataApp(MayanAppConfig): ) document_page_search.add_model_field( - field='document_version__document__metadata__metadata_type__name', + field='document__metadata__metadata_type__name', label=_('Metadata type') ) document_page_search.add_model_field( + field='document__metadata__value', + label=_('Metadata value') + ) + document_version_page_search.add_model_field( + field='document_version__document__metadata__metadata_type__name', + label=_('Metadata type') + ) + document_version_page_search.add_model_field( field='document_version__document__metadata__value', label=_('Metadata value') ) diff --git a/mayan/apps/ocr/apps.py b/mayan/apps/ocr/apps.py index ab33bc3bee..2d9b3dce3a 100644 --- a/mayan/apps/ocr/apps.py +++ b/mayan/apps/ocr/apps.py @@ -12,7 +12,9 @@ from mayan.apps.common.classes import ModelField from mayan.apps.common.menus import ( menu_facet, menu_list_facet, menu_multi_item, menu_secondary, menu_tools ) -from mayan.apps.documents.search import document_search, document_page_search +from mayan.apps.documents.search import ( + document_search, document_page_search, document_version_page_search +) from mayan.apps.documents.signals import post_version_upload from mayan.apps.events.classes import ModelEventType from mayan.apps.navigation.classes import SourceColumn @@ -128,12 +130,14 @@ class OCRApp(MayanAppConfig): ) document_search.add_model_field( - field='pages__ocr_content__content', label=_('OCR') + field='versions__pages__ocr_content__content', label=_('OCR') ) - - document_page_search.add_model_field( + document_version_page_search.add_model_field( field='ocr_content__content', label=_('OCR') ) + #document_page_search.add_model_field( + # field='ocr_content__content', label=_('OCR') + #) menu_facet.bind_links( links=(link_document_ocr_content,), sources=(Document,) diff --git a/mayan/apps/tags/apps.py b/mayan/apps/tags/apps.py index f3d36c1611..2828e06049 100644 --- a/mayan/apps/tags/apps.py +++ b/mayan/apps/tags/apps.py @@ -13,7 +13,9 @@ from mayan.apps.common.menus import ( menu_facet, menu_list_facet, menu_main, menu_multi_item, menu_object, menu_secondary ) -from mayan.apps.documents.search import document_page_search, document_search +from mayan.apps.documents.search import ( + document_page_search, document_search, document_version_page_search +) from mayan.apps.events.classes import ModelEventType from mayan.apps.events.links import ( link_events_for_object, link_object_event_types_user_subcriptions_list, @@ -133,9 +135,12 @@ class TagsApp(MayanAppConfig): ) document_page_search.add_model_field( - field='document_version__document__tags__label', label=_('Tags') + field='document__tags__label', label=_('Tags') ) document_search.add_model_field(field='tags__label', label=_('Tags')) + document_version_page_search.add_model_field( + field='document_version__document__tags__label', label=_('Tags') + ) menu_facet.bind_links( links=(link_document_tag_list,), sources=(Document,) From a9077cb47aa25ebf6aad60dbc2927dfece7e3acc Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 9 Oct 2019 19:40:08 -0400 Subject: [PATCH 19/26] Fix document search tests Signed-off-by: Roberto Rosario --- mayan/apps/documents/search.py | 20 ++++++++------------ mayan/apps/documents/tests/test_search.py | 12 +++++++++--- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/mayan/apps/documents/search.py b/mayan/apps/documents/search.py index b42228a277..393ab3b4e6 100644 --- a/mayan/apps/documents/search.py +++ b/mayan/apps/documents/search.py @@ -38,7 +38,6 @@ document_search = SearchModel( model_name='Document', permission=permission_document_view, serializer_path='mayan.apps.documents.serializers.DocumentSerializer' ) -''' document_search.add_model_field( field='document_type__label', label=_('Document type') ) @@ -51,10 +50,10 @@ document_search.add_model_field( field='uuid', label=_('UUID'), transformation_function=transformation_format_uuid ) -#document_search.add_model_field( -# field='versions__checksum', label=_('Checksum') -#) -''' +document_search.add_model_field( + field='versions__checksum', label=_('Checksum') +) + document_page_search = SearchModel( app_label='documents', list_mode=LIST_MODE_CHOICE_ITEM, model_name='DocumentPage', permission=permission_document_view, @@ -68,7 +67,7 @@ document_version_page_search = SearchModel( #queryset=get_queryset_document_version_page_search_queryset, serializer_path='mayan.apps.documents.serializers.DocumentPageVersionSerializer' ) -''' + document_page_search.add_model_field( field='document__document_type__label', label=_('Document type') @@ -83,12 +82,10 @@ document_page_search.add_model_field( document_page_search.add_model_field( field='document__description', label=_('Description') ) -''' -#document_page_search.add_model_field( -# field='document__document_version__checksum', label=_('Checksum') -#) +document_page_search.add_model_field( + field='document__versions__checksum', label=_('Checksum') +) -''' document_version_page_search.add_model_field( field='document_version__document__document_type__label', label=_('Document type') @@ -106,4 +103,3 @@ document_version_page_search.add_model_field( document_version_page_search.add_model_field( field='document_version__checksum', label=_('Checksum') ) -''' diff --git a/mayan/apps/documents/tests/test_search.py b/mayan/apps/documents/tests/test_search.py index 07f85fdb00..1db040b174 100644 --- a/mayan/apps/documents/tests/test_search.py +++ b/mayan/apps/documents/tests/test_search.py @@ -8,17 +8,23 @@ from ..search import document_search, document_page_search from .mixins import DocumentTestMixin -class DocumentSearchTestCase(DocumentTestMixin, BaseTestCase): +class DocumentSearchTestMixin(object): def _perform_document_page_search(self): return document_page_search.search( - query_string={'q': self.test_document.label}, user=self._test_case_user + query_string={'q': self.test_document.label}, + user=self._test_case_user ) def _perform_document_search(self): return document_search.search( - query_string={'q': self.test_document.label}, user=self._test_case_user + query_string={'q': self.test_document.label}, + user=self._test_case_user ) + +class DocumentSearchTestCase( + DocumentSearchTestMixin, DocumentTestMixin, BaseTestCase +): def test_document_page_search_no_access(self): queryset = self._perform_document_page_search() self.assertFalse(self.test_document.pages.first() in queryset) From cf697d3ea7afe352e087b0f0d5e5804295ddfae1 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 9 Oct 2019 21:06:38 -0400 Subject: [PATCH 20/26] Fix tests Signed-off-by: Roberto Rosario --- .../migrations/0005_rename_page_content.py | 16 ++++++- mayan/apps/documents/search.py | 4 +- mayan/apps/ocr/admin.py | 3 +- mayan/apps/ocr/apps.py | 26 ++++++++--- mayan/apps/ocr/forms.py | 37 +++++++++------ mayan/apps/ocr/icons.py | 6 +-- mayan/apps/ocr/links.py | 16 +++++-- mayan/apps/ocr/managers.py | 42 ++++++++++------- mayan/apps/ocr/methods.py | 12 +++++ .../migrations/0009_rename_page_content.py | 16 ++++++- mayan/apps/ocr/tests/test_api.py | 46 +++++++++++-------- mayan/apps/ocr/tests/test_models.py | 2 +- mayan/apps/ocr/tests/test_views.py | 6 +-- mayan/apps/ocr/urls.py | 12 +++-- mayan/apps/ocr/utils.py | 26 ++++++++--- mayan/apps/ocr/views.py | 33 +++++++++++-- 16 files changed, 215 insertions(+), 88 deletions(-) diff --git a/mayan/apps/document_parsing/migrations/0005_rename_page_content.py b/mayan/apps/document_parsing/migrations/0005_rename_page_content.py index 2a78b99f73..11ad6cdc8b 100644 --- a/mayan/apps/document_parsing/migrations/0005_rename_page_content.py +++ b/mayan/apps/document_parsing/migrations/0005_rename_page_content.py @@ -19,10 +19,24 @@ class Migration(migrations.Migration): name='document_page', field=models.OneToOneField( on_delete=django.db.models.deletion.CASCADE, - name='document_version_page', related_name='content', to='documents.DocumentVersionPage', verbose_name='Document version page' ), ), + migrations.RenameField( + model_name='documentversionpagecontent', + old_name='document_page', + new_name='document_version_page', + ), + migrations.AlterModelOptions( + name='documentversionpagecontent', + options={ + 'verbose_name': 'Document version page content', + 'verbose_name_plural': 'Document version pages contents' + }, + ), ] + + + diff --git a/mayan/apps/documents/search.py b/mayan/apps/documents/search.py index 393ab3b4e6..e631099d3c 100644 --- a/mayan/apps/documents/search.py +++ b/mayan/apps/documents/search.py @@ -64,8 +64,8 @@ document_page_search = SearchModel( document_version_page_search = SearchModel( app_label='documents', list_mode=LIST_MODE_CHOICE_ITEM, model_name='DocumentVersionPage', permission=permission_document_view, - #queryset=get_queryset_document_version_page_search_queryset, - serializer_path='mayan.apps.documents.serializers.DocumentPageVersionSerializer' + queryset=get_queryset_document_version_page_search_queryset, + serializer_path='mayan.apps.documents.serializers.DocumentVersionPageSerializer' ) document_page_search.add_model_field( diff --git a/mayan/apps/ocr/admin.py b/mayan/apps/ocr/admin.py index ded8af09ad..9a927c5260 100644 --- a/mayan/apps/ocr/admin.py +++ b/mayan/apps/ocr/admin.py @@ -10,7 +10,8 @@ from .models import ( @admin.register(DocumentVersionPageOCRContent) class DocumentVersionPageOCRContentAdmin(admin.ModelAdmin): - list_display = ('document_version_page',) + pass + #list_display = ('document_page',) @admin.register(DocumentTypeSettings) diff --git a/mayan/apps/ocr/apps.py b/mayan/apps/ocr/apps.py index 2d9b3dce3a..1d0410bd44 100644 --- a/mayan/apps/ocr/apps.py +++ b/mayan/apps/ocr/apps.py @@ -34,17 +34,19 @@ from .links import ( link_document_ocr_content_delete_multiple, link_document_ocr_download, link_document_ocr_errors_list, link_document_submit, link_document_submit_multiple, link_document_type_ocr_settings, - link_document_type_submit, link_entry_list + link_document_type_submit, link_document_version_page_ocr_content, + link_entry_list ) from .methods import ( - method_document_ocr_submit, method_document_version_ocr_submit + method_document_ocr_submit, method_document_page_get_ocr_content, + method_document_version_ocr_submit ) from .permissions import ( permission_document_type_ocr_setup, permission_ocr_document, permission_ocr_content_view ) from .signals import post_document_version_ocr -from .utils import get_document_ocr_content +from .utils import get_document_ocr_content, get_document_version_ocr_content logger = logging.getLogger(__name__) @@ -75,6 +77,9 @@ class OCRApp(MayanAppConfig): DocumentVersion = apps.get_model( app_label='documents', model_name='DocumentVersion' ) + DocumentVersionPage = apps.get_model( + app_label='documents', model_name='DocumentVersionPage' + ) DocumentVersionOCRError = self.get_model( model_name='DocumentVersionOCRError' @@ -83,8 +88,11 @@ class OCRApp(MayanAppConfig): Document.add_to_class( name='submit_for_ocr', value=method_document_ocr_submit ) + DocumentPage.add_to_class( + name='get_ocr_content', value=method_document_page_get_ocr_content + ) DocumentVersion.add_to_class( - name='ocr_content', value=get_document_ocr_content + name='ocr_content', value=get_document_version_ocr_content ) DocumentVersion.add_to_class( name='submit_for_ocr', value=method_document_version_ocr_submit @@ -98,9 +106,9 @@ class OCRApp(MayanAppConfig): ) ) - #ModelField( - # model=Document, name='versions__pages__ocr_content__content' - #) + ModelField( + model=Document, name='versions__pages__ocr_content__content' + ) ModelPermission.register( model=Document, permissions=( @@ -145,6 +153,10 @@ class OCRApp(MayanAppConfig): menu_list_facet.bind_links( links=(link_document_page_ocr_content,), sources=(DocumentPage,) ) + menu_list_facet.bind_links( + links=(link_document_version_page_ocr_content,), + sources=(DocumentVersionPage,) + ) menu_list_facet.bind_links( links=(link_document_type_ocr_settings,), sources=(DocumentType,) ) diff --git a/mayan/apps/ocr/forms.py b/mayan/apps/ocr/forms.py index bd0b346b60..5379bce4ae 100644 --- a/mayan/apps/ocr/forms.py +++ b/mayan/apps/ocr/forms.py @@ -28,15 +28,26 @@ class DocumentPageOCRContentForm(forms.Form): content = '' self.fields['contents'].initial = '' - try: - page_content = page.ocr_content.content - except DocumentVersionPageOCRContent.DoesNotExist: - pass - else: - content = conditional_escape(force_text(page_content)) + content = conditional_escape( + force_text(self.get_instance_ocr_content(instance=page)) + ) self.fields['contents'].initial = mark_safe(content) + def get_instance_ocr_content(self, instance): + try: + return instance.content_object.ocr_content.content + except DocumentVersionPageOCRContent.DoesNotExist: + return '' + + +class DocumentVersionPageOCRContentForm(DocumentPageOCRContentForm): + def get_instance_ocr_content(self, instance): + try: + return instance.ocr_content.content + except (AttributeError, DocumentVersionPageOCRContent.DoesNotExist): + return '' + class DocumentOCRContentForm(forms.Form): """ @@ -54,19 +65,15 @@ class DocumentOCRContentForm(forms.Form): ) def __init__(self, *args, **kwargs): - self.document = kwargs.pop('instance', None) + document = kwargs.pop('instance', None) super(DocumentOCRContentForm, self).__init__(*args, **kwargs) content = [] self.fields['contents'].initial = '' - try: - document_pages = self.document.pages.all() - except AttributeError: - document_pages = [] - for page in document_pages: + for document_page in document.pages.all(): try: - page_content = page.ocr_content.content - except DocumentVersionPageOCRContent.DoesNotExist: + page_content = document_page.content_object.ocr_content.content + except (AttributeError, DocumentVersionPageOCRContent.DoesNotExist): pass else: content.append(conditional_escape(force_text(page_content))) @@ -74,7 +81,7 @@ class DocumentOCRContentForm(forms.Form): '\n\n\n
- %s -
\n\n\n' % ( ugettext( 'Page %(page_number)d' - ) % {'page_number': page.page_number} + ) % {'page_number': document_page.page_number} ) ) diff --git a/mayan/apps/ocr/icons.py b/mayan/apps/ocr/icons.py index f6e1ce3141..f3f7984992 100644 --- a/mayan/apps/ocr/icons.py +++ b/mayan/apps/ocr/icons.py @@ -19,7 +19,7 @@ icon_document_ocr_errors_list = Icon( icon_document_type_ocr_settings = Icon( driver_name='fontawesome', symbol='font' ) -icon_document_type_submit = Icon(driver_name='fontawesome', symbol='font') -icon_entry_list = Icon(driver_name='fontawesome', symbol='font') - icon_document_submit = icon_document_multiple_submit +icon_document_type_submit = Icon(driver_name='fontawesome', symbol='font') +icon_document_version_page_ocr_content = Icon(driver_name='fontawesome', symbol='font') +icon_entry_list = Icon(driver_name='fontawesome', symbol='font') diff --git a/mayan/apps/ocr/links.py b/mayan/apps/ocr/links.py index 343bf26d18..ed01601ae0 100644 --- a/mayan/apps/ocr/links.py +++ b/mayan/apps/ocr/links.py @@ -15,7 +15,7 @@ def is_document_page_disabled(context): link_document_page_ocr_content = Link( - args='resolved_object.id', conditional_disable=is_document_page_disabled, + args='resolved_object.id',# conditional_disable=is_document_page_disabled, icon_class_path='mayan.apps.ocr.icons.icon_document_page_ocr_content', permissions=(permission_ocr_content_view,), text=_('OCR'), view='ocr:document_page_ocr_content', @@ -58,10 +58,11 @@ link_document_type_submit = Link( permissions=(permission_ocr_document,), text=_('OCR documents per type'), view='ocr:document_type_submit' ) -link_entry_list = Link( - icon_class_path='mayan.apps.ocr.icons.icon_entry_list', - permissions=(permission_ocr_document,), text=_('OCR errors'), - view='ocr:entry_list' +link_document_version_page_ocr_content = Link( + args='resolved_object.id', + icon_class_path='mayan.apps.ocr.icons.icon_document_version_page_ocr_content', + permissions=(permission_ocr_content_view,), text=_('OCR'), + view='ocr:document_version_page_ocr_content', ) link_document_ocr_errors_list = Link( args='resolved_object.id', @@ -75,3 +76,8 @@ link_document_ocr_download = Link( permissions=(permission_ocr_content_view,), text=_('Download OCR text'), view='ocr:document_ocr_download' ) +link_entry_list = Link( + icon_class_path='mayan.apps.ocr.icons.icon_entry_list', + permissions=(permission_ocr_document,), text=_('OCR errors'), + view='ocr:entry_list' +) diff --git a/mayan/apps/ocr/managers.py b/mayan/apps/ocr/managers.py index f4631275d5..4cbe31e713 100644 --- a/mayan/apps/ocr/managers.py +++ b/mayan/apps/ocr/managers.py @@ -9,7 +9,9 @@ from django.conf import settings from django.db import models, transaction from mayan.apps.documents.literals import DOCUMENT_IMAGE_TASK_TIMEOUT -from mayan.apps.documents.tasks import task_generate_document_page_image +from mayan.apps.documents.tasks import ( + task_generate_document_version_page_image +) from .events import ( event_ocr_document_content_deleted, event_ocr_document_version_finish @@ -24,25 +26,28 @@ class DocumentPageOCRContentManager(models.Manager): def delete_content_for(self, document, user=None): with transaction.atomic(): for document_page in document.pages.all(): - self.filter(document_page=document_page).delete() + self.filter( + document_version_page=document_page.content_object + ).delete() event_ocr_document_content_deleted.commit( actor=user, target=document ) - def process_document_page(self, document_page): + def process_document_version_page(self, document_version_page): logger.info( 'Processing page: %d of document version: %s', - document_page.page_number, document_page.document_version + document_version_page.page_number, + document_version_page.document_version ) - DocumentPageOCRContent = apps.get_model( - app_label='ocr', model_name='DocumentPageOCRContent' + DocumentVersionPageOCRContent = apps.get_model( + app_label='ocr', model_name='DocumentVersionPageOCRContent' ) - task = task_generate_document_page_image.apply_async( + task = task_generate_document_version_page_image.apply_async( kwargs=dict( - document_page_id=document_page.pk + document_version_page_id=document_version_page.pk ) ) @@ -50,19 +55,20 @@ class DocumentPageOCRContentManager(models.Manager): timeout=DOCUMENT_IMAGE_TASK_TIMEOUT, disable_sync_subtasks=False ) - with document_page.cache_partition.get_file(filename=cache_filename).open() as file_object: - document_page_content, created = DocumentPageOCRContent.objects.get_or_create( - document_page=document_page + with document_version_page.cache_partition.get_file(filename=cache_filename).open() as file_object: + document_version_page_content, created = DocumentVersionPageOCRContent.objects.get_or_create( + document_version_page=document_version_page ) - document_page_content.content = ocr_backend.execute( + document_version_page_content.content = ocr_backend.execute( file_object=file_object, - language=document_page.document.language + language=document_version_page.document.language ) - document_page_content.save() + document_version_page_content.save() logger.info( 'Finished processing page: %d of document version: %s', - document_page.page_number, document_page.document_version + document_version_page.page_number, + document_version_page.document_version ) def process_document_version(self, document_version): @@ -70,8 +76,10 @@ class DocumentPageOCRContentManager(models.Manager): logger.debug('document version: %d', document_version.pk) try: - for document_page in document_version.pages.all(): - self.process_document_page(document_page=document_page) + for document_version_page in document_version.pages.all(): + self.process_document_version_page( + document_version_page=document_version_page + ) except Exception as exception: logger.error( 'OCR error for document version: %d; %s', document_version.pk, diff --git a/mayan/apps/ocr/methods.py b/mayan/apps/ocr/methods.py index b1c11a8265..567fa4b79a 100644 --- a/mayan/apps/ocr/methods.py +++ b/mayan/apps/ocr/methods.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals from datetime import timedelta +from django.apps import apps from django.utils.timezone import now from mayan.apps.common.settings import settings_db_sync_task_delay @@ -17,6 +18,17 @@ def method_document_ocr_submit(self): latest_version.submit_for_ocr() +def method_document_page_get_ocr_content(self): + DocumentVersionPageOCRContent = apps.get_model( + app_label='ocr', model_name='DocumentVersionPageOCRContent' + ) + + try: + return self.content_object.ocr_content.content + except (AttributeError, DocumentVersionPageOCRContent.DoesNotExist): + return None + + def method_document_version_ocr_submit(self): event_ocr_document_version_submit.commit( action_object=self.document, target=self diff --git a/mayan/apps/ocr/migrations/0009_rename_page_content.py b/mayan/apps/ocr/migrations/0009_rename_page_content.py index 20f6e3babb..f98ee91251 100644 --- a/mayan/apps/ocr/migrations/0009_rename_page_content.py +++ b/mayan/apps/ocr/migrations/0009_rename_page_content.py @@ -19,10 +19,24 @@ class Migration(migrations.Migration): name='document_page', field=models.OneToOneField( on_delete=django.db.models.deletion.CASCADE, - name='document_version_page', + #name='document_version_page', related_name='ocr_content', to='documents.DocumentVersionPage', verbose_name='Document version page' ), ), + migrations.RenameField( + model_name='documentversionpageocrcontent', + old_name='document_page', + new_name='document_version_page', + ), + migrations.AlterModelOptions( + name='documentversionpageocrcontent', + options={ + 'verbose_name': 'Document version page OCR content', + 'verbose_name_plural': 'Document version pages OCR contents' + }, + ), ] + + diff --git a/mayan/apps/ocr/tests/test_api.py b/mayan/apps/ocr/tests/test_api.py index 95c03e288a..7534594ce4 100644 --- a/mayan/apps/ocr/tests/test_api.py +++ b/mayan/apps/ocr/tests/test_api.py @@ -12,13 +12,34 @@ from ..permissions import ( from .literals import TEST_DOCUMENT_CONTENT -class OCRAPITestCase(DocumentTestMixin, BaseAPITestCase): +class OCRAPIViewTestMixin(object): def _request_document_ocr_submit_view(self): return self.post( viewname='rest_api:document-ocr-submit-view', kwargs={'pk': self.test_document.pk} ) + def _request_document_version_ocr_submit_view(self): + return self.post( + viewname='rest_api:document-version-ocr-submit-view', kwargs={ + 'document_pk': self.test_document.pk, + 'version_pk': self.test_document.latest_version.pk + } + ) + + def _request_document_version_page_content_view(self): + return self.get( + viewname='rest_api:document-page-ocr-content-view', kwargs={ + 'document_pk': self.test_document.pk, + 'version_pk': self.test_document.latest_version.pk, + 'page_pk': self.test_document.latest_version.pages.first().pk, + } + ) + + +class OCRAPIViewTestCase( + OCRAPIViewTestMixin, DocumentTestMixin, BaseAPITestCase +): def test_submit_document_no_access(self): response = self._request_document_ocr_submit_view() self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) @@ -35,15 +56,9 @@ class OCRAPITestCase(DocumentTestMixin, BaseAPITestCase): self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) self.assertTrue( - hasattr(self.test_document.pages.first(), 'ocr_content') - ) - - def _request_document_version_ocr_submit_view(self): - return self.post( - viewname='rest_api:document-version-ocr-submit-view', kwargs={ - 'document_pk': self.test_document.pk, - 'version_pk': self.test_document.latest_version.pk - } + hasattr( + self.test_document.pages.first().content_type, 'ocr_content' + ) ) def test_submit_document_version_no_access(self): @@ -62,16 +77,7 @@ class OCRAPITestCase(DocumentTestMixin, BaseAPITestCase): self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) self.assertTrue( - hasattr(self.test_document.pages.first(), 'ocr_content') - ) - - def _request_document_version_page_content_view(self): - return self.get( - viewname='rest_api:document-page-ocr-content-view', kwargs={ - 'document_pk': self.test_document.pk, - 'version_pk': self.test_document.latest_version.pk, - 'page_pk': self.test_document.latest_version.pages.first().pk, - } + hasattr(self.test_document_version.pages.first(), 'ocr_content') ) def test_get_document_version_page_content_no_access(self): diff --git a/mayan/apps/ocr/tests/test_models.py b/mayan/apps/ocr/tests/test_models.py index cd4992d0d4..9bff1738a2 100644 --- a/mayan/apps/ocr/tests/test_models.py +++ b/mayan/apps/ocr/tests/test_models.py @@ -19,7 +19,7 @@ class DocumentOCRTestCase(DocumentTestMixin, BaseTestCase): _skip_file_descriptor_test = True def test_ocr_language_backends_end(self): - content = self.test_document.pages.first().ocr_content.content + content = self.test_document.pages.first().get_ocr_content() self.assertTrue(TEST_DOCUMENT_CONTENT in content) diff --git a/mayan/apps/ocr/tests/test_views.py b/mayan/apps/ocr/tests/test_views.py index d7759fbfc1..03a7ec7baf 100644 --- a/mayan/apps/ocr/tests/test_views.py +++ b/mayan/apps/ocr/tests/test_views.py @@ -30,7 +30,7 @@ class OCRViewTestMixin(object): def _request_document_version_page_content_view(self): return self.get( viewname='ocr:document_version_page_ocr_content', kwargs={ - 'pk': self.test_document.pages.first().pk + 'pk': self.test_document_version.pages.first().pk } ) @@ -87,7 +87,7 @@ class OCRViewsTestCase(OCRViewTestMixin, GenericDocumentViewTestCase): self.assertTrue( DocumentVersionPageOCRContent.objects.filter( - document_version_page=self.test_document.pages.first() + document_version_page=self.test_document.pages.first().content_object ).exists() ) @@ -102,7 +102,7 @@ class OCRViewsTestCase(OCRViewTestMixin, GenericDocumentViewTestCase): self.assertFalse( DocumentVersionPageOCRContent.objects.filter( - document_version_page=self.test_document.pages.first() + document_version_page=self.test_document.pages.first().content_object ).exists() ) diff --git a/mayan/apps/ocr/urls.py b/mayan/apps/ocr/urls.py index 3d2d9b0b25..a8b9d9791e 100644 --- a/mayan/apps/ocr/urls.py +++ b/mayan/apps/ocr/urls.py @@ -8,9 +8,10 @@ from .api_views import ( ) from .views import ( DocumentOCRContentDeleteView, DocumentOCRContentView, - DocumentOCRDownloadView, - DocumentOCRErrorsListView, DocumentPageOCRContentView, DocumentSubmitView, - DocumentTypeSettingsEditView, DocumentTypeSubmitView, EntryListView + DocumentOCRDownloadView, DocumentOCRErrorsListView, + DocumentPageOCRContentView, DocumentSubmitView, + DocumentTypeSettingsEditView, DocumentTypeSubmitView, + DocumentVersionPageOCRContentView, EntryListView ) urlpatterns = [ @@ -50,6 +51,11 @@ urlpatterns = [ view=DocumentPageOCRContentView.as_view(), name='document_page_ocr_content' ), + url( + regex=r'^documents/versions/pages/(?P\d+)/content/$', + view=DocumentVersionPageOCRContentView.as_view(), + name='document_version_page_ocr_content' + ), url( regex=r'^document_types/submit/$', view=DocumentTypeSubmitView.as_view(), name='document_type_submit' diff --git a/mayan/apps/ocr/utils.py b/mayan/apps/ocr/utils.py index 0ac5de5fb3..a192613999 100644 --- a/mayan/apps/ocr/utils.py +++ b/mayan/apps/ocr/utils.py @@ -4,15 +4,29 @@ from django.apps import apps from django.utils.encoding import force_text -def get_document_ocr_content(document): - DocumentPageOCRContent = apps.get_model( - app_label='ocr', model_name='DocumentPageOCRContent' +def get_document_version_ocr_content(document_version): + DocumentVersionPageOCRContent = apps.get_model( + app_label='ocr', model_name='DocumentVersionPageOCRContent' ) - for page in document.pages.all(): + for document_version_page in document_version.pages.all(): try: - page_content = page.ocr_content.content - except DocumentPageOCRContent.DoesNotExist: + page_content = document_version_page.ocr_content.content + except DocumentVersionPageOCRContent.DoesNotExist: + pass + else: + yield force_text(page_content) + + +def get_document_ocr_content(document): + DocumentVersionPageOCRContent = apps.get_model( + app_label='ocr', model_name='DocumentVersionPageOCRContent' + ) + + for document_page in document.pages.all(): + try: + page_content = document_page.content_object.ocr_content.content + except (AttributeError, DocumentVersionPageOCRContent.DoesNotExist): pass else: yield force_text(page_content) diff --git a/mayan/apps/ocr/views.py b/mayan/apps/ocr/views.py index 7541784886..21e7c00fb0 100644 --- a/mayan/apps/ocr/views.py +++ b/mayan/apps/ocr/views.py @@ -13,10 +13,13 @@ from mayan.apps.common.generics import ( from mayan.apps.common.mixins import ExternalObjectMixin from mayan.apps.documents.forms import DocumentTypeFilteredSelectForm from mayan.apps.documents.models import ( - Document, DocumentType, DocumentVersionPage + Document, DocumentPage, DocumentType, DocumentVersionPage ) -from .forms import DocumentPageOCRContentForm, DocumentOCRContentForm +from .forms import ( + DocumentPageOCRContentForm, DocumentOCRContentForm, + DocumentVersionPageOCRContentForm +) from .models import DocumentVersionPageOCRContent, DocumentVersionOCRError from .permissions import ( permission_ocr_content_view, permission_ocr_document, @@ -76,7 +79,7 @@ class DocumentOCRContentView(SingleObjectDetailView): class DocumentPageOCRContentView(SingleObjectDetailView): form_class = DocumentPageOCRContentForm - model = DocumentVersionPage + model = DocumentPage object_permission = permission_ocr_content_view def dispatch(self, request, *args, **kwargs): @@ -96,6 +99,30 @@ class DocumentPageOCRContentView(SingleObjectDetailView): } +class DocumentVersionPageOCRContentView(SingleObjectDetailView): + form_class = DocumentVersionPageOCRContentForm + model = DocumentVersionPage + object_permission = permission_ocr_content_view + + def dispatch(self, request, *args, **kwargs): + result = super(DocumentVersionPageOCRContentView, self).dispatch( + request, *args, **kwargs + ) + self.get_object().document.add_as_recent_document_for_user( + user=request.user + ) + return result + + def get_extra_context(self): + return { + 'hide_labels': True, + 'object': self.get_object(), + 'title': _( + 'OCR result for document version page: %s' + ) % self.get_object(), + } + + class DocumentSubmitView(MultipleObjectConfirmActionView): model = Document object_permission = permission_ocr_document From a4a12b0cfe9e47d181ec5b47145b438ac2b29130 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 9 Oct 2019 21:09:36 -0400 Subject: [PATCH 21/26] Fix tests Signed-off-by: Roberto Rosario --- mayan/apps/ocr/tests/test_api.py | 2 +- mayan/apps/ocr/tests/test_models.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mayan/apps/ocr/tests/test_api.py b/mayan/apps/ocr/tests/test_api.py index 7534594ce4..578e8a94b8 100644 --- a/mayan/apps/ocr/tests/test_api.py +++ b/mayan/apps/ocr/tests/test_api.py @@ -57,7 +57,7 @@ class OCRAPIViewTestCase( self.assertTrue( hasattr( - self.test_document.pages.first().content_type, 'ocr_content' + self.test_document.pages.first().content_object, 'ocr_content' ) ) diff --git a/mayan/apps/ocr/tests/test_models.py b/mayan/apps/ocr/tests/test_models.py index 9bff1738a2..fe31995990 100644 --- a/mayan/apps/ocr/tests/test_models.py +++ b/mayan/apps/ocr/tests/test_models.py @@ -19,7 +19,7 @@ class DocumentOCRTestCase(DocumentTestMixin, BaseTestCase): _skip_file_descriptor_test = True def test_ocr_language_backends_end(self): - content = self.test_document.pages.first().get_ocr_content() + content = self.test_document.pages.first().content_object.ocr_content.content self.assertTrue(TEST_DOCUMENT_CONTENT in content) @@ -40,7 +40,7 @@ class GermanOCRSupportTestCase(DocumentTestMixin, BaseTestCase): ) def test_ocr_language_backends_end(self): - content = self.test_document.pages.first().ocr_content.content + content = self.test_document.pages.first().content_object.ocr_content.content self.assertTrue( TEST_DOCUMENT_CONTENT_DEU_1 in content From 03379ab8ec93aacd9f67b39ec2b1e00d2034dce5 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Thu, 10 Oct 2019 01:14:46 -0400 Subject: [PATCH 22/26] Fix parsing tests Signed-off-by: Roberto Rosario --- mayan/apps/document_parsing/apps.py | 9 ++-- mayan/apps/document_parsing/forms.py | 8 +-- mayan/apps/document_parsing/managers.py | 4 +- mayan/apps/document_parsing/models.py | 52 +++++++++---------- mayan/apps/document_parsing/parsers.py | 38 ++++++++------ mayan/apps/document_parsing/tasks.py | 6 +-- .../document_parsing/tests/test_events.py | 4 +- .../document_parsing/tests/test_parsers.py | 2 +- .../apps/document_parsing/tests/test_views.py | 10 ++-- mayan/apps/document_parsing/utils.py | 24 +++++++-- mayan/apps/documents/apps.py | 2 +- mayan/apps/ocr/apps.py | 2 +- 12 files changed, 94 insertions(+), 67 deletions(-) diff --git a/mayan/apps/document_parsing/apps.py b/mayan/apps/document_parsing/apps.py index 7def216213..504e5efc3d 100644 --- a/mayan/apps/document_parsing/apps.py +++ b/mayan/apps/document_parsing/apps.py @@ -45,7 +45,7 @@ from .permissions import ( permission_parse_document ) from .signals import post_document_version_parsing -from .utils import get_document_content +from .utils import get_document_content, get_document_version_content logger = logging.getLogger(__name__) @@ -65,7 +65,7 @@ class DocumentParsingApp(MayanAppConfig): app_label='documents', model_name='Document' ) DocumentPage = apps.get_model( - app_label='documents', model_name='DocumentVersionPage' + app_label='documents', model_name='DocumentPage' ) DocumentType = apps.get_model( app_label='documents', model_name='DocumentType' @@ -76,6 +76,9 @@ class DocumentParsingApp(MayanAppConfig): DocumentVersion = apps.get_model( app_label='documents', model_name='DocumentVersion' ) + DocumentVersionPage = apps.get_model( + app_label='documents', model_name='DocumentVersionPage' + ) DocumentVersionParseError = self.get_model( model_name='DocumentVersionParseError' ) @@ -87,7 +90,7 @@ class DocumentParsingApp(MayanAppConfig): name='content', value=get_document_content ) DocumentVersion.add_to_class( - name='content', value=get_document_content + name='content', value=get_document_version_content ) DocumentVersion.add_to_class( name='submit_for_parsing', diff --git a/mayan/apps/document_parsing/forms.py b/mayan/apps/document_parsing/forms.py index 12778c6acb..2803c9cd59 100644 --- a/mayan/apps/document_parsing/forms.py +++ b/mayan/apps/document_parsing/forms.py @@ -26,9 +26,9 @@ class DocumentContentForm(forms.Form): except AttributeError: document_pages = [] - for page in document_pages: + for document_page in document_pages: try: - page_content = page.content.content + page_content = document_page.content_object.content.content except DocumentVersionPageContent.DoesNotExist: pass else: @@ -37,7 +37,7 @@ class DocumentContentForm(forms.Form): '\n\n\n
- %s -
\n\n\n' % ( ugettext( 'Page %(page_number)d' - ) % {'page_number': page.page_number} + ) % {'page_number': document_page.page_number} ) ) @@ -72,7 +72,7 @@ class DocumentPageContentForm(forms.Form): self.fields['contents'].initial = '' try: - page_content = document_page.content.content + page_content = document_page.content_object.content.content except DocumentVersionPageContent.DoesNotExist: pass else: diff --git a/mayan/apps/document_parsing/managers.py b/mayan/apps/document_parsing/managers.py index 1669f89e70..7748596f2e 100644 --- a/mayan/apps/document_parsing/managers.py +++ b/mayan/apps/document_parsing/managers.py @@ -22,7 +22,9 @@ class DocumentPageContentManager(models.Manager): def delete_content_for(self, document, user=None): with transaction.atomic(): for document_page in document.pages.all(): - self.filter(document_page=document_page).delete() + self.filter( + document_version_page=document_page.content_object + ).delete() event_parsing_document_content_deleted.commit( actor=user, target=document diff --git a/mayan/apps/document_parsing/models.py b/mayan/apps/document_parsing/models.py index 9c7483f448..05811b63c1 100644 --- a/mayan/apps/document_parsing/models.py +++ b/mayan/apps/document_parsing/models.py @@ -11,32 +11,6 @@ from mayan.apps.documents.models import ( from .managers import DocumentPageContentManager, DocumentTypeSettingsManager -@python_2_unicode_compatible -class DocumentVersionPageContent(models.Model): - """ - This model store's the parsed content of a document page. - """ - document_version_page = models.OneToOneField( - on_delete=models.CASCADE, related_name='content', - to=DocumentVersionPage, verbose_name=_('Document version page') - ) - content = models.TextField( - blank=True, help_text=_( - 'The actual text content as extracted by the document ' - 'parsing backend.' - ), verbose_name=_('Content') - ) - - objects = DocumentPageContentManager() - - class Meta: - verbose_name = _('Document version page content') - verbose_name_plural = _('Document version pages contents') - - def __str__(self): - return force_text(self.document_page) - - class DocumentTypeSettings(models.Model): """ This model stores the parsing settings for a document type. @@ -62,6 +36,32 @@ class DocumentTypeSettings(models.Model): verbose_name_plural = _('Document types settings') +@python_2_unicode_compatible +class DocumentVersionPageContent(models.Model): + """ + This model store's the parsed content of a document page. + """ + document_version_page = models.OneToOneField( + on_delete=models.CASCADE, related_name='content', + to=DocumentVersionPage, verbose_name=_('Document version page') + ) + content = models.TextField( + blank=True, help_text=_( + 'The actual text content as extracted by the document ' + 'parsing backend.' + ), verbose_name=_('Content') + ) + + objects = DocumentPageContentManager() + + class Meta: + verbose_name = _('Document version page content') + verbose_name_plural = _('Document version pages contents') + + def __str__(self): + return force_text(self.document_page) + + @python_2_unicode_compatible class DocumentVersionParseError(models.Model): """ diff --git a/mayan/apps/document_parsing/parsers.py b/mayan/apps/document_parsing/parsers.py index 8ad24b4115..469b974377 100644 --- a/mayan/apps/document_parsing/parsers.py +++ b/mayan/apps/document_parsing/parsers.py @@ -23,11 +23,13 @@ class Parser(object): _registry = {} @classmethod - def parse_document_page(cls, document_page): - for parser_class in cls._registry.get(document_page.document_version.mimetype, ()): + def parse_document_version_page(cls, document_version_page): + for parser_class in cls._registry.get(document_version_page.document_version.mimetype, ()): try: parser = parser_class() - parser.process_document_page(document_page) + parser.process_document_page( + document_version_page=document_version_page + ) except ParserError: # If parser raises error, try next parser in the list pass @@ -41,7 +43,9 @@ class Parser(object): for parser_class in cls._registry.get(document_version.mimetype, ()): try: parser = parser_class() - parser.process_document_version(document_version) + parser.process_document_version( + document_version=document_version + ) except ParserError: # If parser raises error, try next parser in the list pass @@ -64,10 +68,12 @@ class Parser(object): ) logger.debug('document version: %d', document_version.pk) - for document_page in document_version.pages.all(): - self.process_document_page(document_page=document_page) + for document_version_page in document_version.pages.all(): + self.process_document_version_page( + document_version_page=document_version_page + ) - def process_document_page(self, document_page): + def process_document_version_page(self, document_version_page): DocumentVersionPageContent = apps.get_model( app_label='document_parsing', model_name='DocumentVersionPageContent' @@ -75,19 +81,20 @@ class Parser(object): logger.info( 'Processing page: %d of document version: %s', - document_page.page_number, document_page.document_version + document_version_page.page_number, + document_version_page.document_version ) - file_object = document_page.document_version.get_intermediate_file() + file_object = document_version_page.document_version.get_intermediate_file() try: - document_page_content, created = DocumentVersionPageContent.objects.get_or_create( - document_page=document_page + document_version_page_content, created = DocumentVersionPageContent.objects.get_or_create( + document_version_page=document_version_page ) - document_page_content.content = self.execute( - file_object=file_object, page_number=document_page.page_number + document_version_page_content.content = self.execute( + file_object=file_object, page_number=document_version_page.page_number ) - document_page_content.save() + document_version_page_content.save() except Exception as exception: error_message = _('Exception parsing page; %s') % exception logger.error(error_message) @@ -97,7 +104,8 @@ class Parser(object): logger.info( 'Finished processing page: %d of document version: %s', - document_page.page_number, document_page.document_version + document_version_page.page_number, + document_version_page.document_version ) def execute(self, file_object, page_number): diff --git a/mayan/apps/document_parsing/tasks.py b/mayan/apps/document_parsing/tasks.py index 4debffbc60..653552f741 100644 --- a/mayan/apps/document_parsing/tasks.py +++ b/mayan/apps/document_parsing/tasks.py @@ -14,8 +14,8 @@ def task_parse_document_version(document_version_pk): DocumentVersion = apps.get_model( app_label='documents', model_name='DocumentVersion' ) - DocumentPageContent = apps.get_model( - app_label='document_parsing', model_name='DocumentPageContent' + DocumentVersionPageContent = apps.get_model( + app_label='document_parsing', model_name='DocumentVersionPageContent' ) document_version = DocumentVersion.objects.get( @@ -24,6 +24,6 @@ def task_parse_document_version(document_version_pk): logger.info( 'Starting parsing for document version: %s', document_version ) - DocumentPageContent.objects.process_document_version( + DocumentVersionPageContent.objects.process_document_version( document_version=document_version ) diff --git a/mayan/apps/document_parsing/tests/test_events.py b/mayan/apps/document_parsing/tests/test_events.py index 1a2860305d..17bbe198fb 100644 --- a/mayan/apps/document_parsing/tests/test_events.py +++ b/mayan/apps/document_parsing/tests/test_events.py @@ -10,7 +10,7 @@ from ..events import ( event_parsing_document_version_submit, event_parsing_document_version_finish ) -from ..models import DocumentPageContent +from ..models import DocumentVersionPageContent class DocumentParsingEventsTestCase(GenericDocumentTestCase): @@ -19,7 +19,7 @@ class DocumentParsingEventsTestCase(GenericDocumentTestCase): def test_document_content_deleted_event(self): Action.objects.all().delete() - DocumentPageContent.objects.delete_content_for( + DocumentVersionPageContent.objects.delete_content_for( document=self.test_document ) diff --git a/mayan/apps/document_parsing/tests/test_parsers.py b/mayan/apps/document_parsing/tests/test_parsers.py index 237bccc567..2eeb8703a6 100644 --- a/mayan/apps/document_parsing/tests/test_parsers.py +++ b/mayan/apps/document_parsing/tests/test_parsers.py @@ -18,5 +18,5 @@ class ParserTestCase(DocumentTestMixin, BaseTestCase): parser.process_document_version(self.test_document.latest_version) self.assertTrue( - TEST_DOCUMENT_CONTENT in self.test_document.pages.first().content.content + TEST_DOCUMENT_CONTENT in self.test_document.pages.first().content_object.content.content ) diff --git a/mayan/apps/document_parsing/tests/test_views.py b/mayan/apps/document_parsing/tests/test_views.py index bb88c1817b..21ca0ef808 100644 --- a/mayan/apps/document_parsing/tests/test_views.py +++ b/mayan/apps/document_parsing/tests/test_views.py @@ -5,7 +5,7 @@ from django.test import override_settings from mayan.apps.documents.tests.base import GenericDocumentViewTestCase from mayan.apps.documents.tests.literals import TEST_HYBRID_DOCUMENT -from ..models import DocumentPageContent +from ..models import DocumentVersionPageContent from ..permissions import ( permission_content_view, permission_document_type_parsing_setup, permission_parse_document @@ -72,8 +72,8 @@ class DocumentContentViewsTestCase( self.assertEqual(response.status_code, 404) self.assertTrue( - DocumentPageContent.objects.filter( - document_page=self.test_document.pages.first() + DocumentVersionPageContent.objects.filter( + document_version_page=self.test_document.pages.first().content_object ).exists() ) @@ -86,8 +86,8 @@ class DocumentContentViewsTestCase( self.assertEqual(response.status_code, 302) self.assertFalse( - DocumentPageContent.objects.filter( - document_page=self.test_document.pages.first() + DocumentVersionPageContent.objects.filter( + document_version_page=self.test_document.pages.first().content_object ).exists() ) diff --git a/mayan/apps/document_parsing/utils.py b/mayan/apps/document_parsing/utils.py index ab8e049450..5086d5cf02 100644 --- a/mayan/apps/document_parsing/utils.py +++ b/mayan/apps/document_parsing/utils.py @@ -6,14 +6,28 @@ from django.utils.html import conditional_escape def get_document_content(document): - DocumentPageContent = apps.get_model( - app_label='document_parsing', model_name='DocumentPageContent' + DocumentVersionPageContent = apps.get_model( + app_label='document_parsing', model_name='DocumentVersionPageContent' ) - for page in document.pages.all(): + for document_page in document.pages.all(): try: - page_content = page.content.content - except DocumentPageContent.DoesNotExist: + page_content = document_page.content_object.content.content + except DocumentVersionPageContent.DoesNotExist: + pass + else: + yield conditional_escape(force_text(page_content)) + + +def get_document_version_content(document_version): + DocumentVersionPageContent = apps.get_model( + app_label='document_parsing', model_name='DocumentVersionPageContent' + ) + + for document_version_page in document_version.pages.all(): + try: + page_content = document_version_page.content.content + except DocumentVersionPageContent.DoesNotExist: pass else: yield conditional_escape(force_text(page_content)) diff --git a/mayan/apps/documents/apps.py b/mayan/apps/documents/apps.py index 4faae2f8bd..541134fb4b 100644 --- a/mayan/apps/documents/apps.py +++ b/mayan/apps/documents/apps.py @@ -235,7 +235,7 @@ class DocumentsApp(MayanAppConfig): model=DocumentPage, manager_name='passthrough' ) ModelPermission.register_inheritance( - model=DocumentPageResult, related='document_version__document', + model=DocumentPageResult, related='document', ) ModelPermission.register_manager( model=DocumentPageResult, manager_name='passthrough' diff --git a/mayan/apps/ocr/apps.py b/mayan/apps/ocr/apps.py index 1d0410bd44..772ec287fa 100644 --- a/mayan/apps/ocr/apps.py +++ b/mayan/apps/ocr/apps.py @@ -66,7 +66,7 @@ class OCRApp(MayanAppConfig): app_label='documents', model_name='Document' ) DocumentPage = apps.get_model( - app_label='documents', model_name='DocumentVersionPage' + app_label='documents', model_name='DocumentPage' ) DocumentType = apps.get_model( app_label='documents', model_name='DocumentType' From ff03ea07cab89601a5739e0684df5983c3a14096 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Thu, 10 Oct 2019 02:40:25 -0400 Subject: [PATCH 23/26] Add support for appending pages Add version upload form checkbox. Add the append_pages keyword argument. Signed-off-by: Roberto Rosario --- mayan/apps/document_parsing/apps.py | 2 +- mayan/apps/document_parsing/icons.py | 3 +++ mayan/apps/document_parsing/links.py | 14 +++++++------- mayan/apps/document_parsing/managers.py | 2 +- mayan/apps/document_parsing/models.py | 6 ++++-- mayan/apps/documents/models/document_models.py | 4 ++-- .../models/document_version_models.py | 11 +++++++++-- mayan/apps/documents/tasks.py | 4 ++-- mayan/apps/ocr/links.py | 2 +- mayan/apps/ocr/managers.py | 2 +- mayan/apps/ocr/models.py | 4 ++-- mayan/apps/sources/forms.py | 4 ++++ mayan/apps/sources/views.py | 18 +++++++++++------- 13 files changed, 48 insertions(+), 28 deletions(-) diff --git a/mayan/apps/document_parsing/apps.py b/mayan/apps/document_parsing/apps.py index 504e5efc3d..c7bd4e4773 100644 --- a/mayan/apps/document_parsing/apps.py +++ b/mayan/apps/document_parsing/apps.py @@ -148,7 +148,7 @@ class DocumentParsingApp(MayanAppConfig): menu_facet.bind_links( links=(link_document_content,), sources=(Document,) ) - menu_facet.bind_links( + menu_list_facet.bind_links( links=(link_document_page_content,), sources=(DocumentPage,) ) menu_list_facet.bind_links( diff --git a/mayan/apps/document_parsing/icons.py b/mayan/apps/document_parsing/icons.py index 6aad4a8ea2..9fb872dc1f 100644 --- a/mayan/apps/document_parsing/icons.py +++ b/mayan/apps/document_parsing/icons.py @@ -17,6 +17,9 @@ icon_document_content_download = Icon( icon_document_multiple_submit = Icon( driver_name='fontawesome', symbol='font' ) +icon_document_page_content = Icon( + driver_name='fontawesome', symbol='font' +) icon_document_submit = Icon( driver_name='fontawesome', symbol='font' ) diff --git a/mayan/apps/document_parsing/links.py b/mayan/apps/document_parsing/links.py index 2cfe95fab9..a7de474581 100644 --- a/mayan/apps/document_parsing/links.py +++ b/mayan/apps/document_parsing/links.py @@ -32,9 +32,15 @@ link_document_content_delete_multiple = Link( text=_('Delete parsed content'), view='document_parsing:document_content_delete_multiple', ) +link_document_content_download = Link( + args='resolved_object.id', + icon_class_path='mayan.apps.document_parsing.icons.icon_document_content_download', + permissions=(permission_content_view,), text=_('Download content'), + view='document_parsing:document_content_download' +) link_document_page_content = Link( args='resolved_object.id', conditional_disable=is_document_page_disabled, - icon_class_path='mayan.apps.document_parsing.icons.icon_document_content', + icon_class_path='mayan.apps.document_parsing.icons.icon_document_page_content', permissions=(permission_content_view,), text=_('Content'), view='document_parsing:document_page_content' ) @@ -44,12 +50,6 @@ link_document_parsing_errors_list = Link( permissions=(permission_content_view,), text=_('Parsing errors'), view='document_parsing:document_parsing_error_list' ) -link_document_content_download = Link( - args='resolved_object.id', - icon_class_path='mayan.apps.document_parsing.icons.icon_document_content_download', - permissions=(permission_content_view,), text=_('Download content'), - view='document_parsing:document_content_download' -) link_document_submit_multiple = Link( icon_class_path='mayan.apps.document_parsing.icons.icon_document_submit', text=_('Submit for parsing'), diff --git a/mayan/apps/document_parsing/managers.py b/mayan/apps/document_parsing/managers.py index 7748596f2e..8ff1d5d231 100644 --- a/mayan/apps/document_parsing/managers.py +++ b/mayan/apps/document_parsing/managers.py @@ -18,7 +18,7 @@ from .signals import post_document_version_parsing logger = logging.getLogger(__name__) -class DocumentPageContentManager(models.Manager): +class DocumentVersionPageContentManager(models.Manager): def delete_content_for(self, document, user=None): with transaction.atomic(): for document_page in document.pages.all(): diff --git a/mayan/apps/document_parsing/models.py b/mayan/apps/document_parsing/models.py index 05811b63c1..48c2dc79af 100644 --- a/mayan/apps/document_parsing/models.py +++ b/mayan/apps/document_parsing/models.py @@ -8,7 +8,9 @@ from mayan.apps.documents.models import ( DocumentPage, DocumentType, DocumentVersion, DocumentVersionPage ) -from .managers import DocumentPageContentManager, DocumentTypeSettingsManager +from .managers import ( + DocumentVersionPageContentManager, DocumentTypeSettingsManager +) class DocumentTypeSettings(models.Model): @@ -52,7 +54,7 @@ class DocumentVersionPageContent(models.Model): ), verbose_name=_('Content') ) - objects = DocumentPageContentManager() + objects = DocumentVersionPageContentManager() class Meta: verbose_name = _('Document version page content') diff --git a/mayan/apps/documents/models/document_models.py b/mayan/apps/documents/models/document_models.py index 2177119e14..95cbba506d 100644 --- a/mayan/apps/documents/models/document_models.py +++ b/mayan/apps/documents/models/document_models.py @@ -178,7 +178,7 @@ class Document(models.Model): return (self.uuid,) natural_key.dependencies = ['documents.DocumentType'] - def new_version(self, file_object, comment=None, _user=None): + def new_version(self, file_object, append_pages=False, comment=None, _user=None): logger.info('Creating new document version for document: %s', self) DocumentVersion = apps.get_model( app_label='documents', model_name='DocumentVersion' @@ -187,7 +187,7 @@ class Document(models.Model): document_version = DocumentVersion( document=self, comment=comment or '', file=File(file_object) ) - document_version.save(_user=_user) + document_version.save(append_pages=append_pages, _user=_user) logger.info('New document version queued for document: %s', self) diff --git a/mayan/apps/documents/models/document_version_models.py b/mayan/apps/documents/models/document_version_models.py index 7d8cca828a..d491ca5bd8 100644 --- a/mayan/apps/documents/models/document_version_models.py +++ b/mayan/apps/documents/models/document_version_models.py @@ -274,6 +274,7 @@ class DocumentVersion(models.Model): Overloaded save method that updates the document version's checksum, mimetype, and page count when created """ + append_pages = kwargs.pop('append_pages', False) user = kwargs.pop('_user', None) new_document_version = not self.pk @@ -293,7 +294,7 @@ class DocumentVersion(models.Model): # Only do this for new documents self.update_checksum(save=False) self.update_mimetype(save=False) - self.save() + self.save(append_pages=append_pages, _user=user) self.update_page_count(save=False) if setting_fix_orientation.value: self.fix_orientation() @@ -326,7 +327,13 @@ class DocumentVersion(models.Model): sender=Document, instance=self.document ) - self.document.pages_reset(update_page_count=False) + if append_pages: + for version_page in self.pages.all(): + self.document.pages.create( + content_object = version_page + ) + else: + self.document.pages_reset(update_page_count=False) def save_to_file(self, file_object): """ diff --git a/mayan/apps/documents/tasks.py b/mayan/apps/documents/tasks.py index a2215ae69f..7d805ea327 100644 --- a/mayan/apps/documents/tasks.py +++ b/mayan/apps/documents/tasks.py @@ -158,7 +158,7 @@ def task_update_page_count(self, version_id): @app.task(bind=True, default_retry_delay=UPLOAD_NEW_VERSION_RETRY_DELAY, ignore_result=True) -def task_upload_new_version(self, document_id, shared_uploaded_file_id, user_id, comment=None): +def task_upload_new_version(self, document_id, shared_uploaded_file_id, user_id, append_pages=False, comment=None): SharedUploadedFile = apps.get_model( app_label='common', model_name='SharedUploadedFile' ) @@ -193,7 +193,7 @@ def task_upload_new_version(self, document_id, shared_uploaded_file_id, user_id, document=document, comment=comment or '', file=file_object ) try: - document_version.save(_user=user) + document_version.save(append_pages=append_pages, _user=user) except Warning as warning: # New document version are blocked logger.info( diff --git a/mayan/apps/ocr/links.py b/mayan/apps/ocr/links.py index ed01601ae0..68284d26ef 100644 --- a/mayan/apps/ocr/links.py +++ b/mayan/apps/ocr/links.py @@ -15,7 +15,7 @@ def is_document_page_disabled(context): link_document_page_ocr_content = Link( - args='resolved_object.id',# conditional_disable=is_document_page_disabled, + args='resolved_object.id', conditional_disable=is_document_page_disabled, icon_class_path='mayan.apps.ocr.icons.icon_document_page_ocr_content', permissions=(permission_ocr_content_view,), text=_('OCR'), view='ocr:document_page_ocr_content', diff --git a/mayan/apps/ocr/managers.py b/mayan/apps/ocr/managers.py index 4cbe31e713..869c454001 100644 --- a/mayan/apps/ocr/managers.py +++ b/mayan/apps/ocr/managers.py @@ -22,7 +22,7 @@ from .signals import post_document_version_ocr logger = logging.getLogger(__name__) -class DocumentPageOCRContentManager(models.Manager): +class DocumentVesionPageOCRContentManager(models.Manager): def delete_content_for(self, document, user=None): with transaction.atomic(): for document_page in document.pages.all(): diff --git a/mayan/apps/ocr/models.py b/mayan/apps/ocr/models.py index c54a2e12a7..adaf142a0c 100644 --- a/mayan/apps/ocr/models.py +++ b/mayan/apps/ocr/models.py @@ -9,7 +9,7 @@ from mayan.apps.documents.models import ( ) from .managers import ( - DocumentPageOCRContentManager, DocumentTypeSettingsManager + DocumentVesionPageOCRContentManager, DocumentTypeSettingsManager ) @@ -52,7 +52,7 @@ class DocumentVersionPageOCRContent(models.Model): ), verbose_name=_('Content') ) - objects = DocumentPageOCRContentManager() + objects = DocumentVesionPageOCRContentManager() class Meta: verbose_name = _('Document version page OCR content') diff --git a/mayan/apps/sources/forms.py b/mayan/apps/sources/forms.py index a6431f71e5..652e474613 100644 --- a/mayan/apps/sources/forms.py +++ b/mayan/apps/sources/forms.py @@ -31,6 +31,10 @@ class NewVersionForm(forms.Form): required=False, widget=forms.widgets.Textarea(attrs={'rows': 4}), ) + self.fields['append_pages'] = forms.BooleanField( + initial=False, label=_('Append pages?'), + required=False, + ) class UploadBaseForm(forms.Form): diff --git a/mayan/apps/sources/views.py b/mayan/apps/sources/views.py index be55047b7d..304edfa85f 100644 --- a/mayan/apps/sources/views.py +++ b/mayan/apps/sources/views.py @@ -362,7 +362,6 @@ class UploadInteractiveView(UploadBaseView): class UploadInteractiveVersionView(UploadBaseView): def dispatch(self, request, *args, **kwargs): - self.subtemplates_list = [] self.document = get_object_or_404( @@ -417,12 +416,17 @@ class UploadInteractiveVersionView(UploadBaseView): else: user_id = None - task_upload_new_version.apply_async(kwargs=dict( - shared_uploaded_file_id=shared_uploaded_file.pk, - document_id=self.document.pk, - user_id=user_id, - comment=forms['document_form'].cleaned_data.get('comment') - )) + task_upload_new_version.apply_async( + kwargs=dict( + append_pages=forms['document_form'].cleaned_data.get( + 'append_pages', False + ), + shared_uploaded_file_id=shared_uploaded_file.pk, + document_id=self.document.pk, + user_id=user_id, + comment=forms['document_form'].cleaned_data.get('comment') + ) + ) messages.success( message=_( From 739d49679991113874c9445b0268d9cb6c598dc4 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Thu, 10 Oct 2019 11:53:40 -0400 Subject: [PATCH 24/26] Add dedicated pages append action Signed-off-by: Roberto Rosario --- mayan/apps/common/generics.py | 12 ++++++------ mayan/apps/sources/apps.py | 15 +++++++++++---- mayan/apps/sources/forms.py | 29 +++++++++++++++++----------- mayan/apps/sources/icons.py | 4 ++++ mayan/apps/sources/links.py | 7 +++++++ mayan/apps/sources/urls.py | 15 ++++++++++++--- mayan/apps/sources/views.py | 36 ++++++++++++++++++++++++++--------- 7 files changed, 85 insertions(+), 33 deletions(-) diff --git a/mayan/apps/common/generics.py b/mayan/apps/common/generics.py index 37f5c268ea..310717743a 100644 --- a/mayan/apps/common/generics.py +++ b/mayan/apps/common/generics.py @@ -53,8 +53,8 @@ class MultiFormView(DjangoFormView): template_name = 'appearance/generic_form.html' def _create_form(self, form_name, klass): - form_kwargs = self.get_form_kwargs(form_name) - form_create_method = 'create_%s_form' % form_name + form_kwargs = self.get_form_kwargs(form_name=form_name) + form_create_method = 'create_{}_form'.format(form_name) if hasattr(self, form_create_method): form = getattr(self, form_create_method)(**form_kwargs) else: @@ -71,7 +71,7 @@ class MultiFormView(DjangoFormView): def forms_valid(self, forms): for form_name, form in forms.items(): - form_valid_method = '%s_form_valid' % form_name + form_valid_method = '{}_form_valid'.format(form_name) if hasattr(self, form_valid_method): return getattr(self, form_valid_method)(form) @@ -98,8 +98,8 @@ class MultiFormView(DjangoFormView): def get_form_kwargs(self, form_name): kwargs = {} - kwargs.update({'initial': self.get_initial(form_name)}) - kwargs.update({'prefix': self.get_prefix(form_name)}) + kwargs.update({'initial': self.get_initial(form_name=form_name)}) + kwargs.update({'prefix': self.get_prefix(form_name=form_name)}) if self.request.method in ('POST', 'PUT'): kwargs.update({ @@ -124,7 +124,7 @@ class MultiFormView(DjangoFormView): ) def get_initial(self, form_name): - initial_method = 'get_%s_initial' % form_name + initial_method = 'get_{}_initial'.format(form_name) if hasattr(self, initial_method): return getattr(self, initial_method)() else: diff --git a/mayan/apps/sources/apps.py b/mayan/apps/sources/apps.py index 7139e5a846..d948a54fa7 100644 --- a/mayan/apps/sources/apps.py +++ b/mayan/apps/sources/apps.py @@ -21,9 +21,10 @@ from .handlers import ( handler_create_default_document_source, handler_initialize_periodic_tasks ) from .links import ( - link_document_create_multiple, link_setup_sources, - link_setup_source_check_now, link_setup_source_create_imap_email, - link_setup_source_create_pop3_email, link_setup_source_create_sane_scanner, + link_document_create_multiple, link_document_pages_append, + link_setup_sources, link_setup_source_check_now, + link_setup_source_create_imap_email, link_setup_source_create_pop3_email, + link_setup_source_create_sane_scanner, link_setup_source_create_watch_folder, link_setup_source_create_webform, link_setup_source_create_staging_folder, link_setup_source_delete, link_setup_source_edit, link_setup_source_logs, link_staging_file_delete, @@ -145,10 +146,16 @@ class SourcesApp(MayanAppConfig): menu_secondary.bind_links( links=(link_document_version_upload,), sources=( - 'documents:document_version_list', 'documents:upload_version', + 'documents:document_version_list', 'sources:upload_version', 'documents:document_version_revert' ) ) + menu_secondary.bind_links( + links=(link_document_pages_append,), + sources=( + 'documents:document_pages', 'sources:document_pages_append' + ) + ) post_upgrade.connect( receiver=handler_initialize_periodic_tasks, diff --git a/mayan/apps/sources/forms.py b/mayan/apps/sources/forms.py index 652e474613..e4eed9de64 100644 --- a/mayan/apps/sources/forms.py +++ b/mayan/apps/sources/forms.py @@ -23,18 +23,25 @@ class NewDocumentForm(DocumentForm): class NewVersionForm(forms.Form): - def __init__(self, *args, **kwargs): - super(NewVersionForm, self).__init__(*args, **kwargs) + comment = forms.CharField( + help_text=_('An optional comment to explain the upload.'), + label=_('Comment'), required=False, + widget=forms.widgets.Textarea(attrs={'rows': 4}), + ) - self.fields['comment'] = forms.CharField( - label=_('Comment'), - required=False, - widget=forms.widgets.Textarea(attrs={'rows': 4}), - ) - self.fields['append_pages'] = forms.BooleanField( - initial=False, label=_('Append pages?'), - required=False, - ) + append_pages = forms.BooleanField( + help_text=_( + 'If selected, the pages of the file uploaded will be appended ' + 'to the existing document pages. Otherwise the pages of the ' + 'upload will replace the existing pages of the document.' + ), label=_('Append pages?'), required=False, + ) + + def __init__(self, *args, **kwargs): + hide_append_pages = kwargs.pop('hide_append_pages', False) + super(NewVersionForm, self).__init__(*args, **kwargs) + if hide_append_pages: + self.fields['append_pages'].widget = forms.widgets.HiddenInput() class UploadBaseForm(forms.Form): diff --git a/mayan/apps/sources/icons.py b/mayan/apps/sources/icons.py index 1b0a6c8c84..a0a59fcded 100644 --- a/mayan/apps/sources/icons.py +++ b/mayan/apps/sources/icons.py @@ -9,6 +9,10 @@ icon_document_version_upload = Icon( driver_name='fontawesome', symbol='upload' ) icon_log = Icon(driver_name='fontawesome', symbol='exclamation-triangle') +icon_document_pages_append = Icon( + driver_name='fontawesome-dual', primary_symbol='copy', + secondary_symbol='plus' +) icon_setup_sources = Icon(driver_name='fontawesome', symbol='upload') icon_setup_source_check_now = Icon(driver_name='fontawesome', symbol='check') icon_setup_source_delete = Icon(driver_name='fontawesome', symbol='times') diff --git a/mayan/apps/sources/links.py b/mayan/apps/sources/links.py index b1793547f5..da5d3300d6 100644 --- a/mayan/apps/sources/links.py +++ b/mayan/apps/sources/links.py @@ -113,6 +113,13 @@ link_staging_file_delete = Link( permissions=(permission_document_new_version, permission_document_create), tags='dangerous', text=_('Delete'), view='sources:staging_file_delete', ) +link_document_pages_append = Link( + args='resolved_object.pk', + icon_class_path='mayan.apps.sources.icons.icon_document_pages_append', + permissions=(permission_document_new_version,), + text=_('Append pages'), + view='sources:document_pages_append' +) link_document_version_upload = Link( args='resolved_object.pk', condition=document_new_version_not_blocked, icon_class_path='mayan.apps.sources.icons.icon_document_version_upload', diff --git a/mayan/apps/sources/urls.py b/mayan/apps/sources/urls.py index fdc7adc330..173891f5f5 100644 --- a/mayan/apps/sources/urls.py +++ b/mayan/apps/sources/urls.py @@ -7,9 +7,10 @@ from .api_views import ( APIStagingSourceListView, APIStagingSourceView ) from .views import ( - SetupSourceCheckView, SetupSourceCreateView, SetupSourceDeleteView, - SetupSourceEditView, SetupSourceListView, SourceLogListView, - StagingFileDeleteView, UploadInteractiveVersionView, UploadInteractiveView + DocumentPagesAppendView, SetupSourceCheckView, SetupSourceCreateView, + SetupSourceDeleteView, SetupSourceEditView, SetupSourceListView, + SourceLogListView, StagingFileDeleteView, UploadInteractiveVersionView, + UploadInteractiveView ) from .wizards import DocumentCreateWizard @@ -41,6 +42,14 @@ urlpatterns = [ regex=r'^documents/(?P\d+)/versions/upload/interactive/$', view=UploadInteractiveVersionView.as_view(), name='upload_version' ), + url( + regex=r'^documents/(?P\d+)/pages/append/interactive/(?P\d+)/$', + view=DocumentPagesAppendView.as_view(), name='document_pages_append' + ), + url( + regex=r'^documents/(?P\d+)/pages/append/interactive/$', + view=DocumentPagesAppendView.as_view(), name='document_pages_append' + ), # Setup views diff --git a/mayan/apps/sources/views.py b/mayan/apps/sources/views.py index 304edfa85f..3f26a650c2 100644 --- a/mayan/apps/sources/views.py +++ b/mayan/apps/sources/views.py @@ -452,13 +452,6 @@ class UploadInteractiveVersionView(UploadBaseView): files=kwargs.get('files', None), ) - def create_document_form_form(self, **kwargs): - return self.get_form_classes()['document_form']( - prefix=kwargs['prefix'], - data=kwargs.get('data', None), - files=kwargs.get('files', None), - ) - def get_form_classes(self): return { 'document_form': NewVersionForm, @@ -471,8 +464,33 @@ class UploadInteractiveVersionView(UploadBaseView): ).get_context_data(**kwargs) context['object'] = self.document context['title'] = _( - 'Upload a new version from source: %s' - ) % self.source.label + 'Upload a new version for document "%(document)s" from source: %(source)s' + ) % {'document': self.document, 'source': self.source.label} + + context['submit_label'] = _('Submit') + + return context + + +class DocumentPagesAppendView(UploadInteractiveVersionView): + def get_document_form_initial(self): + return { + 'append_pages': True, + } + + def get_form_extra_kwargs(self, form_name): + if form_name == 'document_form': + return { + 'hide_append_pages': True + } + + def get_context_data(self, **kwargs): + context = super( + DocumentPagesAppendView, self + ).get_context_data(**kwargs) + context['title'] = _( + 'Append pages to document "%(document)s" from source: %(source)s' + ) % {'document': self.document, 'source': self.source.label} return context From bd0d298be3f8f4f6a56c8c093fb6823b1089319c Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Thu, 10 Oct 2019 14:34:50 -0400 Subject: [PATCH 25/26] New document version improvements from clients/bc - Comment field help text. - Remove create_document_form_form. - Use static NewVersionForm. - Update sources document upload and new version upload view names. Signed-off-by: Roberto Rosario --- .../apps/cabinets/tests/test_wizard_steps.py | 2 +- mayan/apps/checkouts/tests/test_views.py | 2 +- .../apps/metadata/tests/test_wizard_steps.py | 4 +-- mayan/apps/sources/apps.py | 5 ++-- mayan/apps/sources/forms.py | 13 ++++----- mayan/apps/sources/links.py | 2 +- mayan/apps/sources/tests/test_views.py | 10 +++---- mayan/apps/sources/urls.py | 14 ++++++---- mayan/apps/sources/views.py | 27 ++++++++----------- mayan/apps/sources/wizards.py | 2 +- mayan/apps/tags/tests/test_wizard_steps.py | 2 +- 11 files changed, 40 insertions(+), 43 deletions(-) diff --git a/mayan/apps/cabinets/tests/test_wizard_steps.py b/mayan/apps/cabinets/tests/test_wizard_steps.py index 84f7fabd82..3f93b768fa 100644 --- a/mayan/apps/cabinets/tests/test_wizard_steps.py +++ b/mayan/apps/cabinets/tests/test_wizard_steps.py @@ -33,7 +33,7 @@ class CabinetDocumentUploadTestCase(CabinetTestMixin, GenericDocumentViewTestCas def _request_upload_interactive_document_create_view(self): with open(TEST_SMALL_DOCUMENT_PATH, mode='rb') as file_object: return self.post( - viewname='sources:upload_interactive', kwargs={ + viewname='sources:document_upload_interactive', kwargs={ 'source_id': self.test_source.pk }, data={ 'document_type_id': self.test_document_type.pk, diff --git a/mayan/apps/checkouts/tests/test_views.py b/mayan/apps/checkouts/tests/test_views.py index 128e1643e0..c701618511 100644 --- a/mayan/apps/checkouts/tests/test_views.py +++ b/mayan/apps/checkouts/tests/test_views.py @@ -376,7 +376,7 @@ class NewVersionBlockViewTestCase( self.login_superuser() response = self.post( - viewname='sources:upload_version', kwargs={ + viewname='sources:document_version_upload', kwargs={ 'document_pk': self.test_document.pk }, follow=True ) diff --git a/mayan/apps/metadata/tests/test_wizard_steps.py b/mayan/apps/metadata/tests/test_wizard_steps.py index a2384fdd83..853d735085 100644 --- a/mayan/apps/metadata/tests/test_wizard_steps.py +++ b/mayan/apps/metadata/tests/test_wizard_steps.py @@ -34,7 +34,7 @@ class DocumentUploadMetadataTestCase(MetadataTypeTestMixin, GenericDocumentViewT def test_upload_interactive_with_unicode_metadata(self): url = URL( - path=reverse(viewname='sources:upload_interactive') + path=reverse(viewname='sources:document_upload_interactive') ) url.args['metadata0_id'] = self.test_metadata_type.pk url.args['metadata0_value'] = TEST_METADATA_VALUE_UNICODE @@ -61,7 +61,7 @@ class DocumentUploadMetadataTestCase(MetadataTypeTestMixin, GenericDocumentViewT def test_upload_interactive_with_ampersand_metadata(self): url = URL( - path=reverse(viewname='sources:upload_interactive') + path=reverse(viewname='sources:document_upload_interactive') ) url.args['metadata0_id'] = self.test_metadata_type.pk url.args['metadata0_value'] = TEST_METADATA_VALUE_WITH_AMPERSAND diff --git a/mayan/apps/sources/apps.py b/mayan/apps/sources/apps.py index 7139e5a846..bd86e07bd6 100644 --- a/mayan/apps/sources/apps.py +++ b/mayan/apps/sources/apps.py @@ -145,8 +145,9 @@ class SourcesApp(MayanAppConfig): menu_secondary.bind_links( links=(link_document_version_upload,), sources=( - 'documents:document_version_list', 'documents:upload_version', - 'documents:document_version_revert' + 'documents:document_version_list', + 'documents:document_version_revert', + 'sources:document_version_upload' ) ) diff --git a/mayan/apps/sources/forms.py b/mayan/apps/sources/forms.py index a6431f71e5..e555967cd9 100644 --- a/mayan/apps/sources/forms.py +++ b/mayan/apps/sources/forms.py @@ -23,14 +23,11 @@ class NewDocumentForm(DocumentForm): class NewVersionForm(forms.Form): - def __init__(self, *args, **kwargs): - super(NewVersionForm, self).__init__(*args, **kwargs) - - self.fields['comment'] = forms.CharField( - label=_('Comment'), - required=False, - widget=forms.widgets.Textarea(attrs={'rows': 4}), - ) + comment = forms.CharField( + help_text=_('An optional comment to explain the upload.'), + label=_('Comment'), required=False, + widget=forms.widgets.Textarea(attrs={'rows': 4}), + ) class UploadBaseForm(forms.Form): diff --git a/mayan/apps/sources/links.py b/mayan/apps/sources/links.py index b1793547f5..35146f38a1 100644 --- a/mayan/apps/sources/links.py +++ b/mayan/apps/sources/links.py @@ -117,7 +117,7 @@ link_document_version_upload = Link( args='resolved_object.pk', condition=document_new_version_not_blocked, icon_class_path='mayan.apps.sources.icons.icon_document_version_upload', permissions=(permission_document_new_version,), - text=_('Upload new version'), view='sources:upload_version', + text=_('Upload new version'), view='sources:document_version_upload', ) link_setup_source_logs = Link( args=('resolved_object.pk',), diff --git a/mayan/apps/sources/tests/test_views.py b/mayan/apps/sources/tests/test_views.py index ee64dfdffd..a5484201f1 100644 --- a/mayan/apps/sources/tests/test_views.py +++ b/mayan/apps/sources/tests/test_views.py @@ -32,7 +32,7 @@ class DocumentUploadWizardViewTestMixin(object): def _request_upload_wizard_view(self, document_path=TEST_SMALL_DOCUMENT_PATH): with open(document_path, mode='rb') as file_object: return self.post( - viewname='sources:upload_interactive', kwargs={ + viewname='sources:document_upload_interactive', kwargs={ 'source_id': self.test_source.pk }, data={ 'source-file': file_object, @@ -42,7 +42,7 @@ class DocumentUploadWizardViewTestMixin(object): def _request_upload_interactive_view(self): return self.get( - viewname='sources:upload_interactive', data={ + viewname='sources:document_upload_interactive', data={ 'document_type_id': self.test_document_type.pk, } ) @@ -113,7 +113,7 @@ class DocumentUploadWizardViewTestCase( with open(TEST_SMALL_DOCUMENT_PATH, mode='rb') as file_object: response = self.post( - viewname='sources:upload_interactive', kwargs={ + viewname='sources:document_upload_interactive', kwargs={ 'source_id': self.test_source.pk }, data={ 'source-file': file_object, @@ -157,7 +157,7 @@ class DocumentUploadIssueTestCase(GenericDocumentViewTestCase): # Upload the test document with open(TEST_SMALL_DOCUMENT_PATH, mode='rb') as file_object: self.post( - viewname='sources:upload_interactive', data={ + viewname='sources:document_upload_interactive', data={ 'document-language': 'eng', 'source-file': file_object, 'document_type_id': self.test_document_type.pk @@ -207,7 +207,7 @@ class NewDocumentVersionViewTestCase(GenericDocumentViewTestCase): NewVersionBlock.objects.block(self.test_document) response = self.post( - viewname='sources:upload_version', kwargs={ + viewname='sources:document_version_upload', kwargs={ 'document_pk': self.test_document.pk }, follow=True ) diff --git a/mayan/apps/sources/urls.py b/mayan/apps/sources/urls.py index fdc7adc330..cffeb4bf29 100644 --- a/mayan/apps/sources/urls.py +++ b/mayan/apps/sources/urls.py @@ -9,7 +9,7 @@ from .api_views import ( from .views import ( SetupSourceCheckView, SetupSourceCreateView, SetupSourceDeleteView, SetupSourceEditView, SetupSourceListView, SourceLogListView, - StagingFileDeleteView, UploadInteractiveVersionView, UploadInteractiveView + StagingFileDeleteView, DocumentVersionUploadInteractiveView, UploadInteractiveView ) from .wizards import DocumentCreateWizard @@ -27,19 +27,23 @@ urlpatterns = [ ), url( regex=r'^documents/upload/new/interactive/(?P\d+)/$', - view=UploadInteractiveView.as_view(), name='upload_interactive' + view=UploadInteractiveView.as_view(), + name='document_upload_interactive' ), url( regex=r'^documents/upload/new/interactive/$', - view=UploadInteractiveView.as_view(), name='upload_interactive' + view=UploadInteractiveView.as_view(), + name='document_upload_interactive' ), url( regex=r'^documents/(?P\d+)/versions/upload/interactive/(?P\d+)/$', - view=UploadInteractiveVersionView.as_view(), name='upload_version' + view=DocumentVersionUploadInteractiveView.as_view(), + name='document_version_upload' ), url( regex=r'^documents/(?P\d+)/versions/upload/interactive/$', - view=UploadInteractiveVersionView.as_view(), name='upload_version' + view=DocumentVersionUploadInteractiveView.as_view(), + name='document_version_upload' ), # Setup views diff --git a/mayan/apps/sources/views.py b/mayan/apps/sources/views.py index be55047b7d..4e24c01be3 100644 --- a/mayan/apps/sources/views.py +++ b/mayan/apps/sources/views.py @@ -84,10 +84,10 @@ class UploadBaseView(MultiFormView): @staticmethod def get_tab_link_for_source(source, document=None): if document: - view = 'sources:upload_version' + view = 'sources:document_version_upload' args = ('"{}"'.format(document.pk), '"{}"'.format(source.pk),) else: - view = 'sources:upload_interactive' + view = 'sources:document_upload_interactive' args = ('"{}"'.format(source.pk),) return Link( @@ -180,8 +180,8 @@ class UploadBaseView(MultiFormView): }, }) - menu_facet.bound_links['sources:upload_interactive'] = self.tab_links - menu_facet.bound_links['sources:upload_version'] = self.tab_links + menu_facet.bound_links['sources:document_upload_interactive'] = self.tab_links + menu_facet.bound_links['sources:document_version_upload'] = self.tab_links context.update( { @@ -360,7 +360,7 @@ class UploadInteractiveView(UploadBaseView): return context -class UploadInteractiveVersionView(UploadBaseView): +class DocumentVersionUploadInteractiveView(UploadBaseView): def dispatch(self, request, *args, **kwargs): self.subtemplates_list = [] @@ -392,7 +392,7 @@ class UploadInteractiveVersionView(UploadBaseView): self.tab_links = UploadBaseView.get_active_tab_links(self.document) return super( - UploadInteractiveVersionView, self + DocumentVersionUploadInteractiveView, self ).dispatch(request, *args, **kwargs) def forms_valid(self, forms): @@ -448,13 +448,6 @@ class UploadInteractiveVersionView(UploadBaseView): files=kwargs.get('files', None), ) - def create_document_form_form(self, **kwargs): - return self.get_form_classes()['document_form']( - prefix=kwargs['prefix'], - data=kwargs.get('data', None), - files=kwargs.get('files', None), - ) - def get_form_classes(self): return { 'document_form': NewVersionForm, @@ -463,12 +456,14 @@ class UploadInteractiveVersionView(UploadBaseView): def get_context_data(self, **kwargs): context = super( - UploadInteractiveVersionView, self + DocumentVersionUploadInteractiveView, self ).get_context_data(**kwargs) context['object'] = self.document context['title'] = _( - 'Upload a new version from source: %s' - ) % self.source.label + 'Upload a new version for document "%(document)s" ' + 'from source: %(source)s' + ) % {'document': self.document, 'source': self.source.label} + context['submit_label'] = _('Submit') return context diff --git a/mayan/apps/sources/wizards.py b/mayan/apps/sources/wizards.py index c88c7c0822..0523442b15 100644 --- a/mayan/apps/sources/wizards.py +++ b/mayan/apps/sources/wizards.py @@ -201,7 +201,7 @@ class DocumentCreateWizard(SessionWizardView): for step in WizardStep.get_all(): query_dict.update(step.done(wizard=self) or {}) - url = furl(reverse(viewname='sources:upload_interactive')) + url = furl(reverse(viewname='sources:document_upload_interactive')) # Use equal and not .update() to get the same result as using # urlencode(doseq=True) url.args = query_dict diff --git a/mayan/apps/tags/tests/test_wizard_steps.py b/mayan/apps/tags/tests/test_wizard_steps.py index 46824a83b1..b93a8edf10 100644 --- a/mayan/apps/tags/tests/test_wizard_steps.py +++ b/mayan/apps/tags/tests/test_wizard_steps.py @@ -27,7 +27,7 @@ class TaggedDocumentUploadTestCase(TagTestMixin, GenericDocumentViewTestCase): def _request_upload_interactive_document_create_view(self): with open(TEST_SMALL_DOCUMENT_PATH, mode='rb') as file_object: return self.post( - viewname='sources:upload_interactive', kwargs={ + viewname='sources:document_upload_interactive', kwargs={ 'source_id': self.test_source.pk }, data={ 'document_type_id': self.test_document_type.pk, From c731ab70502cb58a440a24064d84dddb780db6ce Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Thu, 10 Oct 2019 14:50:26 -0400 Subject: [PATCH 26/26] Add kwargs and update string formatting Signed-off-by: Roberto Rosario --- mayan/apps/common/generics.py | 44 +++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/mayan/apps/common/generics.py b/mayan/apps/common/generics.py index 37f5c268ea..7de15ca51f 100644 --- a/mayan/apps/common/generics.py +++ b/mayan/apps/common/generics.py @@ -53,8 +53,8 @@ class MultiFormView(DjangoFormView): template_name = 'appearance/generic_form.html' def _create_form(self, form_name, klass): - form_kwargs = self.get_form_kwargs(form_name) - form_create_method = 'create_%s_form' % form_name + form_kwargs = self.get_form_kwargs(form_name=form_name) + form_create_method = 'create_{}_form'.format(form_name) if hasattr(self, form_create_method): form = getattr(self, form_create_method)(**form_kwargs) else: @@ -66,17 +66,17 @@ class MultiFormView(DjangoFormView): def dispatch(self, request, *args, **kwargs): form_classes = self.get_form_classes() - self.forms = self.get_forms(form_classes) + self.forms = self.get_forms(form_classes=form_classes) return super(MultiFormView, self).dispatch(request, *args, **kwargs) def forms_valid(self, forms): for form_name, form in forms.items(): - form_valid_method = '%s_form_valid' % form_name + form_valid_method = '{}_form_valid'.format(form_name) if hasattr(self, form_valid_method): - return getattr(self, form_valid_method)(form) + return getattr(self, form_valid_method)(form=form) - self.all_forms_valid(forms) + self.all_forms_valid(forms=forms) return HttpResponseRedirect(redirect_to=self.get_success_url()) @@ -98,14 +98,16 @@ class MultiFormView(DjangoFormView): def get_form_kwargs(self, form_name): kwargs = {} - kwargs.update({'initial': self.get_initial(form_name)}) - kwargs.update({'prefix': self.get_prefix(form_name)}) + kwargs.update({'initial': self.get_initial(form_name=form_name)}) + kwargs.update({'prefix': self.get_prefix(form_name=form_name)}) if self.request.method in ('POST', 'PUT'): - kwargs.update({ - 'data': self.request.POST, - 'files': self.request.FILES, - }) + kwargs.update( + { + 'data': self.request.POST, + 'files': self.request.FILES, + } + ) kwargs.update(self.get_form_extra_kwargs(form_name=form_name) or {}) @@ -118,13 +120,13 @@ class MultiFormView(DjangoFormView): return dict( [ ( - key, self._create_form(key, klass) + key, self._create_form(form_name=key, klass=klass) ) for key, klass in form_classes.items() ] ) def get_initial(self, form_name): - initial_method = 'get_%s_initial' % form_name + initial_method = 'get_{}_initial'.format(form_name) if hasattr(self, initial_method): return getattr(self, initial_method)() else: @@ -206,9 +208,9 @@ class AddRemoveView( getattr(self.main_object, self.related_field).add(*queryset) else: raise ImproperlyConfigured( - 'View %s must be called with a main_object_method_add, a ' + 'View {} must be called with a main_object_method_add, a ' 'related_field, or an action_add ' - 'method.' % self.__class__.__name__ + 'method.'.format(self.__class__.__name__) ) def _action_remove(self, queryset): @@ -225,9 +227,9 @@ class AddRemoveView( getattr(self.main_object, self.related_field).remove(*queryset) else: raise ImproperlyConfigured( - 'View %s must be called with a main_object_method_remove, a ' + 'View {} must be called with a main_object_method_remove, a ' 'related_field, or an action_remove ' - 'method.' % self.__class__.__name__ + 'method.'.format(self.__class__.__name__) ) def dispatch(self, request, *args, **kwargs): @@ -348,8 +350,10 @@ class AddRemoveView( def get_list_added_queryset(self): if not self.related_field: raise ImproperlyConfigured( - 'View %s must be called with either a related_field or ' - 'override .get_list_added_queryset().' % self.__class__.__name__ + 'View {} must be called with either a related_field or ' + 'override .get_list_added_queryset().'.format( + self.__class__.__name__ + ) ) return self.get_secondary_object_list().filter(