From 0699ad0556153f3db56b5b2f83fe98a406b1b0dc Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Thu, 10 Oct 2019 11:55:42 -0400 Subject: [PATCH] Add support for new document page structure Documents now have their own dedicated DocumentPage submodel. The old DocumentPage is now called DocumentVersionPage. This allows mappings between document pages and document version pages, allowing renumbering, appending pages. DocumentPages have a content_object to map them to any other object. For now they only map to DocumentVersionPages. New option added to the version upload form to append the pages of the new version. A new view was added to just append new pages with wraps the new document version upload form and hides the append pages checkbox set to True. Add a new action, reset_pages to reset the pages of the document to those of the latest version. Missing: appending tests, checks for proper content_object in OCR and document parsing. Author: Roberto Rosario Date: Thu Oct 11 12:00:25 2019 -0400 --- docs/releases/3.3.rst | 4 +- mayan/__init__.py | 4 +- mayan/apps/cabinets/apps.py | 10 +- mayan/apps/common/generics.py | 12 +- mayan/apps/document_comments/apps.py | 10 +- mayan/apps/document_parsing/admin.py | 8 +- mayan/apps/document_parsing/api_views.py | 6 +- mayan/apps/document_parsing/apps.py | 23 +- mayan/apps/document_parsing/forms.py | 14 +- mayan/apps/document_parsing/icons.py | 3 + mayan/apps/document_parsing/links.py | 14 +- mayan/apps/document_parsing/managers.py | 6 +- .../migrations/0005_rename_page_content.py | 42 +++ mayan/apps/document_parsing/models.py | 58 ++-- mayan/apps/document_parsing/parsers.py | 43 +-- mayan/apps/document_parsing/serializers.py | 4 +- mayan/apps/document_parsing/tasks.py | 6 +- .../document_parsing/tests/test_events.py | 4 +- .../document_parsing/tests/test_parsers.py | 2 +- .../apps/document_parsing/tests/test_views.py | 10 +- mayan/apps/document_parsing/urls.py | 9 +- mayan/apps/document_parsing/utils.py | 24 +- mayan/apps/document_parsing/views.py | 32 +- mayan/apps/documents/admin.py | 19 +- mayan/apps/documents/api_views.py | 136 ++++++++- mayan/apps/documents/apps.py | 49 ++- mayan/apps/documents/dashboard_widgets.py | 6 +- mayan/apps/documents/icons.py | 5 +- mayan/apps/documents/links.py | 28 +- mayan/apps/documents/literals.py | 2 + mayan/apps/documents/managers.py | 23 +- .../migrations/0052_rename_document_page.py | 38 +++ ..._create_document_page_and_result_models.py | 57 ++++ .../migrations/0054_reset_document_pages.py | 62 ++++ mayan/apps/documents/models/__init__.py | 1 + .../apps/documents/models/document_models.py | 125 ++++---- .../documents/models/document_page_models.py | 118 +++++--- .../models/document_version_models.py | 36 ++- .../models/document_version_page_models.py | 282 ++++++++++++++++++ mayan/apps/documents/queues.py | 8 + mayan/apps/documents/search.py | 48 ++- mayan/apps/documents/serializers.py | 53 +++- mayan/apps/documents/statistics.py | 6 +- mayan/apps/documents/tasks.py | 43 ++- mayan/apps/documents/tests/mixins.py | 50 +++- mayan/apps/documents/tests/test_api.py | 37 ++- .../tests/test_document_page_views.py | 79 ++--- .../tests/test_document_version_views.py | 44 ++- .../documents/tests/test_document_views.py | 30 +- mayan/apps/documents/tests/test_search.py | 20 +- .../tests/test_trashed_document_views.py | 94 +++--- mayan/apps/documents/urls.py | 55 +++- .../documents/views/document_page_views.py | 10 +- .../documents/views/document_version_views.py | 49 ++- mayan/apps/documents/views/document_views.py | 101 +++---- mayan/apps/dynamic_search/classes.py | 9 +- .../migrations/0003_auto_20191008_1510.py | 20 ++ mayan/apps/file_metadata/apps.py | 16 +- mayan/apps/metadata/apps.py | 16 +- mayan/apps/ocr/admin.py | 10 +- mayan/apps/ocr/api_views.py | 6 +- mayan/apps/ocr/apps.py | 34 ++- mayan/apps/ocr/forms.py | 39 ++- mayan/apps/ocr/icons.py | 6 +- mayan/apps/ocr/links.py | 14 +- mayan/apps/ocr/managers.py | 48 +-- mayan/apps/ocr/methods.py | 12 + .../migrations/0009_rename_page_content.py | 42 +++ mayan/apps/ocr/models.py | 18 +- mayan/apps/ocr/serializers.py | 4 +- mayan/apps/ocr/tasks.py | 6 +- mayan/apps/ocr/tests/test_api.py | 50 ++-- mayan/apps/ocr/tests/test_events.py | 4 +- mayan/apps/ocr/tests/test_models.py | 4 +- mayan/apps/ocr/tests/test_views.py | 50 ++-- mayan/apps/ocr/urls.py | 12 +- mayan/apps/ocr/utils.py | 26 +- mayan/apps/ocr/views.py | 37 ++- mayan/apps/sources/apps.py | 15 +- mayan/apps/sources/forms.py | 25 +- mayan/apps/sources/icons.py | 4 + mayan/apps/sources/links.py | 7 + mayan/apps/sources/urls.py | 15 +- mayan/apps/sources/views.py | 54 +++- mayan/apps/tags/apps.py | 11 +- requirements/base.txt | 2 - setup.py | 2 - 87 files changed, 1973 insertions(+), 707 deletions(-) create mode 100644 mayan/apps/document_parsing/migrations/0005_rename_page_content.py create mode 100644 mayan/apps/documents/migrations/0052_rename_document_page.py create mode 100644 mayan/apps/documents/migrations/0053_create_document_page_and_result_models.py create mode 100644 mayan/apps/documents/migrations/0054_reset_document_pages.py create mode 100644 mayan/apps/documents/models/document_version_page_models.py create mode 100644 mayan/apps/file_caching/migrations/0003_auto_20191008_1510.py create mode 100644 mayan/apps/ocr/migrations/0009_rename_page_content.py diff --git a/docs/releases/3.3.rst b/docs/releases/3.3.rst index 17ff89a513..47878877f9 100644 --- a/docs/releases/3.3.rst +++ b/docs/releases/3.3.rst @@ -93,7 +93,7 @@ Changes Removals -------- -- Database conversion. Reason for removal. The database conversions support +- Database conversion. Reason for removal: The database conversions support provided by this feature (SQLite to PostgreSQL) was being confused with database migrations and upgrades. @@ -108,7 +108,7 @@ Removals Continued confusion about the purpose of the feature and confusion about how errors with this feature were a reflexion of the code quality of - Mayannecessitated the removal of the database conversion feature. + Mayan necessitated the removal of the database conversion feature. - Django environ diff --git a/mayan/__init__.py b/mayan/__init__.py index 9bd0dcd8fd..9401b78d20 100644 --- a/mayan/__init__.py +++ b/mayan/__init__.py @@ -2,8 +2,8 @@ from __future__ import unicode_literals __title__ = 'Mayan EDMS' __version__ = '3.3beta1' -__build__ = 0x030208 -__build_string__ = 'v3.2.8-255-g69086d87dd_Tue Oct 8 09:43:10 2019 -0400' +__build__ = 0x030300 +__build_string__ = 'v3.3beta1_Sat Oct 5 15:08:53 2019 -0400' __django_version__ = '1.11' __author__ = 'Roberto Rosario' __author_email__ = 'roberto.rosario@mayan-edms.com' diff --git a/mayan/apps/cabinets/apps.py b/mayan/apps/cabinets/apps.py index 6011bd4874..a2a0dc24bd 100644 --- a/mayan/apps/cabinets/apps.py +++ b/mayan/apps/cabinets/apps.py @@ -10,12 +10,14 @@ from mayan.apps.common.menus import ( menu_facet, menu_list_facet, menu_main, menu_multi_item, menu_object, menu_secondary ) +from mayan.apps.documents.search import ( + document_page_search, document_search, document_version_page_search +) from mayan.apps.events.classes import ModelEventType from mayan.apps.events.links import ( link_events_for_object, link_object_event_types_user_subcriptions_list, ) from mayan.apps.events.permissions import permission_events_view -from mayan.apps.documents.search import document_page_search, document_search from mayan.apps.navigation.classes import SourceColumn from .dependencies import * # NOQA @@ -115,12 +117,16 @@ class CabinetsApp(MayanAppConfig): ) document_page_search.add_model_field( - field='document_version__document__cabinets__label', + field='document__cabinets__label', label=_('Cabinets') ) document_search.add_model_field( field='cabinets__label', label=_('Cabinets') ) + document_version_page_search.add_model_field( + field='document_version__document__cabinets__label', + label=_('Cabinets') + ) menu_facet.bind_links( links=(link_document_cabinet_list,), sources=(Document,) diff --git a/mayan/apps/common/generics.py b/mayan/apps/common/generics.py index 37f5c268ea..310717743a 100644 --- a/mayan/apps/common/generics.py +++ b/mayan/apps/common/generics.py @@ -53,8 +53,8 @@ class MultiFormView(DjangoFormView): template_name = 'appearance/generic_form.html' def _create_form(self, form_name, klass): - form_kwargs = self.get_form_kwargs(form_name) - form_create_method = 'create_%s_form' % form_name + form_kwargs = self.get_form_kwargs(form_name=form_name) + form_create_method = 'create_{}_form'.format(form_name) if hasattr(self, form_create_method): form = getattr(self, form_create_method)(**form_kwargs) else: @@ -71,7 +71,7 @@ class MultiFormView(DjangoFormView): def forms_valid(self, forms): for form_name, form in forms.items(): - form_valid_method = '%s_form_valid' % form_name + form_valid_method = '{}_form_valid'.format(form_name) if hasattr(self, form_valid_method): return getattr(self, form_valid_method)(form) @@ -98,8 +98,8 @@ class MultiFormView(DjangoFormView): def get_form_kwargs(self, form_name): kwargs = {} - kwargs.update({'initial': self.get_initial(form_name)}) - kwargs.update({'prefix': self.get_prefix(form_name)}) + kwargs.update({'initial': self.get_initial(form_name=form_name)}) + kwargs.update({'prefix': self.get_prefix(form_name=form_name)}) if self.request.method in ('POST', 'PUT'): kwargs.update({ @@ -124,7 +124,7 @@ class MultiFormView(DjangoFormView): ) def get_initial(self, form_name): - initial_method = 'get_%s_initial' % form_name + initial_method = 'get_{}_initial'.format(form_name) if hasattr(self, initial_method): return getattr(self, initial_method)() else: diff --git a/mayan/apps/document_comments/apps.py b/mayan/apps/document_comments/apps.py index 371d7d9be4..ee34bb8038 100644 --- a/mayan/apps/document_comments/apps.py +++ b/mayan/apps/document_comments/apps.py @@ -8,7 +8,9 @@ from mayan.apps.common.apps import MayanAppConfig from mayan.apps.common.menus import ( menu_facet, menu_list_facet, menu_object, menu_secondary ) -from mayan.apps.documents.search import document_page_search, document_search +from mayan.apps.documents.search import ( + document_page_search, document_search, document_version_page_search +) from mayan.apps.events.classes import ModelEventType from mayan.apps.events.links import ( link_events_for_object, link_object_event_types_user_subcriptions_list @@ -80,13 +82,17 @@ class DocumentCommentsApp(MayanAppConfig): SourceColumn(attribute='comment', source=Comment) document_page_search.add_model_field( - field='document_version__document__comments__comment', + field='document__comments__comment', label=_('Comments') ) document_search.add_model_field( field='comments__comment', label=_('Comments') ) + document_version_page_search.add_model_field( + field='document_version__document__comments__comment', + label=_('Comments') + ) menu_facet.bind_links( links=(link_comments_for_document,), sources=(Document,) diff --git a/mayan/apps/document_parsing/admin.py b/mayan/apps/document_parsing/admin.py index 258da5ec3d..5fb8ca6a37 100644 --- a/mayan/apps/document_parsing/admin.py +++ b/mayan/apps/document_parsing/admin.py @@ -3,13 +3,13 @@ from __future__ import unicode_literals from django.contrib import admin from .models import ( - DocumentPageContent, DocumentVersionParseError + DocumentVersionPageContent, DocumentVersionParseError ) -@admin.register(DocumentPageContent) -class DocumentPageContentAdmin(admin.ModelAdmin): - list_display = ('document_page',) +@admin.register(DocumentVersionPageContent) +class DocumentVersionPageContentAdmin(admin.ModelAdmin): + list_display = ('document_version_page',) @admin.register(DocumentVersionParseError) diff --git a/mayan/apps/document_parsing/api_views.py b/mayan/apps/document_parsing/api_views.py index 7d29b61d94..ce024f34be 100644 --- a/mayan/apps/document_parsing/api_views.py +++ b/mayan/apps/document_parsing/api_views.py @@ -8,7 +8,7 @@ from rest_framework.response import Response from mayan.apps.documents.models import Document from mayan.apps.rest_api.permissions import MayanPermission -from .models import DocumentPageContent +from .models import DocumentVersionPageContent from .permissions import permission_content_view from .serializers import DocumentPageContentSerializer @@ -41,8 +41,8 @@ class APIDocumentPageContentView(generics.RetrieveAPIView): try: content = instance.content - except DocumentPageContent.DoesNotExist: - content = DocumentPageContent.objects.none() + except DocumentVersionPageContent.DoesNotExist: + content = DocumentVersionPageContent.objects.none() serializer = self.get_serializer(content) return Response(serializer.data) diff --git a/mayan/apps/document_parsing/apps.py b/mayan/apps/document_parsing/apps.py index ea4355c5a8..c7bd4e4773 100644 --- a/mayan/apps/document_parsing/apps.py +++ b/mayan/apps/document_parsing/apps.py @@ -12,7 +12,9 @@ from mayan.apps.common.classes import ModelField from mayan.apps.common.menus import ( menu_facet, menu_list_facet, menu_multi_item, menu_secondary, menu_tools ) -from mayan.apps.documents.search import document_search, document_page_search +from mayan.apps.documents.search import ( + document_search, document_page_search, document_version_page_search +) from mayan.apps.documents.signals import post_version_upload from mayan.apps.events.classes import ModelEventType from mayan.apps.navigation.classes import SourceColumn @@ -43,7 +45,7 @@ from .permissions import ( permission_parse_document ) from .signals import post_document_version_parsing -from .utils import get_document_content +from .utils import get_document_content, get_document_version_content logger = logging.getLogger(__name__) @@ -74,6 +76,9 @@ class DocumentParsingApp(MayanAppConfig): DocumentVersion = apps.get_model( app_label='documents', model_name='DocumentVersion' ) + DocumentVersionPage = apps.get_model( + app_label='documents', model_name='DocumentVersionPage' + ) DocumentVersionParseError = self.get_model( model_name='DocumentVersionParseError' ) @@ -85,7 +90,7 @@ class DocumentParsingApp(MayanAppConfig): name='content', value=get_document_content ) DocumentVersion.add_to_class( - name='content', value=get_document_content + name='content', value=get_document_version_content ) DocumentVersion.add_to_class( name='submit_for_parsing', @@ -100,9 +105,9 @@ class DocumentParsingApp(MayanAppConfig): ) ) - ModelField( - model=Document, name='versions__version_pages__content__content' - ) + #ModelField( + # model=Document, name='versions__pages__content__content' + #) ModelPermission.register( model=Document, permissions=( @@ -133,17 +138,17 @@ class DocumentParsingApp(MayanAppConfig): ) document_search.add_model_field( - field='versions__version_pages__content__content', label=_('Content') + field='versions__pages__content__content', label=_('Content') ) - document_page_search.add_model_field( + document_version_page_search.add_model_field( field='content__content', label=_('Content') ) menu_facet.bind_links( links=(link_document_content,), sources=(Document,) ) - menu_facet.bind_links( + menu_list_facet.bind_links( links=(link_document_page_content,), sources=(DocumentPage,) ) menu_list_facet.bind_links( diff --git a/mayan/apps/document_parsing/forms.py b/mayan/apps/document_parsing/forms.py index 5c4d6af26d..2803c9cd59 100644 --- a/mayan/apps/document_parsing/forms.py +++ b/mayan/apps/document_parsing/forms.py @@ -8,7 +8,7 @@ from django.utils.translation import ugettext_lazy as _, ugettext from mayan.apps.common.widgets import TextAreaDiv -from .models import DocumentPageContent +from .models import DocumentVersionPageContent class DocumentContentForm(forms.Form): @@ -26,10 +26,10 @@ class DocumentContentForm(forms.Form): except AttributeError: document_pages = [] - for page in document_pages: + for document_page in document_pages: try: - page_content = page.content.content - except DocumentPageContent.DoesNotExist: + page_content = document_page.content_object.content.content + except DocumentVersionPageContent.DoesNotExist: pass else: content.append(conditional_escape(force_text(page_content))) @@ -37,7 +37,7 @@ class DocumentContentForm(forms.Form): '\n\n\n
- %s -
\n\n\n' % ( ugettext( 'Page %(page_number)d' - ) % {'page_number': page.page_number} + ) % {'page_number': document_page.page_number} ) ) @@ -72,8 +72,8 @@ class DocumentPageContentForm(forms.Form): self.fields['contents'].initial = '' try: - page_content = document_page.content.content - except DocumentPageContent.DoesNotExist: + page_content = document_page.content_object.content.content + except DocumentVersionPageContent.DoesNotExist: pass else: content = conditional_escape(force_text(page_content)) diff --git a/mayan/apps/document_parsing/icons.py b/mayan/apps/document_parsing/icons.py index 6aad4a8ea2..9fb872dc1f 100644 --- a/mayan/apps/document_parsing/icons.py +++ b/mayan/apps/document_parsing/icons.py @@ -17,6 +17,9 @@ icon_document_content_download = Icon( icon_document_multiple_submit = Icon( driver_name='fontawesome', symbol='font' ) +icon_document_page_content = Icon( + driver_name='fontawesome', symbol='font' +) icon_document_submit = Icon( driver_name='fontawesome', symbol='font' ) diff --git a/mayan/apps/document_parsing/links.py b/mayan/apps/document_parsing/links.py index 2cfe95fab9..a7de474581 100644 --- a/mayan/apps/document_parsing/links.py +++ b/mayan/apps/document_parsing/links.py @@ -32,9 +32,15 @@ link_document_content_delete_multiple = Link( text=_('Delete parsed content'), view='document_parsing:document_content_delete_multiple', ) +link_document_content_download = Link( + args='resolved_object.id', + icon_class_path='mayan.apps.document_parsing.icons.icon_document_content_download', + permissions=(permission_content_view,), text=_('Download content'), + view='document_parsing:document_content_download' +) link_document_page_content = Link( args='resolved_object.id', conditional_disable=is_document_page_disabled, - icon_class_path='mayan.apps.document_parsing.icons.icon_document_content', + icon_class_path='mayan.apps.document_parsing.icons.icon_document_page_content', permissions=(permission_content_view,), text=_('Content'), view='document_parsing:document_page_content' ) @@ -44,12 +50,6 @@ link_document_parsing_errors_list = Link( permissions=(permission_content_view,), text=_('Parsing errors'), view='document_parsing:document_parsing_error_list' ) -link_document_content_download = Link( - args='resolved_object.id', - icon_class_path='mayan.apps.document_parsing.icons.icon_document_content_download', - permissions=(permission_content_view,), text=_('Download content'), - view='document_parsing:document_content_download' -) link_document_submit_multiple = Link( icon_class_path='mayan.apps.document_parsing.icons.icon_document_submit', text=_('Submit for parsing'), diff --git a/mayan/apps/document_parsing/managers.py b/mayan/apps/document_parsing/managers.py index 1669f89e70..8ff1d5d231 100644 --- a/mayan/apps/document_parsing/managers.py +++ b/mayan/apps/document_parsing/managers.py @@ -18,11 +18,13 @@ from .signals import post_document_version_parsing logger = logging.getLogger(__name__) -class DocumentPageContentManager(models.Manager): +class DocumentVersionPageContentManager(models.Manager): def delete_content_for(self, document, user=None): with transaction.atomic(): for document_page in document.pages.all(): - self.filter(document_page=document_page).delete() + self.filter( + document_version_page=document_page.content_object + ).delete() event_parsing_document_content_deleted.commit( actor=user, target=document diff --git a/mayan/apps/document_parsing/migrations/0005_rename_page_content.py b/mayan/apps/document_parsing/migrations/0005_rename_page_content.py new file mode 100644 index 0000000000..11ad6cdc8b --- /dev/null +++ b/mayan/apps/document_parsing/migrations/0005_rename_page_content.py @@ -0,0 +1,42 @@ +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + dependencies = [ + ('document_parsing', '0004_auto_20180917_0645'), + ('documents', '0052_rename_document_page'), + ] + + operations = [ + migrations.RenameModel( + 'DocumentPageContent', 'DocumentVersionPageContent' + ), + migrations.AlterField( + model_name='documentversionpagecontent', + name='document_page', + field=models.OneToOneField( + on_delete=django.db.models.deletion.CASCADE, + related_name='content', + to='documents.DocumentVersionPage', + verbose_name='Document version page' + ), + ), + migrations.RenameField( + model_name='documentversionpagecontent', + old_name='document_page', + new_name='document_version_page', + ), + migrations.AlterModelOptions( + name='documentversionpagecontent', + options={ + 'verbose_name': 'Document version page content', + 'verbose_name_plural': 'Document version pages contents' + }, + ), + ] + + + diff --git a/mayan/apps/document_parsing/models.py b/mayan/apps/document_parsing/models.py index 5e9b52320e..48c2dc79af 100644 --- a/mayan/apps/document_parsing/models.py +++ b/mayan/apps/document_parsing/models.py @@ -5,36 +5,12 @@ from django.utils.encoding import force_text, python_2_unicode_compatible from django.utils.translation import ugettext_lazy as _ from mayan.apps.documents.models import ( - DocumentPage, DocumentType, DocumentVersion + DocumentPage, DocumentType, DocumentVersion, DocumentVersionPage ) -from .managers import DocumentPageContentManager, DocumentTypeSettingsManager - - -@python_2_unicode_compatible -class DocumentPageContent(models.Model): - """ - This model store's the parsed content of a document page. - """ - document_page = models.OneToOneField( - on_delete=models.CASCADE, related_name='content', to=DocumentPage, - verbose_name=_('Document page') - ) - content = models.TextField( - blank=True, help_text=_( - 'The actual text content as extracted by the document ' - 'parsing backend.' - ), verbose_name=_('Content') - ) - - objects = DocumentPageContentManager() - - class Meta: - verbose_name = _('Document page content') - verbose_name_plural = _('Document pages contents') - - def __str__(self): - return force_text(self.document_page) +from .managers import ( + DocumentVersionPageContentManager, DocumentTypeSettingsManager +) class DocumentTypeSettings(models.Model): @@ -62,6 +38,32 @@ class DocumentTypeSettings(models.Model): verbose_name_plural = _('Document types settings') +@python_2_unicode_compatible +class DocumentVersionPageContent(models.Model): + """ + This model store's the parsed content of a document page. + """ + document_version_page = models.OneToOneField( + on_delete=models.CASCADE, related_name='content', + to=DocumentVersionPage, verbose_name=_('Document version page') + ) + content = models.TextField( + blank=True, help_text=_( + 'The actual text content as extracted by the document ' + 'parsing backend.' + ), verbose_name=_('Content') + ) + + objects = DocumentVersionPageContentManager() + + class Meta: + verbose_name = _('Document version page content') + verbose_name_plural = _('Document version pages contents') + + def __str__(self): + return force_text(self.document_page) + + @python_2_unicode_compatible class DocumentVersionParseError(models.Model): """ diff --git a/mayan/apps/document_parsing/parsers.py b/mayan/apps/document_parsing/parsers.py index 0e4d0cff70..469b974377 100644 --- a/mayan/apps/document_parsing/parsers.py +++ b/mayan/apps/document_parsing/parsers.py @@ -23,11 +23,13 @@ class Parser(object): _registry = {} @classmethod - def parse_document_page(cls, document_page): - for parser_class in cls._registry.get(document_page.document_version.mimetype, ()): + def parse_document_version_page(cls, document_version_page): + for parser_class in cls._registry.get(document_version_page.document_version.mimetype, ()): try: parser = parser_class() - parser.process_document_page(document_page) + parser.process_document_page( + document_version_page=document_version_page + ) except ParserError: # If parser raises error, try next parser in the list pass @@ -41,7 +43,9 @@ class Parser(object): for parser_class in cls._registry.get(document_version.mimetype, ()): try: parser = parser_class() - parser.process_document_version(document_version) + parser.process_document_version( + document_version=document_version + ) except ParserError: # If parser raises error, try next parser in the list pass @@ -64,29 +68,33 @@ class Parser(object): ) logger.debug('document version: %d', document_version.pk) - for document_page in document_version.pages.all(): - self.process_document_page(document_page=document_page) + for document_version_page in document_version.pages.all(): + self.process_document_version_page( + document_version_page=document_version_page + ) - def process_document_page(self, document_page): - DocumentPageContent = apps.get_model( - app_label='document_parsing', model_name='DocumentPageContent' + def process_document_version_page(self, document_version_page): + DocumentVersionPageContent = apps.get_model( + app_label='document_parsing', + model_name='DocumentVersionPageContent' ) logger.info( 'Processing page: %d of document version: %s', - document_page.page_number, document_page.document_version + document_version_page.page_number, + document_version_page.document_version ) - file_object = document_page.document_version.get_intermediate_file() + file_object = document_version_page.document_version.get_intermediate_file() try: - document_page_content, created = DocumentPageContent.objects.get_or_create( - document_page=document_page + document_version_page_content, created = DocumentVersionPageContent.objects.get_or_create( + document_version_page=document_version_page ) - document_page_content.content = self.execute( - file_object=file_object, page_number=document_page.page_number + document_version_page_content.content = self.execute( + file_object=file_object, page_number=document_version_page.page_number ) - document_page_content.save() + document_version_page_content.save() except Exception as exception: error_message = _('Exception parsing page; %s') % exception logger.error(error_message) @@ -96,7 +104,8 @@ class Parser(object): logger.info( 'Finished processing page: %d of document version: %s', - document_page.page_number, document_page.document_version + document_version_page.page_number, + document_version_page.document_version ) def execute(self, file_object, page_number): diff --git a/mayan/apps/document_parsing/serializers.py b/mayan/apps/document_parsing/serializers.py index 7161d2fc40..53d90d354e 100644 --- a/mayan/apps/document_parsing/serializers.py +++ b/mayan/apps/document_parsing/serializers.py @@ -2,10 +2,10 @@ from __future__ import unicode_literals from rest_framework import serializers -from .models import DocumentPageContent +from .models import DocumentVersionPageContent class DocumentPageContentSerializer(serializers.ModelSerializer): class Meta: fields = ('content',) - model = DocumentPageContent + model = DocumentVersionPageContent diff --git a/mayan/apps/document_parsing/tasks.py b/mayan/apps/document_parsing/tasks.py index 4debffbc60..653552f741 100644 --- a/mayan/apps/document_parsing/tasks.py +++ b/mayan/apps/document_parsing/tasks.py @@ -14,8 +14,8 @@ def task_parse_document_version(document_version_pk): DocumentVersion = apps.get_model( app_label='documents', model_name='DocumentVersion' ) - DocumentPageContent = apps.get_model( - app_label='document_parsing', model_name='DocumentPageContent' + DocumentVersionPageContent = apps.get_model( + app_label='document_parsing', model_name='DocumentVersionPageContent' ) document_version = DocumentVersion.objects.get( @@ -24,6 +24,6 @@ def task_parse_document_version(document_version_pk): logger.info( 'Starting parsing for document version: %s', document_version ) - DocumentPageContent.objects.process_document_version( + DocumentVersionPageContent.objects.process_document_version( document_version=document_version ) diff --git a/mayan/apps/document_parsing/tests/test_events.py b/mayan/apps/document_parsing/tests/test_events.py index 1a2860305d..17bbe198fb 100644 --- a/mayan/apps/document_parsing/tests/test_events.py +++ b/mayan/apps/document_parsing/tests/test_events.py @@ -10,7 +10,7 @@ from ..events import ( event_parsing_document_version_submit, event_parsing_document_version_finish ) -from ..models import DocumentPageContent +from ..models import DocumentVersionPageContent class DocumentParsingEventsTestCase(GenericDocumentTestCase): @@ -19,7 +19,7 @@ class DocumentParsingEventsTestCase(GenericDocumentTestCase): def test_document_content_deleted_event(self): Action.objects.all().delete() - DocumentPageContent.objects.delete_content_for( + DocumentVersionPageContent.objects.delete_content_for( document=self.test_document ) diff --git a/mayan/apps/document_parsing/tests/test_parsers.py b/mayan/apps/document_parsing/tests/test_parsers.py index 237bccc567..2eeb8703a6 100644 --- a/mayan/apps/document_parsing/tests/test_parsers.py +++ b/mayan/apps/document_parsing/tests/test_parsers.py @@ -18,5 +18,5 @@ class ParserTestCase(DocumentTestMixin, BaseTestCase): parser.process_document_version(self.test_document.latest_version) self.assertTrue( - TEST_DOCUMENT_CONTENT in self.test_document.pages.first().content.content + TEST_DOCUMENT_CONTENT in self.test_document.pages.first().content_object.content.content ) diff --git a/mayan/apps/document_parsing/tests/test_views.py b/mayan/apps/document_parsing/tests/test_views.py index bb88c1817b..21ca0ef808 100644 --- a/mayan/apps/document_parsing/tests/test_views.py +++ b/mayan/apps/document_parsing/tests/test_views.py @@ -5,7 +5,7 @@ from django.test import override_settings from mayan.apps.documents.tests.base import GenericDocumentViewTestCase from mayan.apps.documents.tests.literals import TEST_HYBRID_DOCUMENT -from ..models import DocumentPageContent +from ..models import DocumentVersionPageContent from ..permissions import ( permission_content_view, permission_document_type_parsing_setup, permission_parse_document @@ -72,8 +72,8 @@ class DocumentContentViewsTestCase( self.assertEqual(response.status_code, 404) self.assertTrue( - DocumentPageContent.objects.filter( - document_page=self.test_document.pages.first() + DocumentVersionPageContent.objects.filter( + document_version_page=self.test_document.pages.first().content_object ).exists() ) @@ -86,8 +86,8 @@ class DocumentContentViewsTestCase( self.assertEqual(response.status_code, 302) self.assertFalse( - DocumentPageContent.objects.filter( - document_page=self.test_document.pages.first() + DocumentVersionPageContent.objects.filter( + document_version_page=self.test_document.pages.first().content_object ).exists() ) diff --git a/mayan/apps/document_parsing/urls.py b/mayan/apps/document_parsing/urls.py index f590236176..877ae67a4e 100644 --- a/mayan/apps/document_parsing/urls.py +++ b/mayan/apps/document_parsing/urls.py @@ -7,7 +7,9 @@ from .views import ( DocumentContentView, DocumentContentDeleteView, DocumentContentDownloadView, DocumentPageContentView, DocumentParsingErrorsListView, DocumentSubmitView, - DocumentTypeSettingsEditView, DocumentTypeSubmitView, ParseErrorListView + DocumentTypeSettingsEditView, DocumentTypeSubmitView, + DocumentVersionPageContentView, + ParseErrorListView ) urlpatterns = [ @@ -34,6 +36,11 @@ urlpatterns = [ regex=r'^documents/pages/(?P\d+)/content/$', view=DocumentPageContentView.as_view(), name='document_page_content' ), + url( + regex=r'^documents/versions/pages/(?P\d+)/content/$', + view=DocumentVersionPageContentView.as_view(), + name='document_version_page_content' + ), url( regex=r'^documents/(?P\d+)/submit/$', view=DocumentSubmitView.as_view(), name='document_submit' diff --git a/mayan/apps/document_parsing/utils.py b/mayan/apps/document_parsing/utils.py index ab8e049450..5086d5cf02 100644 --- a/mayan/apps/document_parsing/utils.py +++ b/mayan/apps/document_parsing/utils.py @@ -6,14 +6,28 @@ from django.utils.html import conditional_escape def get_document_content(document): - DocumentPageContent = apps.get_model( - app_label='document_parsing', model_name='DocumentPageContent' + DocumentVersionPageContent = apps.get_model( + app_label='document_parsing', model_name='DocumentVersionPageContent' ) - for page in document.pages.all(): + for document_page in document.pages.all(): try: - page_content = page.content.content - except DocumentPageContent.DoesNotExist: + page_content = document_page.content_object.content.content + except DocumentVersionPageContent.DoesNotExist: + pass + else: + yield conditional_escape(force_text(page_content)) + + +def get_document_version_content(document_version): + DocumentVersionPageContent = apps.get_model( + app_label='document_parsing', model_name='DocumentVersionPageContent' + ) + + for document_version_page in document_version.pages.all(): + try: + page_content = document_version_page.content.content + except DocumentVersionPageContent.DoesNotExist: pass else: yield conditional_escape(force_text(page_content)) diff --git a/mayan/apps/document_parsing/views.py b/mayan/apps/document_parsing/views.py index de9cb3ef0b..7201253c18 100644 --- a/mayan/apps/document_parsing/views.py +++ b/mayan/apps/document_parsing/views.py @@ -12,10 +12,12 @@ from mayan.apps.common.generics import ( ) from mayan.apps.common.mixins import ExternalObjectMixin from mayan.apps.documents.forms import DocumentTypeFilteredSelectForm -from mayan.apps.documents.models import Document, DocumentPage, DocumentType +from mayan.apps.documents.models import ( + Document, DocumentPage, DocumentType, DocumentVersionPage +) from .forms import DocumentContentForm, DocumentPageContentForm -from .models import DocumentPageContent, DocumentVersionParseError +from .models import DocumentVersionPageContent, DocumentVersionParseError from .permissions import ( permission_content_view, permission_document_type_parsing_setup, permission_parse_document @@ -46,7 +48,7 @@ class DocumentContentDeleteView(MultipleObjectConfirmActionView): return result def object_action(self, form, instance): - DocumentPageContent.objects.delete_content_for( + DocumentVersionPageContent.objects.delete_content_for( document=instance, user=self.request.user ) @@ -107,6 +109,30 @@ class DocumentPageContentView(SingleObjectDetailView): } +class DocumentVersionPageContentView(SingleObjectDetailView): + form_class = DocumentPageContentForm + model = DocumentVersionPage + object_permission = permission_content_view + + def dispatch(self, request, *args, **kwargs): + result = super(DocumentPageContentView, self).dispatch( + request, *args, **kwargs + ) + self.get_object().document.add_as_recent_document_for_user( + request.user + ) + return result + + def get_extra_context(self): + return { + 'hide_labels': True, + 'object': self.get_object(), + 'title': _( + 'Content for document version page: %s' + ) % self.get_object(), + } + + class DocumentParsingErrorsListView(SingleObjectListView): view_permission = permission_content_view diff --git a/mayan/apps/documents/admin.py b/mayan/apps/documents/admin.py index 62d30c0e1f..59b6004f31 100644 --- a/mayan/apps/documents/admin.py +++ b/mayan/apps/documents/admin.py @@ -3,18 +3,12 @@ from __future__ import unicode_literals from django.contrib import admin from .models import ( - DeletedDocument, Document, DocumentPage, DocumentType, - DocumentTypeFilename, DocumentVersion, DuplicatedDocument, RecentDocument + DeletedDocument, Document, DocumentType, DocumentTypeFilename, + DocumentVersion, DocumentVersionPage, DuplicatedDocument, + RecentDocument ) -class DocumentPageInline(admin.StackedInline): - model = DocumentPage - extra = 1 - classes = ('collapse-open',) - allow_add = True - - class DocumentTypeFilenameInline(admin.StackedInline): model = DocumentTypeFilename extra = 1 @@ -29,6 +23,13 @@ class DocumentVersionInline(admin.StackedInline): allow_add = True +class DocumentVersionPageInline(admin.StackedInline): + model = DocumentVersionPage + extra = 1 + classes = ('collapse-open',) + allow_add = True + + @admin.register(DeletedDocument) class DeletedDocumentAdmin(admin.ModelAdmin): date_hierarchy = 'deleted_date_time' diff --git a/mayan/apps/documents/api_views.py b/mayan/apps/documents/api_views.py index 4d17038722..87c4c9bd3d 100644 --- a/mayan/apps/documents/api_views.py +++ b/mayan/apps/documents/api_views.py @@ -33,10 +33,14 @@ from .serializers import ( DocumentTypeSerializer, DocumentVersionSerializer, NewDocumentSerializer, NewDocumentVersionSerializer, RecentDocumentSerializer, WritableDocumentSerializer, - WritableDocumentTypeSerializer, WritableDocumentVersionSerializer + WritableDocumentTypeSerializer, WritableDocumentVersionSerializer, + DocumentVersionPageSerializer ) from .settings import settings_document_page_image_cache_time -from .tasks import task_generate_document_page_image +from .tasks import ( + task_generate_document_page_image, + task_generate_document_version_page_image +) logger = logging.getLogger(__name__) @@ -168,13 +172,8 @@ class APIDocumentPageImageView(generics.RetrieveAPIView): ) return document - def get_document_version(self): - return get_object_or_404( - self.get_document().versions.all(), pk=self.kwargs['version_pk'] - ) - def get_queryset(self): - return self.get_document_version().pages_all.all() + return self.get_document().pages_all.all() def get_serializer(self, *args, **kwargs): return None @@ -221,6 +220,95 @@ class APIDocumentPageImageView(generics.RetrieveAPIView): return response +class APIDocumentVersionPageImageView(generics.RetrieveAPIView): + """ + get: Returns an image representation of the selected document version page. + """ + lookup_url_kwarg = 'page_pk' + + def get_document(self): + if self.request.method == 'GET': + permission_required = permission_document_view + else: + permission_required = permission_document_edit + + document = get_object_or_404(Document.passthrough, pk=self.kwargs['pk']) + + AccessControlList.objects.check_access( + obj=document, permissions=(permission_required,), + user=self.request.user + ) + return document + + def get_document_version(self): + return get_object_or_404( + self.get_document().versions.all(), pk=self.kwargs['version_pk'] + ) + + def get_queryset(self): + return self.get_document_version().pages.all() + + def get_serializer(self, *args, **kwargs): + return None + + def get_serializer_class(self): + return None + + @cache_control(private=True) + def retrieve(self, request, *args, **kwargs): + width = request.GET.get('width') + height = request.GET.get('height') + zoom = request.GET.get('zoom') + + if zoom: + zoom = int(zoom) + + rotation = request.GET.get('rotation') + + if rotation: + rotation = int(rotation) + + maximum_layer_order = request.GET.get('maximum_layer_order') + if maximum_layer_order: + maximum_layer_order = int(maximum_layer_order) + + task = task_generate_document_version_page_image.apply_async( + kwargs=dict( + document_version_page_id=self.get_object().pk, width=width, + height=height, zoom=zoom, rotation=rotation, + maximum_layer_order=maximum_layer_order, + user_id=request.user.pk + ) + ) + + cache_filename = task.get(timeout=DOCUMENT_IMAGE_TASK_TIMEOUT) + cache_file = self.get_object().cache_partition.get_file(filename=cache_filename) + with cache_file.open() as file_object: + response = HttpResponse(file_object.read(), content_type='image') + if '_hash' in request.GET: + patch_cache_control( + response=response, + max_age=settings_document_page_image_cache_time.value + ) + return response + + +class APIDocumentPageListView(generics.ListAPIView): + serializer_class = DocumentPageSerializer + + def get_document(self): + document = get_object_or_404(Document, pk=self.kwargs['pk']) + + AccessControlList.objects.check_access( + obj=document, permissions=(permission_document_view,), + user=self.request.user + ) + return document + + def get_queryset(self): + return self.get_document().pages.all() + + class APIDocumentPageView(generics.RetrieveUpdateAPIView): """ get: Returns the selected document page details. @@ -230,6 +318,33 @@ class APIDocumentPageView(generics.RetrieveUpdateAPIView): lookup_url_kwarg = 'page_pk' serializer_class = DocumentPageSerializer + def get_document(self): + if self.request.method == 'GET': + permission_required = permission_document_view + else: + permission_required = permission_document_edit + + document = get_object_or_404(Document, pk=self.kwargs['pk']) + + AccessControlList.objects.check_access( + obj=document, permissions=(permission_required,), + user=self.request.user + ) + return document + + def get_queryset(self): + return self.get_document().pages.all() + + +class APIDocumentVersionPageView(generics.RetrieveUpdateAPIView): + """ + get: Returns the selected document verion page details. + patch: Edit the selected document version page. + put: Edit the selected document version page. + """ + lookup_url_kwarg = 'page_pk' + serializer_class = DocumentVersionPageSerializer + def get_document(self): if self.request.method == 'GET': permission_required = permission_document_view @@ -289,8 +404,7 @@ class APIDocumentTypeView(generics.RetrieveUpdateDestroyAPIView): 'GET': (permission_document_type_view,), 'PUT': (permission_document_type_edit,), 'PATCH': (permission_document_type_edit,), - 'DELETE': (permission_document_type_delete,) - } + 'DELETE': (permission_document_type_delete,) } permission_classes = (MayanPermission,) queryset = DocumentType.objects.all() @@ -423,7 +537,7 @@ class APIRecentDocumentListView(generics.ListAPIView): class APIDocumentVersionPageListView(generics.ListAPIView): - serializer_class = DocumentPageSerializer + serializer_class = DocumentVersionPageSerializer def get_document(self): document = get_object_or_404(Document, pk=self.kwargs['pk']) diff --git a/mayan/apps/documents/apps.py b/mayan/apps/documents/apps.py index 09e85928c7..541134fb4b 100644 --- a/mayan/apps/documents/apps.py +++ b/mayan/apps/documents/apps.py @@ -59,7 +59,7 @@ from .links import ( link_document_multiple_delete, link_document_multiple_document_type_edit, link_document_multiple_download, link_document_multiple_favorites_add, link_document_multiple_favorites_remove, link_document_multiple_restore, - link_document_multiple_trash, link_document_multiple_update_page_count, + link_document_multiple_trash, link_document_multiple_pages_reset, link_document_page_disable, link_document_page_multiple_disable, link_document_page_enable, link_document_page_multiple_enable, link_document_page_navigation_first, link_document_page_navigation_last, @@ -74,8 +74,10 @@ from .links import ( link_document_type_filename_create, link_document_type_filename_delete, link_document_type_filename_edit, link_document_type_filename_list, link_document_type_list, link_document_type_policies, - link_document_type_setup, link_document_update_page_count, + link_document_type_setup, link_document_pages_reset, link_document_version_download, link_document_version_list, + link_document_version_multiple_page_count_update, + link_document_version_page_count_update, link_document_version_return_document, link_document_version_return_list, link_document_version_revert, link_document_version_view, link_duplicated_document_list, link_duplicated_document_scan, @@ -87,10 +89,10 @@ from .permissions import ( permission_document_download, permission_document_edit, permission_document_new_version, permission_document_print, permission_document_properties_edit, permission_document_restore, - permission_document_trash, permission_document_type_delete, - permission_document_type_edit, permission_document_type_view, - permission_document_version_revert, permission_document_version_view, - permission_document_view + permission_document_tools, permission_document_trash, + permission_document_type_delete, permission_document_type_edit, + permission_document_type_view, permission_document_version_revert, + permission_document_version_view, permission_document_view, ) # Just import to initialize the search models from .search import document_search, document_page_search # NOQA @@ -121,10 +123,11 @@ class DocumentsApp(MayanAppConfig): DeletedDocument = self.get_model(model_name='DeletedDocument') Document = self.get_model(model_name='Document') DocumentPage = self.get_model(model_name='DocumentPage') - DocumentPageResult = self.get_model(model_name='DocumentPageResult') + DocumentPageResult = self.get_model(model_name='DocumentVersionPageResult') DocumentType = self.get_model(model_name='DocumentType') DocumentTypeFilename = self.get_model(model_name='DocumentTypeFilename') DocumentVersion = self.get_model(model_name='DocumentVersion') + DocumentVersionPage = self.get_model(model_name='DocumentVersionPage') DuplicatedDocument = self.get_model(model_name='DuplicatedDocument') DynamicSerializerField.add_serializer( @@ -190,13 +193,15 @@ class DocumentsApp(MayanAppConfig): permission_acl_edit, permission_acl_view, permission_document_delete, permission_document_download, permission_document_edit, permission_document_new_version, - permission_document_print, permission_document_properties_edit, - permission_document_restore, permission_document_trash, - permission_document_version_revert, + permission_document_print, + permission_document_properties_edit, + permission_document_restore, permission_document_tools, + permission_document_trash, permission_document_version_revert, permission_document_version_view, permission_document_view, permission_events_view, permission_transformation_create, permission_transformation_delete, - permission_transformation_edit, permission_transformation_view, + permission_transformation_edit, + permission_transformation_view, ) ) @@ -224,13 +229,13 @@ class DocumentsApp(MayanAppConfig): model=Document, manager_name='passthrough' ) ModelPermission.register_inheritance( - model=DocumentPage, related='document_version__document', + model=DocumentPage, related='document', ) ModelPermission.register_manager( model=DocumentPage, manager_name='passthrough' ) ModelPermission.register_inheritance( - model=DocumentPageResult, related='document_version__document', + model=DocumentPageResult, related='document', ) ModelPermission.register_manager( model=DocumentPageResult, manager_name='passthrough' @@ -241,6 +246,9 @@ class DocumentsApp(MayanAppConfig): ModelPermission.register_inheritance( model=DocumentVersion, related='document', ) + ModelPermission.register_inheritance( + model=DocumentVersionPage, related='document_version', + ) # Document and document page thumbnail widget document_page_thumbnail_widget = DocumentPageThumbnailWidget() @@ -454,7 +462,7 @@ class DocumentsApp(MayanAppConfig): link_document_quick_download, link_document_download, link_document_clear_transformations, link_document_clone_transformations, - link_document_update_page_count, + link_document_pages_reset, ), sources=(Document,) ) menu_object.bind_links( @@ -495,7 +503,7 @@ class DocumentsApp(MayanAppConfig): link_document_multiple_favorites_remove, link_document_multiple_clear_transformations, link_document_multiple_trash, link_document_multiple_download, - link_document_multiple_update_page_count, + link_document_multiple_pages_reset, link_document_multiple_document_type_edit, ), sources=(Document,) ) @@ -547,6 +555,17 @@ class DocumentsApp(MayanAppConfig): link_document_version_return_list ), sources=(DocumentVersion,) ) + menu_multi_item.bind_links( + links=( + link_document_version_multiple_page_count_update, + ), sources=(DocumentVersion,) + ) + menu_object.bind_links( + links=( + link_document_version_page_count_update, + ), sources=(DocumentVersion,) + ) + menu_list_facet.bind_links( links=(link_document_version_view,), sources=(DocumentVersion,) ) diff --git a/mayan/apps/documents/dashboard_widgets.py b/mayan/apps/documents/dashboard_widgets.py index 2ca9dd1b89..221e2cfbbd 100644 --- a/mayan/apps/documents/dashboard_widgets.py +++ b/mayan/apps/documents/dashboard_widgets.py @@ -32,12 +32,12 @@ class DashboardWidgetDocumentPagesTotal(DashboardWidgetNumeric): AccessControlList = apps.get_model( app_label='acls', model_name='AccessControlList' ) - DocumentPage = apps.get_model( - app_label='documents', model_name='DocumentPage' + DocumentVersionPage = apps.get_model( + app_label='documents', model_name='DocumentVersionPage' ) self.count = AccessControlList.objects.restrict_queryset( permission=permission_document_view, user=request.user, - queryset=DocumentPage.objects.all() + queryset=DocumentVersionPage.objects.all() ).count() return super(DashboardWidgetDocumentPagesTotal, self).render(request) diff --git a/mayan/apps/documents/icons.py b/mayan/apps/documents/icons.py index fb891fd585..ce365c8678 100644 --- a/mayan/apps/documents/icons.py +++ b/mayan/apps/documents/icons.py @@ -36,7 +36,10 @@ icon_document_edit = Icon( ) icon_document = Icon(driver_name='fontawesome', symbol='book') icon_document_list = icon_document -icon_document_page_count_update = Icon( +icon_document_pages_reset = Icon( + driver_name='fontawesome', symbol='copy' +) +icon_document_version_page_count_update = Icon( driver_name='fontawesome', symbol='copy' ) icon_document_preview = Icon(driver_name='fontawesome', symbol='eye') diff --git a/mayan/apps/documents/links.py b/mayan/apps/documents/links.py index 271aa16579..89cbd31782 100644 --- a/mayan/apps/documents/links.py +++ b/mayan/apps/documents/links.py @@ -168,12 +168,12 @@ link_document_quick_download = Link( permissions=(permission_document_download,), text=_('Quick download'), view='documents:document_download', ) -link_document_update_page_count = Link( +link_document_pages_reset = Link( args='resolved_object.pk', - icon_class_path='mayan.apps.documents.icons.icon_document_page_count_update', + icon_class_path='mayan.apps.documents.icons.icon_document_pages_reset', permissions=(permission_document_tools,), - text=_('Recalculate page count'), - view='documents:document_update_page_count' + text=_('Reset pages'), + view='documents:document_pages_reset' ) link_document_restore = Link( permissions=(permission_document_restore,), @@ -217,10 +217,10 @@ link_document_multiple_download = Link( text=_('Advanced download'), view='documents:document_multiple_download_form' ) -link_document_multiple_update_page_count = Link( - icon_class_path='mayan.apps.documents.icons.icon_document_page_count_update', - text=_('Recalculate page count'), - view='documents:document_multiple_update_page_count' +link_document_multiple_pages_reset = Link( + icon_class_path='mayan.apps.documents.icons.icon_document_pages_reset', + text=_('Reset pages'), + view='documents:document_multiple_pages_reset' ) link_document_multiple_restore = Link( icon_class_path='mayan.apps.documents.icons.icon_trashed_document_restore', @@ -246,6 +246,18 @@ link_document_version_return_list = Link( permissions=(permission_document_version_view,), text=_('Versions'), view='documents:document_version_list', ) +link_document_version_page_count_update = Link( + args='resolved_object.pk', + icon_class_path='mayan.apps.documents.icons.icon_document_version_page_count_update', + permissions=(permission_document_tools,), + text=_('Update page count'), + view='documents:document_version_page_count_update' +) +link_document_version_multiple_page_count_update = Link( + icon_class_path='mayan.apps.documents.icons.icon_document_version_page_count_update', + text=_('Update page count'), + view='documents:document_version_multiple_page_count_update' +) link_document_version_view = Link( args='resolved_object.pk', icon_class_path='mayan.apps.documents.icons.icon_document_version_view', diff --git a/mayan/apps/documents/literals.py b/mayan/apps/documents/literals.py index d639d1980f..35ef6b2f36 100644 --- a/mayan/apps/documents/literals.py +++ b/mayan/apps/documents/literals.py @@ -43,3 +43,5 @@ PAGE_RANGE_RANGE = 'range' PAGE_RANGE_CHOICES = ( (PAGE_RANGE_ALL, _('All pages')), (PAGE_RANGE_RANGE, _('Page range')) ) + +RETRY_DELAY_DOCUMENT_RESET_PAGES = 30 diff --git a/mayan/apps/documents/managers.py b/mayan/apps/documents/managers.py index 6c9dd92640..28236f109c 100644 --- a/mayan/apps/documents/managers.py +++ b/mayan/apps/documents/managers.py @@ -28,15 +28,15 @@ class DocumentManager(models.Manager): class DocumentPageManager(models.Manager): def get_by_natural_key(self, page_number, document_version_natural_key): - DocumentVersion = apps.get_model( - app_label='documents', model_name='DocumentVersion' + Document = apps.get_model( + app_label='documents', model_name='Document' ) try: - document_version = DocumentVersion.objects.get_by_natural_key(*document_version_natural_key) - except DocumentVersion.DoesNotExist: + document = Document.objects.get_by_natural_key(*document_version_natural_key) + except Document.DoesNotExist: raise self.model.DoesNotExist - return self.get(document_version__pk=document_version.pk, page_number=page_number) + return self.get(document__pk=document.pk, page_number=page_number) def get_queryset(self): return models.QuerySet( @@ -124,6 +124,19 @@ class DocumentVersionManager(models.Manager): return self.get(document__pk=document.pk, checksum=checksum) +class DocumentVersionPageManager(models.Manager): + def get_by_natural_key(self, page_number, document_version_natural_key): + DocumentVersion = apps.get_model( + app_label='documents', model_name='DocumentVersion' + ) + try: + document_version = DocumentVersion.objects.get_by_natural_key(*document_version_natural_key) + except DocumentVersion.DoesNotExist: + raise self.model.DoesNotExist + + return self.get(document_version__pk=document_version.pk, page_number=page_number) + + class DuplicatedDocumentManager(models.Manager): def clean_empty_duplicate_lists(self): self.filter(documents=None).delete() diff --git a/mayan/apps/documents/migrations/0052_rename_document_page.py b/mayan/apps/documents/migrations/0052_rename_document_page.py new file mode 100644 index 0000000000..d6751c3810 --- /dev/null +++ b/mayan/apps/documents/migrations/0052_rename_document_page.py @@ -0,0 +1,38 @@ +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + dependencies = [ + ('documents', '0051_documentpage_enabled'), + ] + + operations = [ + migrations.DeleteModel( + name='DocumentPageResult', + ), + migrations.RenameModel('DocumentPage', 'DocumentVersionPage'), + migrations.AlterField( + model_name='documentversionpage', + name='document_version', + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name='pages', to='documents.DocumentVersion', + verbose_name='Document version' + ), + ), + migrations.RemoveField( + model_name='documentversionpage', + name='enabled', + ), + migrations.AlterModelOptions( + name='documentversionpage', + options={ + 'ordering': ('page_number',), + 'verbose_name': 'Document version page', + 'verbose_name_plural': 'Document version pages' + }, + ), + ] diff --git a/mayan/apps/documents/migrations/0053_create_document_page_and_result_models.py b/mayan/apps/documents/migrations/0053_create_document_page_and_result_models.py new file mode 100644 index 0000000000..f7f4081c6e --- /dev/null +++ b/mayan/apps/documents/migrations/0053_create_document_page_and_result_models.py @@ -0,0 +1,57 @@ +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + dependencies = [ + ('contenttypes', '0002_remove_content_type_name'), + ('documents', '0052_rename_document_page'), + ] + + operations = [ + migrations.CreateModel( + name='DocumentPage', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('enabled', models.BooleanField(default=True, verbose_name='Enabled')), + ('page_number', models.PositiveIntegerField(blank=True, db_index=True, null=True, verbose_name='Page number')), + ('object_id', models.PositiveIntegerField()), + ('content_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType')), + ('document', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='pages', to='documents.Document', verbose_name='Document')), + ], + options={ + 'unique_together': set([('document', 'page_number')]), + 'verbose_name': 'Document page', + 'verbose_name_plural': 'Document pages', + 'ordering': ('page_number',), + }, + ), + migrations.CreateModel( + name='DocumentPageResult', + fields=[ + ], + options={ + 'verbose_name': 'Document page result', + 'verbose_name_plural': 'Document pages result', + 'ordering': ('document', 'page_number'), + 'proxy': True, + 'indexes': [], + }, + bases=('documents.documentpage',), + ), + migrations.CreateModel( + name='DocumentVersionPageResult', + fields=[ + ], + options={ + 'verbose_name': 'Document version page', + 'verbose_name_plural': 'Document version pages', + 'ordering': ('document_version__document', 'page_number'), + 'proxy': True, + 'indexes': [], + }, + bases=('documents.documentversionpage',), + ), + ] diff --git a/mayan/apps/documents/migrations/0054_reset_document_pages.py b/mayan/apps/documents/migrations/0054_reset_document_pages.py new file mode 100644 index 0000000000..413f423e3d --- /dev/null +++ b/mayan/apps/documents/migrations/0054_reset_document_pages.py @@ -0,0 +1,62 @@ +from __future__ import unicode_literals + +from django.db import migrations + + +def get_latest_version(document): + return document.versions.order_by('timestamp').last() + + +def operation_reset_document_pages(apps, schema_editor): + Document = apps.get_model(app_label='documents', model_name='Document') + DocumentPage = apps.get_model( + app_label='documents', model_name='DocumentPage' + ) + + # Define inside the function to use the migration's apps instance + def pages_reset(document): + ContentType = apps.get_model('contenttypes', 'ContentType') + DocumentVersionPage = apps.get_model( + app_label='documents', model_name='DocumentVersionPage' + ) + + content_type = ContentType.objects.get_for_model( + model=DocumentVersionPage + ) + + for document_page in document.pages.all(): + document_page.delete() + + for version_page in get_latest_version(document=document).pages.all(): + document_page = document.pages.create( + content_type=content_type, + page_number=version_page.page_number, + object_id=version_page.pk, + ) + + for document in Document.objects.using(schema_editor.connection.alias).all(): + pages_reset(document=document) + + +def operation_reset_document_pages_reverse(apps, schema_editor): + Document = apps.get_model(app_label='documents', model_name='Document') + DocumentPage = apps.get_model( + app_label='documents', model_name='DocumentPage' + ) + + for document in Document.objects.using(schema_editor.connection.alias).all(): + for document_page in document.pages.all(): + document_page.delete() + + +class Migration(migrations.Migration): + dependencies = [ + ('documents', '0053_create_document_page_and_result_models'), + ] + + operations = [ + migrations.RunPython( + code=operation_reset_document_pages, + reverse_code=operation_reset_document_pages_reverse + ), + ] diff --git a/mayan/apps/documents/models/__init__.py b/mayan/apps/documents/models/__init__.py index bb4b55b82a..69063382c8 100644 --- a/mayan/apps/documents/models/__init__.py +++ b/mayan/apps/documents/models/__init__.py @@ -2,4 +2,5 @@ from .document_models import * # NOQA from .document_page_models import * # NOQA from .document_type_models import * # NOQA from .document_version_models import * # NOQA +from .document_version_page_models import * # NOQA from .misc_models import * # NOQA diff --git a/mayan/apps/documents/models/document_models.py b/mayan/apps/documents/models/document_models.py index 8b6fef2400..95cbba506d 100644 --- a/mayan/apps/documents/models/document_models.py +++ b/mayan/apps/documents/models/document_models.py @@ -5,9 +5,10 @@ import uuid from django.apps import apps from django.core.files import File -from django.db import models +from django.db import models, transaction from django.urls import reverse from django.utils.encoding import python_2_unicode_compatible +from django.utils.functional import cached_property from django.utils.timezone import now from django.utils.translation import ugettext, ugettext_lazy as _ @@ -15,6 +16,7 @@ from ..events import ( event_document_create, event_document_properties_edit, event_document_type_change, ) +from ..literals import DOCUMENT_IMAGES_CACHE_NAME from ..managers import DocumentManager, PassthroughManager, TrashCanManager from ..settings import setting_language from ..signals import post_document_type_change @@ -102,6 +104,26 @@ class Document(models.Model): ) return RecentDocument.objects.add_document_for_user(user, self) + @cached_property + def cache(self): + Cache = apps.get_model(app_label='file_caching', model_name='Cache') + return Cache.objects.get(name=DOCUMENT_IMAGES_CACHE_NAME) + + @cached_property + def cache_partition(self): + partition, created = self.cache.partitions.get_or_create( + name='document-{}'.format(self.uuid) + ) + return partition + + @property + def checksum(self): + return self.latest_version.checksum + + @property + def date_updated(self): + return self.latest_version.timestamp + def delete(self, *args, **kwargs): to_trash = kwargs.pop('to_trash', True) @@ -126,25 +148,37 @@ class Document(models.Model): else: return False + @property + def file_mime_encoding(self): + return self.latest_version.encoding + + @property + def file_mimetype(self): + return self.latest_version.mimetype + def get_absolute_url(self): return reverse( viewname='documents:document_preview', kwargs={'pk': self.pk} ) def get_api_image_url(self, *args, **kwargs): - latest_version = self.latest_version - if latest_version: - return latest_version.get_api_image_url(*args, **kwargs) + first_page = self.pages.first() + if first_page: + return first_page.get_api_image_url(*args, **kwargs) @property def is_in_trash(self): return self.in_trash + @property + def latest_version(self): + return self.versions.order_by('timestamp').last() + def natural_key(self): return (self.uuid,) natural_key.dependencies = ['documents.DocumentType'] - def new_version(self, file_object, comment=None, _user=None): + def new_version(self, file_object, append_pages=False, comment=None, _user=None): logger.info('Creating new document version for document: %s', self) DocumentVersion = apps.get_model( app_label='documents', model_name='DocumentVersion' @@ -153,9 +187,10 @@ class Document(models.Model): document_version = DocumentVersion( document=self, comment=comment or '', file=File(file_object) ) - document_version.save(_user=_user) + document_version.save(append_pages=append_pages, _user=_user) logger.info('New document version queued for document: %s', self) + return document_version def open(self, *args, **kwargs): @@ -165,6 +200,34 @@ class Document(models.Model): """ return self.latest_version.open(*args, **kwargs) + @property + def page_count(self): + return self.pages.count() + + @property + def pages(self): + return self.pages.all() + + @property + def pages_all(self): + DocumentPage = apps.get_model( + app_label='documents', model_name='DocumentPage' + ) + return DocumentPage.passthrough.filter(document=self) + + def pages_reset(self, update_page_count=True): + with transaction.atomic(): + for page in self.pages.all(): + page.delete() + + if update_page_count: + self.latest_version.update_page_count() + + for version_page in self.latest_version.pages.all(): + document_page = self.pages.create( + content_object = version_page + ) + def restore(self): self.in_trash = False self.save() @@ -209,53 +272,3 @@ class Document(models.Model): @property def size(self): return self.latest_version.size - - # Compatibility methods - - @property - def checksum(self): - return self.latest_version.checksum - - @property - def date_updated(self): - return self.latest_version.timestamp - - @property - def file_mime_encoding(self): - return self.latest_version.encoding - - @property - def file_mimetype(self): - return self.latest_version.mimetype - - @property - def latest_version(self): - return self.versions.order_by('timestamp').last() - - @property - def page_count(self): - return self.latest_version.page_count - - @property - def pages_all(self): - try: - return self.latest_version.pages_all - except AttributeError: - # Document has no version yet - DocumentPage = apps.get_model( - app_label='documents', model_name='DocumentPage' - ) - - return DocumentPage.objects.none() - - @property - def pages(self): - try: - return self.latest_version.pages - except AttributeError: - # Document has no version yet - DocumentPage = apps.get_model( - app_label='documents', model_name='DocumentPage' - ) - - return DocumentPage.objects.none() diff --git a/mayan/apps/documents/models/document_page_models.py b/mayan/apps/documents/models/document_page_models.py index 00be66ab4b..b6784a1ff8 100644 --- a/mayan/apps/documents/models/document_page_models.py +++ b/mayan/apps/documents/models/document_page_models.py @@ -4,14 +4,16 @@ import logging from furl import furl +from django.contrib.contenttypes.fields import GenericForeignKey +from django.contrib.contenttypes.models import ContentType from django.db import models +from django.db.models import Max from django.urls import reverse from django.utils.encoding import force_text, python_2_unicode_compatible from django.utils.functional import cached_property from django.utils.translation import ugettext_lazy as _ from mayan.apps.converter.literals import DEFAULT_ZOOM_LEVEL, DEFAULT_ROTATION - from mayan.apps.converter.models import LayerTransformation from mayan.apps.converter.transformations import ( BaseTransformation, TransformationResize, TransformationRotate, @@ -26,7 +28,7 @@ from ..settings import ( setting_zoom_min_level ) -from .document_version_models import DocumentVersion +from .document_models import Document __all__ = ('DocumentPage', 'DocumentPageResult') logger = logging.getLogger(__name__) @@ -35,16 +37,22 @@ logger = logging.getLogger(__name__) @python_2_unicode_compatible class DocumentPage(models.Model): """ - Model that describes a document version page + Model that describes a document page """ - document_version = models.ForeignKey( - on_delete=models.CASCADE, related_name='version_pages', to=DocumentVersion, - verbose_name=_('Document version') + document = models.ForeignKey( + on_delete=models.CASCADE, related_name='pages', to=Document, + verbose_name=_('Document') ) enabled = models.BooleanField(default=True, verbose_name=_('Enabled')) page_number = models.PositiveIntegerField( - db_index=True, default=1, editable=False, - verbose_name=_('Page number') + db_index=True, blank=True, null=True, verbose_name=_('Page number') + ) + content_type = models.ForeignKey( + on_delete=models.CASCADE, to=ContentType + ) + object_id = models.PositiveIntegerField() + content_object = GenericForeignKey( + ct_field='content_type', fk_field='object_id' ) objects = DocumentPageManager() @@ -52,6 +60,7 @@ class DocumentPage(models.Model): class Meta: ordering = ('page_number',) + unique_together = ('document', 'page_number') verbose_name = _('Document page') verbose_name_plural = _('Document pages') @@ -60,7 +69,7 @@ class DocumentPage(models.Model): @cached_property def cache_partition(self): - partition, created = self.document_version.cache.partitions.get_or_create( + partition, created = self.document.cache.partitions.get_or_create( name=self.uuid ) return partition @@ -69,19 +78,15 @@ class DocumentPage(models.Model): self.cache_partition.delete() super(DocumentPage, self).delete(*args, **kwargs) - def detect_orientation(self): - with self.document_version.open() as file_object: - converter = get_converter_class()( - file_object=file_object, - mime_type=self.document_version.mimetype - ) - return converter.detect_orientation( - page_number=self.page_number - ) - - @property - def document(self): - return self.document_version.document + #def detect_orientation(self): + # with self.document_version.open() as file_object: + # converter = get_converter_class()( + # file_object=file_object, + # mime_type=self.document_version.mimetype + # ) + # return converter.detect_orientation( + # page_number=self.page_number + # ) def generate_image(self, user=None, **kwargs): transformation_list = self.get_combined_transformation_list(user=user, **kwargs) @@ -90,7 +95,7 @@ class DocumentPage(models.Model): # Check is transformed image is available logger.debug('transformations cache filename: %s', combined_cache_filename) - if not setting_disable_transformed_image_cache.value and self.cache_partition.get_file(filename=combined_cache_filename): + if self.cache_partition.get_file(filename=combined_cache_filename): logger.debug( 'transformations cache file "%s" found', combined_cache_filename ) @@ -128,8 +133,7 @@ class DocumentPage(models.Model): final_url.args = kwargs final_url.path = reverse( viewname='rest_api:documentpage-image', kwargs={ - 'pk': self.document.pk, 'version_pk': self.document_version.pk, - 'page_pk': self.pk + 'pk': self.document.pk, 'page_pk': self.pk } ) final_url.args['_hash'] = transformations_hash @@ -190,12 +194,12 @@ class DocumentPage(models.Model): return transformation_list def get_image(self, transformations=None): - cache_filename = 'base_image' + cache_filename = 'document_page' logger.debug('Page cache filename: %s', cache_filename) cache_file = self.cache_partition.get_file(filename=cache_filename) - if not setting_disable_base_image_cache.value and cache_file: + if cache_file: logger.debug('Page cache file "%s" found', cache_filename) with cache_file.open() as file_object: @@ -216,14 +220,25 @@ class DocumentPage(models.Model): logger.debug('Page cache file "%s" not found', cache_filename) try: - with self.document_version.get_intermediate_file() as file_object: + #with self.document_version.get_intermediate_file() as file_object: + #Render or get cached document version page + + #self.content_object.generate_image() + self.content_object.get_image() + cache_filename = 'base_image' + cache_file = self.content_object.cache_partition.get_file(filename=cache_filename) + + with cache_file.open() as file_object: converter = get_converter_class()( file_object=file_object ) - converter.seek_page(page_number=self.page_number - 1) + converter.seek_page(page_number=0) + #self.page_number - 1) page_image = converter.get_page() + cache_filename = 'document_page' + # Since open "wb+" doesn't create files, create it explicitly with self.cache_partition.create_file(filename=cache_filename) as file_object: file_object.write(page_image.getvalue()) @@ -241,28 +256,39 @@ class DocumentPage(models.Model): ) raise + def get_label(self): + return _( + 'Page %(page_number)d out of %(total_pages)d of %(document)s' + ) % { + 'document': force_text(self.document), + 'page_number': self.page_number, + 'total_pages': self.document.pages_all.count() + } + get_label.short_description = _('Label') + @property def is_in_trash(self): return self.document.is_in_trash - def get_label(self): - return _( - 'Page %(page_num)d out of %(total_pages)d of %(document)s' - ) % { - 'document': force_text(self.document), - 'page_num': self.page_number, - 'total_pages': self.document_version.pages_all.count() - } - get_label.short_description = _('Label') - def natural_key(self): - return (self.page_number, self.document_version.natural_key()) - natural_key.dependencies = ['documents.DocumentVersion'] + return (self.page_number, self.document.natural_key()) + natural_key.dependencies = ['documents.Document'] + + def save(self, *args, **kwargs): + if not self.page_number: + last_page_number = DocumentPage.objects.filter( + document=self.document + ).aggregate(Max('page_number'))['page_number__max'] + if last_page_number is not None: + self.page_number = last_page_number + 1 + else: + self.page_number = 1 + super(DocumentPage, self).save(*args, **kwargs) @property def siblings(self): return DocumentPage.objects.filter( - document_version=self.document_version + document=self.document ) @property @@ -271,12 +297,12 @@ class DocumentPage(models.Model): Make cache UUID a mix of version ID and page ID to avoid using stale images """ - return '{}-{}'.format(self.document_version.uuid, self.pk) + return '{}-{}'.format(self.document.uuid, self.pk) class DocumentPageResult(DocumentPage): class Meta: - ordering = ('document_version__document', 'page_number') + ordering = ('document', 'page_number') proxy = True - verbose_name = _('Document page') - verbose_name_plural = _('Document pages') + verbose_name = _('Document page result') + verbose_name_plural = _('Document pages result') diff --git a/mayan/apps/documents/models/document_version_models.py b/mayan/apps/documents/models/document_version_models.py index 1c24fbfeda..d491ca5bd8 100644 --- a/mayan/apps/documents/models/document_version_models.py +++ b/mayan/apps/documents/models/document_version_models.py @@ -246,23 +246,12 @@ class DocumentVersion(models.Model): return result - @property - def pages_all(self): - DocumentPage = apps.get_model( - app_label='documents', model_name='DocumentPage' - ) - return DocumentPage.passthrough.filter(document_version=self) - - @property - def pages(self): - return self.version_pages.all() - - @property - def page_count(self): - """ - The number of pages that the document posses. - """ - return self.pages.count() + #@property + #def page_count(self): + # """ + # The number of pages that the document posses. + # """ + # return self.pages.count() def revert(self, _user=None): """ @@ -285,6 +274,7 @@ class DocumentVersion(models.Model): Overloaded save method that updates the document version's checksum, mimetype, and page count when created """ + append_pages = kwargs.pop('append_pages', False) user = kwargs.pop('_user', None) new_document_version = not self.pk @@ -304,7 +294,7 @@ class DocumentVersion(models.Model): # Only do this for new documents self.update_checksum(save=False) self.update_mimetype(save=False) - self.save() + self.save(append_pages=append_pages, _user=user) self.update_page_count(save=False) if setting_fix_orientation.value: self.fix_orientation() @@ -337,6 +327,14 @@ class DocumentVersion(models.Model): sender=Document, instance=self.document ) + if append_pages: + for version_page in self.pages.all(): + self.document.pages.create( + content_object = version_page + ) + else: + self.document.pages_reset(update_page_count=False) + def save_to_file(self, file_object): """ Save a copy of the document from the document storage backend @@ -410,7 +408,7 @@ class DocumentVersion(models.Model): pass else: DocumentPage = apps.get_model( - app_label='documents', model_name='DocumentPage' + app_label='documents', model_name='DocumentVersionPage' ) with transaction.atomic(): diff --git a/mayan/apps/documents/models/document_version_page_models.py b/mayan/apps/documents/models/document_version_page_models.py new file mode 100644 index 0000000000..c0c5b7f8f3 --- /dev/null +++ b/mayan/apps/documents/models/document_version_page_models.py @@ -0,0 +1,282 @@ +from __future__ import absolute_import, unicode_literals + +import logging + +from furl import furl + +from django.db import models +from django.urls import reverse +from django.utils.encoding import force_text, python_2_unicode_compatible +from django.utils.functional import cached_property +from django.utils.translation import ugettext_lazy as _ + +from mayan.apps.converter.literals import DEFAULT_ZOOM_LEVEL, DEFAULT_ROTATION + +from mayan.apps.converter.models import LayerTransformation +from mayan.apps.converter.transformations import ( + BaseTransformation, TransformationResize, TransformationRotate, + TransformationZoom +) +from mayan.apps.converter.utils import get_converter_class + +from ..managers import DocumentVersionPageManager +from ..settings import ( + setting_disable_base_image_cache, setting_disable_transformed_image_cache, + setting_display_width, setting_display_height, setting_zoom_max_level, + setting_zoom_min_level +) + +from .document_version_models import DocumentVersion + +__all__ = ('DocumentVersionPage', 'DocumentVersionPageResult') +logger = logging.getLogger(__name__) + + +@python_2_unicode_compatible +class DocumentVersionPage(models.Model): + """ + Model that describes a document version page + """ + document_version = models.ForeignKey( + on_delete=models.CASCADE, related_name='pages', to=DocumentVersion, + verbose_name=_('Document version') + ) + page_number = models.PositiveIntegerField( + db_index=True, default=1, editable=False, + verbose_name=_('Page number') + ) + + objects = DocumentVersionPageManager() + + class Meta: + ordering = ('page_number',) + verbose_name = _('Document version page') + verbose_name_plural = _('Document version pages') + + def __str__(self): + return self.get_label() + + @cached_property + def cache_partition(self): + partition, created = self.document_version.cache.partitions.get_or_create( + name=self.uuid + ) + return partition + + def delete(self, *args, **kwargs): + self.cache_partition.delete() + super(DocumentVersionPage, self).delete(*args, **kwargs) + + #def detect_orientation(self): + # with self.document_version.open() as file_object: + # converter = get_converter_class()( + # file_object=file_object, + # mime_type=self.document_version.mimetype + # ) + # return converter.detect_orientation( + # page_number=self.page_number + # ) + + @property + def document(self): + return self.document_version.document + + def generate_image(self, user=None, **kwargs): + transformation_list = self.get_combined_transformation_list(user=user, **kwargs) + combined_cache_filename = BaseTransformation.combine(transformation_list) + + # Check is transformed image is available + logger.debug('transformations cache filename: %s', combined_cache_filename) + + if self.cache_partition.get_file(filename=combined_cache_filename): + logger.debug( + 'transformations cache file "%s" found', combined_cache_filename + ) + else: + logger.debug( + 'transformations cache file "%s" not found', combined_cache_filename + ) + image = self.get_image(transformations=transformation_list) + with self.cache_partition.create_file(filename=combined_cache_filename) as file_object: + file_object.write(image.getvalue()) + + return combined_cache_filename + + #def get_absolute_url(self): + # return reverse( + # viewname='documents:document_version_page_view', kwargs={ + # 'pk': self.pk + # } + # ) + + def get_api_image_url(self, *args, **kwargs): + """ + Create an unique URL combining: + - the page's image URL + - the interactive argument + - a hash from the server side and interactive transformations + The purpose of this unique URL is to allow client side caching + if document page images. + """ + transformations_hash = BaseTransformation.combine( + self.get_combined_transformation_list(*args, **kwargs) + ) + + kwargs.pop('transformations', None) + + final_url = furl() + final_url.args = kwargs + final_url.path = reverse( + viewname='rest_api:documentversionpage-image', kwargs={ + 'pk': self.document.pk, 'version_pk': self.document_version.pk, + 'page_pk': self.pk + } + ) + final_url.args['_hash'] = transformations_hash + + return final_url.tostr() + + def get_combined_transformation_list(self, user=None, *args, **kwargs): + """ + Return a list of transformation containing the server side + document page transformation as well as tranformations created + from the arguments as transient interactive transformation. + """ + # Convert arguments into transformations + transformations = kwargs.get('transformations', []) + + # Set sensible defaults if the argument is not specified or if the + # argument is None + width = kwargs.get('width', setting_display_width.value) or setting_display_width.value + height = kwargs.get('height', setting_display_height.value) or setting_display_height.value + rotation = kwargs.get('rotation', DEFAULT_ROTATION) or DEFAULT_ROTATION + zoom_level = kwargs.get('zoom', DEFAULT_ZOOM_LEVEL) or DEFAULT_ZOOM_LEVEL + + if zoom_level < setting_zoom_min_level.value: + zoom_level = setting_zoom_min_level.value + + if zoom_level > setting_zoom_max_level.value: + zoom_level = setting_zoom_max_level.value + + # Generate transformation hash + transformation_list = [] + + maximum_layer_order = kwargs.get('maximum_layer_order', None) + + # Stored transformations first + for stored_transformation in LayerTransformation.objects.get_for_object( + self, maximum_layer_order=maximum_layer_order, as_classes=True, + user=user + ): + transformation_list.append(stored_transformation) + + # Interactive transformations second + for transformation in transformations: + transformation_list.append(transformation) + + if rotation: + transformation_list.append( + TransformationRotate(degrees=rotation) + ) + + if width: + transformation_list.append( + TransformationResize(width=width, height=height) + ) + + if zoom_level: + transformation_list.append(TransformationZoom(percent=zoom_level)) + + return transformation_list + + def get_image(self, transformations=None): + cache_filename = 'base_image' + logger.debug('Page cache filename: %s', cache_filename) + + cache_file = self.cache_partition.get_file(filename=cache_filename) + + if cache_file: + logger.debug('Page cache file "%s" found', cache_filename) + + with cache_file.open() as file_object: + converter = get_converter_class()( + file_object=file_object + ) + + converter.seek_page(page_number=0) + + # This code is also repeated below to allow using a context + # manager with cache_file.open and close it automatically. + # Apply runtime transformations + for transformation in transformations or []: + converter.transform(transformation=transformation) + + return converter.get_page() + else: + logger.debug('Page cache file "%s" not found', cache_filename) + + try: + with self.document_version.get_intermediate_file() as file_object: + converter = get_converter_class()( + file_object=file_object + ) + converter.seek_page(page_number=self.page_number - 1) + + page_image = converter.get_page() + + # Since open "wb+" doesn't create files, create it explicitly + with self.cache_partition.create_file(filename=cache_filename) as file_object: + file_object.write(page_image.getvalue()) + + # Apply runtime transformations + for transformation in transformations or []: + converter.transform(transformation=transformation) + + return converter.get_page() + except Exception as exception: + # Cleanup in case of error + logger.error( + 'Error creating page cache file "%s"; %s', + cache_filename, exception + ) + raise + + #@property + #def is_in_trash(self): + # return self.document.is_in_trash + + def get_label(self): + return _( + 'Version page %(page_number)d out of %(total_pages)d of %(document)s' + ) % { + 'document': force_text(self.document), + 'page_number': self.page_number, + 'total_pages': self.document_version.pages.count() + } + get_label.short_description = _('Label') + + def natural_key(self): + return (self.page_number, self.document_version.natural_key()) + natural_key.dependencies = ['documents.DocumentVersion'] + + @property + def siblings(self): + return DocumentVersionPage.objects.filter( + document_version=self.document_version + ) + + @property + def uuid(self): + """ + Make cache UUID a mix of version ID and page ID to avoid using stale + images + """ + return '{}-{}'.format(self.document_version.uuid, self.pk) + + +class DocumentVersionPageResult(DocumentVersionPage): + class Meta: + ordering = ('document_version__document', 'page_number') + proxy = True + verbose_name = _('Document version page') + verbose_name_plural = _('Document version pages') diff --git a/mayan/apps/documents/queues.py b/mayan/apps/documents/queues.py index 11280f17d8..e01c62d8e3 100644 --- a/mayan/apps/documents/queues.py +++ b/mayan/apps/documents/queues.py @@ -30,6 +30,10 @@ queue_converter.add_task_type( dotted_path='mayan.apps.documents.tasks.task_generate_document_page_image', label=_('Generate document page image') ) +queue_converter.add_task_type( + dotted_path='mayan.apps.documents.tasks.task_generate_document_version_page_image', + label=_('Generate document version page image') +) queue_documents.add_task_type( dotted_path='mayan.apps.documents.tasks.task_delete_document', @@ -66,6 +70,10 @@ queue_tools.add_task_type( label=_('Duplicated document scan') ) +queue_uploads.add_task_type( + dotted_path='mayan.apps.documents.tasks.task_document_pages_reset', + label=_('Reset document pages') +) queue_uploads.add_task_type( dotted_path='mayan.apps.documents.tasks.task_update_page_count', label=_('Update document page count') diff --git a/mayan/apps/documents/search.py b/mayan/apps/documents/search.py index de03a36dd0..e631099d3c 100644 --- a/mayan/apps/documents/search.py +++ b/mayan/apps/documents/search.py @@ -17,12 +17,20 @@ def transformation_format_uuid(term_string): return term_string -def get_queryset_page_search_queryset(): +def get_queryset_document_page_search_queryset(): # Ignore documents in trash can DocumentPage = apps.get_model( app_label='documents', model_name='DocumentPage' ) - return DocumentPage.objects.filter(document_version__document__in_trash=False) + return DocumentPage.objects.filter(document__in_trash=False) + + +def get_queryset_document_version_page_search_queryset(): + # Ignore documents in trash can + DocumentVersionPage = apps.get_model( + app_label='documents', model_name='DocumentVersionPage' + ) + return DocumentVersionPage.objects.filter(document_version__document__in_trash=False) document_search = SearchModel( @@ -30,7 +38,6 @@ document_search = SearchModel( model_name='Document', permission=permission_document_view, serializer_path='mayan.apps.documents.serializers.DocumentSerializer' ) - document_search.add_model_field( field='document_type__label', label=_('Document type') ) @@ -50,24 +57,49 @@ document_search.add_model_field( document_page_search = SearchModel( app_label='documents', list_mode=LIST_MODE_CHOICE_ITEM, model_name='DocumentPage', permission=permission_document_view, - queryset=get_queryset_page_search_queryset, + queryset=get_queryset_document_page_search_queryset, serializer_path='mayan.apps.documents.serializers.DocumentPageSerializer' ) +document_version_page_search = SearchModel( + app_label='documents', list_mode=LIST_MODE_CHOICE_ITEM, + model_name='DocumentVersionPage', permission=permission_document_view, + queryset=get_queryset_document_version_page_search_queryset, + serializer_path='mayan.apps.documents.serializers.DocumentVersionPageSerializer' +) + document_page_search.add_model_field( - field='document_version__document__document_type__label', + field='document__document_type__label', label=_('Document type') ) document_page_search.add_model_field( - field='document_version__document__versions__mimetype', + field='document__versions__mimetype', label=_('MIME type') ) document_page_search.add_model_field( + field='document__label', label=_('Label') +) +document_page_search.add_model_field( + field='document__description', label=_('Description') +) +document_page_search.add_model_field( + field='document__versions__checksum', label=_('Checksum') +) + +document_version_page_search.add_model_field( + field='document_version__document__document_type__label', + label=_('Document type') +) +document_version_page_search.add_model_field( + field='document_version__document__versions__mimetype', + label=_('MIME type') +) +document_version_page_search.add_model_field( field='document_version__document__label', label=_('Label') ) -document_page_search.add_model_field( +document_version_page_search.add_model_field( field='document_version__document__description', label=_('Description') ) -document_page_search.add_model_field( +document_version_page_search.add_model_field( field='document_version__checksum', label=_('Checksum') ) diff --git a/mayan/apps/documents/serializers.py b/mayan/apps/documents/serializers.py index 9632a546cb..5a6c172f7a 100644 --- a/mayan/apps/documents/serializers.py +++ b/mayan/apps/documents/serializers.py @@ -8,42 +8,40 @@ from rest_framework.reverse import reverse from mayan.apps.common.models import SharedUploadedFile from .models import ( - Document, DocumentVersion, DocumentPage, DocumentType, - DocumentTypeFilename, RecentDocument + Document, DocumentPage, DocumentType, DocumentTypeFilename, + DocumentVersion, DocumentVersionPage, RecentDocument ) from .settings import setting_language from .tasks import task_upload_new_version class DocumentPageSerializer(serializers.HyperlinkedModelSerializer): - document_version_url = serializers.SerializerMethodField() + document_url = serializers.SerializerMethodField() image_url = serializers.SerializerMethodField() url = serializers.SerializerMethodField() class Meta: - fields = ('document_version_url', 'image_url', 'page_number', 'url') + fields = ('document_url', 'image_url', 'page_number', 'url') model = DocumentPage - def get_document_version_url(self, instance): + def get_document_url(self, instance): return reverse( - viewname='rest_api:documentversion-detail', args=( - instance.document.pk, instance.document_version.pk, + viewname='rest_api:document-detail', args=( + instance.document.pk, ), request=self.context['request'], format=self.context['format'] ) def get_image_url(self, instance): return reverse( viewname='rest_api:documentpage-image', args=( - instance.document.pk, instance.document_version.pk, - instance.pk, + instance.document.pk, instance.pk, ), request=self.context['request'], format=self.context['format'] ) def get_url(self, instance): return reverse( viewname='rest_api:documentpage-detail', args=( - instance.document.pk, instance.document_version.pk, - instance.pk, + instance.document.pk, instance.pk, ), request=self.context['request'], format=self.context['format'] ) @@ -97,6 +95,39 @@ class WritableDocumentTypeSerializer(serializers.ModelSerializer): return obj.documents.count() +class DocumentVersionPageSerializer(serializers.HyperlinkedModelSerializer): + document_version_url = serializers.SerializerMethodField() + image_url = serializers.SerializerMethodField() + url = serializers.SerializerMethodField() + + class Meta: + fields = ('document_version_url', 'image_url', 'page_number', 'url') + model = DocumentVersionPage + + def get_document_version_url(self, instance): + return reverse( + viewname='rest_api:documentversion-detail', args=( + instance.document.pk, instance.document_version.pk, + ), request=self.context['request'], format=self.context['format'] + ) + + def get_image_url(self, instance): + return reverse( + viewname='rest_api:documentversionpage-image', args=( + instance.document.pk, instance.document_version.pk, + instance.pk, + ), request=self.context['request'], format=self.context['format'] + ) + + def get_url(self, instance): + return reverse( + viewname='rest_api:documentversionpage-detail', args=( + instance.document.pk, instance.document_version.pk, + instance.pk, + ), request=self.context['request'], format=self.context['format'] + ) + + class DocumentVersionSerializer(serializers.HyperlinkedModelSerializer): document_url = serializers.SerializerMethodField() download_url = serializers.SerializerMethodField() diff --git a/mayan/apps/documents/statistics.py b/mayan/apps/documents/statistics.py index 2f1059e7d1..94d91be994 100644 --- a/mayan/apps/documents/statistics.py +++ b/mayan/apps/documents/statistics.py @@ -41,7 +41,7 @@ def new_documents_per_month(): def new_document_pages_per_month(): DocumentPage = apps.get_model( - app_label='documents', model_name='DocumentPage' + app_label='documents', model_name='DocumentVersionPage' ) qss = qsstats.QuerySetStats( @@ -106,7 +106,7 @@ def new_document_pages_this_month(user=None): app_label='acls', model_name='AccessControlList' ) DocumentPage = apps.get_model( - app_label='documents', model_name='DocumentPage' + app_label='documents', model_name='DocumentVersionPage' ) queryset = DocumentPage.objects.all() @@ -195,7 +195,7 @@ def total_document_version_per_month(): def total_document_page_per_month(): DocumentPage = apps.get_model( - app_label='documents', model_name='DocumentPage' + app_label='documents', model_name='DocumentVersionPage' ) qss = qsstats.QuerySetStats( diff --git a/mayan/apps/documents/tasks.py b/mayan/apps/documents/tasks.py index 34956086bc..a2d4fe9fc7 100644 --- a/mayan/apps/documents/tasks.py +++ b/mayan/apps/documents/tasks.py @@ -9,8 +9,8 @@ from django.db import OperationalError from mayan.celery import app from .literals import ( - UPDATE_PAGE_COUNT_RETRY_DELAY, UPLOAD_NEW_DOCUMENT_RETRY_DELAY, - UPLOAD_NEW_VERSION_RETRY_DELAY + RETRY_DELAY_DOCUMENT_RESET_PAGES, UPDATE_PAGE_COUNT_RETRY_DELAY, + UPLOAD_NEW_DOCUMENT_RETRY_DELAY, ) logger = logging.getLogger(__name__) @@ -65,6 +65,25 @@ def task_delete_stubs(): logger.info(msg='Finshed') +@app.task(bind=True, default_retry_delay=RETRY_DELAY_DOCUMENT_RESET_PAGES, ignore_result=True) +def task_document_pages_reset(self, document_id): + Document = apps.get_model( + app_label='documents', model_name='Document' + ) + + document = Document.objects.get(pk=document_id) + + try: + document.pages_reset() + except OperationalError as exception: + logger.warning( + 'Operational error during attempt to reset pages for ' + 'document: %s; %s. Retrying.', document, + exception + ) + raise self.retry(exc=exception) + + @app.task() def task_generate_document_page_image(document_page_id, user_id=None, **kwargs): DocumentPage = apps.get_model( @@ -81,6 +100,22 @@ def task_generate_document_page_image(document_page_id, user_id=None, **kwargs): return document_page.generate_image(user=user, **kwargs) +@app.task() +def task_generate_document_version_page_image(document_version_page_id, user_id=None, **kwargs): + DocumentVersionPage = apps.get_model( + app_label='documents', model_name='DocumentVersionPage' + ) + User = get_user_model() + + if user_id: + user = User.objects.get(pk=user_id) + else: + user = None + + document_version_page = DocumentVersionPage.objects.get(pk=document_version_page_id) + return document_version_page.generate_image(user=user, **kwargs) + + @app.task(ignore_result=True) def task_scan_duplicates_all(): DuplicatedDocument = apps.get_model( @@ -177,7 +212,7 @@ def task_upload_new_document(self, document_type_id, shared_uploaded_file_id): @app.task(bind=True, default_retry_delay=UPLOAD_NEW_VERSION_RETRY_DELAY, ignore_result=True) -def task_upload_new_version(self, document_id, shared_uploaded_file_id, user_id, comment=None): +def task_upload_new_version(self, document_id, shared_uploaded_file_id, user_id, append_pages=False, comment=None): SharedUploadedFile = apps.get_model( app_label='common', model_name='SharedUploadedFile' ) @@ -212,7 +247,7 @@ def task_upload_new_version(self, document_id, shared_uploaded_file_id, user_id, document=document, comment=comment or '', file=file_object ) try: - document_version.save(_user=user) + document_version.save(append_pages=append_pages, _user=user) except Warning as warning: # New document version are blocked logger.info( diff --git a/mayan/apps/documents/tests/mixins.py b/mayan/apps/documents/tests/mixins.py index bb996c6cde..d11afaa921 100644 --- a/mayan/apps/documents/tests/mixins.py +++ b/mayan/apps/documents/tests/mixins.py @@ -69,6 +69,8 @@ class DocumentTestMixin(object): self.test_document = document self.test_documents.append(document) + self.test_document_version = document.latest_version + self.test_document_page = document.pages_all.first() class DocumentTypeViewTestMixin(object): @@ -148,6 +150,26 @@ class DocumentVersionTestMixin(object): ) +class DocumentVersionViewTestMixin(object): + def _request_document_version_list_view(self): + return self.get( + viewname='documents:document_version_list', + kwargs={'pk': self.test_document.pk} + ) + + def _request_document_version_revert_view(self, document_version): + return self.post( + viewname='documents:document_version_revert', + kwargs={'pk': document_version.pk} + ) + + def _request_test_document_version_page_count_update_view(self): + return self.post( + viewname='documents:document_version_page_count_update', + kwargs={'pk': self.test_document_version.pk} + ) + + class DocumentViewTestMixin(object): def _request_document_properties_view(self): return self.get( @@ -200,6 +222,12 @@ class DocumentViewTestMixin(object): data={'id_list': self.test_document.pk} ) + def _request_document_pages_reset_view(self): + return self.post( + viewname='documents:document_pages_reset', + kwargs={'pk': self.test_document.pk} + ) + def _request_document_version_download(self, data=None): data = data or {} return self.get( @@ -208,18 +236,6 @@ class DocumentViewTestMixin(object): }, data=data ) - def _request_document_update_page_count_view(self): - return self.post( - viewname='documents:document_update_page_count', - kwargs={'pk': self.test_document.pk} - ) - - def _request_document_multiple_update_page_count_view(self): - return self.post( - viewname='documents:document_multiple_update_page_count', - data={'id_list': self.test_document.pk} - ) - def _request_document_clear_transformations_view(self): return self.post( viewname='documents:document_clear_transformations', @@ -232,8 +248,11 @@ class DocumentViewTestMixin(object): data={'id_list': self.test_document.pk} ) - def _request_empty_trash_view(self): - return self.post(viewname='documents:trash_can_empty') + def _request_document_multiple_pages_reset_view(self): + return self.post( + viewname='documents:document_multiple_pages_reset', + data={'id_list': self.test_document.pk} + ) def _request_document_print_view(self): return self.get( @@ -243,3 +262,6 @@ class DocumentViewTestMixin(object): 'page_group': PAGE_RANGE_ALL } ) + + def _request_empty_trash_view(self): + return self.post(viewname='documents:trash_can_empty') diff --git a/mayan/apps/documents/tests/test_api.py b/mayan/apps/documents/tests/test_api.py index dea50b2618..60694de79e 100644 --- a/mayan/apps/documents/tests/test_api.py +++ b/mayan/apps/documents/tests/test_api.py @@ -530,8 +530,7 @@ class DocumentPageAPIViewTestMixin(object): page = self.test_document.pages.first() return self.get( viewname='rest_api:documentpage-image', kwargs={ - 'pk': page.document.pk, 'version_pk': page.document_version.pk, - 'page_pk': page.pk + 'pk': page.document.pk, 'page_pk': page.pk } ) @@ -552,6 +551,33 @@ class DocumentPageAPIViewTestCase( self.assertEqual(response.status_code, status.HTTP_200_OK) +class DocumentVersionPageAPIViewTestMixin(object): + def _request_document_version_page_image(self): + page = self.test_document_version.pages.first() + return self.get( + viewname='rest_api:documentversionpage-image', kwargs={ + 'pk': page.document.pk, 'version_pk': page.document_version.pk, + 'page_pk': page.pk + } + ) + + +class DocumentVersionPageAPIViewTestCase( + DocumentVersionPageAPIViewTestMixin, DocumentTestMixin, BaseAPITestCase +): + def test_document_version_page_api_image_view_no_access(self): + response = self._request_document_version_page_image() + self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) + + def test_document_version_page_api_image_view_with_access(self): + self.grant_access( + obj=self.test_document, permission=permission_document_view + ) + + response = self._request_document_version_page_image() + self.assertEqual(response.status_code, status.HTTP_200_OK) + + class TrashedDocumentAPIViewTestMixin(object): def _request_test_document_api_trash_view(self): return self.delete( @@ -575,13 +601,10 @@ class TrashedDocumentAPIViewTestMixin(object): ) def _request_test_trashed_document_api_image_view(self): - latest_version = self.test_document.latest_version - return self.get( viewname='rest_api:documentpage-image', kwargs={ - 'pk': latest_version.document.pk, - 'version_pk': latest_version.pk, - 'page_pk': latest_version.pages.first().pk + 'pk': self.test_document.pk, + 'page_pk': self.test_document.pages.first().pk } ) diff --git a/mayan/apps/documents/tests/test_document_page_views.py b/mayan/apps/documents/tests/test_document_page_views.py index 29ee28cec3..96b2702968 100644 --- a/mayan/apps/documents/tests/test_document_page_views.py +++ b/mayan/apps/documents/tests/test_document_page_views.py @@ -9,10 +9,10 @@ from ..permissions import ( from .base import GenericDocumentViewTestCase -class DocumentPageDisableViewTestCase(GenericDocumentViewTestCase): - def setUp(self): - super(DocumentPageDisableViewTestCase, self).setUp() - self.test_document_page = self.test_document.pages_all.first() +class DocumentPageDisableViewTestMixin(object): + def _disable_test_document_page(self): + self.test_document_page.enabled = False + self.test_document_page.save() def _request_test_document_page_disable_view(self): return self.post( @@ -21,6 +21,31 @@ class DocumentPageDisableViewTestCase(GenericDocumentViewTestCase): } ) + def _request_test_document_page_enable_view(self): + return self.post( + viewname='documents:document_page_enable', kwargs={ + 'pk': self.test_document_page.pk + } + ) + + def _request_test_document_page_multiple_disable_view(self): + return self.post( + viewname='documents:document_page_multiple_disable', data={ + 'id_list': self.test_document_page.pk + } + ) + + def _request_test_document_page_multiple_enable_view(self): + return self.post( + viewname='documents:document_page_multiple_enable', data={ + 'id_list': self.test_document_page.pk + } + ) + + +class DocumentPageDisableViewTestCase( + DocumentPageDisableViewTestMixin, GenericDocumentViewTestCase +): def test_document_page_disable_view_no_permission(self): test_document_page_count = self.test_document.pages.count() @@ -45,13 +70,6 @@ class DocumentPageDisableViewTestCase(GenericDocumentViewTestCase): test_document_page_count, self.test_document.pages.count() ) - def _request_test_document_page_multiple_disable_view(self): - return self.post( - viewname='documents:document_page_multiple_disable', data={ - 'id_list': self.test_document_page.pk - } - ) - def test_document_page_multiple_disable_view_no_permission(self): test_document_page_count = self.test_document.pages.count() @@ -76,17 +94,6 @@ class DocumentPageDisableViewTestCase(GenericDocumentViewTestCase): test_document_page_count, self.test_document.pages.count() ) - def _disable_test_document_page(self): - self.test_document_page.enabled = False - self.test_document_page.save() - - def _request_test_document_page_enable_view(self): - return self.post( - viewname='documents:document_page_enable', kwargs={ - 'pk': self.test_document_page.pk - } - ) - def test_document_page_enable_view_no_permission(self): self._disable_test_document_page() @@ -114,13 +121,6 @@ class DocumentPageDisableViewTestCase(GenericDocumentViewTestCase): test_document_page_count, self.test_document.pages.count() ) - def _request_test_document_page_multiple_enable_view(self): - return self.post( - viewname='documents:document_page_multiple_enable', data={ - 'id_list': self.test_document_page.pk - } - ) - def test_document_page_multiple_enable_view_no_permission(self): self._disable_test_document_page() test_document_page_count = self.test_document.pages.count() @@ -148,7 +148,7 @@ class DocumentPageDisableViewTestCase(GenericDocumentViewTestCase): ) -class DocumentPageViewTestCase(GenericDocumentViewTestCase): +class DocumentPageViewTestMixin(object): def _request_test_document_page_list_view(self): return self.get( viewname='documents:document_pages', kwargs={ @@ -156,6 +156,18 @@ class DocumentPageViewTestCase(GenericDocumentViewTestCase): } ) + def _request_test_document_page_view(self, document_page): + return self.get( + viewname='documents:document_page_view', kwargs={ + 'pk': document_page.pk, + } + ) + + +class DocumentPageViewTestCase( + DocumentPageViewTestMixin, GenericDocumentViewTestCase +): + def test_document_page_list_view_no_permission(self): response = self._request_test_document_page_list_view() self.assertEqual(response.status_code, 404) @@ -170,13 +182,6 @@ class DocumentPageViewTestCase(GenericDocumentViewTestCase): response=response, text=self.test_document.label, status_code=200 ) - def _request_test_document_page_view(self, document_page): - return self.get( - viewname='documents:document_page_view', kwargs={ - 'pk': document_page.pk, - } - ) - def test_document_page_view_no_permissions(self): response = self._request_test_document_page_view( document_page=self.test_document.pages.first() diff --git a/mayan/apps/documents/tests/test_document_version_views.py b/mayan/apps/documents/tests/test_document_version_views.py index 4f712a638b..d2ab419378 100644 --- a/mayan/apps/documents/tests/test_document_version_views.py +++ b/mayan/apps/documents/tests/test_document_version_views.py @@ -1,21 +1,19 @@ from __future__ import unicode_literals from ..permissions import ( - permission_document_version_revert, permission_document_version_view, + permission_document_tools, permission_document_version_revert, + permission_document_version_view, ) from .base import GenericDocumentViewTestCase from .literals import TEST_VERSION_COMMENT -from .mixins import DocumentVersionTestMixin +from .mixins import DocumentVersionTestMixin, DocumentVersionViewTestMixin -class DocumentVersionTestCase(DocumentVersionTestMixin, GenericDocumentViewTestCase): - def _request_document_version_list_view(self): - return self.get( - viewname='documents:document_version_list', - kwargs={'pk': self.test_document.pk} - ) - +class DocumentVersionTestCase( + DocumentVersionViewTestMixin, DocumentVersionTestMixin, + GenericDocumentViewTestCase +): def test_document_version_list_no_permission(self): self._upload_new_version() @@ -33,12 +31,6 @@ class DocumentVersionTestCase(DocumentVersionTestMixin, GenericDocumentViewTestC response=response, text=TEST_VERSION_COMMENT, status_code=200 ) - def _request_document_version_revert_view(self, document_version): - return self.post( - viewname='documents:document_version_revert', - kwargs={'pk': document_version.pk} - ) - def test_document_version_revert_no_permission(self): first_version = self.test_document.latest_version self._upload_new_version() @@ -64,3 +56,25 @@ class DocumentVersionTestCase(DocumentVersionTestMixin, GenericDocumentViewTestC self.assertEqual(response.status_code, 302) self.assertEqual(self.test_document.versions.count(), 1) + + def test_document_version_page_count_update_view_no_permission(self): + self.test_document_version.pages.all().delete() + + response = self._request_test_document_version_page_count_update_view() + self.assertEqual(response.status_code, 404) + + self.assertEqual(self.test_document_version.pages.count(), 0) + + def test_document_version_page_count_update_view_with_access(self): + page_count = self.test_document_version.pages.count() + + self.test_document_version.pages.all().delete() + + self.grant_access( + obj=self.test_document, permission=permission_document_tools + ) + + response = self._request_test_document_version_page_count_update_view() + self.assertEqual(response.status_code, 302) + + self.assertEqual(self.test_document_version.pages.count(), page_count) diff --git a/mayan/apps/documents/tests/test_document_views.py b/mayan/apps/documents/tests/test_document_views.py index 381a71211d..45c96f6786 100644 --- a/mayan/apps/documents/tests/test_document_views.py +++ b/mayan/apps/documents/tests/test_document_views.py @@ -292,46 +292,44 @@ class DocumentsViewsTestCase( ) ) - def test_document_update_page_count_view_no_permission(self): + def test_document_pages_reset_view_no_permission(self): self.test_document.pages.all().delete() - self.assertEqual(self.test_document.pages.count(), 0) - response = self._request_document_update_page_count_view() + response = self._request_document_pages_reset_view() self.assertEqual(response.status_code, 404) self.assertEqual(self.test_document.pages.count(), 0) - def test_document_update_page_count_view_with_permission(self): - # TODO: Revise permission association - + def test_document_pages_reset_view_with_access(self): page_count = self.test_document.pages.count() self.test_document.pages.all().delete() - self.assertEqual(self.test_document.pages.count(), 0) - self.grant_permission(permission=permission_document_tools) + self.grant_access( + obj=self.test_document, permission=permission_document_tools + ) - response = self._request_document_update_page_count_view() + response = self._request_document_pages_reset_view() self.assertEqual(response.status_code, 302) self.assertEqual(self.test_document.pages.count(), page_count) - def test_document_multiple_update_page_count_view_no_permission(self): + def test_document_multiple_pages_reset_view_no_permission(self): self.test_document.pages.all().delete() - self.assertEqual(self.test_document.pages.count(), 0) - response = self._request_document_multiple_update_page_count_view() + response = self._request_document_multiple_pages_reset_view() self.assertEqual(response.status_code, 404) self.assertEqual(self.test_document.pages.count(), 0) - def test_document_multiple_update_page_count_view_with_permission(self): + def test_document_multiple_pages_reset_view_with_access(self): page_count = self.test_document.pages.count() self.test_document.pages.all().delete() - self.assertEqual(self.test_document.pages.count(), 0) - self.grant_permission(permission=permission_document_tools) + self.grant_access( + obj=self.test_document, permission=permission_document_tools + ) - response = self._request_document_multiple_update_page_count_view() + response = self._request_document_multiple_pages_reset_view() self.assertEqual(response.status_code, 302) self.assertEqual(self.test_document.pages.count(), page_count) diff --git a/mayan/apps/documents/tests/test_search.py b/mayan/apps/documents/tests/test_search.py index 531f6e8d2f..1db040b174 100644 --- a/mayan/apps/documents/tests/test_search.py +++ b/mayan/apps/documents/tests/test_search.py @@ -1,22 +1,30 @@ from __future__ import unicode_literals from mayan.apps.common.tests.base import BaseTestCase -from mayan.apps.documents.permissions import permission_document_view -from mayan.apps.documents.search import document_search, document_page_search -from mayan.apps.documents.tests.mixins import DocumentTestMixin + +from ..permissions import permission_document_view +from ..search import document_search, document_page_search + +from .mixins import DocumentTestMixin -class DocumentSearchTestCase(DocumentTestMixin, BaseTestCase): +class DocumentSearchTestMixin(object): def _perform_document_page_search(self): return document_page_search.search( - query_string={'q': self.test_document.label}, user=self._test_case_user + query_string={'q': self.test_document.label}, + user=self._test_case_user ) def _perform_document_search(self): return document_search.search( - query_string={'q': self.test_document.label}, user=self._test_case_user + query_string={'q': self.test_document.label}, + user=self._test_case_user ) + +class DocumentSearchTestCase( + DocumentSearchTestMixin, DocumentTestMixin, BaseTestCase +): def test_document_page_search_no_access(self): queryset = self._perform_document_page_search() self.assertFalse(self.test_document.pages.first() in queryset) diff --git a/mayan/apps/documents/tests/test_trashed_document_views.py b/mayan/apps/documents/tests/test_trashed_document_views.py index 8be806f36e..775bef0f38 100644 --- a/mayan/apps/documents/tests/test_trashed_document_views.py +++ b/mayan/apps/documents/tests/test_trashed_document_views.py @@ -9,7 +9,7 @@ from ..permissions import ( from .base import GenericDocumentViewTestCase -class TrashedDocumentTestCase(GenericDocumentViewTestCase): +class TrashedDocumentTestMixin(object): def _request_document_restore_get_view(self): return self.get( viewname='documents:document_restore', kwargs={ @@ -17,6 +17,48 @@ class TrashedDocumentTestCase(GenericDocumentViewTestCase): } ) + def _request_document_restore_post_view(self): + return self.post( + viewname='documents:document_restore', kwargs={ + 'pk': self.test_document.pk + } + ) + + def _request_document_trash_get_view(self): + return self.get( + viewname='documents:document_trash', kwargs={ + 'pk': self.test_document.pk + } + ) + + def _request_document_trash_post_view(self): + return self.post( + viewname='documents:document_trash', kwargs={ + 'pk': self.test_document.pk + } + ) + + def _request_trashed_document_delete_get_view(self): + return self.get( + viewname='documents:document_delete', kwargs={ + 'pk': self.test_document.pk + } + ) + + def _request_trashed_document_delete_post_view(self): + return self.post( + viewname='documents:document_delete', kwargs={ + 'pk': self.test_document.pk + } + ) + + def _request_trashed_document_list_view(self): + return self.get(viewname='documents:document_list_deleted') + + +class TrashedDocumentTestCase( + TrashedDocumentTestMixin, GenericDocumentViewTestCase +): def test_document_restore_get_view_no_permission(self): self.test_document.delete() self.assertEqual(Document.objects.count(), 0) @@ -43,13 +85,6 @@ class TrashedDocumentTestCase(GenericDocumentViewTestCase): self.assertEqual(Document.objects.count(), document_count) - def _request_document_restore_post_view(self): - return self.post( - viewname='documents:document_restore', kwargs={ - 'pk': self.test_document.pk - } - ) - def test_document_restore_post_view_no_permission(self): self.test_document.delete() self.assertEqual(Document.objects.count(), 0) @@ -74,13 +109,6 @@ class TrashedDocumentTestCase(GenericDocumentViewTestCase): self.assertEqual(DeletedDocument.objects.count(), 0) self.assertEqual(Document.objects.count(), 1) - def _request_document_trash_get_view(self): - return self.get( - viewname='documents:document_trash', kwargs={ - 'pk': self.test_document.pk - } - ) - def test_document_trash_get_view_no_permissions(self): document_count = Document.objects.count() @@ -101,13 +129,6 @@ class TrashedDocumentTestCase(GenericDocumentViewTestCase): self.assertEqual(Document.objects.count(), document_count) - def _request_document_trash_post_view(self): - return self.post( - viewname='documents:document_trash', kwargs={ - 'pk': self.test_document.pk - } - ) - def test_document_trash_post_view_no_permissions(self): response = self._request_document_trash_post_view() self.assertEqual(response.status_code, 404) @@ -126,13 +147,6 @@ class TrashedDocumentTestCase(GenericDocumentViewTestCase): self.assertEqual(DeletedDocument.objects.count(), 1) self.assertEqual(Document.objects.count(), 0) - def _request_document_delete_get_view(self): - return self.get( - viewname='documents:document_delete', kwargs={ - 'pk': self.test_document.pk - } - ) - def test_document_delete_get_view_no_permissions(self): self.test_document.delete() self.assertEqual(Document.objects.count(), 0) @@ -140,7 +154,7 @@ class TrashedDocumentTestCase(GenericDocumentViewTestCase): trashed_document_count = DeletedDocument.objects.count() - response = self._request_document_delete_get_view() + response = self._request_trashed_document_delete_get_view() self.assertEqual(response.status_code, 404) self.assertEqual( @@ -158,26 +172,19 @@ class TrashedDocumentTestCase(GenericDocumentViewTestCase): trashed_document_count = DeletedDocument.objects.count() - response = self._request_document_delete_get_view() + response = self._request_trashed_document_delete_get_view() self.assertEqual(response.status_code, 200) self.assertEqual( DeletedDocument.objects.count(), trashed_document_count ) - def _request_document_delete_post_view(self): - return self.post( - viewname='documents:document_delete', kwargs={ - 'pk': self.test_document.pk - } - ) - def test_document_delete_post_view_no_permissions(self): self.test_document.delete() self.assertEqual(Document.objects.count(), 0) self.assertEqual(DeletedDocument.objects.count(), 1) - response = self._request_document_delete_post_view() + response = self._request_trashed_document_delete_post_view() self.assertEqual(response.status_code, 404) self.assertEqual(Document.objects.count(), 0) @@ -192,19 +199,16 @@ class TrashedDocumentTestCase(GenericDocumentViewTestCase): obj=self.test_document, permission=permission_document_delete ) - response = self._request_document_delete_post_view() + response = self._request_trashed_document_delete_post_view() self.assertEqual(response.status_code, 302) self.assertEqual(DeletedDocument.objects.count(), 0) self.assertEqual(Document.objects.count(), 0) - def _request_document_list_deleted_view(self): - return self.get(viewname='documents:document_list_deleted') - def test_deleted_document_list_view_no_permissions(self): self.test_document.delete() - response = self._request_document_list_deleted_view() + response = self._request_trashed_document_list_view() self.assertNotContains( response=response, text=self.test_document.label, status_code=200 ) @@ -216,7 +220,7 @@ class TrashedDocumentTestCase(GenericDocumentViewTestCase): obj=self.test_document, permission=permission_document_view ) - response = self._request_document_list_deleted_view() + response = self._request_trashed_document_list_view() self.assertContains( response=response, text=self.test_document.label, status_code=200 ) diff --git a/mayan/apps/documents/urls.py b/mayan/apps/documents/urls.py index 84aec25094..a0e25cc670 100644 --- a/mayan/apps/documents/urls.py +++ b/mayan/apps/documents/urls.py @@ -4,21 +4,24 @@ from django.conf.urls import url from .api_views import ( APITrashedDocumentListView, APIDeletedDocumentRestoreView, - APIDeletedDocumentView, APIDocumentDownloadView, APIDocumentView, - APIDocumentListView, APIDocumentVersionDownloadView, + APIDeletedDocumentView, APIDocumentDownloadView, APIDocumentPageListView, + APIDocumentView, APIDocumentListView, APIDocumentVersionDownloadView, APIDocumentPageImageView, APIDocumentPageView, APIDocumentTypeDocumentListView, APIDocumentTypeListView, APIDocumentTypeView, APIDocumentVersionsListView, APIDocumentVersionPageListView, APIDocumentVersionView, - APIRecentDocumentListView + APIRecentDocumentListView, + APIDocumentVersionPageView, + APIDocumentVersionPageImageView ) from .views.document_views import ( DocumentDocumentTypeEditView, DocumentDownloadFormView, DocumentDownloadView, DocumentDuplicatesListView, DocumentEditView, DocumentListView, DocumentPreviewView, DocumentPrint, - DocumentTransformationsClearView, DocumentTransformationsCloneView, - DocumentUpdatePageCountView, DocumentView, DuplicatedDocumentListView, - RecentAccessDocumentListView, RecentAddedDocumentListView + DocumentPagesResetView, DocumentTransformationsClearView, + DocumentTransformationsCloneView, DocumentView, + DuplicatedDocumentListView, RecentAccessDocumentListView, + RecentAddedDocumentListView ) from .views.document_page_views import ( DocumentPageDisable, DocumentPageEnable, DocumentPageListView, @@ -30,7 +33,8 @@ from .views.document_page_views import ( ) from .views.document_version_views import ( DocumentVersionDownloadFormView, DocumentVersionDownloadView, - DocumentVersionListView, DocumentVersionRevertView, DocumentVersionView, + DocumentVersionListView, DocumentVersionRevertView, + DocumentVersionUpdatePageCountView, DocumentVersionView, ) from .views.document_type_views import ( DocumentTypeCreateView, DocumentTypeDeleteView, @@ -172,14 +176,14 @@ urlpatterns_documents = [ name='document_print' ), url( - regex=r'^documents/(?P\d+)/reset_page_count/$', - view=DocumentUpdatePageCountView.as_view(), - name='document_update_page_count' + regex=r'^documents/(?P\d+)/pages/reset/$', + view=DocumentPagesResetView.as_view(), + name='document_pages_reset' ), url( - regex=r'^documents/multiple/reset_page_count/$', - view=DocumentUpdatePageCountView.as_view(), - name='document_multiple_update_page_count' + regex=r'^documents/multiple/pages/reset/$', + view=DocumentPagesResetView.as_view(), + name='document_multiple_pages_reset' ), url( regex=r'^documents/(?P\d+)/download/form/$', @@ -305,6 +309,16 @@ urlpatterns_document_versions = [ view=DocumentVersionDownloadView.as_view(), name='document_version_download' ), + url( + regex=r'^documents/versions/(?P\d+)/pages/update/$', + view=DocumentVersionUpdatePageCountView.as_view(), + name='document_version_page_count_update' + ), + url( + regex=r'^documents/versions/multiple/pages/update/$', + view=DocumentVersionUpdatePageCountView.as_view(), + name='document_version_multiple_page_count_update' + ), url( regex=r'^documents/versions/(?P\d+)/revert/$', view=DocumentVersionRevertView.as_view(), @@ -405,6 +419,11 @@ api_urls = [ view=APIDocumentVersionPageListView.as_view(), name='documentversion-page-list' ), + url( + regex=r'^documents/(?P[0-9]+)/pages/$', + view=APIDocumentPageListView.as_view(), + name='document-page-list' + ), url( regex=r'^documents/(?P[0-9]+)/versions/(?P[0-9]+)/download/$', view=APIDocumentVersionDownloadView.as_view(), @@ -416,12 +435,20 @@ api_urls = [ ), url( regex=r'^documents/(?P[0-9]+)/versions/(?P[0-9]+)/pages/(?P[0-9]+)$', + view=APIDocumentVersionPageView.as_view(), name='documentversionpage-detail' + ), + url( + regex=r'^documents/(?P[0-9]+)/pages/(?P[0-9]+)$', view=APIDocumentPageView.as_view(), name='documentpage-detail' ), url( - regex=r'^documents/(?P[0-9]+)/versions/(?P[0-9]+)/pages/(?P[0-9]+)/image/$', + regex=r'^documents/(?P[0-9]+)/pages/(?P[0-9]+)/image/$', view=APIDocumentPageImageView.as_view(), name='documentpage-image' ), + url( + regex=r'^documents/(?P[0-9]+)/versions/(?P[0-9]+)/pages/(?P[0-9]+)/image/$', + view=APIDocumentVersionPageImageView.as_view(), name='documentversionpage-image' + ), url( regex=r'^trashed_documents/$', view=APITrashedDocumentListView.as_view(), name='trasheddocument-list' diff --git a/mayan/apps/documents/views/document_page_views.py b/mayan/apps/documents/views/document_page_views.py index f05767d627..46fcacd9e7 100644 --- a/mayan/apps/documents/views/document_page_views.py +++ b/mayan/apps/documents/views/document_page_views.py @@ -20,8 +20,8 @@ from mayan.apps.converter.literals import DEFAULT_ROTATION, DEFAULT_ZOOM_LEVEL from ..forms import DocumentPageForm from ..icons import icon_document_pages -from ..links import link_document_update_page_count -from ..models import Document, DocumentPage +from ..links import link_document_pages_reset +from ..models import Document, DocumentPage, DocumentVersionPage from ..permissions import permission_document_edit, permission_document_view from ..settings import ( setting_rotation_step, setting_zoom_percent_step, setting_zoom_max_level, @@ -50,13 +50,13 @@ class DocumentPageListView(ExternalObjectMixin, SingleObjectListView): 'hide_object': True, 'list_as_items': True, 'no_results_icon': icon_document_pages, - 'no_results_main_link': link_document_update_page_count.resolve( + 'no_results_main_link': link_document_pages_reset.resolve( request=self.request, resolved_object=self.external_object ), 'no_results_text': _( 'This could mean that the document is of a format that is ' - 'not supported, that it is corrupted or that the upload ' - 'process was interrupted. Use the document page recalculation ' + 'not supported, that it is corrupted, or that the upload ' + 'process was interrupted. Use the document page reset ' 'action to attempt to introspect the page count again.' ), 'no_results_title': _('No document pages available'), diff --git a/mayan/apps/documents/views/document_version_views.py b/mayan/apps/documents/views/document_version_views.py index 4c53bbff36..8a2c3cd1ed 100644 --- a/mayan/apps/documents/views/document_version_views.py +++ b/mayan/apps/documents/views/document_version_views.py @@ -3,10 +3,11 @@ from __future__ import absolute_import, unicode_literals import logging from django.contrib import messages -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import ugettext_lazy as _, ungettext from mayan.apps.common.generics import ( - ConfirmView, SingleObjectDetailView, SingleObjectListView + ConfirmView, MultipleObjectConfirmActionView, SingleObjectDetailView, + SingleObjectListView ) from mayan.apps.common.mixins import ExternalObjectMixin @@ -14,9 +15,10 @@ from ..events import event_document_view from ..forms import DocumentVersionDownloadForm, DocumentVersionPreviewForm from ..models import Document, DocumentVersion from ..permissions import ( - permission_document_download, permission_document_version_revert, - permission_document_version_view + permission_document_download, permission_document_tools, + permission_document_version_revert, permission_document_version_view ) +from ..tasks import task_update_page_count from .document_views import DocumentDownloadFormView, DocumentDownloadView @@ -142,6 +144,45 @@ class DocumentVersionRevertView(ExternalObjectMixin, ConfirmView): ) +class DocumentVersionUpdatePageCountView(MultipleObjectConfirmActionView): + model = DocumentVersion + object_permission = permission_document_tools + success_message = _( + '%(count)d document version queued for page count recalculation' + ) + success_message_plural = _( + '%(count)d documents version queued for page count recalculation' + ) + + def get_extra_context(self): + queryset = self.object_list + + result = { + 'title': ungettext( + singular='Recalculate the page count of the selected document version?', + plural='Recalculate the page count of the selected document versions?', + number=queryset.count() + ) + } + + if queryset.count() == 1: + result.update( + { + 'object': queryset.first(), + 'title': _( + 'Recalculate the page count of the document version: %s?' + ) % queryset.first() + } + ) + + return result + + def object_action(self, form, instance): + task_update_page_count.apply_async( + kwargs={'version_id': instance.pk} + ) + + class DocumentVersionView(SingleObjectDetailView): form_class = DocumentVersionPreviewForm model = DocumentVersion diff --git a/mayan/apps/documents/views/document_views.py b/mayan/apps/documents/views/document_views.py index 566077c99c..dab008a75c 100644 --- a/mayan/apps/documents/views/document_views.py +++ b/mayan/apps/documents/views/document_views.py @@ -44,14 +44,14 @@ from ..permissions import ( from ..settings import ( setting_print_width, setting_print_height, setting_recent_added_count ) -from ..tasks import task_update_page_count +from ..tasks import task_document_pages_reset from ..utils import parse_range __all__ = ( 'DocumentListView', 'DocumentDocumentTypeEditView', 'DocumentDuplicatesListView', 'DocumentEditView', 'DocumentPreviewView', 'DocumentView', 'DocumentDownloadFormView', 'DocumentDownloadView', - 'DocumentUpdatePageCountView', 'DocumentTransformationsClearView', + 'DocumentPagesResetView', 'DocumentTransformationsClearView', 'DocumentTransformationsCloneView', 'DocumentPrint', 'DuplicatedDocumentListView', 'RecentAccessDocumentListView', 'RecentAddedDocumentListView' @@ -418,6 +418,52 @@ class DocumentPreviewView(SingleObjectDetailView): } +class DocumentPagesResetView(MultipleObjectConfirmActionView): + model = Document + object_permission = permission_document_tools + success_message = _('%(count)d document queued for pages reset') + success_message_plural = _('%(count)d documents queued for pages reset') + + def get_extra_context(self): + queryset = self.object_list + + result = { + 'title': ungettext( + singular='Reset the pages of the selected document?', + plural='Reset the pages of the selected documents?', + number=queryset.count() + ) + } + + if queryset.count() == 1: + result.update( + { + 'object': queryset.first(), + 'title': _( + 'Reset the pages of the document: %s?' + ) % queryset.first() + } + ) + + return result + + def object_action(self, form, instance): + latest_version = instance.latest_version + if latest_version: + task_document_pages_reset.apply_async( + kwargs={'document_id': instance.pk} + ) + else: + messages.error( + self.request, _( + 'Document "%(document)s" is empty. Upload at least one ' + 'document version before attempting to reset the pages. ' + ) % { + 'document': instance, + } + ) + + class DocumentView(SingleObjectDetailView): form_class = DocumentPropertiesForm model = Document @@ -436,57 +482,6 @@ class DocumentView(SingleObjectDetailView): } -class DocumentUpdatePageCountView(MultipleObjectConfirmActionView): - model = Document - object_permission = permission_document_tools - success_message = _( - '%(count)d document queued for page count recalculation' - ) - success_message_plural = _( - '%(count)d documents queued for page count recalculation' - ) - - def get_extra_context(self): - queryset = self.object_list - - result = { - 'title': ungettext( - singular='Recalculate the page count of the selected document?', - plural='Recalculate the page count of the selected documents?', - number=queryset.count() - ) - } - - if queryset.count() == 1: - result.update( - { - 'object': queryset.first(), - 'title': _( - 'Recalculate the page count of the document: %s?' - ) % queryset.first() - } - ) - - return result - - def object_action(self, form, instance): - latest_version = instance.latest_version - if latest_version: - task_update_page_count.apply_async( - kwargs={'version_id': latest_version.pk} - ) - else: - messages.error( - self.request, _( - 'Document "%(document)s" is empty. Upload at least one ' - 'document version before attempting to detect the ' - 'page count.' - ) % { - 'document': instance, - } - ) - - class DocumentTransformationsClearView(MultipleObjectConfirmActionView): model = Document object_permission = permission_transformation_delete diff --git a/mayan/apps/dynamic_search/classes.py b/mayan/apps/dynamic_search/classes.py index 2476f07a15..1630bcc7fb 100644 --- a/mayan/apps/dynamic_search/classes.py +++ b/mayan/apps/dynamic_search/classes.py @@ -184,7 +184,14 @@ class SearchModel(object): query_string=query_string, global_and_search=global_and_search ) - queryset = self.get_queryset().filter(search_query.query).distinct() + try: + queryset = self.get_queryset().filter(search_query.query).distinct() + except Exception: + logger.error( + 'Error filtering model %s with queryset: %s', self.model, + search_query.query + ) + raise if self.permission: queryset = AccessControlList.objects.restrict_queryset( diff --git a/mayan/apps/file_caching/migrations/0003_auto_20191008_1510.py b/mayan/apps/file_caching/migrations/0003_auto_20191008_1510.py new file mode 100644 index 0000000000..14382e0479 --- /dev/null +++ b/mayan/apps/file_caching/migrations/0003_auto_20191008_1510.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.22 on 2019-10-08 15:10 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('file_caching', '0002_auto_20190729_0236'), + ] + + operations = [ + migrations.AlterField( + model_name='cache', + name='name', + field=models.CharField(db_index=True, help_text='Internal name of the cache.', max_length=128, unique=True, verbose_name='Name'), + ), + ] diff --git a/mayan/apps/file_metadata/apps.py b/mayan/apps/file_metadata/apps.py index a9830d467d..675ebe3365 100644 --- a/mayan/apps/file_metadata/apps.py +++ b/mayan/apps/file_metadata/apps.py @@ -12,7 +12,9 @@ from mayan.apps.common.menus import ( menu_tools ) from mayan.apps.document_indexing.handlers import handler_index_document -from mayan.apps.documents.search import document_page_search, document_search +from mayan.apps.documents.search import ( + document_page_search, document_search, document_version_page_search +) from mayan.apps.documents.signals import post_version_upload from mayan.apps.events.classes import ModelEventType from mayan.apps.navigation.classes import SourceColumn @@ -156,11 +158,19 @@ class FileMetadataApp(MayanAppConfig): label=_('File metadata value') ) - document_page_search.add_model_field( + #document_page_search.add_model_field( + # field='document__document_version__file_metadata_drivers__entries__key', + # label=_('File metadata key') + #) + #document_page_search.add_model_field( + # field='document__document_version__file_metadata_drivers__entries__value', + # label=_('File metadata value') + #) + document_version_page_search.add_model_field( field='document_version__file_metadata_drivers__entries__key', label=_('File metadata key') ) - document_page_search.add_model_field( + document_version_page_search.add_model_field( field='document_version__file_metadata_drivers__entries__value', label=_('File metadata value') ) diff --git a/mayan/apps/metadata/apps.py b/mayan/apps/metadata/apps.py index 44fd6bb546..b0224e9ceb 100644 --- a/mayan/apps/metadata/apps.py +++ b/mayan/apps/metadata/apps.py @@ -16,7 +16,9 @@ from mayan.apps.common.menus import ( menu_facet, menu_list_facet, menu_multi_item, menu_object, menu_secondary, menu_setup ) -from mayan.apps.documents.search import document_page_search, document_search +from mayan.apps.documents.search import ( + document_page_search, document_search, document_version_page_search +) from mayan.apps.documents.signals import post_document_type_change from mayan.apps.events.classes import ModelEventType from mayan.apps.events.links import ( @@ -76,7 +78,7 @@ class MetadataApp(MayanAppConfig): app_label='documents', model_name='Document' ) DocumentPageResult = apps.get_model( - app_label='documents', model_name='DocumentPageResult' + app_label='documents', model_name='DocumentVersionPageResult' ) DocumentType = apps.get_model( @@ -188,10 +190,18 @@ class MetadataApp(MayanAppConfig): ) document_page_search.add_model_field( - field='document_version__document__metadata__metadata_type__name', + field='document__metadata__metadata_type__name', label=_('Metadata type') ) document_page_search.add_model_field( + field='document__metadata__value', + label=_('Metadata value') + ) + document_version_page_search.add_model_field( + field='document_version__document__metadata__metadata_type__name', + label=_('Metadata type') + ) + document_version_page_search.add_model_field( field='document_version__document__metadata__value', label=_('Metadata value') ) diff --git a/mayan/apps/ocr/admin.py b/mayan/apps/ocr/admin.py index 865481d602..9a927c5260 100644 --- a/mayan/apps/ocr/admin.py +++ b/mayan/apps/ocr/admin.py @@ -3,13 +3,15 @@ from __future__ import unicode_literals from django.contrib import admin from .models import ( - DocumentPageOCRContent, DocumentTypeSettings, DocumentVersionOCRError + DocumentTypeSettings, DocumentVersionPageOCRContent, + DocumentVersionOCRError ) -@admin.register(DocumentPageOCRContent) -class DocumentPageOCRContentAdmin(admin.ModelAdmin): - list_display = ('document_page',) +@admin.register(DocumentVersionPageOCRContent) +class DocumentVersionPageOCRContentAdmin(admin.ModelAdmin): + pass + #list_display = ('document_page',) @admin.register(DocumentTypeSettings) diff --git a/mayan/apps/ocr/api_views.py b/mayan/apps/ocr/api_views.py index 0dea74beb3..20a4fe833e 100644 --- a/mayan/apps/ocr/api_views.py +++ b/mayan/apps/ocr/api_views.py @@ -8,7 +8,7 @@ from rest_framework.response import Response from mayan.apps.documents.models import Document, DocumentVersion from mayan.apps.rest_api.permissions import MayanPermission -from .models import DocumentPageOCRContent +from .models import DocumentVersionPageOCRContent from .permissions import permission_ocr_content_view, permission_ocr_document from .serializers import DocumentPageOCRContentSerializer @@ -90,8 +90,8 @@ class APIDocumentPageOCRContentView(generics.RetrieveAPIView): try: ocr_content = instance.ocr_content - except DocumentPageOCRContent.DoesNotExist: - ocr_content = DocumentPageOCRContent.objects.none() + except DocumentVersionPageOCRContent.DoesNotExist: + ocr_content = DocumentVersionPageOCRContent.objects.none() serializer = self.get_serializer(ocr_content) return Response(serializer.data) diff --git a/mayan/apps/ocr/apps.py b/mayan/apps/ocr/apps.py index 33b4f1893b..772ec287fa 100644 --- a/mayan/apps/ocr/apps.py +++ b/mayan/apps/ocr/apps.py @@ -12,7 +12,9 @@ from mayan.apps.common.classes import ModelField from mayan.apps.common.menus import ( menu_facet, menu_list_facet, menu_multi_item, menu_secondary, menu_tools ) -from mayan.apps.documents.search import document_search, document_page_search +from mayan.apps.documents.search import ( + document_search, document_page_search, document_version_page_search +) from mayan.apps.documents.signals import post_version_upload from mayan.apps.events.classes import ModelEventType from mayan.apps.navigation.classes import SourceColumn @@ -32,17 +34,19 @@ from .links import ( link_document_ocr_content_delete_multiple, link_document_ocr_download, link_document_ocr_errors_list, link_document_submit, link_document_submit_multiple, link_document_type_ocr_settings, - link_document_type_submit, link_entry_list + link_document_type_submit, link_document_version_page_ocr_content, + link_entry_list ) from .methods import ( - method_document_ocr_submit, method_document_version_ocr_submit + method_document_ocr_submit, method_document_page_get_ocr_content, + method_document_version_ocr_submit ) from .permissions import ( permission_document_type_ocr_setup, permission_ocr_document, permission_ocr_content_view ) from .signals import post_document_version_ocr -from .utils import get_document_ocr_content +from .utils import get_document_ocr_content, get_document_version_ocr_content logger = logging.getLogger(__name__) @@ -73,6 +77,9 @@ class OCRApp(MayanAppConfig): DocumentVersion = apps.get_model( app_label='documents', model_name='DocumentVersion' ) + DocumentVersionPage = apps.get_model( + app_label='documents', model_name='DocumentVersionPage' + ) DocumentVersionOCRError = self.get_model( model_name='DocumentVersionOCRError' @@ -81,8 +88,11 @@ class OCRApp(MayanAppConfig): Document.add_to_class( name='submit_for_ocr', value=method_document_ocr_submit ) + DocumentPage.add_to_class( + name='get_ocr_content', value=method_document_page_get_ocr_content + ) DocumentVersion.add_to_class( - name='ocr_content', value=get_document_ocr_content + name='ocr_content', value=get_document_version_ocr_content ) DocumentVersion.add_to_class( name='submit_for_ocr', value=method_document_version_ocr_submit @@ -97,7 +107,7 @@ class OCRApp(MayanAppConfig): ) ModelField( - model=Document, name='versions__version_pages__ocr_content__content' + model=Document, name='versions__pages__ocr_content__content' ) ModelPermission.register( @@ -128,12 +138,14 @@ class OCRApp(MayanAppConfig): ) document_search.add_model_field( - field='versions__version_pages__ocr_content__content', label=_('OCR') + field='versions__pages__ocr_content__content', label=_('OCR') ) - - document_page_search.add_model_field( + document_version_page_search.add_model_field( field='ocr_content__content', label=_('OCR') ) + #document_page_search.add_model_field( + # field='ocr_content__content', label=_('OCR') + #) menu_facet.bind_links( links=(link_document_ocr_content,), sources=(Document,) @@ -141,6 +153,10 @@ class OCRApp(MayanAppConfig): menu_list_facet.bind_links( links=(link_document_page_ocr_content,), sources=(DocumentPage,) ) + menu_list_facet.bind_links( + links=(link_document_version_page_ocr_content,), + sources=(DocumentVersionPage,) + ) menu_list_facet.bind_links( links=(link_document_type_ocr_settings,), sources=(DocumentType,) ) diff --git a/mayan/apps/ocr/forms.py b/mayan/apps/ocr/forms.py index 6951861ecf..5379bce4ae 100644 --- a/mayan/apps/ocr/forms.py +++ b/mayan/apps/ocr/forms.py @@ -8,7 +8,7 @@ from django.utils.translation import ugettext_lazy as _, ugettext from mayan.apps.common.widgets import TextAreaDiv -from .models import DocumentPageOCRContent +from .models import DocumentVersionPageOCRContent class DocumentPageOCRContentForm(forms.Form): @@ -28,15 +28,26 @@ class DocumentPageOCRContentForm(forms.Form): content = '' self.fields['contents'].initial = '' - try: - page_content = page.ocr_content.content - except DocumentPageOCRContent.DoesNotExist: - pass - else: - content = conditional_escape(force_text(page_content)) + content = conditional_escape( + force_text(self.get_instance_ocr_content(instance=page)) + ) self.fields['contents'].initial = mark_safe(content) + def get_instance_ocr_content(self, instance): + try: + return instance.content_object.ocr_content.content + except DocumentVersionPageOCRContent.DoesNotExist: + return '' + + +class DocumentVersionPageOCRContentForm(DocumentPageOCRContentForm): + def get_instance_ocr_content(self, instance): + try: + return instance.ocr_content.content + except (AttributeError, DocumentVersionPageOCRContent.DoesNotExist): + return '' + class DocumentOCRContentForm(forms.Form): """ @@ -54,19 +65,15 @@ class DocumentOCRContentForm(forms.Form): ) def __init__(self, *args, **kwargs): - self.document = kwargs.pop('instance', None) + document = kwargs.pop('instance', None) super(DocumentOCRContentForm, self).__init__(*args, **kwargs) content = [] self.fields['contents'].initial = '' - try: - document_pages = self.document.pages.all() - except AttributeError: - document_pages = [] - for page in document_pages: + for document_page in document.pages.all(): try: - page_content = page.ocr_content.content - except DocumentPageOCRContent.DoesNotExist: + page_content = document_page.content_object.ocr_content.content + except (AttributeError, DocumentVersionPageOCRContent.DoesNotExist): pass else: content.append(conditional_escape(force_text(page_content))) @@ -74,7 +81,7 @@ class DocumentOCRContentForm(forms.Form): '\n\n\n
- %s -
\n\n\n' % ( ugettext( 'Page %(page_number)d' - ) % {'page_number': page.page_number} + ) % {'page_number': document_page.page_number} ) ) diff --git a/mayan/apps/ocr/icons.py b/mayan/apps/ocr/icons.py index f6e1ce3141..f3f7984992 100644 --- a/mayan/apps/ocr/icons.py +++ b/mayan/apps/ocr/icons.py @@ -19,7 +19,7 @@ icon_document_ocr_errors_list = Icon( icon_document_type_ocr_settings = Icon( driver_name='fontawesome', symbol='font' ) -icon_document_type_submit = Icon(driver_name='fontawesome', symbol='font') -icon_entry_list = Icon(driver_name='fontawesome', symbol='font') - icon_document_submit = icon_document_multiple_submit +icon_document_type_submit = Icon(driver_name='fontawesome', symbol='font') +icon_document_version_page_ocr_content = Icon(driver_name='fontawesome', symbol='font') +icon_entry_list = Icon(driver_name='fontawesome', symbol='font') diff --git a/mayan/apps/ocr/links.py b/mayan/apps/ocr/links.py index 343bf26d18..68284d26ef 100644 --- a/mayan/apps/ocr/links.py +++ b/mayan/apps/ocr/links.py @@ -58,10 +58,11 @@ link_document_type_submit = Link( permissions=(permission_ocr_document,), text=_('OCR documents per type'), view='ocr:document_type_submit' ) -link_entry_list = Link( - icon_class_path='mayan.apps.ocr.icons.icon_entry_list', - permissions=(permission_ocr_document,), text=_('OCR errors'), - view='ocr:entry_list' +link_document_version_page_ocr_content = Link( + args='resolved_object.id', + icon_class_path='mayan.apps.ocr.icons.icon_document_version_page_ocr_content', + permissions=(permission_ocr_content_view,), text=_('OCR'), + view='ocr:document_version_page_ocr_content', ) link_document_ocr_errors_list = Link( args='resolved_object.id', @@ -75,3 +76,8 @@ link_document_ocr_download = Link( permissions=(permission_ocr_content_view,), text=_('Download OCR text'), view='ocr:document_ocr_download' ) +link_entry_list = Link( + icon_class_path='mayan.apps.ocr.icons.icon_entry_list', + permissions=(permission_ocr_document,), text=_('OCR errors'), + view='ocr:entry_list' +) diff --git a/mayan/apps/ocr/managers.py b/mayan/apps/ocr/managers.py index ee8e1bd95a..869c454001 100644 --- a/mayan/apps/ocr/managers.py +++ b/mayan/apps/ocr/managers.py @@ -9,7 +9,9 @@ from django.conf import settings from django.db import models, transaction from mayan.apps.documents.literals import DOCUMENT_IMAGE_TASK_TIMEOUT -from mayan.apps.documents.tasks import task_generate_document_page_image +from mayan.apps.documents.tasks import ( + task_generate_document_version_page_image +) from .events import ( event_ocr_document_content_deleted, event_ocr_document_version_finish @@ -20,47 +22,53 @@ from .signals import post_document_version_ocr logger = logging.getLogger(__name__) -class DocumentPageOCRContentManager(models.Manager): +class DocumentVesionPageOCRContentManager(models.Manager): def delete_content_for(self, document, user=None): with transaction.atomic(): for document_page in document.pages.all(): - self.filter(document_page=document_page).delete() + self.filter( + document_version_page=document_page.content_object + ).delete() event_ocr_document_content_deleted.commit( actor=user, target=document ) - def process_document_page(self, document_page): + def process_document_version_page(self, document_version_page): logger.info( 'Processing page: %d of document version: %s', - document_page.page_number, document_page.document_version + document_version_page.page_number, + document_version_page.document_version ) - DocumentPageOCRContent = apps.get_model( - app_label='ocr', model_name='DocumentPageOCRContent' + DocumentVersionPageOCRContent = apps.get_model( + app_label='ocr', model_name='DocumentVersionPageOCRContent' ) - task = task_generate_document_page_image.apply_async( + task = task_generate_document_version_page_image.apply_async( kwargs=dict( - document_page_id=document_page.pk + document_version_page_id=document_version_page.pk ) ) - cache_filename = task.get(timeout=DOCUMENT_IMAGE_TASK_TIMEOUT, disable_sync_subtasks=False) + cache_filename = task.get( + timeout=DOCUMENT_IMAGE_TASK_TIMEOUT, disable_sync_subtasks=False + ) - with document_page.cache_partition.get_file(filename=cache_filename).open() as file_object: - document_page_content, created = DocumentPageOCRContent.objects.get_or_create( - document_page=document_page + with document_version_page.cache_partition.get_file(filename=cache_filename).open() as file_object: + document_version_page_content, created = DocumentVersionPageOCRContent.objects.get_or_create( + document_version_page=document_version_page ) - document_page_content.content = ocr_backend.execute( + document_version_page_content.content = ocr_backend.execute( file_object=file_object, - language=document_page.document.language + language=document_version_page.document.language ) - document_page_content.save() + document_version_page_content.save() logger.info( 'Finished processing page: %d of document version: %s', - document_page.page_number, document_page.document_version + document_version_page.page_number, + document_version_page.document_version ) def process_document_version(self, document_version): @@ -68,8 +76,10 @@ class DocumentPageOCRContentManager(models.Manager): logger.debug('document version: %d', document_version.pk) try: - for document_page in document_version.pages.all(): - self.process_document_page(document_page=document_page) + for document_version_page in document_version.pages.all(): + self.process_document_version_page( + document_version_page=document_version_page + ) except Exception as exception: logger.error( 'OCR error for document version: %d; %s', document_version.pk, diff --git a/mayan/apps/ocr/methods.py b/mayan/apps/ocr/methods.py index b1c11a8265..567fa4b79a 100644 --- a/mayan/apps/ocr/methods.py +++ b/mayan/apps/ocr/methods.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals from datetime import timedelta +from django.apps import apps from django.utils.timezone import now from mayan.apps.common.settings import settings_db_sync_task_delay @@ -17,6 +18,17 @@ def method_document_ocr_submit(self): latest_version.submit_for_ocr() +def method_document_page_get_ocr_content(self): + DocumentVersionPageOCRContent = apps.get_model( + app_label='ocr', model_name='DocumentVersionPageOCRContent' + ) + + try: + return self.content_object.ocr_content.content + except (AttributeError, DocumentVersionPageOCRContent.DoesNotExist): + return None + + def method_document_version_ocr_submit(self): event_ocr_document_version_submit.commit( action_object=self.document, target=self diff --git a/mayan/apps/ocr/migrations/0009_rename_page_content.py b/mayan/apps/ocr/migrations/0009_rename_page_content.py new file mode 100644 index 0000000000..f98ee91251 --- /dev/null +++ b/mayan/apps/ocr/migrations/0009_rename_page_content.py @@ -0,0 +1,42 @@ +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + dependencies = [ + ('ocr', '0008_auto_20180917_0646'), + ('documents', '0052_rename_document_page'), + ] + + operations = [ + migrations.RenameModel( + 'DocumentPageOCRContent', 'DocumentVersionPageOCRContent' + ), + migrations.AlterField( + model_name='documentversionpageocrcontent', + name='document_page', + field=models.OneToOneField( + on_delete=django.db.models.deletion.CASCADE, + #name='document_version_page', + related_name='ocr_content', + to='documents.DocumentVersionPage', + verbose_name='Document version page' + ), + ), + migrations.RenameField( + model_name='documentversionpageocrcontent', + old_name='document_page', + new_name='document_version_page', + ), + migrations.AlterModelOptions( + name='documentversionpageocrcontent', + options={ + 'verbose_name': 'Document version page OCR content', + 'verbose_name_plural': 'Document version pages OCR contents' + }, + ), + ] + + diff --git a/mayan/apps/ocr/models.py b/mayan/apps/ocr/models.py index 9f42843a7e..adaf142a0c 100644 --- a/mayan/apps/ocr/models.py +++ b/mayan/apps/ocr/models.py @@ -4,10 +4,12 @@ from django.db import models from django.utils.encoding import force_text, python_2_unicode_compatible from django.utils.translation import ugettext_lazy as _ -from mayan.apps.documents.models import DocumentPage, DocumentType, DocumentVersion +from mayan.apps.documents.models import ( + DocumentPage, DocumentType, DocumentVersion, DocumentVersionPage +) from .managers import ( - DocumentPageOCRContentManager, DocumentTypeSettingsManager + DocumentVesionPageOCRContentManager, DocumentTypeSettingsManager ) @@ -36,13 +38,13 @@ class DocumentTypeSettings(models.Model): @python_2_unicode_compatible -class DocumentPageOCRContent(models.Model): +class DocumentVersionPageOCRContent(models.Model): """ This model stores the OCR results for a document page. """ - document_page = models.OneToOneField( + document_version_page = models.OneToOneField( on_delete=models.CASCADE, related_name='ocr_content', - to=DocumentPage, verbose_name=_('Document page') + to=DocumentVersionPage, verbose_name=_('Document version page') ) content = models.TextField( blank=True, help_text=_( @@ -50,11 +52,11 @@ class DocumentPageOCRContent(models.Model): ), verbose_name=_('Content') ) - objects = DocumentPageOCRContentManager() + objects = DocumentVesionPageOCRContentManager() class Meta: - verbose_name = _('Document page OCR content') - verbose_name_plural = _('Document pages OCR contents') + verbose_name = _('Document version page OCR content') + verbose_name_plural = _('Document version pages OCR contents') def __str__(self): return force_text(self.document_page) diff --git a/mayan/apps/ocr/serializers.py b/mayan/apps/ocr/serializers.py index 3d9c06c18d..780e9a48a6 100644 --- a/mayan/apps/ocr/serializers.py +++ b/mayan/apps/ocr/serializers.py @@ -2,10 +2,10 @@ from __future__ import unicode_literals from rest_framework import serializers -from .models import DocumentPageOCRContent +from .models import DocumentVersionPageOCRContent class DocumentPageOCRContentSerializer(serializers.ModelSerializer): class Meta: fields = ('content',) - model = DocumentPageOCRContent + model = DocumentVersionPageOCRContent diff --git a/mayan/apps/ocr/tasks.py b/mayan/apps/ocr/tasks.py index a6d9c82efb..4ba1f5268f 100644 --- a/mayan/apps/ocr/tasks.py +++ b/mayan/apps/ocr/tasks.py @@ -19,8 +19,8 @@ def task_do_ocr(self, document_version_pk): DocumentVersion = apps.get_model( app_label='documents', model_name='DocumentVersion' ) - DocumentPageOCRContent = apps.get_model( - app_label='ocr', model_name='DocumentPageOCRContent' + DocumentVersionPageOCRContent = apps.get_model( + app_label='ocr', model_name='DocumentVersionPageOCRContent' ) lock_id = 'task_do_ocr_doc_version-%d' % document_version_pk @@ -39,7 +39,7 @@ def task_do_ocr(self, document_version_pk): 'Starting document OCR for document version: %s', document_version ) - DocumentPageOCRContent.objects.process_document_version( + DocumentVersionPageOCRContent.objects.process_document_version( document_version=document_version ) except OperationalError as exception: diff --git a/mayan/apps/ocr/tests/test_api.py b/mayan/apps/ocr/tests/test_api.py index e847550141..578e8a94b8 100644 --- a/mayan/apps/ocr/tests/test_api.py +++ b/mayan/apps/ocr/tests/test_api.py @@ -12,13 +12,34 @@ from ..permissions import ( from .literals import TEST_DOCUMENT_CONTENT -class OCRAPITestCase(DocumentTestMixin, BaseAPITestCase): +class OCRAPIViewTestMixin(object): def _request_document_ocr_submit_view(self): return self.post( viewname='rest_api:document-ocr-submit-view', kwargs={'pk': self.test_document.pk} ) + def _request_document_version_ocr_submit_view(self): + return self.post( + viewname='rest_api:document-version-ocr-submit-view', kwargs={ + 'document_pk': self.test_document.pk, + 'version_pk': self.test_document.latest_version.pk + } + ) + + def _request_document_version_page_content_view(self): + return self.get( + viewname='rest_api:document-page-ocr-content-view', kwargs={ + 'document_pk': self.test_document.pk, + 'version_pk': self.test_document.latest_version.pk, + 'page_pk': self.test_document.latest_version.pages.first().pk, + } + ) + + +class OCRAPIViewTestCase( + OCRAPIViewTestMixin, DocumentTestMixin, BaseAPITestCase +): def test_submit_document_no_access(self): response = self._request_document_ocr_submit_view() self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) @@ -35,15 +56,9 @@ class OCRAPITestCase(DocumentTestMixin, BaseAPITestCase): self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) self.assertTrue( - hasattr(self.test_document.pages.first(), 'ocr_content') - ) - - def _request_document_version_ocr_submit_view(self): - return self.post( - viewname='rest_api:document-version-ocr-submit-view', kwargs={ - 'document_pk': self.test_document.pk, - 'version_pk': self.test_document.latest_version.pk - } + hasattr( + self.test_document.pages.first().content_object, 'ocr_content' + ) ) def test_submit_document_version_no_access(self): @@ -62,20 +77,11 @@ class OCRAPITestCase(DocumentTestMixin, BaseAPITestCase): self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) self.assertTrue( - hasattr(self.test_document.pages.first(), 'ocr_content') - ) - - def _request_document_page_content_view(self): - return self.get( - viewname='rest_api:document-page-ocr-content-view', kwargs={ - 'document_pk': self.test_document.pk, - 'version_pk': self.test_document.latest_version.pk, - 'page_pk': self.test_document.latest_version.pages.first().pk, - } + hasattr(self.test_document_version.pages.first(), 'ocr_content') ) def test_get_document_version_page_content_no_access(self): - response = self._request_document_page_content_view() + response = self._request_document_version_page_content_view() self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) def test_get_document_version_page_content_with_access(self): @@ -83,7 +89,7 @@ class OCRAPITestCase(DocumentTestMixin, BaseAPITestCase): self.grant_access( permission=permission_ocr_content_view, obj=self.test_document ) - response = self._request_document_page_content_view() + response = self._request_document_version_page_content_view() self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertTrue( TEST_DOCUMENT_CONTENT in response.data['content'] diff --git a/mayan/apps/ocr/tests/test_events.py b/mayan/apps/ocr/tests/test_events.py index 4a4318ec64..c51cee8fad 100644 --- a/mayan/apps/ocr/tests/test_events.py +++ b/mayan/apps/ocr/tests/test_events.py @@ -8,13 +8,13 @@ from ..events import ( event_ocr_document_content_deleted, event_ocr_document_version_submit, event_ocr_document_version_finish ) -from ..models import DocumentPageOCRContent +from ..models import DocumentVersionPageOCRContent class OCREventsTestCase(GenericDocumentTestCase): def test_document_content_deleted_event(self): Action.objects.all().delete() - DocumentPageOCRContent.objects.delete_content_for( + DocumentVersionPageOCRContent.objects.delete_content_for( document=self.test_document ) diff --git a/mayan/apps/ocr/tests/test_models.py b/mayan/apps/ocr/tests/test_models.py index cd4992d0d4..fe31995990 100644 --- a/mayan/apps/ocr/tests/test_models.py +++ b/mayan/apps/ocr/tests/test_models.py @@ -19,7 +19,7 @@ class DocumentOCRTestCase(DocumentTestMixin, BaseTestCase): _skip_file_descriptor_test = True def test_ocr_language_backends_end(self): - content = self.test_document.pages.first().ocr_content.content + content = self.test_document.pages.first().content_object.ocr_content.content self.assertTrue(TEST_DOCUMENT_CONTENT in content) @@ -40,7 +40,7 @@ class GermanOCRSupportTestCase(DocumentTestMixin, BaseTestCase): ) def test_ocr_language_backends_end(self): - content = self.test_document.pages.first().ocr_content.content + content = self.test_document.pages.first().content_object.ocr_content.content self.assertTrue( TEST_DOCUMENT_CONTENT_DEU_1 in content diff --git a/mayan/apps/ocr/tests/test_views.py b/mayan/apps/ocr/tests/test_views.py index 6ed278c54a..03a7ec7baf 100644 --- a/mayan/apps/ocr/tests/test_views.py +++ b/mayan/apps/ocr/tests/test_views.py @@ -2,7 +2,7 @@ from __future__ import unicode_literals from mayan.apps.documents.tests.base import GenericDocumentViewTestCase -from ..models import DocumentPageOCRContent +from ..models import DocumentVersionPageOCRContent from ..permissions import ( permission_ocr_content_view, permission_ocr_document, permission_document_type_ocr_setup @@ -27,10 +27,10 @@ class OCRViewTestMixin(object): } ) - def _request_document_page_content_view(self): + def _request_document_version_page_content_view(self): return self.get( - viewname='ocr:document_page_ocr_content', kwargs={ - 'pk': self.test_document.pages.first().pk + viewname='ocr:document_version_page_ocr_content', kwargs={ + 'pk': self.test_document_version.pages.first().pk } ) @@ -86,8 +86,8 @@ class OCRViewsTestCase(OCRViewTestMixin, GenericDocumentViewTestCase): self.assertEqual(response.status_code, 404) self.assertTrue( - DocumentPageOCRContent.objects.filter( - document_page=self.test_document.pages.first() + DocumentVersionPageOCRContent.objects.filter( + document_version_page=self.test_document.pages.first().content_object ).exists() ) @@ -101,28 +101,11 @@ class OCRViewsTestCase(OCRViewTestMixin, GenericDocumentViewTestCase): self.assertEqual(response.status_code, 302) self.assertFalse( - DocumentPageOCRContent.objects.filter( - document_page=self.test_document.pages.first() + DocumentVersionPageOCRContent.objects.filter( + document_version_page=self.test_document.pages.first().content_object ).exists() ) - def test_document_page_content_view_no_permissions(self): - self.test_document.submit_for_ocr() - - response = self._request_document_page_content_view() - self.assertEqual(response.status_code, 404) - - def test_document_page_content_view_with_access(self): - self.test_document.submit_for_ocr() - self.grant_access( - obj=self.test_document, permission=permission_ocr_content_view - ) - - response = self._request_document_page_content_view() - self.assertContains( - response=response, text=TEST_DOCUMENT_CONTENT, status_code=200 - ) - def test_document_submit_view_no_permission(self): response = self._request_document_submit_view() self.assertEqual(response.status_code, 404) @@ -188,6 +171,23 @@ class OCRViewsTestCase(OCRViewTestMixin, GenericDocumentViewTestCase): ), ) + def test_document_version_page_content_view_no_permissions(self): + self.test_document.submit_for_ocr() + + response = self._request_document_version_page_content_view() + self.assertEqual(response.status_code, 404) + + def test_document_version_page_content_view_with_access(self): + self.test_document.submit_for_ocr() + self.grant_access( + obj=self.test_document, permission=permission_ocr_content_view + ) + + response = self._request_document_version_page_content_view() + self.assertContains( + response=response, text=TEST_DOCUMENT_CONTENT, status_code=200 + ) + class DocumentTypeOCRViewTestMixin(object): def _request_document_type_ocr_settings_view(self): diff --git a/mayan/apps/ocr/urls.py b/mayan/apps/ocr/urls.py index 3d2d9b0b25..a8b9d9791e 100644 --- a/mayan/apps/ocr/urls.py +++ b/mayan/apps/ocr/urls.py @@ -8,9 +8,10 @@ from .api_views import ( ) from .views import ( DocumentOCRContentDeleteView, DocumentOCRContentView, - DocumentOCRDownloadView, - DocumentOCRErrorsListView, DocumentPageOCRContentView, DocumentSubmitView, - DocumentTypeSettingsEditView, DocumentTypeSubmitView, EntryListView + DocumentOCRDownloadView, DocumentOCRErrorsListView, + DocumentPageOCRContentView, DocumentSubmitView, + DocumentTypeSettingsEditView, DocumentTypeSubmitView, + DocumentVersionPageOCRContentView, EntryListView ) urlpatterns = [ @@ -50,6 +51,11 @@ urlpatterns = [ view=DocumentPageOCRContentView.as_view(), name='document_page_ocr_content' ), + url( + regex=r'^documents/versions/pages/(?P\d+)/content/$', + view=DocumentVersionPageOCRContentView.as_view(), + name='document_version_page_ocr_content' + ), url( regex=r'^document_types/submit/$', view=DocumentTypeSubmitView.as_view(), name='document_type_submit' diff --git a/mayan/apps/ocr/utils.py b/mayan/apps/ocr/utils.py index 0ac5de5fb3..a192613999 100644 --- a/mayan/apps/ocr/utils.py +++ b/mayan/apps/ocr/utils.py @@ -4,15 +4,29 @@ from django.apps import apps from django.utils.encoding import force_text -def get_document_ocr_content(document): - DocumentPageOCRContent = apps.get_model( - app_label='ocr', model_name='DocumentPageOCRContent' +def get_document_version_ocr_content(document_version): + DocumentVersionPageOCRContent = apps.get_model( + app_label='ocr', model_name='DocumentVersionPageOCRContent' ) - for page in document.pages.all(): + for document_version_page in document_version.pages.all(): try: - page_content = page.ocr_content.content - except DocumentPageOCRContent.DoesNotExist: + page_content = document_version_page.ocr_content.content + except DocumentVersionPageOCRContent.DoesNotExist: + pass + else: + yield force_text(page_content) + + +def get_document_ocr_content(document): + DocumentVersionPageOCRContent = apps.get_model( + app_label='ocr', model_name='DocumentVersionPageOCRContent' + ) + + for document_page in document.pages.all(): + try: + page_content = document_page.content_object.ocr_content.content + except (AttributeError, DocumentVersionPageOCRContent.DoesNotExist): pass else: yield force_text(page_content) diff --git a/mayan/apps/ocr/views.py b/mayan/apps/ocr/views.py index 41b978388e..21e7c00fb0 100644 --- a/mayan/apps/ocr/views.py +++ b/mayan/apps/ocr/views.py @@ -12,10 +12,15 @@ from mayan.apps.common.generics import ( ) from mayan.apps.common.mixins import ExternalObjectMixin from mayan.apps.documents.forms import DocumentTypeFilteredSelectForm -from mayan.apps.documents.models import Document, DocumentPage, DocumentType +from mayan.apps.documents.models import ( + Document, DocumentPage, DocumentType, DocumentVersionPage +) -from .forms import DocumentPageOCRContentForm, DocumentOCRContentForm -from .models import DocumentPageOCRContent, DocumentVersionOCRError +from .forms import ( + DocumentPageOCRContentForm, DocumentOCRContentForm, + DocumentVersionPageOCRContentForm +) +from .models import DocumentVersionPageOCRContent, DocumentVersionOCRError from .permissions import ( permission_ocr_content_view, permission_ocr_document, permission_document_type_ocr_setup @@ -46,7 +51,7 @@ class DocumentOCRContentDeleteView(MultipleObjectConfirmActionView): return result def object_action(self, form, instance): - DocumentPageOCRContent.objects.delete_content_for( + DocumentVersionPageOCRContent.objects.delete_content_for( document=instance, user=self.request.user ) @@ -94,6 +99,30 @@ class DocumentPageOCRContentView(SingleObjectDetailView): } +class DocumentVersionPageOCRContentView(SingleObjectDetailView): + form_class = DocumentVersionPageOCRContentForm + model = DocumentVersionPage + object_permission = permission_ocr_content_view + + def dispatch(self, request, *args, **kwargs): + result = super(DocumentVersionPageOCRContentView, self).dispatch( + request, *args, **kwargs + ) + self.get_object().document.add_as_recent_document_for_user( + user=request.user + ) + return result + + def get_extra_context(self): + return { + 'hide_labels': True, + 'object': self.get_object(), + 'title': _( + 'OCR result for document version page: %s' + ) % self.get_object(), + } + + class DocumentSubmitView(MultipleObjectConfirmActionView): model = Document object_permission = permission_ocr_document diff --git a/mayan/apps/sources/apps.py b/mayan/apps/sources/apps.py index 7139e5a846..d948a54fa7 100644 --- a/mayan/apps/sources/apps.py +++ b/mayan/apps/sources/apps.py @@ -21,9 +21,10 @@ from .handlers import ( handler_create_default_document_source, handler_initialize_periodic_tasks ) from .links import ( - link_document_create_multiple, link_setup_sources, - link_setup_source_check_now, link_setup_source_create_imap_email, - link_setup_source_create_pop3_email, link_setup_source_create_sane_scanner, + link_document_create_multiple, link_document_pages_append, + link_setup_sources, link_setup_source_check_now, + link_setup_source_create_imap_email, link_setup_source_create_pop3_email, + link_setup_source_create_sane_scanner, link_setup_source_create_watch_folder, link_setup_source_create_webform, link_setup_source_create_staging_folder, link_setup_source_delete, link_setup_source_edit, link_setup_source_logs, link_staging_file_delete, @@ -145,10 +146,16 @@ class SourcesApp(MayanAppConfig): menu_secondary.bind_links( links=(link_document_version_upload,), sources=( - 'documents:document_version_list', 'documents:upload_version', + 'documents:document_version_list', 'sources:upload_version', 'documents:document_version_revert' ) ) + menu_secondary.bind_links( + links=(link_document_pages_append,), + sources=( + 'documents:document_pages', 'sources:document_pages_append' + ) + ) post_upgrade.connect( receiver=handler_initialize_periodic_tasks, diff --git a/mayan/apps/sources/forms.py b/mayan/apps/sources/forms.py index a6431f71e5..e4eed9de64 100644 --- a/mayan/apps/sources/forms.py +++ b/mayan/apps/sources/forms.py @@ -23,14 +23,25 @@ class NewDocumentForm(DocumentForm): class NewVersionForm(forms.Form): - def __init__(self, *args, **kwargs): - super(NewVersionForm, self).__init__(*args, **kwargs) + comment = forms.CharField( + help_text=_('An optional comment to explain the upload.'), + label=_('Comment'), required=False, + widget=forms.widgets.Textarea(attrs={'rows': 4}), + ) - self.fields['comment'] = forms.CharField( - label=_('Comment'), - required=False, - widget=forms.widgets.Textarea(attrs={'rows': 4}), - ) + append_pages = forms.BooleanField( + help_text=_( + 'If selected, the pages of the file uploaded will be appended ' + 'to the existing document pages. Otherwise the pages of the ' + 'upload will replace the existing pages of the document.' + ), label=_('Append pages?'), required=False, + ) + + def __init__(self, *args, **kwargs): + hide_append_pages = kwargs.pop('hide_append_pages', False) + super(NewVersionForm, self).__init__(*args, **kwargs) + if hide_append_pages: + self.fields['append_pages'].widget = forms.widgets.HiddenInput() class UploadBaseForm(forms.Form): diff --git a/mayan/apps/sources/icons.py b/mayan/apps/sources/icons.py index 1b0a6c8c84..a0a59fcded 100644 --- a/mayan/apps/sources/icons.py +++ b/mayan/apps/sources/icons.py @@ -9,6 +9,10 @@ icon_document_version_upload = Icon( driver_name='fontawesome', symbol='upload' ) icon_log = Icon(driver_name='fontawesome', symbol='exclamation-triangle') +icon_document_pages_append = Icon( + driver_name='fontawesome-dual', primary_symbol='copy', + secondary_symbol='plus' +) icon_setup_sources = Icon(driver_name='fontawesome', symbol='upload') icon_setup_source_check_now = Icon(driver_name='fontawesome', symbol='check') icon_setup_source_delete = Icon(driver_name='fontawesome', symbol='times') diff --git a/mayan/apps/sources/links.py b/mayan/apps/sources/links.py index b1793547f5..da5d3300d6 100644 --- a/mayan/apps/sources/links.py +++ b/mayan/apps/sources/links.py @@ -113,6 +113,13 @@ link_staging_file_delete = Link( permissions=(permission_document_new_version, permission_document_create), tags='dangerous', text=_('Delete'), view='sources:staging_file_delete', ) +link_document_pages_append = Link( + args='resolved_object.pk', + icon_class_path='mayan.apps.sources.icons.icon_document_pages_append', + permissions=(permission_document_new_version,), + text=_('Append pages'), + view='sources:document_pages_append' +) link_document_version_upload = Link( args='resolved_object.pk', condition=document_new_version_not_blocked, icon_class_path='mayan.apps.sources.icons.icon_document_version_upload', diff --git a/mayan/apps/sources/urls.py b/mayan/apps/sources/urls.py index fdc7adc330..173891f5f5 100644 --- a/mayan/apps/sources/urls.py +++ b/mayan/apps/sources/urls.py @@ -7,9 +7,10 @@ from .api_views import ( APIStagingSourceListView, APIStagingSourceView ) from .views import ( - SetupSourceCheckView, SetupSourceCreateView, SetupSourceDeleteView, - SetupSourceEditView, SetupSourceListView, SourceLogListView, - StagingFileDeleteView, UploadInteractiveVersionView, UploadInteractiveView + DocumentPagesAppendView, SetupSourceCheckView, SetupSourceCreateView, + SetupSourceDeleteView, SetupSourceEditView, SetupSourceListView, + SourceLogListView, StagingFileDeleteView, UploadInteractiveVersionView, + UploadInteractiveView ) from .wizards import DocumentCreateWizard @@ -41,6 +42,14 @@ urlpatterns = [ regex=r'^documents/(?P\d+)/versions/upload/interactive/$', view=UploadInteractiveVersionView.as_view(), name='upload_version' ), + url( + regex=r'^documents/(?P\d+)/pages/append/interactive/(?P\d+)/$', + view=DocumentPagesAppendView.as_view(), name='document_pages_append' + ), + url( + regex=r'^documents/(?P\d+)/pages/append/interactive/$', + view=DocumentPagesAppendView.as_view(), name='document_pages_append' + ), # Setup views diff --git a/mayan/apps/sources/views.py b/mayan/apps/sources/views.py index be55047b7d..3f26a650c2 100644 --- a/mayan/apps/sources/views.py +++ b/mayan/apps/sources/views.py @@ -362,7 +362,6 @@ class UploadInteractiveView(UploadBaseView): class UploadInteractiveVersionView(UploadBaseView): def dispatch(self, request, *args, **kwargs): - self.subtemplates_list = [] self.document = get_object_or_404( @@ -417,12 +416,17 @@ class UploadInteractiveVersionView(UploadBaseView): else: user_id = None - task_upload_new_version.apply_async(kwargs=dict( - shared_uploaded_file_id=shared_uploaded_file.pk, - document_id=self.document.pk, - user_id=user_id, - comment=forms['document_form'].cleaned_data.get('comment') - )) + task_upload_new_version.apply_async( + kwargs=dict( + append_pages=forms['document_form'].cleaned_data.get( + 'append_pages', False + ), + shared_uploaded_file_id=shared_uploaded_file.pk, + document_id=self.document.pk, + user_id=user_id, + comment=forms['document_form'].cleaned_data.get('comment') + ) + ) messages.success( message=_( @@ -448,13 +452,6 @@ class UploadInteractiveVersionView(UploadBaseView): files=kwargs.get('files', None), ) - def create_document_form_form(self, **kwargs): - return self.get_form_classes()['document_form']( - prefix=kwargs['prefix'], - data=kwargs.get('data', None), - files=kwargs.get('files', None), - ) - def get_form_classes(self): return { 'document_form': NewVersionForm, @@ -467,8 +464,33 @@ class UploadInteractiveVersionView(UploadBaseView): ).get_context_data(**kwargs) context['object'] = self.document context['title'] = _( - 'Upload a new version from source: %s' - ) % self.source.label + 'Upload a new version for document "%(document)s" from source: %(source)s' + ) % {'document': self.document, 'source': self.source.label} + + context['submit_label'] = _('Submit') + + return context + + +class DocumentPagesAppendView(UploadInteractiveVersionView): + def get_document_form_initial(self): + return { + 'append_pages': True, + } + + def get_form_extra_kwargs(self, form_name): + if form_name == 'document_form': + return { + 'hide_append_pages': True + } + + def get_context_data(self, **kwargs): + context = super( + DocumentPagesAppendView, self + ).get_context_data(**kwargs) + context['title'] = _( + 'Append pages to document "%(document)s" from source: %(source)s' + ) % {'document': self.document, 'source': self.source.label} return context diff --git a/mayan/apps/tags/apps.py b/mayan/apps/tags/apps.py index c64811558a..2828e06049 100644 --- a/mayan/apps/tags/apps.py +++ b/mayan/apps/tags/apps.py @@ -13,7 +13,9 @@ from mayan.apps.common.menus import ( menu_facet, menu_list_facet, menu_main, menu_multi_item, menu_object, menu_secondary ) -from mayan.apps.documents.search import document_page_search, document_search +from mayan.apps.documents.search import ( + document_page_search, document_search, document_version_page_search +) from mayan.apps.events.classes import ModelEventType from mayan.apps.events.links import ( link_events_for_object, link_object_event_types_user_subcriptions_list, @@ -62,7 +64,7 @@ class TagsApp(MayanAppConfig): ) DocumentPageResult = apps.get_model( - app_label='documents', model_name='DocumentPageResult' + app_label='documents', model_name='DocumentVersionPageResult' ) DocumentTag = self.get_model(model_name='DocumentTag') @@ -133,9 +135,12 @@ class TagsApp(MayanAppConfig): ) document_page_search.add_model_field( - field='document_version__document__tags__label', label=_('Tags') + field='document__tags__label', label=_('Tags') ) document_search.add_model_field(field='tags__label', label=_('Tags')) + document_version_page_search.add_model_field( + field='document_version__document__tags__label', label=_('Tags') + ) menu_facet.bind_links( links=(link_document_tag_list,), sources=(Document,) diff --git a/requirements/base.txt b/requirements/base.txt index 337001f564..87dd80811c 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,9 +1,7 @@ Pillow==6.0.0 PyPDF2==1.26.0 PyYAML==5.1.1 -celery==4.3.0 django-activity-stream==0.7.0 -django-celery-beat==1.5.0 django-colorful==1.3 django-cors-headers==2.5.2 django-downloadview==1.9 diff --git a/setup.py b/setup.py index 5b4c273ef0..a8ab7ec567 100644 --- a/setup.py +++ b/setup.py @@ -60,9 +60,7 @@ django==1.11.24 Pillow==6.0.0 PyPDF2==1.26.0 PyYAML==5.1.1 -celery==4.3.0 django-activity-stream==0.7.0 -django-celery-beat==1.5.0 django-colorful==1.3 django-cors-headers==2.5.2 django-downloadview==1.9