diff --git a/docs/releases/3.3.rst b/docs/releases/3.3.rst index 17ff89a513..47878877f9 100644 --- a/docs/releases/3.3.rst +++ b/docs/releases/3.3.rst @@ -93,7 +93,7 @@ Changes Removals -------- -- Database conversion. Reason for removal. The database conversions support +- Database conversion. Reason for removal: The database conversions support provided by this feature (SQLite to PostgreSQL) was being confused with database migrations and upgrades. @@ -108,7 +108,7 @@ Removals Continued confusion about the purpose of the feature and confusion about how errors with this feature were a reflexion of the code quality of - Mayannecessitated the removal of the database conversion feature. + Mayan necessitated the removal of the database conversion feature. - Django environ diff --git a/mayan/__init__.py b/mayan/__init__.py index 9bd0dcd8fd..9401b78d20 100644 --- a/mayan/__init__.py +++ b/mayan/__init__.py @@ -2,8 +2,8 @@ from __future__ import unicode_literals __title__ = 'Mayan EDMS' __version__ = '3.3beta1' -__build__ = 0x030208 -__build_string__ = 'v3.2.8-255-g69086d87dd_Tue Oct 8 09:43:10 2019 -0400' +__build__ = 0x030300 +__build_string__ = 'v3.3beta1_Sat Oct 5 15:08:53 2019 -0400' __django_version__ = '1.11' __author__ = 'Roberto Rosario' __author_email__ = 'roberto.rosario@mayan-edms.com' diff --git a/mayan/apps/cabinets/apps.py b/mayan/apps/cabinets/apps.py index 6011bd4874..a2a0dc24bd 100644 --- a/mayan/apps/cabinets/apps.py +++ b/mayan/apps/cabinets/apps.py @@ -10,12 +10,14 @@ from mayan.apps.common.menus import ( menu_facet, menu_list_facet, menu_main, menu_multi_item, menu_object, menu_secondary ) +from mayan.apps.documents.search import ( + document_page_search, document_search, document_version_page_search +) from mayan.apps.events.classes import ModelEventType from mayan.apps.events.links import ( link_events_for_object, link_object_event_types_user_subcriptions_list, ) from mayan.apps.events.permissions import permission_events_view -from mayan.apps.documents.search import document_page_search, document_search from mayan.apps.navigation.classes import SourceColumn from .dependencies import * # NOQA @@ -115,12 +117,16 @@ class CabinetsApp(MayanAppConfig): ) document_page_search.add_model_field( - field='document_version__document__cabinets__label', + field='document__cabinets__label', label=_('Cabinets') ) document_search.add_model_field( field='cabinets__label', label=_('Cabinets') ) + document_version_page_search.add_model_field( + field='document_version__document__cabinets__label', + label=_('Cabinets') + ) menu_facet.bind_links( links=(link_document_cabinet_list,), sources=(Document,) diff --git a/mayan/apps/common/generics.py b/mayan/apps/common/generics.py index 37f5c268ea..310717743a 100644 --- a/mayan/apps/common/generics.py +++ b/mayan/apps/common/generics.py @@ -53,8 +53,8 @@ class MultiFormView(DjangoFormView): template_name = 'appearance/generic_form.html' def _create_form(self, form_name, klass): - form_kwargs = self.get_form_kwargs(form_name) - form_create_method = 'create_%s_form' % form_name + form_kwargs = self.get_form_kwargs(form_name=form_name) + form_create_method = 'create_{}_form'.format(form_name) if hasattr(self, form_create_method): form = getattr(self, form_create_method)(**form_kwargs) else: @@ -71,7 +71,7 @@ class MultiFormView(DjangoFormView): def forms_valid(self, forms): for form_name, form in forms.items(): - form_valid_method = '%s_form_valid' % form_name + form_valid_method = '{}_form_valid'.format(form_name) if hasattr(self, form_valid_method): return getattr(self, form_valid_method)(form) @@ -98,8 +98,8 @@ class MultiFormView(DjangoFormView): def get_form_kwargs(self, form_name): kwargs = {} - kwargs.update({'initial': self.get_initial(form_name)}) - kwargs.update({'prefix': self.get_prefix(form_name)}) + kwargs.update({'initial': self.get_initial(form_name=form_name)}) + kwargs.update({'prefix': self.get_prefix(form_name=form_name)}) if self.request.method in ('POST', 'PUT'): kwargs.update({ @@ -124,7 +124,7 @@ class MultiFormView(DjangoFormView): ) def get_initial(self, form_name): - initial_method = 'get_%s_initial' % form_name + initial_method = 'get_{}_initial'.format(form_name) if hasattr(self, initial_method): return getattr(self, initial_method)() else: diff --git a/mayan/apps/document_comments/apps.py b/mayan/apps/document_comments/apps.py index 371d7d9be4..ee34bb8038 100644 --- a/mayan/apps/document_comments/apps.py +++ b/mayan/apps/document_comments/apps.py @@ -8,7 +8,9 @@ from mayan.apps.common.apps import MayanAppConfig from mayan.apps.common.menus import ( menu_facet, menu_list_facet, menu_object, menu_secondary ) -from mayan.apps.documents.search import document_page_search, document_search +from mayan.apps.documents.search import ( + document_page_search, document_search, document_version_page_search +) from mayan.apps.events.classes import ModelEventType from mayan.apps.events.links import ( link_events_for_object, link_object_event_types_user_subcriptions_list @@ -80,13 +82,17 @@ class DocumentCommentsApp(MayanAppConfig): SourceColumn(attribute='comment', source=Comment) document_page_search.add_model_field( - field='document_version__document__comments__comment', + field='document__comments__comment', label=_('Comments') ) document_search.add_model_field( field='comments__comment', label=_('Comments') ) + document_version_page_search.add_model_field( + field='document_version__document__comments__comment', + label=_('Comments') + ) menu_facet.bind_links( links=(link_comments_for_document,), sources=(Document,) diff --git a/mayan/apps/document_parsing/admin.py b/mayan/apps/document_parsing/admin.py index 258da5ec3d..5fb8ca6a37 100644 --- a/mayan/apps/document_parsing/admin.py +++ b/mayan/apps/document_parsing/admin.py @@ -3,13 +3,13 @@ from __future__ import unicode_literals from django.contrib import admin from .models import ( - DocumentPageContent, DocumentVersionParseError + DocumentVersionPageContent, DocumentVersionParseError ) -@admin.register(DocumentPageContent) -class DocumentPageContentAdmin(admin.ModelAdmin): - list_display = ('document_page',) +@admin.register(DocumentVersionPageContent) +class DocumentVersionPageContentAdmin(admin.ModelAdmin): + list_display = ('document_version_page',) @admin.register(DocumentVersionParseError) diff --git a/mayan/apps/document_parsing/api_views.py b/mayan/apps/document_parsing/api_views.py index 7d29b61d94..ce024f34be 100644 --- a/mayan/apps/document_parsing/api_views.py +++ b/mayan/apps/document_parsing/api_views.py @@ -8,7 +8,7 @@ from rest_framework.response import Response from mayan.apps.documents.models import Document from mayan.apps.rest_api.permissions import MayanPermission -from .models import DocumentPageContent +from .models import DocumentVersionPageContent from .permissions import permission_content_view from .serializers import DocumentPageContentSerializer @@ -41,8 +41,8 @@ class APIDocumentPageContentView(generics.RetrieveAPIView): try: content = instance.content - except DocumentPageContent.DoesNotExist: - content = DocumentPageContent.objects.none() + except DocumentVersionPageContent.DoesNotExist: + content = DocumentVersionPageContent.objects.none() serializer = self.get_serializer(content) return Response(serializer.data) diff --git a/mayan/apps/document_parsing/apps.py b/mayan/apps/document_parsing/apps.py index ea4355c5a8..c7bd4e4773 100644 --- a/mayan/apps/document_parsing/apps.py +++ b/mayan/apps/document_parsing/apps.py @@ -12,7 +12,9 @@ from mayan.apps.common.classes import ModelField from mayan.apps.common.menus import ( menu_facet, menu_list_facet, menu_multi_item, menu_secondary, menu_tools ) -from mayan.apps.documents.search import document_search, document_page_search +from mayan.apps.documents.search import ( + document_search, document_page_search, document_version_page_search +) from mayan.apps.documents.signals import post_version_upload from mayan.apps.events.classes import ModelEventType from mayan.apps.navigation.classes import SourceColumn @@ -43,7 +45,7 @@ from .permissions import ( permission_parse_document ) from .signals import post_document_version_parsing -from .utils import get_document_content +from .utils import get_document_content, get_document_version_content logger = logging.getLogger(__name__) @@ -74,6 +76,9 @@ class DocumentParsingApp(MayanAppConfig): DocumentVersion = apps.get_model( app_label='documents', model_name='DocumentVersion' ) + DocumentVersionPage = apps.get_model( + app_label='documents', model_name='DocumentVersionPage' + ) DocumentVersionParseError = self.get_model( model_name='DocumentVersionParseError' ) @@ -85,7 +90,7 @@ class DocumentParsingApp(MayanAppConfig): name='content', value=get_document_content ) DocumentVersion.add_to_class( - name='content', value=get_document_content + name='content', value=get_document_version_content ) DocumentVersion.add_to_class( name='submit_for_parsing', @@ -100,9 +105,9 @@ class DocumentParsingApp(MayanAppConfig): ) ) - ModelField( - model=Document, name='versions__version_pages__content__content' - ) + #ModelField( + # model=Document, name='versions__pages__content__content' + #) ModelPermission.register( model=Document, permissions=( @@ -133,17 +138,17 @@ class DocumentParsingApp(MayanAppConfig): ) document_search.add_model_field( - field='versions__version_pages__content__content', label=_('Content') + field='versions__pages__content__content', label=_('Content') ) - document_page_search.add_model_field( + document_version_page_search.add_model_field( field='content__content', label=_('Content') ) menu_facet.bind_links( links=(link_document_content,), sources=(Document,) ) - menu_facet.bind_links( + menu_list_facet.bind_links( links=(link_document_page_content,), sources=(DocumentPage,) ) menu_list_facet.bind_links( diff --git a/mayan/apps/document_parsing/forms.py b/mayan/apps/document_parsing/forms.py index 5c4d6af26d..2803c9cd59 100644 --- a/mayan/apps/document_parsing/forms.py +++ b/mayan/apps/document_parsing/forms.py @@ -8,7 +8,7 @@ from django.utils.translation import ugettext_lazy as _, ugettext from mayan.apps.common.widgets import TextAreaDiv -from .models import DocumentPageContent +from .models import DocumentVersionPageContent class DocumentContentForm(forms.Form): @@ -26,10 +26,10 @@ class DocumentContentForm(forms.Form): except AttributeError: document_pages = [] - for page in document_pages: + for document_page in document_pages: try: - page_content = page.content.content - except DocumentPageContent.DoesNotExist: + page_content = document_page.content_object.content.content + except DocumentVersionPageContent.DoesNotExist: pass else: content.append(conditional_escape(force_text(page_content))) @@ -37,7 +37,7 @@ class DocumentContentForm(forms.Form): '\n\n\n
- %s -
\n\n\n' % ( ugettext( 'Page %(page_number)d' - ) % {'page_number': page.page_number} + ) % {'page_number': document_page.page_number} ) ) @@ -72,8 +72,8 @@ class DocumentPageContentForm(forms.Form): self.fields['contents'].initial = '' try: - page_content = document_page.content.content - except DocumentPageContent.DoesNotExist: + page_content = document_page.content_object.content.content + except DocumentVersionPageContent.DoesNotExist: pass else: content = conditional_escape(force_text(page_content)) diff --git a/mayan/apps/document_parsing/icons.py b/mayan/apps/document_parsing/icons.py index 6aad4a8ea2..9fb872dc1f 100644 --- a/mayan/apps/document_parsing/icons.py +++ b/mayan/apps/document_parsing/icons.py @@ -17,6 +17,9 @@ icon_document_content_download = Icon( icon_document_multiple_submit = Icon( driver_name='fontawesome', symbol='font' ) +icon_document_page_content = Icon( + driver_name='fontawesome', symbol='font' +) icon_document_submit = Icon( driver_name='fontawesome', symbol='font' ) diff --git a/mayan/apps/document_parsing/links.py b/mayan/apps/document_parsing/links.py index 2cfe95fab9..a7de474581 100644 --- a/mayan/apps/document_parsing/links.py +++ b/mayan/apps/document_parsing/links.py @@ -32,9 +32,15 @@ link_document_content_delete_multiple = Link( text=_('Delete parsed content'), view='document_parsing:document_content_delete_multiple', ) +link_document_content_download = Link( + args='resolved_object.id', + icon_class_path='mayan.apps.document_parsing.icons.icon_document_content_download', + permissions=(permission_content_view,), text=_('Download content'), + view='document_parsing:document_content_download' +) link_document_page_content = Link( args='resolved_object.id', conditional_disable=is_document_page_disabled, - icon_class_path='mayan.apps.document_parsing.icons.icon_document_content', + icon_class_path='mayan.apps.document_parsing.icons.icon_document_page_content', permissions=(permission_content_view,), text=_('Content'), view='document_parsing:document_page_content' ) @@ -44,12 +50,6 @@ link_document_parsing_errors_list = Link( permissions=(permission_content_view,), text=_('Parsing errors'), view='document_parsing:document_parsing_error_list' ) -link_document_content_download = Link( - args='resolved_object.id', - icon_class_path='mayan.apps.document_parsing.icons.icon_document_content_download', - permissions=(permission_content_view,), text=_('Download content'), - view='document_parsing:document_content_download' -) link_document_submit_multiple = Link( icon_class_path='mayan.apps.document_parsing.icons.icon_document_submit', text=_('Submit for parsing'), diff --git a/mayan/apps/document_parsing/managers.py b/mayan/apps/document_parsing/managers.py index 1669f89e70..8ff1d5d231 100644 --- a/mayan/apps/document_parsing/managers.py +++ b/mayan/apps/document_parsing/managers.py @@ -18,11 +18,13 @@ from .signals import post_document_version_parsing logger = logging.getLogger(__name__) -class DocumentPageContentManager(models.Manager): +class DocumentVersionPageContentManager(models.Manager): def delete_content_for(self, document, user=None): with transaction.atomic(): for document_page in document.pages.all(): - self.filter(document_page=document_page).delete() + self.filter( + document_version_page=document_page.content_object + ).delete() event_parsing_document_content_deleted.commit( actor=user, target=document diff --git a/mayan/apps/document_parsing/migrations/0005_rename_page_content.py b/mayan/apps/document_parsing/migrations/0005_rename_page_content.py new file mode 100644 index 0000000000..11ad6cdc8b --- /dev/null +++ b/mayan/apps/document_parsing/migrations/0005_rename_page_content.py @@ -0,0 +1,42 @@ +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + dependencies = [ + ('document_parsing', '0004_auto_20180917_0645'), + ('documents', '0052_rename_document_page'), + ] + + operations = [ + migrations.RenameModel( + 'DocumentPageContent', 'DocumentVersionPageContent' + ), + migrations.AlterField( + model_name='documentversionpagecontent', + name='document_page', + field=models.OneToOneField( + on_delete=django.db.models.deletion.CASCADE, + related_name='content', + to='documents.DocumentVersionPage', + verbose_name='Document version page' + ), + ), + migrations.RenameField( + model_name='documentversionpagecontent', + old_name='document_page', + new_name='document_version_page', + ), + migrations.AlterModelOptions( + name='documentversionpagecontent', + options={ + 'verbose_name': 'Document version page content', + 'verbose_name_plural': 'Document version pages contents' + }, + ), + ] + + + diff --git a/mayan/apps/document_parsing/models.py b/mayan/apps/document_parsing/models.py index 5e9b52320e..48c2dc79af 100644 --- a/mayan/apps/document_parsing/models.py +++ b/mayan/apps/document_parsing/models.py @@ -5,36 +5,12 @@ from django.utils.encoding import force_text, python_2_unicode_compatible from django.utils.translation import ugettext_lazy as _ from mayan.apps.documents.models import ( - DocumentPage, DocumentType, DocumentVersion + DocumentPage, DocumentType, DocumentVersion, DocumentVersionPage ) -from .managers import DocumentPageContentManager, DocumentTypeSettingsManager - - -@python_2_unicode_compatible -class DocumentPageContent(models.Model): - """ - This model store's the parsed content of a document page. - """ - document_page = models.OneToOneField( - on_delete=models.CASCADE, related_name='content', to=DocumentPage, - verbose_name=_('Document page') - ) - content = models.TextField( - blank=True, help_text=_( - 'The actual text content as extracted by the document ' - 'parsing backend.' - ), verbose_name=_('Content') - ) - - objects = DocumentPageContentManager() - - class Meta: - verbose_name = _('Document page content') - verbose_name_plural = _('Document pages contents') - - def __str__(self): - return force_text(self.document_page) +from .managers import ( + DocumentVersionPageContentManager, DocumentTypeSettingsManager +) class DocumentTypeSettings(models.Model): @@ -62,6 +38,32 @@ class DocumentTypeSettings(models.Model): verbose_name_plural = _('Document types settings') +@python_2_unicode_compatible +class DocumentVersionPageContent(models.Model): + """ + This model store's the parsed content of a document page. + """ + document_version_page = models.OneToOneField( + on_delete=models.CASCADE, related_name='content', + to=DocumentVersionPage, verbose_name=_('Document version page') + ) + content = models.TextField( + blank=True, help_text=_( + 'The actual text content as extracted by the document ' + 'parsing backend.' + ), verbose_name=_('Content') + ) + + objects = DocumentVersionPageContentManager() + + class Meta: + verbose_name = _('Document version page content') + verbose_name_plural = _('Document version pages contents') + + def __str__(self): + return force_text(self.document_page) + + @python_2_unicode_compatible class DocumentVersionParseError(models.Model): """ diff --git a/mayan/apps/document_parsing/parsers.py b/mayan/apps/document_parsing/parsers.py index 0e4d0cff70..469b974377 100644 --- a/mayan/apps/document_parsing/parsers.py +++ b/mayan/apps/document_parsing/parsers.py @@ -23,11 +23,13 @@ class Parser(object): _registry = {} @classmethod - def parse_document_page(cls, document_page): - for parser_class in cls._registry.get(document_page.document_version.mimetype, ()): + def parse_document_version_page(cls, document_version_page): + for parser_class in cls._registry.get(document_version_page.document_version.mimetype, ()): try: parser = parser_class() - parser.process_document_page(document_page) + parser.process_document_page( + document_version_page=document_version_page + ) except ParserError: # If parser raises error, try next parser in the list pass @@ -41,7 +43,9 @@ class Parser(object): for parser_class in cls._registry.get(document_version.mimetype, ()): try: parser = parser_class() - parser.process_document_version(document_version) + parser.process_document_version( + document_version=document_version + ) except ParserError: # If parser raises error, try next parser in the list pass @@ -64,29 +68,33 @@ class Parser(object): ) logger.debug('document version: %d', document_version.pk) - for document_page in document_version.pages.all(): - self.process_document_page(document_page=document_page) + for document_version_page in document_version.pages.all(): + self.process_document_version_page( + document_version_page=document_version_page + ) - def process_document_page(self, document_page): - DocumentPageContent = apps.get_model( - app_label='document_parsing', model_name='DocumentPageContent' + def process_document_version_page(self, document_version_page): + DocumentVersionPageContent = apps.get_model( + app_label='document_parsing', + model_name='DocumentVersionPageContent' ) logger.info( 'Processing page: %d of document version: %s', - document_page.page_number, document_page.document_version + document_version_page.page_number, + document_version_page.document_version ) - file_object = document_page.document_version.get_intermediate_file() + file_object = document_version_page.document_version.get_intermediate_file() try: - document_page_content, created = DocumentPageContent.objects.get_or_create( - document_page=document_page + document_version_page_content, created = DocumentVersionPageContent.objects.get_or_create( + document_version_page=document_version_page ) - document_page_content.content = self.execute( - file_object=file_object, page_number=document_page.page_number + document_version_page_content.content = self.execute( + file_object=file_object, page_number=document_version_page.page_number ) - document_page_content.save() + document_version_page_content.save() except Exception as exception: error_message = _('Exception parsing page; %s') % exception logger.error(error_message) @@ -96,7 +104,8 @@ class Parser(object): logger.info( 'Finished processing page: %d of document version: %s', - document_page.page_number, document_page.document_version + document_version_page.page_number, + document_version_page.document_version ) def execute(self, file_object, page_number): diff --git a/mayan/apps/document_parsing/serializers.py b/mayan/apps/document_parsing/serializers.py index 7161d2fc40..53d90d354e 100644 --- a/mayan/apps/document_parsing/serializers.py +++ b/mayan/apps/document_parsing/serializers.py @@ -2,10 +2,10 @@ from __future__ import unicode_literals from rest_framework import serializers -from .models import DocumentPageContent +from .models import DocumentVersionPageContent class DocumentPageContentSerializer(serializers.ModelSerializer): class Meta: fields = ('content',) - model = DocumentPageContent + model = DocumentVersionPageContent diff --git a/mayan/apps/document_parsing/tasks.py b/mayan/apps/document_parsing/tasks.py index 4debffbc60..653552f741 100644 --- a/mayan/apps/document_parsing/tasks.py +++ b/mayan/apps/document_parsing/tasks.py @@ -14,8 +14,8 @@ def task_parse_document_version(document_version_pk): DocumentVersion = apps.get_model( app_label='documents', model_name='DocumentVersion' ) - DocumentPageContent = apps.get_model( - app_label='document_parsing', model_name='DocumentPageContent' + DocumentVersionPageContent = apps.get_model( + app_label='document_parsing', model_name='DocumentVersionPageContent' ) document_version = DocumentVersion.objects.get( @@ -24,6 +24,6 @@ def task_parse_document_version(document_version_pk): logger.info( 'Starting parsing for document version: %s', document_version ) - DocumentPageContent.objects.process_document_version( + DocumentVersionPageContent.objects.process_document_version( document_version=document_version ) diff --git a/mayan/apps/document_parsing/tests/test_events.py b/mayan/apps/document_parsing/tests/test_events.py index 1a2860305d..17bbe198fb 100644 --- a/mayan/apps/document_parsing/tests/test_events.py +++ b/mayan/apps/document_parsing/tests/test_events.py @@ -10,7 +10,7 @@ from ..events import ( event_parsing_document_version_submit, event_parsing_document_version_finish ) -from ..models import DocumentPageContent +from ..models import DocumentVersionPageContent class DocumentParsingEventsTestCase(GenericDocumentTestCase): @@ -19,7 +19,7 @@ class DocumentParsingEventsTestCase(GenericDocumentTestCase): def test_document_content_deleted_event(self): Action.objects.all().delete() - DocumentPageContent.objects.delete_content_for( + DocumentVersionPageContent.objects.delete_content_for( document=self.test_document ) diff --git a/mayan/apps/document_parsing/tests/test_parsers.py b/mayan/apps/document_parsing/tests/test_parsers.py index 237bccc567..2eeb8703a6 100644 --- a/mayan/apps/document_parsing/tests/test_parsers.py +++ b/mayan/apps/document_parsing/tests/test_parsers.py @@ -18,5 +18,5 @@ class ParserTestCase(DocumentTestMixin, BaseTestCase): parser.process_document_version(self.test_document.latest_version) self.assertTrue( - TEST_DOCUMENT_CONTENT in self.test_document.pages.first().content.content + TEST_DOCUMENT_CONTENT in self.test_document.pages.first().content_object.content.content ) diff --git a/mayan/apps/document_parsing/tests/test_views.py b/mayan/apps/document_parsing/tests/test_views.py index bb88c1817b..21ca0ef808 100644 --- a/mayan/apps/document_parsing/tests/test_views.py +++ b/mayan/apps/document_parsing/tests/test_views.py @@ -5,7 +5,7 @@ from django.test import override_settings from mayan.apps.documents.tests.base import GenericDocumentViewTestCase from mayan.apps.documents.tests.literals import TEST_HYBRID_DOCUMENT -from ..models import DocumentPageContent +from ..models import DocumentVersionPageContent from ..permissions import ( permission_content_view, permission_document_type_parsing_setup, permission_parse_document @@ -72,8 +72,8 @@ class DocumentContentViewsTestCase( self.assertEqual(response.status_code, 404) self.assertTrue( - DocumentPageContent.objects.filter( - document_page=self.test_document.pages.first() + DocumentVersionPageContent.objects.filter( + document_version_page=self.test_document.pages.first().content_object ).exists() ) @@ -86,8 +86,8 @@ class DocumentContentViewsTestCase( self.assertEqual(response.status_code, 302) self.assertFalse( - DocumentPageContent.objects.filter( - document_page=self.test_document.pages.first() + DocumentVersionPageContent.objects.filter( + document_version_page=self.test_document.pages.first().content_object ).exists() ) diff --git a/mayan/apps/document_parsing/urls.py b/mayan/apps/document_parsing/urls.py index f590236176..877ae67a4e 100644 --- a/mayan/apps/document_parsing/urls.py +++ b/mayan/apps/document_parsing/urls.py @@ -7,7 +7,9 @@ from .views import ( DocumentContentView, DocumentContentDeleteView, DocumentContentDownloadView, DocumentPageContentView, DocumentParsingErrorsListView, DocumentSubmitView, - DocumentTypeSettingsEditView, DocumentTypeSubmitView, ParseErrorListView + DocumentTypeSettingsEditView, DocumentTypeSubmitView, + DocumentVersionPageContentView, + ParseErrorListView ) urlpatterns = [ @@ -34,6 +36,11 @@ urlpatterns = [ regex=r'^documents/pages/(?P\d+)/content/$', view=DocumentPageContentView.as_view(), name='document_page_content' ), + url( + regex=r'^documents/versions/pages/(?P\d+)/content/$', + view=DocumentVersionPageContentView.as_view(), + name='document_version_page_content' + ), url( regex=r'^documents/(?P\d+)/submit/$', view=DocumentSubmitView.as_view(), name='document_submit' diff --git a/mayan/apps/document_parsing/utils.py b/mayan/apps/document_parsing/utils.py index ab8e049450..5086d5cf02 100644 --- a/mayan/apps/document_parsing/utils.py +++ b/mayan/apps/document_parsing/utils.py @@ -6,14 +6,28 @@ from django.utils.html import conditional_escape def get_document_content(document): - DocumentPageContent = apps.get_model( - app_label='document_parsing', model_name='DocumentPageContent' + DocumentVersionPageContent = apps.get_model( + app_label='document_parsing', model_name='DocumentVersionPageContent' ) - for page in document.pages.all(): + for document_page in document.pages.all(): try: - page_content = page.content.content - except DocumentPageContent.DoesNotExist: + page_content = document_page.content_object.content.content + except DocumentVersionPageContent.DoesNotExist: + pass + else: + yield conditional_escape(force_text(page_content)) + + +def get_document_version_content(document_version): + DocumentVersionPageContent = apps.get_model( + app_label='document_parsing', model_name='DocumentVersionPageContent' + ) + + for document_version_page in document_version.pages.all(): + try: + page_content = document_version_page.content.content + except DocumentVersionPageContent.DoesNotExist: pass else: yield conditional_escape(force_text(page_content)) diff --git a/mayan/apps/document_parsing/views.py b/mayan/apps/document_parsing/views.py index de9cb3ef0b..7201253c18 100644 --- a/mayan/apps/document_parsing/views.py +++ b/mayan/apps/document_parsing/views.py @@ -12,10 +12,12 @@ from mayan.apps.common.generics import ( ) from mayan.apps.common.mixins import ExternalObjectMixin from mayan.apps.documents.forms import DocumentTypeFilteredSelectForm -from mayan.apps.documents.models import Document, DocumentPage, DocumentType +from mayan.apps.documents.models import ( + Document, DocumentPage, DocumentType, DocumentVersionPage +) from .forms import DocumentContentForm, DocumentPageContentForm -from .models import DocumentPageContent, DocumentVersionParseError +from .models import DocumentVersionPageContent, DocumentVersionParseError from .permissions import ( permission_content_view, permission_document_type_parsing_setup, permission_parse_document @@ -46,7 +48,7 @@ class DocumentContentDeleteView(MultipleObjectConfirmActionView): return result def object_action(self, form, instance): - DocumentPageContent.objects.delete_content_for( + DocumentVersionPageContent.objects.delete_content_for( document=instance, user=self.request.user ) @@ -107,6 +109,30 @@ class DocumentPageContentView(SingleObjectDetailView): } +class DocumentVersionPageContentView(SingleObjectDetailView): + form_class = DocumentPageContentForm + model = DocumentVersionPage + object_permission = permission_content_view + + def dispatch(self, request, *args, **kwargs): + result = super(DocumentPageContentView, self).dispatch( + request, *args, **kwargs + ) + self.get_object().document.add_as_recent_document_for_user( + request.user + ) + return result + + def get_extra_context(self): + return { + 'hide_labels': True, + 'object': self.get_object(), + 'title': _( + 'Content for document version page: %s' + ) % self.get_object(), + } + + class DocumentParsingErrorsListView(SingleObjectListView): view_permission = permission_content_view diff --git a/mayan/apps/documents/admin.py b/mayan/apps/documents/admin.py index 62d30c0e1f..59b6004f31 100644 --- a/mayan/apps/documents/admin.py +++ b/mayan/apps/documents/admin.py @@ -3,18 +3,12 @@ from __future__ import unicode_literals from django.contrib import admin from .models import ( - DeletedDocument, Document, DocumentPage, DocumentType, - DocumentTypeFilename, DocumentVersion, DuplicatedDocument, RecentDocument + DeletedDocument, Document, DocumentType, DocumentTypeFilename, + DocumentVersion, DocumentVersionPage, DuplicatedDocument, + RecentDocument ) -class DocumentPageInline(admin.StackedInline): - model = DocumentPage - extra = 1 - classes = ('collapse-open',) - allow_add = True - - class DocumentTypeFilenameInline(admin.StackedInline): model = DocumentTypeFilename extra = 1 @@ -29,6 +23,13 @@ class DocumentVersionInline(admin.StackedInline): allow_add = True +class DocumentVersionPageInline(admin.StackedInline): + model = DocumentVersionPage + extra = 1 + classes = ('collapse-open',) + allow_add = True + + @admin.register(DeletedDocument) class DeletedDocumentAdmin(admin.ModelAdmin): date_hierarchy = 'deleted_date_time' diff --git a/mayan/apps/documents/api_views.py b/mayan/apps/documents/api_views.py index 4d17038722..87c4c9bd3d 100644 --- a/mayan/apps/documents/api_views.py +++ b/mayan/apps/documents/api_views.py @@ -33,10 +33,14 @@ from .serializers import ( DocumentTypeSerializer, DocumentVersionSerializer, NewDocumentSerializer, NewDocumentVersionSerializer, RecentDocumentSerializer, WritableDocumentSerializer, - WritableDocumentTypeSerializer, WritableDocumentVersionSerializer + WritableDocumentTypeSerializer, WritableDocumentVersionSerializer, + DocumentVersionPageSerializer ) from .settings import settings_document_page_image_cache_time -from .tasks import task_generate_document_page_image +from .tasks import ( + task_generate_document_page_image, + task_generate_document_version_page_image +) logger = logging.getLogger(__name__) @@ -168,13 +172,8 @@ class APIDocumentPageImageView(generics.RetrieveAPIView): ) return document - def get_document_version(self): - return get_object_or_404( - self.get_document().versions.all(), pk=self.kwargs['version_pk'] - ) - def get_queryset(self): - return self.get_document_version().pages_all.all() + return self.get_document().pages_all.all() def get_serializer(self, *args, **kwargs): return None @@ -221,6 +220,95 @@ class APIDocumentPageImageView(generics.RetrieveAPIView): return response +class APIDocumentVersionPageImageView(generics.RetrieveAPIView): + """ + get: Returns an image representation of the selected document version page. + """ + lookup_url_kwarg = 'page_pk' + + def get_document(self): + if self.request.method == 'GET': + permission_required = permission_document_view + else: + permission_required = permission_document_edit + + document = get_object_or_404(Document.passthrough, pk=self.kwargs['pk']) + + AccessControlList.objects.check_access( + obj=document, permissions=(permission_required,), + user=self.request.user + ) + return document + + def get_document_version(self): + return get_object_or_404( + self.get_document().versions.all(), pk=self.kwargs['version_pk'] + ) + + def get_queryset(self): + return self.get_document_version().pages.all() + + def get_serializer(self, *args, **kwargs): + return None + + def get_serializer_class(self): + return None + + @cache_control(private=True) + def retrieve(self, request, *args, **kwargs): + width = request.GET.get('width') + height = request.GET.get('height') + zoom = request.GET.get('zoom') + + if zoom: + zoom = int(zoom) + + rotation = request.GET.get('rotation') + + if rotation: + rotation = int(rotation) + + maximum_layer_order = request.GET.get('maximum_layer_order') + if maximum_layer_order: + maximum_layer_order = int(maximum_layer_order) + + task = task_generate_document_version_page_image.apply_async( + kwargs=dict( + document_version_page_id=self.get_object().pk, width=width, + height=height, zoom=zoom, rotation=rotation, + maximum_layer_order=maximum_layer_order, + user_id=request.user.pk + ) + ) + + cache_filename = task.get(timeout=DOCUMENT_IMAGE_TASK_TIMEOUT) + cache_file = self.get_object().cache_partition.get_file(filename=cache_filename) + with cache_file.open() as file_object: + response = HttpResponse(file_object.read(), content_type='image') + if '_hash' in request.GET: + patch_cache_control( + response=response, + max_age=settings_document_page_image_cache_time.value + ) + return response + + +class APIDocumentPageListView(generics.ListAPIView): + serializer_class = DocumentPageSerializer + + def get_document(self): + document = get_object_or_404(Document, pk=self.kwargs['pk']) + + AccessControlList.objects.check_access( + obj=document, permissions=(permission_document_view,), + user=self.request.user + ) + return document + + def get_queryset(self): + return self.get_document().pages.all() + + class APIDocumentPageView(generics.RetrieveUpdateAPIView): """ get: Returns the selected document page details. @@ -230,6 +318,33 @@ class APIDocumentPageView(generics.RetrieveUpdateAPIView): lookup_url_kwarg = 'page_pk' serializer_class = DocumentPageSerializer + def get_document(self): + if self.request.method == 'GET': + permission_required = permission_document_view + else: + permission_required = permission_document_edit + + document = get_object_or_404(Document, pk=self.kwargs['pk']) + + AccessControlList.objects.check_access( + obj=document, permissions=(permission_required,), + user=self.request.user + ) + return document + + def get_queryset(self): + return self.get_document().pages.all() + + +class APIDocumentVersionPageView(generics.RetrieveUpdateAPIView): + """ + get: Returns the selected document verion page details. + patch: Edit the selected document version page. + put: Edit the selected document version page. + """ + lookup_url_kwarg = 'page_pk' + serializer_class = DocumentVersionPageSerializer + def get_document(self): if self.request.method == 'GET': permission_required = permission_document_view @@ -289,8 +404,7 @@ class APIDocumentTypeView(generics.RetrieveUpdateDestroyAPIView): 'GET': (permission_document_type_view,), 'PUT': (permission_document_type_edit,), 'PATCH': (permission_document_type_edit,), - 'DELETE': (permission_document_type_delete,) - } + 'DELETE': (permission_document_type_delete,) } permission_classes = (MayanPermission,) queryset = DocumentType.objects.all() @@ -423,7 +537,7 @@ class APIRecentDocumentListView(generics.ListAPIView): class APIDocumentVersionPageListView(generics.ListAPIView): - serializer_class = DocumentPageSerializer + serializer_class = DocumentVersionPageSerializer def get_document(self): document = get_object_or_404(Document, pk=self.kwargs['pk']) diff --git a/mayan/apps/documents/apps.py b/mayan/apps/documents/apps.py index 09e85928c7..541134fb4b 100644 --- a/mayan/apps/documents/apps.py +++ b/mayan/apps/documents/apps.py @@ -59,7 +59,7 @@ from .links import ( link_document_multiple_delete, link_document_multiple_document_type_edit, link_document_multiple_download, link_document_multiple_favorites_add, link_document_multiple_favorites_remove, link_document_multiple_restore, - link_document_multiple_trash, link_document_multiple_update_page_count, + link_document_multiple_trash, link_document_multiple_pages_reset, link_document_page_disable, link_document_page_multiple_disable, link_document_page_enable, link_document_page_multiple_enable, link_document_page_navigation_first, link_document_page_navigation_last, @@ -74,8 +74,10 @@ from .links import ( link_document_type_filename_create, link_document_type_filename_delete, link_document_type_filename_edit, link_document_type_filename_list, link_document_type_list, link_document_type_policies, - link_document_type_setup, link_document_update_page_count, + link_document_type_setup, link_document_pages_reset, link_document_version_download, link_document_version_list, + link_document_version_multiple_page_count_update, + link_document_version_page_count_update, link_document_version_return_document, link_document_version_return_list, link_document_version_revert, link_document_version_view, link_duplicated_document_list, link_duplicated_document_scan, @@ -87,10 +89,10 @@ from .permissions import ( permission_document_download, permission_document_edit, permission_document_new_version, permission_document_print, permission_document_properties_edit, permission_document_restore, - permission_document_trash, permission_document_type_delete, - permission_document_type_edit, permission_document_type_view, - permission_document_version_revert, permission_document_version_view, - permission_document_view + permission_document_tools, permission_document_trash, + permission_document_type_delete, permission_document_type_edit, + permission_document_type_view, permission_document_version_revert, + permission_document_version_view, permission_document_view, ) # Just import to initialize the search models from .search import document_search, document_page_search # NOQA @@ -121,10 +123,11 @@ class DocumentsApp(MayanAppConfig): DeletedDocument = self.get_model(model_name='DeletedDocument') Document = self.get_model(model_name='Document') DocumentPage = self.get_model(model_name='DocumentPage') - DocumentPageResult = self.get_model(model_name='DocumentPageResult') + DocumentPageResult = self.get_model(model_name='DocumentVersionPageResult') DocumentType = self.get_model(model_name='DocumentType') DocumentTypeFilename = self.get_model(model_name='DocumentTypeFilename') DocumentVersion = self.get_model(model_name='DocumentVersion') + DocumentVersionPage = self.get_model(model_name='DocumentVersionPage') DuplicatedDocument = self.get_model(model_name='DuplicatedDocument') DynamicSerializerField.add_serializer( @@ -190,13 +193,15 @@ class DocumentsApp(MayanAppConfig): permission_acl_edit, permission_acl_view, permission_document_delete, permission_document_download, permission_document_edit, permission_document_new_version, - permission_document_print, permission_document_properties_edit, - permission_document_restore, permission_document_trash, - permission_document_version_revert, + permission_document_print, + permission_document_properties_edit, + permission_document_restore, permission_document_tools, + permission_document_trash, permission_document_version_revert, permission_document_version_view, permission_document_view, permission_events_view, permission_transformation_create, permission_transformation_delete, - permission_transformation_edit, permission_transformation_view, + permission_transformation_edit, + permission_transformation_view, ) ) @@ -224,13 +229,13 @@ class DocumentsApp(MayanAppConfig): model=Document, manager_name='passthrough' ) ModelPermission.register_inheritance( - model=DocumentPage, related='document_version__document', + model=DocumentPage, related='document', ) ModelPermission.register_manager( model=DocumentPage, manager_name='passthrough' ) ModelPermission.register_inheritance( - model=DocumentPageResult, related='document_version__document', + model=DocumentPageResult, related='document', ) ModelPermission.register_manager( model=DocumentPageResult, manager_name='passthrough' @@ -241,6 +246,9 @@ class DocumentsApp(MayanAppConfig): ModelPermission.register_inheritance( model=DocumentVersion, related='document', ) + ModelPermission.register_inheritance( + model=DocumentVersionPage, related='document_version', + ) # Document and document page thumbnail widget document_page_thumbnail_widget = DocumentPageThumbnailWidget() @@ -454,7 +462,7 @@ class DocumentsApp(MayanAppConfig): link_document_quick_download, link_document_download, link_document_clear_transformations, link_document_clone_transformations, - link_document_update_page_count, + link_document_pages_reset, ), sources=(Document,) ) menu_object.bind_links( @@ -495,7 +503,7 @@ class DocumentsApp(MayanAppConfig): link_document_multiple_favorites_remove, link_document_multiple_clear_transformations, link_document_multiple_trash, link_document_multiple_download, - link_document_multiple_update_page_count, + link_document_multiple_pages_reset, link_document_multiple_document_type_edit, ), sources=(Document,) ) @@ -547,6 +555,17 @@ class DocumentsApp(MayanAppConfig): link_document_version_return_list ), sources=(DocumentVersion,) ) + menu_multi_item.bind_links( + links=( + link_document_version_multiple_page_count_update, + ), sources=(DocumentVersion,) + ) + menu_object.bind_links( + links=( + link_document_version_page_count_update, + ), sources=(DocumentVersion,) + ) + menu_list_facet.bind_links( links=(link_document_version_view,), sources=(DocumentVersion,) ) diff --git a/mayan/apps/documents/dashboard_widgets.py b/mayan/apps/documents/dashboard_widgets.py index 2ca9dd1b89..221e2cfbbd 100644 --- a/mayan/apps/documents/dashboard_widgets.py +++ b/mayan/apps/documents/dashboard_widgets.py @@ -32,12 +32,12 @@ class DashboardWidgetDocumentPagesTotal(DashboardWidgetNumeric): AccessControlList = apps.get_model( app_label='acls', model_name='AccessControlList' ) - DocumentPage = apps.get_model( - app_label='documents', model_name='DocumentPage' + DocumentVersionPage = apps.get_model( + app_label='documents', model_name='DocumentVersionPage' ) self.count = AccessControlList.objects.restrict_queryset( permission=permission_document_view, user=request.user, - queryset=DocumentPage.objects.all() + queryset=DocumentVersionPage.objects.all() ).count() return super(DashboardWidgetDocumentPagesTotal, self).render(request) diff --git a/mayan/apps/documents/icons.py b/mayan/apps/documents/icons.py index fb891fd585..ce365c8678 100644 --- a/mayan/apps/documents/icons.py +++ b/mayan/apps/documents/icons.py @@ -36,7 +36,10 @@ icon_document_edit = Icon( ) icon_document = Icon(driver_name='fontawesome', symbol='book') icon_document_list = icon_document -icon_document_page_count_update = Icon( +icon_document_pages_reset = Icon( + driver_name='fontawesome', symbol='copy' +) +icon_document_version_page_count_update = Icon( driver_name='fontawesome', symbol='copy' ) icon_document_preview = Icon(driver_name='fontawesome', symbol='eye') diff --git a/mayan/apps/documents/links.py b/mayan/apps/documents/links.py index 271aa16579..89cbd31782 100644 --- a/mayan/apps/documents/links.py +++ b/mayan/apps/documents/links.py @@ -168,12 +168,12 @@ link_document_quick_download = Link( permissions=(permission_document_download,), text=_('Quick download'), view='documents:document_download', ) -link_document_update_page_count = Link( +link_document_pages_reset = Link( args='resolved_object.pk', - icon_class_path='mayan.apps.documents.icons.icon_document_page_count_update', + icon_class_path='mayan.apps.documents.icons.icon_document_pages_reset', permissions=(permission_document_tools,), - text=_('Recalculate page count'), - view='documents:document_update_page_count' + text=_('Reset pages'), + view='documents:document_pages_reset' ) link_document_restore = Link( permissions=(permission_document_restore,), @@ -217,10 +217,10 @@ link_document_multiple_download = Link( text=_('Advanced download'), view='documents:document_multiple_download_form' ) -link_document_multiple_update_page_count = Link( - icon_class_path='mayan.apps.documents.icons.icon_document_page_count_update', - text=_('Recalculate page count'), - view='documents:document_multiple_update_page_count' +link_document_multiple_pages_reset = Link( + icon_class_path='mayan.apps.documents.icons.icon_document_pages_reset', + text=_('Reset pages'), + view='documents:document_multiple_pages_reset' ) link_document_multiple_restore = Link( icon_class_path='mayan.apps.documents.icons.icon_trashed_document_restore', @@ -246,6 +246,18 @@ link_document_version_return_list = Link( permissions=(permission_document_version_view,), text=_('Versions'), view='documents:document_version_list', ) +link_document_version_page_count_update = Link( + args='resolved_object.pk', + icon_class_path='mayan.apps.documents.icons.icon_document_version_page_count_update', + permissions=(permission_document_tools,), + text=_('Update page count'), + view='documents:document_version_page_count_update' +) +link_document_version_multiple_page_count_update = Link( + icon_class_path='mayan.apps.documents.icons.icon_document_version_page_count_update', + text=_('Update page count'), + view='documents:document_version_multiple_page_count_update' +) link_document_version_view = Link( args='resolved_object.pk', icon_class_path='mayan.apps.documents.icons.icon_document_version_view', diff --git a/mayan/apps/documents/literals.py b/mayan/apps/documents/literals.py index d639d1980f..35ef6b2f36 100644 --- a/mayan/apps/documents/literals.py +++ b/mayan/apps/documents/literals.py @@ -43,3 +43,5 @@ PAGE_RANGE_RANGE = 'range' PAGE_RANGE_CHOICES = ( (PAGE_RANGE_ALL, _('All pages')), (PAGE_RANGE_RANGE, _('Page range')) ) + +RETRY_DELAY_DOCUMENT_RESET_PAGES = 30 diff --git a/mayan/apps/documents/managers.py b/mayan/apps/documents/managers.py index 6c9dd92640..28236f109c 100644 --- a/mayan/apps/documents/managers.py +++ b/mayan/apps/documents/managers.py @@ -28,15 +28,15 @@ class DocumentManager(models.Manager): class DocumentPageManager(models.Manager): def get_by_natural_key(self, page_number, document_version_natural_key): - DocumentVersion = apps.get_model( - app_label='documents', model_name='DocumentVersion' + Document = apps.get_model( + app_label='documents', model_name='Document' ) try: - document_version = DocumentVersion.objects.get_by_natural_key(*document_version_natural_key) - except DocumentVersion.DoesNotExist: + document = Document.objects.get_by_natural_key(*document_version_natural_key) + except Document.DoesNotExist: raise self.model.DoesNotExist - return self.get(document_version__pk=document_version.pk, page_number=page_number) + return self.get(document__pk=document.pk, page_number=page_number) def get_queryset(self): return models.QuerySet( @@ -124,6 +124,19 @@ class DocumentVersionManager(models.Manager): return self.get(document__pk=document.pk, checksum=checksum) +class DocumentVersionPageManager(models.Manager): + def get_by_natural_key(self, page_number, document_version_natural_key): + DocumentVersion = apps.get_model( + app_label='documents', model_name='DocumentVersion' + ) + try: + document_version = DocumentVersion.objects.get_by_natural_key(*document_version_natural_key) + except DocumentVersion.DoesNotExist: + raise self.model.DoesNotExist + + return self.get(document_version__pk=document_version.pk, page_number=page_number) + + class DuplicatedDocumentManager(models.Manager): def clean_empty_duplicate_lists(self): self.filter(documents=None).delete() diff --git a/mayan/apps/documents/migrations/0052_rename_document_page.py b/mayan/apps/documents/migrations/0052_rename_document_page.py new file mode 100644 index 0000000000..d6751c3810 --- /dev/null +++ b/mayan/apps/documents/migrations/0052_rename_document_page.py @@ -0,0 +1,38 @@ +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + dependencies = [ + ('documents', '0051_documentpage_enabled'), + ] + + operations = [ + migrations.DeleteModel( + name='DocumentPageResult', + ), + migrations.RenameModel('DocumentPage', 'DocumentVersionPage'), + migrations.AlterField( + model_name='documentversionpage', + name='document_version', + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name='pages', to='documents.DocumentVersion', + verbose_name='Document version' + ), + ), + migrations.RemoveField( + model_name='documentversionpage', + name='enabled', + ), + migrations.AlterModelOptions( + name='documentversionpage', + options={ + 'ordering': ('page_number',), + 'verbose_name': 'Document version page', + 'verbose_name_plural': 'Document version pages' + }, + ), + ] diff --git a/mayan/apps/documents/migrations/0053_create_document_page_and_result_models.py b/mayan/apps/documents/migrations/0053_create_document_page_and_result_models.py new file mode 100644 index 0000000000..f7f4081c6e --- /dev/null +++ b/mayan/apps/documents/migrations/0053_create_document_page_and_result_models.py @@ -0,0 +1,57 @@ +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + dependencies = [ + ('contenttypes', '0002_remove_content_type_name'), + ('documents', '0052_rename_document_page'), + ] + + operations = [ + migrations.CreateModel( + name='DocumentPage', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('enabled', models.BooleanField(default=True, verbose_name='Enabled')), + ('page_number', models.PositiveIntegerField(blank=True, db_index=True, null=True, verbose_name='Page number')), + ('object_id', models.PositiveIntegerField()), + ('content_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType')), + ('document', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='pages', to='documents.Document', verbose_name='Document')), + ], + options={ + 'unique_together': set([('document', 'page_number')]), + 'verbose_name': 'Document page', + 'verbose_name_plural': 'Document pages', + 'ordering': ('page_number',), + }, + ), + migrations.CreateModel( + name='DocumentPageResult', + fields=[ + ], + options={ + 'verbose_name': 'Document page result', + 'verbose_name_plural': 'Document pages result', + 'ordering': ('document', 'page_number'), + 'proxy': True, + 'indexes': [], + }, + bases=('documents.documentpage',), + ), + migrations.CreateModel( + name='DocumentVersionPageResult', + fields=[ + ], + options={ + 'verbose_name': 'Document version page', + 'verbose_name_plural': 'Document version pages', + 'ordering': ('document_version__document', 'page_number'), + 'proxy': True, + 'indexes': [], + }, + bases=('documents.documentversionpage',), + ), + ] diff --git a/mayan/apps/documents/migrations/0054_reset_document_pages.py b/mayan/apps/documents/migrations/0054_reset_document_pages.py new file mode 100644 index 0000000000..413f423e3d --- /dev/null +++ b/mayan/apps/documents/migrations/0054_reset_document_pages.py @@ -0,0 +1,62 @@ +from __future__ import unicode_literals + +from django.db import migrations + + +def get_latest_version(document): + return document.versions.order_by('timestamp').last() + + +def operation_reset_document_pages(apps, schema_editor): + Document = apps.get_model(app_label='documents', model_name='Document') + DocumentPage = apps.get_model( + app_label='documents', model_name='DocumentPage' + ) + + # Define inside the function to use the migration's apps instance + def pages_reset(document): + ContentType = apps.get_model('contenttypes', 'ContentType') + DocumentVersionPage = apps.get_model( + app_label='documents', model_name='DocumentVersionPage' + ) + + content_type = ContentType.objects.get_for_model( + model=DocumentVersionPage + ) + + for document_page in document.pages.all(): + document_page.delete() + + for version_page in get_latest_version(document=document).pages.all(): + document_page = document.pages.create( + content_type=content_type, + page_number=version_page.page_number, + object_id=version_page.pk, + ) + + for document in Document.objects.using(schema_editor.connection.alias).all(): + pages_reset(document=document) + + +def operation_reset_document_pages_reverse(apps, schema_editor): + Document = apps.get_model(app_label='documents', model_name='Document') + DocumentPage = apps.get_model( + app_label='documents', model_name='DocumentPage' + ) + + for document in Document.objects.using(schema_editor.connection.alias).all(): + for document_page in document.pages.all(): + document_page.delete() + + +class Migration(migrations.Migration): + dependencies = [ + ('documents', '0053_create_document_page_and_result_models'), + ] + + operations = [ + migrations.RunPython( + code=operation_reset_document_pages, + reverse_code=operation_reset_document_pages_reverse + ), + ] diff --git a/mayan/apps/documents/models/__init__.py b/mayan/apps/documents/models/__init__.py index bb4b55b82a..69063382c8 100644 --- a/mayan/apps/documents/models/__init__.py +++ b/mayan/apps/documents/models/__init__.py @@ -2,4 +2,5 @@ from .document_models import * # NOQA from .document_page_models import * # NOQA from .document_type_models import * # NOQA from .document_version_models import * # NOQA +from .document_version_page_models import * # NOQA from .misc_models import * # NOQA diff --git a/mayan/apps/documents/models/document_models.py b/mayan/apps/documents/models/document_models.py index 8b6fef2400..95cbba506d 100644 --- a/mayan/apps/documents/models/document_models.py +++ b/mayan/apps/documents/models/document_models.py @@ -5,9 +5,10 @@ import uuid from django.apps import apps from django.core.files import File -from django.db import models +from django.db import models, transaction from django.urls import reverse from django.utils.encoding import python_2_unicode_compatible +from django.utils.functional import cached_property from django.utils.timezone import now from django.utils.translation import ugettext, ugettext_lazy as _ @@ -15,6 +16,7 @@ from ..events import ( event_document_create, event_document_properties_edit, event_document_type_change, ) +from ..literals import DOCUMENT_IMAGES_CACHE_NAME from ..managers import DocumentManager, PassthroughManager, TrashCanManager from ..settings import setting_language from ..signals import post_document_type_change @@ -102,6 +104,26 @@ class Document(models.Model): ) return RecentDocument.objects.add_document_for_user(user, self) + @cached_property + def cache(self): + Cache = apps.get_model(app_label='file_caching', model_name='Cache') + return Cache.objects.get(name=DOCUMENT_IMAGES_CACHE_NAME) + + @cached_property + def cache_partition(self): + partition, created = self.cache.partitions.get_or_create( + name='document-{}'.format(self.uuid) + ) + return partition + + @property + def checksum(self): + return self.latest_version.checksum + + @property + def date_updated(self): + return self.latest_version.timestamp + def delete(self, *args, **kwargs): to_trash = kwargs.pop('to_trash', True) @@ -126,25 +148,37 @@ class Document(models.Model): else: return False + @property + def file_mime_encoding(self): + return self.latest_version.encoding + + @property + def file_mimetype(self): + return self.latest_version.mimetype + def get_absolute_url(self): return reverse( viewname='documents:document_preview', kwargs={'pk': self.pk} ) def get_api_image_url(self, *args, **kwargs): - latest_version = self.latest_version - if latest_version: - return latest_version.get_api_image_url(*args, **kwargs) + first_page = self.pages.first() + if first_page: + return first_page.get_api_image_url(*args, **kwargs) @property def is_in_trash(self): return self.in_trash + @property + def latest_version(self): + return self.versions.order_by('timestamp').last() + def natural_key(self): return (self.uuid,) natural_key.dependencies = ['documents.DocumentType'] - def new_version(self, file_object, comment=None, _user=None): + def new_version(self, file_object, append_pages=False, comment=None, _user=None): logger.info('Creating new document version for document: %s', self) DocumentVersion = apps.get_model( app_label='documents', model_name='DocumentVersion' @@ -153,9 +187,10 @@ class Document(models.Model): document_version = DocumentVersion( document=self, comment=comment or '', file=File(file_object) ) - document_version.save(_user=_user) + document_version.save(append_pages=append_pages, _user=_user) logger.info('New document version queued for document: %s', self) + return document_version def open(self, *args, **kwargs): @@ -165,6 +200,34 @@ class Document(models.Model): """ return self.latest_version.open(*args, **kwargs) + @property + def page_count(self): + return self.pages.count() + + @property + def pages(self): + return self.pages.all() + + @property + def pages_all(self): + DocumentPage = apps.get_model( + app_label='documents', model_name='DocumentPage' + ) + return DocumentPage.passthrough.filter(document=self) + + def pages_reset(self, update_page_count=True): + with transaction.atomic(): + for page in self.pages.all(): + page.delete() + + if update_page_count: + self.latest_version.update_page_count() + + for version_page in self.latest_version.pages.all(): + document_page = self.pages.create( + content_object = version_page + ) + def restore(self): self.in_trash = False self.save() @@ -209,53 +272,3 @@ class Document(models.Model): @property def size(self): return self.latest_version.size - - # Compatibility methods - - @property - def checksum(self): - return self.latest_version.checksum - - @property - def date_updated(self): - return self.latest_version.timestamp - - @property - def file_mime_encoding(self): - return self.latest_version.encoding - - @property - def file_mimetype(self): - return self.latest_version.mimetype - - @property - def latest_version(self): - return self.versions.order_by('timestamp').last() - - @property - def page_count(self): - return self.latest_version.page_count - - @property - def pages_all(self): - try: - return self.latest_version.pages_all - except AttributeError: - # Document has no version yet - DocumentPage = apps.get_model( - app_label='documents', model_name='DocumentPage' - ) - - return DocumentPage.objects.none() - - @property - def pages(self): - try: - return self.latest_version.pages - except AttributeError: - # Document has no version yet - DocumentPage = apps.get_model( - app_label='documents', model_name='DocumentPage' - ) - - return DocumentPage.objects.none() diff --git a/mayan/apps/documents/models/document_page_models.py b/mayan/apps/documents/models/document_page_models.py index 00be66ab4b..b6784a1ff8 100644 --- a/mayan/apps/documents/models/document_page_models.py +++ b/mayan/apps/documents/models/document_page_models.py @@ -4,14 +4,16 @@ import logging from furl import furl +from django.contrib.contenttypes.fields import GenericForeignKey +from django.contrib.contenttypes.models import ContentType from django.db import models +from django.db.models import Max from django.urls import reverse from django.utils.encoding import force_text, python_2_unicode_compatible from django.utils.functional import cached_property from django.utils.translation import ugettext_lazy as _ from mayan.apps.converter.literals import DEFAULT_ZOOM_LEVEL, DEFAULT_ROTATION - from mayan.apps.converter.models import LayerTransformation from mayan.apps.converter.transformations import ( BaseTransformation, TransformationResize, TransformationRotate, @@ -26,7 +28,7 @@ from ..settings import ( setting_zoom_min_level ) -from .document_version_models import DocumentVersion +from .document_models import Document __all__ = ('DocumentPage', 'DocumentPageResult') logger = logging.getLogger(__name__) @@ -35,16 +37,22 @@ logger = logging.getLogger(__name__) @python_2_unicode_compatible class DocumentPage(models.Model): """ - Model that describes a document version page + Model that describes a document page """ - document_version = models.ForeignKey( - on_delete=models.CASCADE, related_name='version_pages', to=DocumentVersion, - verbose_name=_('Document version') + document = models.ForeignKey( + on_delete=models.CASCADE, related_name='pages', to=Document, + verbose_name=_('Document') ) enabled = models.BooleanField(default=True, verbose_name=_('Enabled')) page_number = models.PositiveIntegerField( - db_index=True, default=1, editable=False, - verbose_name=_('Page number') + db_index=True, blank=True, null=True, verbose_name=_('Page number') + ) + content_type = models.ForeignKey( + on_delete=models.CASCADE, to=ContentType + ) + object_id = models.PositiveIntegerField() + content_object = GenericForeignKey( + ct_field='content_type', fk_field='object_id' ) objects = DocumentPageManager() @@ -52,6 +60,7 @@ class DocumentPage(models.Model): class Meta: ordering = ('page_number',) + unique_together = ('document', 'page_number') verbose_name = _('Document page') verbose_name_plural = _('Document pages') @@ -60,7 +69,7 @@ class DocumentPage(models.Model): @cached_property def cache_partition(self): - partition, created = self.document_version.cache.partitions.get_or_create( + partition, created = self.document.cache.partitions.get_or_create( name=self.uuid ) return partition @@ -69,19 +78,15 @@ class DocumentPage(models.Model): self.cache_partition.delete() super(DocumentPage, self).delete(*args, **kwargs) - def detect_orientation(self): - with self.document_version.open() as file_object: - converter = get_converter_class()( - file_object=file_object, - mime_type=self.document_version.mimetype - ) - return converter.detect_orientation( - page_number=self.page_number - ) - - @property - def document(self): - return self.document_version.document + #def detect_orientation(self): + # with self.document_version.open() as file_object: + # converter = get_converter_class()( + # file_object=file_object, + # mime_type=self.document_version.mimetype + # ) + # return converter.detect_orientation( + # page_number=self.page_number + # ) def generate_image(self, user=None, **kwargs): transformation_list = self.get_combined_transformation_list(user=user, **kwargs) @@ -90,7 +95,7 @@ class DocumentPage(models.Model): # Check is transformed image is available logger.debug('transformations cache filename: %s', combined_cache_filename) - if not setting_disable_transformed_image_cache.value and self.cache_partition.get_file(filename=combined_cache_filename): + if self.cache_partition.get_file(filename=combined_cache_filename): logger.debug( 'transformations cache file "%s" found', combined_cache_filename ) @@ -128,8 +133,7 @@ class DocumentPage(models.Model): final_url.args = kwargs final_url.path = reverse( viewname='rest_api:documentpage-image', kwargs={ - 'pk': self.document.pk, 'version_pk': self.document_version.pk, - 'page_pk': self.pk + 'pk': self.document.pk, 'page_pk': self.pk } ) final_url.args['_hash'] = transformations_hash @@ -190,12 +194,12 @@ class DocumentPage(models.Model): return transformation_list def get_image(self, transformations=None): - cache_filename = 'base_image' + cache_filename = 'document_page' logger.debug('Page cache filename: %s', cache_filename) cache_file = self.cache_partition.get_file(filename=cache_filename) - if not setting_disable_base_image_cache.value and cache_file: + if cache_file: logger.debug('Page cache file "%s" found', cache_filename) with cache_file.open() as file_object: @@ -216,14 +220,25 @@ class DocumentPage(models.Model): logger.debug('Page cache file "%s" not found', cache_filename) try: - with self.document_version.get_intermediate_file() as file_object: + #with self.document_version.get_intermediate_file() as file_object: + #Render or get cached document version page + + #self.content_object.generate_image() + self.content_object.get_image() + cache_filename = 'base_image' + cache_file = self.content_object.cache_partition.get_file(filename=cache_filename) + + with cache_file.open() as file_object: converter = get_converter_class()( file_object=file_object ) - converter.seek_page(page_number=self.page_number - 1) + converter.seek_page(page_number=0) + #self.page_number - 1) page_image = converter.get_page() + cache_filename = 'document_page' + # Since open "wb+" doesn't create files, create it explicitly with self.cache_partition.create_file(filename=cache_filename) as file_object: file_object.write(page_image.getvalue()) @@ -241,28 +256,39 @@ class DocumentPage(models.Model): ) raise + def get_label(self): + return _( + 'Page %(page_number)d out of %(total_pages)d of %(document)s' + ) % { + 'document': force_text(self.document), + 'page_number': self.page_number, + 'total_pages': self.document.pages_all.count() + } + get_label.short_description = _('Label') + @property def is_in_trash(self): return self.document.is_in_trash - def get_label(self): - return _( - 'Page %(page_num)d out of %(total_pages)d of %(document)s' - ) % { - 'document': force_text(self.document), - 'page_num': self.page_number, - 'total_pages': self.document_version.pages_all.count() - } - get_label.short_description = _('Label') - def natural_key(self): - return (self.page_number, self.document_version.natural_key()) - natural_key.dependencies = ['documents.DocumentVersion'] + return (self.page_number, self.document.natural_key()) + natural_key.dependencies = ['documents.Document'] + + def save(self, *args, **kwargs): + if not self.page_number: + last_page_number = DocumentPage.objects.filter( + document=self.document + ).aggregate(Max('page_number'))['page_number__max'] + if last_page_number is not None: + self.page_number = last_page_number + 1 + else: + self.page_number = 1 + super(DocumentPage, self).save(*args, **kwargs) @property def siblings(self): return DocumentPage.objects.filter( - document_version=self.document_version + document=self.document ) @property @@ -271,12 +297,12 @@ class DocumentPage(models.Model): Make cache UUID a mix of version ID and page ID to avoid using stale images """ - return '{}-{}'.format(self.document_version.uuid, self.pk) + return '{}-{}'.format(self.document.uuid, self.pk) class DocumentPageResult(DocumentPage): class Meta: - ordering = ('document_version__document', 'page_number') + ordering = ('document', 'page_number') proxy = True - verbose_name = _('Document page') - verbose_name_plural = _('Document pages') + verbose_name = _('Document page result') + verbose_name_plural = _('Document pages result') diff --git a/mayan/apps/documents/models/document_version_models.py b/mayan/apps/documents/models/document_version_models.py index 1c24fbfeda..d491ca5bd8 100644 --- a/mayan/apps/documents/models/document_version_models.py +++ b/mayan/apps/documents/models/document_version_models.py @@ -246,23 +246,12 @@ class DocumentVersion(models.Model): return result - @property - def pages_all(self): - DocumentPage = apps.get_model( - app_label='documents', model_name='DocumentPage' - ) - return DocumentPage.passthrough.filter(document_version=self) - - @property - def pages(self): - return self.version_pages.all() - - @property - def page_count(self): - """ - The number of pages that the document posses. - """ - return self.pages.count() + #@property + #def page_count(self): + # """ + # The number of pages that the document posses. + # """ + # return self.pages.count() def revert(self, _user=None): """ @@ -285,6 +274,7 @@ class DocumentVersion(models.Model): Overloaded save method that updates the document version's checksum, mimetype, and page count when created """ + append_pages = kwargs.pop('append_pages', False) user = kwargs.pop('_user', None) new_document_version = not self.pk @@ -304,7 +294,7 @@ class DocumentVersion(models.Model): # Only do this for new documents self.update_checksum(save=False) self.update_mimetype(save=False) - self.save() + self.save(append_pages=append_pages, _user=user) self.update_page_count(save=False) if setting_fix_orientation.value: self.fix_orientation() @@ -337,6 +327,14 @@ class DocumentVersion(models.Model): sender=Document, instance=self.document ) + if append_pages: + for version_page in self.pages.all(): + self.document.pages.create( + content_object = version_page + ) + else: + self.document.pages_reset(update_page_count=False) + def save_to_file(self, file_object): """ Save a copy of the document from the document storage backend @@ -410,7 +408,7 @@ class DocumentVersion(models.Model): pass else: DocumentPage = apps.get_model( - app_label='documents', model_name='DocumentPage' + app_label='documents', model_name='DocumentVersionPage' ) with transaction.atomic(): diff --git a/mayan/apps/documents/models/document_version_page_models.py b/mayan/apps/documents/models/document_version_page_models.py new file mode 100644 index 0000000000..c0c5b7f8f3 --- /dev/null +++ b/mayan/apps/documents/models/document_version_page_models.py @@ -0,0 +1,282 @@ +from __future__ import absolute_import, unicode_literals + +import logging + +from furl import furl + +from django.db import models +from django.urls import reverse +from django.utils.encoding import force_text, python_2_unicode_compatible +from django.utils.functional import cached_property +from django.utils.translation import ugettext_lazy as _ + +from mayan.apps.converter.literals import DEFAULT_ZOOM_LEVEL, DEFAULT_ROTATION + +from mayan.apps.converter.models import LayerTransformation +from mayan.apps.converter.transformations import ( + BaseTransformation, TransformationResize, TransformationRotate, + TransformationZoom +) +from mayan.apps.converter.utils import get_converter_class + +from ..managers import DocumentVersionPageManager +from ..settings import ( + setting_disable_base_image_cache, setting_disable_transformed_image_cache, + setting_display_width, setting_display_height, setting_zoom_max_level, + setting_zoom_min_level +) + +from .document_version_models import DocumentVersion + +__all__ = ('DocumentVersionPage', 'DocumentVersionPageResult') +logger = logging.getLogger(__name__) + + +@python_2_unicode_compatible +class DocumentVersionPage(models.Model): + """ + Model that describes a document version page + """ + document_version = models.ForeignKey( + on_delete=models.CASCADE, related_name='pages', to=DocumentVersion, + verbose_name=_('Document version') + ) + page_number = models.PositiveIntegerField( + db_index=True, default=1, editable=False, + verbose_name=_('Page number') + ) + + objects = DocumentVersionPageManager() + + class Meta: + ordering = ('page_number',) + verbose_name = _('Document version page') + verbose_name_plural = _('Document version pages') + + def __str__(self): + return self.get_label() + + @cached_property + def cache_partition(self): + partition, created = self.document_version.cache.partitions.get_or_create( + name=self.uuid + ) + return partition + + def delete(self, *args, **kwargs): + self.cache_partition.delete() + super(DocumentVersionPage, self).delete(*args, **kwargs) + + #def detect_orientation(self): + # with self.document_version.open() as file_object: + # converter = get_converter_class()( + # file_object=file_object, + # mime_type=self.document_version.mimetype + # ) + # return converter.detect_orientation( + # page_number=self.page_number + # ) + + @property + def document(self): + return self.document_version.document + + def generate_image(self, user=None, **kwargs): + transformation_list = self.get_combined_transformation_list(user=user, **kwargs) + combined_cache_filename = BaseTransformation.combine(transformation_list) + + # Check is transformed image is available + logger.debug('transformations cache filename: %s', combined_cache_filename) + + if self.cache_partition.get_file(filename=combined_cache_filename): + logger.debug( + 'transformations cache file "%s" found', combined_cache_filename + ) + else: + logger.debug( + 'transformations cache file "%s" not found', combined_cache_filename + ) + image = self.get_image(transformations=transformation_list) + with self.cache_partition.create_file(filename=combined_cache_filename) as file_object: + file_object.write(image.getvalue()) + + return combined_cache_filename + + #def get_absolute_url(self): + # return reverse( + # viewname='documents:document_version_page_view', kwargs={ + # 'pk': self.pk + # } + # ) + + def get_api_image_url(self, *args, **kwargs): + """ + Create an unique URL combining: + - the page's image URL + - the interactive argument + - a hash from the server side and interactive transformations + The purpose of this unique URL is to allow client side caching + if document page images. + """ + transformations_hash = BaseTransformation.combine( + self.get_combined_transformation_list(*args, **kwargs) + ) + + kwargs.pop('transformations', None) + + final_url = furl() + final_url.args = kwargs + final_url.path = reverse( + viewname='rest_api:documentversionpage-image', kwargs={ + 'pk': self.document.pk, 'version_pk': self.document_version.pk, + 'page_pk': self.pk + } + ) + final_url.args['_hash'] = transformations_hash + + return final_url.tostr() + + def get_combined_transformation_list(self, user=None, *args, **kwargs): + """ + Return a list of transformation containing the server side + document page transformation as well as tranformations created + from the arguments as transient interactive transformation. + """ + # Convert arguments into transformations + transformations = kwargs.get('transformations', []) + + # Set sensible defaults if the argument is not specified or if the + # argument is None + width = kwargs.get('width', setting_display_width.value) or setting_display_width.value + height = kwargs.get('height', setting_display_height.value) or setting_display_height.value + rotation = kwargs.get('rotation', DEFAULT_ROTATION) or DEFAULT_ROTATION + zoom_level = kwargs.get('zoom', DEFAULT_ZOOM_LEVEL) or DEFAULT_ZOOM_LEVEL + + if zoom_level < setting_zoom_min_level.value: + zoom_level = setting_zoom_min_level.value + + if zoom_level > setting_zoom_max_level.value: + zoom_level = setting_zoom_max_level.value + + # Generate transformation hash + transformation_list = [] + + maximum_layer_order = kwargs.get('maximum_layer_order', None) + + # Stored transformations first + for stored_transformation in LayerTransformation.objects.get_for_object( + self, maximum_layer_order=maximum_layer_order, as_classes=True, + user=user + ): + transformation_list.append(stored_transformation) + + # Interactive transformations second + for transformation in transformations: + transformation_list.append(transformation) + + if rotation: + transformation_list.append( + TransformationRotate(degrees=rotation) + ) + + if width: + transformation_list.append( + TransformationResize(width=width, height=height) + ) + + if zoom_level: + transformation_list.append(TransformationZoom(percent=zoom_level)) + + return transformation_list + + def get_image(self, transformations=None): + cache_filename = 'base_image' + logger.debug('Page cache filename: %s', cache_filename) + + cache_file = self.cache_partition.get_file(filename=cache_filename) + + if cache_file: + logger.debug('Page cache file "%s" found', cache_filename) + + with cache_file.open() as file_object: + converter = get_converter_class()( + file_object=file_object + ) + + converter.seek_page(page_number=0) + + # This code is also repeated below to allow using a context + # manager with cache_file.open and close it automatically. + # Apply runtime transformations + for transformation in transformations or []: + converter.transform(transformation=transformation) + + return converter.get_page() + else: + logger.debug('Page cache file "%s" not found', cache_filename) + + try: + with self.document_version.get_intermediate_file() as file_object: + converter = get_converter_class()( + file_object=file_object + ) + converter.seek_page(page_number=self.page_number - 1) + + page_image = converter.get_page() + + # Since open "wb+" doesn't create files, create it explicitly + with self.cache_partition.create_file(filename=cache_filename) as file_object: + file_object.write(page_image.getvalue()) + + # Apply runtime transformations + for transformation in transformations or []: + converter.transform(transformation=transformation) + + return converter.get_page() + except Exception as exception: + # Cleanup in case of error + logger.error( + 'Error creating page cache file "%s"; %s', + cache_filename, exception + ) + raise + + #@property + #def is_in_trash(self): + # return self.document.is_in_trash + + def get_label(self): + return _( + 'Version page %(page_number)d out of %(total_pages)d of %(document)s' + ) % { + 'document': force_text(self.document), + 'page_number': self.page_number, + 'total_pages': self.document_version.pages.count() + } + get_label.short_description = _('Label') + + def natural_key(self): + return (self.page_number, self.document_version.natural_key()) + natural_key.dependencies = ['documents.DocumentVersion'] + + @property + def siblings(self): + return DocumentVersionPage.objects.filter( + document_version=self.document_version + ) + + @property + def uuid(self): + """ + Make cache UUID a mix of version ID and page ID to avoid using stale + images + """ + return '{}-{}'.format(self.document_version.uuid, self.pk) + + +class DocumentVersionPageResult(DocumentVersionPage): + class Meta: + ordering = ('document_version__document', 'page_number') + proxy = True + verbose_name = _('Document version page') + verbose_name_plural = _('Document version pages') diff --git a/mayan/apps/documents/queues.py b/mayan/apps/documents/queues.py index 11280f17d8..e01c62d8e3 100644 --- a/mayan/apps/documents/queues.py +++ b/mayan/apps/documents/queues.py @@ -30,6 +30,10 @@ queue_converter.add_task_type( dotted_path='mayan.apps.documents.tasks.task_generate_document_page_image', label=_('Generate document page image') ) +queue_converter.add_task_type( + dotted_path='mayan.apps.documents.tasks.task_generate_document_version_page_image', + label=_('Generate document version page image') +) queue_documents.add_task_type( dotted_path='mayan.apps.documents.tasks.task_delete_document', @@ -66,6 +70,10 @@ queue_tools.add_task_type( label=_('Duplicated document scan') ) +queue_uploads.add_task_type( + dotted_path='mayan.apps.documents.tasks.task_document_pages_reset', + label=_('Reset document pages') +) queue_uploads.add_task_type( dotted_path='mayan.apps.documents.tasks.task_update_page_count', label=_('Update document page count') diff --git a/mayan/apps/documents/search.py b/mayan/apps/documents/search.py index de03a36dd0..e631099d3c 100644 --- a/mayan/apps/documents/search.py +++ b/mayan/apps/documents/search.py @@ -17,12 +17,20 @@ def transformation_format_uuid(term_string): return term_string -def get_queryset_page_search_queryset(): +def get_queryset_document_page_search_queryset(): # Ignore documents in trash can DocumentPage = apps.get_model( app_label='documents', model_name='DocumentPage' ) - return DocumentPage.objects.filter(document_version__document__in_trash=False) + return DocumentPage.objects.filter(document__in_trash=False) + + +def get_queryset_document_version_page_search_queryset(): + # Ignore documents in trash can + DocumentVersionPage = apps.get_model( + app_label='documents', model_name='DocumentVersionPage' + ) + return DocumentVersionPage.objects.filter(document_version__document__in_trash=False) document_search = SearchModel( @@ -30,7 +38,6 @@ document_search = SearchModel( model_name='Document', permission=permission_document_view, serializer_path='mayan.apps.documents.serializers.DocumentSerializer' ) - document_search.add_model_field( field='document_type__label', label=_('Document type') ) @@ -50,24 +57,49 @@ document_search.add_model_field( document_page_search = SearchModel( app_label='documents', list_mode=LIST_MODE_CHOICE_ITEM, model_name='DocumentPage', permission=permission_document_view, - queryset=get_queryset_page_search_queryset, + queryset=get_queryset_document_page_search_queryset, serializer_path='mayan.apps.documents.serializers.DocumentPageSerializer' ) +document_version_page_search = SearchModel( + app_label='documents', list_mode=LIST_MODE_CHOICE_ITEM, + model_name='DocumentVersionPage', permission=permission_document_view, + queryset=get_queryset_document_version_page_search_queryset, + serializer_path='mayan.apps.documents.serializers.DocumentVersionPageSerializer' +) + document_page_search.add_model_field( - field='document_version__document__document_type__label', + field='document__document_type__label', label=_('Document type') ) document_page_search.add_model_field( - field='document_version__document__versions__mimetype', + field='document__versions__mimetype', label=_('MIME type') ) document_page_search.add_model_field( + field='document__label', label=_('Label') +) +document_page_search.add_model_field( + field='document__description', label=_('Description') +) +document_page_search.add_model_field( + field='document__versions__checksum', label=_('Checksum') +) + +document_version_page_search.add_model_field( + field='document_version__document__document_type__label', + label=_('Document type') +) +document_version_page_search.add_model_field( + field='document_version__document__versions__mimetype', + label=_('MIME type') +) +document_version_page_search.add_model_field( field='document_version__document__label', label=_('Label') ) -document_page_search.add_model_field( +document_version_page_search.add_model_field( field='document_version__document__description', label=_('Description') ) -document_page_search.add_model_field( +document_version_page_search.add_model_field( field='document_version__checksum', label=_('Checksum') ) diff --git a/mayan/apps/documents/serializers.py b/mayan/apps/documents/serializers.py index 9632a546cb..5a6c172f7a 100644 --- a/mayan/apps/documents/serializers.py +++ b/mayan/apps/documents/serializers.py @@ -8,42 +8,40 @@ from rest_framework.reverse import reverse from mayan.apps.common.models import SharedUploadedFile from .models import ( - Document, DocumentVersion, DocumentPage, DocumentType, - DocumentTypeFilename, RecentDocument + Document, DocumentPage, DocumentType, DocumentTypeFilename, + DocumentVersion, DocumentVersionPage, RecentDocument ) from .settings import setting_language from .tasks import task_upload_new_version class DocumentPageSerializer(serializers.HyperlinkedModelSerializer): - document_version_url = serializers.SerializerMethodField() + document_url = serializers.SerializerMethodField() image_url = serializers.SerializerMethodField() url = serializers.SerializerMethodField() class Meta: - fields = ('document_version_url', 'image_url', 'page_number', 'url') + fields = ('document_url', 'image_url', 'page_number', 'url') model = DocumentPage - def get_document_version_url(self, instance): + def get_document_url(self, instance): return reverse( - viewname='rest_api:documentversion-detail', args=( - instance.document.pk, instance.document_version.pk, + viewname='rest_api:document-detail', args=( + instance.document.pk, ), request=self.context['request'], format=self.context['format'] ) def get_image_url(self, instance): return reverse( viewname='rest_api:documentpage-image', args=( - instance.document.pk, instance.document_version.pk, - instance.pk, + instance.document.pk, instance.pk, ), request=self.context['request'], format=self.context['format'] ) def get_url(self, instance): return reverse( viewname='rest_api:documentpage-detail', args=( - instance.document.pk, instance.document_version.pk, - instance.pk, + instance.document.pk, instance.pk, ), request=self.context['request'], format=self.context['format'] ) @@ -97,6 +95,39 @@ class WritableDocumentTypeSerializer(serializers.ModelSerializer): return obj.documents.count() +class DocumentVersionPageSerializer(serializers.HyperlinkedModelSerializer): + document_version_url = serializers.SerializerMethodField() + image_url = serializers.SerializerMethodField() + url = serializers.SerializerMethodField() + + class Meta: + fields = ('document_version_url', 'image_url', 'page_number', 'url') + model = DocumentVersionPage + + def get_document_version_url(self, instance): + return reverse( + viewname='rest_api:documentversion-detail', args=( + instance.document.pk, instance.document_version.pk, + ), request=self.context['request'], format=self.context['format'] + ) + + def get_image_url(self, instance): + return reverse( + viewname='rest_api:documentversionpage-image', args=( + instance.document.pk, instance.document_version.pk, + instance.pk, + ), request=self.context['request'], format=self.context['format'] + ) + + def get_url(self, instance): + return reverse( + viewname='rest_api:documentversionpage-detail', args=( + instance.document.pk, instance.document_version.pk, + instance.pk, + ), request=self.context['request'], format=self.context['format'] + ) + + class DocumentVersionSerializer(serializers.HyperlinkedModelSerializer): document_url = serializers.SerializerMethodField() download_url = serializers.SerializerMethodField() diff --git a/mayan/apps/documents/statistics.py b/mayan/apps/documents/statistics.py index 2f1059e7d1..94d91be994 100644 --- a/mayan/apps/documents/statistics.py +++ b/mayan/apps/documents/statistics.py @@ -41,7 +41,7 @@ def new_documents_per_month(): def new_document_pages_per_month(): DocumentPage = apps.get_model( - app_label='documents', model_name='DocumentPage' + app_label='documents', model_name='DocumentVersionPage' ) qss = qsstats.QuerySetStats( @@ -106,7 +106,7 @@ def new_document_pages_this_month(user=None): app_label='acls', model_name='AccessControlList' ) DocumentPage = apps.get_model( - app_label='documents', model_name='DocumentPage' + app_label='documents', model_name='DocumentVersionPage' ) queryset = DocumentPage.objects.all() @@ -195,7 +195,7 @@ def total_document_version_per_month(): def total_document_page_per_month(): DocumentPage = apps.get_model( - app_label='documents', model_name='DocumentPage' + app_label='documents', model_name='DocumentVersionPage' ) qss = qsstats.QuerySetStats( diff --git a/mayan/apps/documents/tasks.py b/mayan/apps/documents/tasks.py index 34956086bc..a2d4fe9fc7 100644 --- a/mayan/apps/documents/tasks.py +++ b/mayan/apps/documents/tasks.py @@ -9,8 +9,8 @@ from django.db import OperationalError from mayan.celery import app from .literals import ( - UPDATE_PAGE_COUNT_RETRY_DELAY, UPLOAD_NEW_DOCUMENT_RETRY_DELAY, - UPLOAD_NEW_VERSION_RETRY_DELAY + RETRY_DELAY_DOCUMENT_RESET_PAGES, UPDATE_PAGE_COUNT_RETRY_DELAY, + UPLOAD_NEW_DOCUMENT_RETRY_DELAY, ) logger = logging.getLogger(__name__) @@ -65,6 +65,25 @@ def task_delete_stubs(): logger.info(msg='Finshed') +@app.task(bind=True, default_retry_delay=RETRY_DELAY_DOCUMENT_RESET_PAGES, ignore_result=True) +def task_document_pages_reset(self, document_id): + Document = apps.get_model( + app_label='documents', model_name='Document' + ) + + document = Document.objects.get(pk=document_id) + + try: + document.pages_reset() + except OperationalError as exception: + logger.warning( + 'Operational error during attempt to reset pages for ' + 'document: %s; %s. Retrying.', document, + exception + ) + raise self.retry(exc=exception) + + @app.task() def task_generate_document_page_image(document_page_id, user_id=None, **kwargs): DocumentPage = apps.get_model( @@ -81,6 +100,22 @@ def task_generate_document_page_image(document_page_id, user_id=None, **kwargs): return document_page.generate_image(user=user, **kwargs) +@app.task() +def task_generate_document_version_page_image(document_version_page_id, user_id=None, **kwargs): + DocumentVersionPage = apps.get_model( + app_label='documents', model_name='DocumentVersionPage' + ) + User = get_user_model() + + if user_id: + user = User.objects.get(pk=user_id) + else: + user = None + + document_version_page = DocumentVersionPage.objects.get(pk=document_version_page_id) + return document_version_page.generate_image(user=user, **kwargs) + + @app.task(ignore_result=True) def task_scan_duplicates_all(): DuplicatedDocument = apps.get_model( @@ -177,7 +212,7 @@ def task_upload_new_document(self, document_type_id, shared_uploaded_file_id): @app.task(bind=True, default_retry_delay=UPLOAD_NEW_VERSION_RETRY_DELAY, ignore_result=True) -def task_upload_new_version(self, document_id, shared_uploaded_file_id, user_id, comment=None): +def task_upload_new_version(self, document_id, shared_uploaded_file_id, user_id, append_pages=False, comment=None): SharedUploadedFile = apps.get_model( app_label='common', model_name='SharedUploadedFile' ) @@ -212,7 +247,7 @@ def task_upload_new_version(self, document_id, shared_uploaded_file_id, user_id, document=document, comment=comment or '', file=file_object ) try: - document_version.save(_user=user) + document_version.save(append_pages=append_pages, _user=user) except Warning as warning: # New document version are blocked logger.info( diff --git a/mayan/apps/documents/tests/mixins.py b/mayan/apps/documents/tests/mixins.py index bb996c6cde..d11afaa921 100644 --- a/mayan/apps/documents/tests/mixins.py +++ b/mayan/apps/documents/tests/mixins.py @@ -69,6 +69,8 @@ class DocumentTestMixin(object): self.test_document = document self.test_documents.append(document) + self.test_document_version = document.latest_version + self.test_document_page = document.pages_all.first() class DocumentTypeViewTestMixin(object): @@ -148,6 +150,26 @@ class DocumentVersionTestMixin(object): ) +class DocumentVersionViewTestMixin(object): + def _request_document_version_list_view(self): + return self.get( + viewname='documents:document_version_list', + kwargs={'pk': self.test_document.pk} + ) + + def _request_document_version_revert_view(self, document_version): + return self.post( + viewname='documents:document_version_revert', + kwargs={'pk': document_version.pk} + ) + + def _request_test_document_version_page_count_update_view(self): + return self.post( + viewname='documents:document_version_page_count_update', + kwargs={'pk': self.test_document_version.pk} + ) + + class DocumentViewTestMixin(object): def _request_document_properties_view(self): return self.get( @@ -200,6 +222,12 @@ class DocumentViewTestMixin(object): data={'id_list': self.test_document.pk} ) + def _request_document_pages_reset_view(self): + return self.post( + viewname='documents:document_pages_reset', + kwargs={'pk': self.test_document.pk} + ) + def _request_document_version_download(self, data=None): data = data or {} return self.get( @@ -208,18 +236,6 @@ class DocumentViewTestMixin(object): }, data=data ) - def _request_document_update_page_count_view(self): - return self.post( - viewname='documents:document_update_page_count', - kwargs={'pk': self.test_document.pk} - ) - - def _request_document_multiple_update_page_count_view(self): - return self.post( - viewname='documents:document_multiple_update_page_count', - data={'id_list': self.test_document.pk} - ) - def _request_document_clear_transformations_view(self): return self.post( viewname='documents:document_clear_transformations', @@ -232,8 +248,11 @@ class DocumentViewTestMixin(object): data={'id_list': self.test_document.pk} ) - def _request_empty_trash_view(self): - return self.post(viewname='documents:trash_can_empty') + def _request_document_multiple_pages_reset_view(self): + return self.post( + viewname='documents:document_multiple_pages_reset', + data={'id_list': self.test_document.pk} + ) def _request_document_print_view(self): return self.get( @@ -243,3 +262,6 @@ class DocumentViewTestMixin(object): 'page_group': PAGE_RANGE_ALL } ) + + def _request_empty_trash_view(self): + return self.post(viewname='documents:trash_can_empty') diff --git a/mayan/apps/documents/tests/test_api.py b/mayan/apps/documents/tests/test_api.py index dea50b2618..60694de79e 100644 --- a/mayan/apps/documents/tests/test_api.py +++ b/mayan/apps/documents/tests/test_api.py @@ -530,8 +530,7 @@ class DocumentPageAPIViewTestMixin(object): page = self.test_document.pages.first() return self.get( viewname='rest_api:documentpage-image', kwargs={ - 'pk': page.document.pk, 'version_pk': page.document_version.pk, - 'page_pk': page.pk + 'pk': page.document.pk, 'page_pk': page.pk } ) @@ -552,6 +551,33 @@ class DocumentPageAPIViewTestCase( self.assertEqual(response.status_code, status.HTTP_200_OK) +class DocumentVersionPageAPIViewTestMixin(object): + def _request_document_version_page_image(self): + page = self.test_document_version.pages.first() + return self.get( + viewname='rest_api:documentversionpage-image', kwargs={ + 'pk': page.document.pk, 'version_pk': page.document_version.pk, + 'page_pk': page.pk + } + ) + + +class DocumentVersionPageAPIViewTestCase( + DocumentVersionPageAPIViewTestMixin, DocumentTestMixin, BaseAPITestCase +): + def test_document_version_page_api_image_view_no_access(self): + response = self._request_document_version_page_image() + self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) + + def test_document_version_page_api_image_view_with_access(self): + self.grant_access( + obj=self.test_document, permission=permission_document_view + ) + + response = self._request_document_version_page_image() + self.assertEqual(response.status_code, status.HTTP_200_OK) + + class TrashedDocumentAPIViewTestMixin(object): def _request_test_document_api_trash_view(self): return self.delete( @@ -575,13 +601,10 @@ class TrashedDocumentAPIViewTestMixin(object): ) def _request_test_trashed_document_api_image_view(self): - latest_version = self.test_document.latest_version - return self.get( viewname='rest_api:documentpage-image', kwargs={ - 'pk': latest_version.document.pk, - 'version_pk': latest_version.pk, - 'page_pk': latest_version.pages.first().pk + 'pk': self.test_document.pk, + 'page_pk': self.test_document.pages.first().pk } ) diff --git a/mayan/apps/documents/tests/test_document_page_views.py b/mayan/apps/documents/tests/test_document_page_views.py index 29ee28cec3..96b2702968 100644 --- a/mayan/apps/documents/tests/test_document_page_views.py +++ b/mayan/apps/documents/tests/test_document_page_views.py @@ -9,10 +9,10 @@ from ..permissions import ( from .base import GenericDocumentViewTestCase -class DocumentPageDisableViewTestCase(GenericDocumentViewTestCase): - def setUp(self): - super(DocumentPageDisableViewTestCase, self).setUp() - self.test_document_page = self.test_document.pages_all.first() +class DocumentPageDisableViewTestMixin(object): + def _disable_test_document_page(self): + self.test_document_page.enabled = False + self.test_document_page.save() def _request_test_document_page_disable_view(self): return self.post( @@ -21,6 +21,31 @@ class DocumentPageDisableViewTestCase(GenericDocumentViewTestCase): } ) + def _request_test_document_page_enable_view(self): + return self.post( + viewname='documents:document_page_enable', kwargs={ + 'pk': self.test_document_page.pk + } + ) + + def _request_test_document_page_multiple_disable_view(self): + return self.post( + viewname='documents:document_page_multiple_disable', data={ + 'id_list': self.test_document_page.pk + } + ) + + def _request_test_document_page_multiple_enable_view(self): + return self.post( + viewname='documents:document_page_multiple_enable', data={ + 'id_list': self.test_document_page.pk + } + ) + + +class DocumentPageDisableViewTestCase( + DocumentPageDisableViewTestMixin, GenericDocumentViewTestCase +): def test_document_page_disable_view_no_permission(self): test_document_page_count = self.test_document.pages.count() @@ -45,13 +70,6 @@ class DocumentPageDisableViewTestCase(GenericDocumentViewTestCase): test_document_page_count, self.test_document.pages.count() ) - def _request_test_document_page_multiple_disable_view(self): - return self.post( - viewname='documents:document_page_multiple_disable', data={ - 'id_list': self.test_document_page.pk - } - ) - def test_document_page_multiple_disable_view_no_permission(self): test_document_page_count = self.test_document.pages.count() @@ -76,17 +94,6 @@ class DocumentPageDisableViewTestCase(GenericDocumentViewTestCase): test_document_page_count, self.test_document.pages.count() ) - def _disable_test_document_page(self): - self.test_document_page.enabled = False - self.test_document_page.save() - - def _request_test_document_page_enable_view(self): - return self.post( - viewname='documents:document_page_enable', kwargs={ - 'pk': self.test_document_page.pk - } - ) - def test_document_page_enable_view_no_permission(self): self._disable_test_document_page() @@ -114,13 +121,6 @@ class DocumentPageDisableViewTestCase(GenericDocumentViewTestCase): test_document_page_count, self.test_document.pages.count() ) - def _request_test_document_page_multiple_enable_view(self): - return self.post( - viewname='documents:document_page_multiple_enable', data={ - 'id_list': self.test_document_page.pk - } - ) - def test_document_page_multiple_enable_view_no_permission(self): self._disable_test_document_page() test_document_page_count = self.test_document.pages.count() @@ -148,7 +148,7 @@ class DocumentPageDisableViewTestCase(GenericDocumentViewTestCase): ) -class DocumentPageViewTestCase(GenericDocumentViewTestCase): +class DocumentPageViewTestMixin(object): def _request_test_document_page_list_view(self): return self.get( viewname='documents:document_pages', kwargs={ @@ -156,6 +156,18 @@ class DocumentPageViewTestCase(GenericDocumentViewTestCase): } ) + def _request_test_document_page_view(self, document_page): + return self.get( + viewname='documents:document_page_view', kwargs={ + 'pk': document_page.pk, + } + ) + + +class DocumentPageViewTestCase( + DocumentPageViewTestMixin, GenericDocumentViewTestCase +): + def test_document_page_list_view_no_permission(self): response = self._request_test_document_page_list_view() self.assertEqual(response.status_code, 404) @@ -170,13 +182,6 @@ class DocumentPageViewTestCase(GenericDocumentViewTestCase): response=response, text=self.test_document.label, status_code=200 ) - def _request_test_document_page_view(self, document_page): - return self.get( - viewname='documents:document_page_view', kwargs={ - 'pk': document_page.pk, - } - ) - def test_document_page_view_no_permissions(self): response = self._request_test_document_page_view( document_page=self.test_document.pages.first() diff --git a/mayan/apps/documents/tests/test_document_version_views.py b/mayan/apps/documents/tests/test_document_version_views.py index 4f712a638b..d2ab419378 100644 --- a/mayan/apps/documents/tests/test_document_version_views.py +++ b/mayan/apps/documents/tests/test_document_version_views.py @@ -1,21 +1,19 @@ from __future__ import unicode_literals from ..permissions import ( - permission_document_version_revert, permission_document_version_view, + permission_document_tools, permission_document_version_revert, + permission_document_version_view, ) from .base import GenericDocumentViewTestCase from .literals import TEST_VERSION_COMMENT -from .mixins import DocumentVersionTestMixin +from .mixins import DocumentVersionTestMixin, DocumentVersionViewTestMixin -class DocumentVersionTestCase(DocumentVersionTestMixin, GenericDocumentViewTestCase): - def _request_document_version_list_view(self): - return self.get( - viewname='documents:document_version_list', - kwargs={'pk': self.test_document.pk} - ) - +class DocumentVersionTestCase( + DocumentVersionViewTestMixin, DocumentVersionTestMixin, + GenericDocumentViewTestCase +): def test_document_version_list_no_permission(self): self._upload_new_version() @@ -33,12 +31,6 @@ class DocumentVersionTestCase(DocumentVersionTestMixin, GenericDocumentViewTestC response=response, text=TEST_VERSION_COMMENT, status_code=200 ) - def _request_document_version_revert_view(self, document_version): - return self.post( - viewname='documents:document_version_revert', - kwargs={'pk': document_version.pk} - ) - def test_document_version_revert_no_permission(self): first_version = self.test_document.latest_version self._upload_new_version() @@ -64,3 +56,25 @@ class DocumentVersionTestCase(DocumentVersionTestMixin, GenericDocumentViewTestC self.assertEqual(response.status_code, 302) self.assertEqual(self.test_document.versions.count(), 1) + + def test_document_version_page_count_update_view_no_permission(self): + self.test_document_version.pages.all().delete() + + response = self._request_test_document_version_page_count_update_view() + self.assertEqual(response.status_code, 404) + + self.assertEqual(self.test_document_version.pages.count(), 0) + + def test_document_version_page_count_update_view_with_access(self): + page_count = self.test_document_version.pages.count() + + self.test_document_version.pages.all().delete() + + self.grant_access( + obj=self.test_document, permission=permission_document_tools + ) + + response = self._request_test_document_version_page_count_update_view() + self.assertEqual(response.status_code, 302) + + self.assertEqual(self.test_document_version.pages.count(), page_count) diff --git a/mayan/apps/documents/tests/test_document_views.py b/mayan/apps/documents/tests/test_document_views.py index 381a71211d..45c96f6786 100644 --- a/mayan/apps/documents/tests/test_document_views.py +++ b/mayan/apps/documents/tests/test_document_views.py @@ -292,46 +292,44 @@ class DocumentsViewsTestCase( ) ) - def test_document_update_page_count_view_no_permission(self): + def test_document_pages_reset_view_no_permission(self): self.test_document.pages.all().delete() - self.assertEqual(self.test_document.pages.count(), 0) - response = self._request_document_update_page_count_view() + response = self._request_document_pages_reset_view() self.assertEqual(response.status_code, 404) self.assertEqual(self.test_document.pages.count(), 0) - def test_document_update_page_count_view_with_permission(self): - # TODO: Revise permission association - + def test_document_pages_reset_view_with_access(self): page_count = self.test_document.pages.count() self.test_document.pages.all().delete() - self.assertEqual(self.test_document.pages.count(), 0) - self.grant_permission(permission=permission_document_tools) + self.grant_access( + obj=self.test_document, permission=permission_document_tools + ) - response = self._request_document_update_page_count_view() + response = self._request_document_pages_reset_view() self.assertEqual(response.status_code, 302) self.assertEqual(self.test_document.pages.count(), page_count) - def test_document_multiple_update_page_count_view_no_permission(self): + def test_document_multiple_pages_reset_view_no_permission(self): self.test_document.pages.all().delete() - self.assertEqual(self.test_document.pages.count(), 0) - response = self._request_document_multiple_update_page_count_view() + response = self._request_document_multiple_pages_reset_view() self.assertEqual(response.status_code, 404) self.assertEqual(self.test_document.pages.count(), 0) - def test_document_multiple_update_page_count_view_with_permission(self): + def test_document_multiple_pages_reset_view_with_access(self): page_count = self.test_document.pages.count() self.test_document.pages.all().delete() - self.assertEqual(self.test_document.pages.count(), 0) - self.grant_permission(permission=permission_document_tools) + self.grant_access( + obj=self.test_document, permission=permission_document_tools + ) - response = self._request_document_multiple_update_page_count_view() + response = self._request_document_multiple_pages_reset_view() self.assertEqual(response.status_code, 302) self.assertEqual(self.test_document.pages.count(), page_count) diff --git a/mayan/apps/documents/tests/test_search.py b/mayan/apps/documents/tests/test_search.py index 531f6e8d2f..1db040b174 100644 --- a/mayan/apps/documents/tests/test_search.py +++ b/mayan/apps/documents/tests/test_search.py @@ -1,22 +1,30 @@ from __future__ import unicode_literals from mayan.apps.common.tests.base import BaseTestCase -from mayan.apps.documents.permissions import permission_document_view -from mayan.apps.documents.search import document_search, document_page_search -from mayan.apps.documents.tests.mixins import DocumentTestMixin + +from ..permissions import permission_document_view +from ..search import document_search, document_page_search + +from .mixins import DocumentTestMixin -class DocumentSearchTestCase(DocumentTestMixin, BaseTestCase): +class DocumentSearchTestMixin(object): def _perform_document_page_search(self): return document_page_search.search( - query_string={'q': self.test_document.label}, user=self._test_case_user + query_string={'q': self.test_document.label}, + user=self._test_case_user ) def _perform_document_search(self): return document_search.search( - query_string={'q': self.test_document.label}, user=self._test_case_user + query_string={'q': self.test_document.label}, + user=self._test_case_user ) + +class DocumentSearchTestCase( + DocumentSearchTestMixin, DocumentTestMixin, BaseTestCase +): def test_document_page_search_no_access(self): queryset = self._perform_document_page_search() self.assertFalse(self.test_document.pages.first() in queryset) diff --git a/mayan/apps/documents/tests/test_trashed_document_views.py b/mayan/apps/documents/tests/test_trashed_document_views.py index 8be806f36e..775bef0f38 100644 --- a/mayan/apps/documents/tests/test_trashed_document_views.py +++ b/mayan/apps/documents/tests/test_trashed_document_views.py @@ -9,7 +9,7 @@ from ..permissions import ( from .base import GenericDocumentViewTestCase -class TrashedDocumentTestCase(GenericDocumentViewTestCase): +class TrashedDocumentTestMixin(object): def _request_document_restore_get_view(self): return self.get( viewname='documents:document_restore', kwargs={ @@ -17,6 +17,48 @@ class TrashedDocumentTestCase(GenericDocumentViewTestCase): } ) + def _request_document_restore_post_view(self): + return self.post( + viewname='documents:document_restore', kwargs={ + 'pk': self.test_document.pk + } + ) + + def _request_document_trash_get_view(self): + return self.get( + viewname='documents:document_trash', kwargs={ + 'pk': self.test_document.pk + } + ) + + def _request_document_trash_post_view(self): + return self.post( + viewname='documents:document_trash', kwargs={ + 'pk': self.test_document.pk + } + ) + + def _request_trashed_document_delete_get_view(self): + return self.get( + viewname='documents:document_delete', kwargs={ + 'pk': self.test_document.pk + } + ) + + def _request_trashed_document_delete_post_view(self): + return self.post( + viewname='documents:document_delete', kwargs={ + 'pk': self.test_document.pk + } + ) + + def _request_trashed_document_list_view(self): + return self.get(viewname='documents:document_list_deleted') + + +class TrashedDocumentTestCase( + TrashedDocumentTestMixin, GenericDocumentViewTestCase +): def test_document_restore_get_view_no_permission(self): self.test_document.delete() self.assertEqual(Document.objects.count(), 0) @@ -43,13 +85,6 @@ class TrashedDocumentTestCase(GenericDocumentViewTestCase): self.assertEqual(Document.objects.count(), document_count) - def _request_document_restore_post_view(self): - return self.post( - viewname='documents:document_restore', kwargs={ - 'pk': self.test_document.pk - } - ) - def test_document_restore_post_view_no_permission(self): self.test_document.delete() self.assertEqual(Document.objects.count(), 0) @@ -74,13 +109,6 @@ class TrashedDocumentTestCase(GenericDocumentViewTestCase): self.assertEqual(DeletedDocument.objects.count(), 0) self.assertEqual(Document.objects.count(), 1) - def _request_document_trash_get_view(self): - return self.get( - viewname='documents:document_trash', kwargs={ - 'pk': self.test_document.pk - } - ) - def test_document_trash_get_view_no_permissions(self): document_count = Document.objects.count() @@ -101,13 +129,6 @@ class TrashedDocumentTestCase(GenericDocumentViewTestCase): self.assertEqual(Document.objects.count(), document_count) - def _request_document_trash_post_view(self): - return self.post( - viewname='documents:document_trash', kwargs={ - 'pk': self.test_document.pk - } - ) - def test_document_trash_post_view_no_permissions(self): response = self._request_document_trash_post_view() self.assertEqual(response.status_code, 404) @@ -126,13 +147,6 @@ class TrashedDocumentTestCase(GenericDocumentViewTestCase): self.assertEqual(DeletedDocument.objects.count(), 1) self.assertEqual(Document.objects.count(), 0) - def _request_document_delete_get_view(self): - return self.get( - viewname='documents:document_delete', kwargs={ - 'pk': self.test_document.pk - } - ) - def test_document_delete_get_view_no_permissions(self): self.test_document.delete() self.assertEqual(Document.objects.count(), 0) @@ -140,7 +154,7 @@ class TrashedDocumentTestCase(GenericDocumentViewTestCase): trashed_document_count = DeletedDocument.objects.count() - response = self._request_document_delete_get_view() + response = self._request_trashed_document_delete_get_view() self.assertEqual(response.status_code, 404) self.assertEqual( @@ -158,26 +172,19 @@ class TrashedDocumentTestCase(GenericDocumentViewTestCase): trashed_document_count = DeletedDocument.objects.count() - response = self._request_document_delete_get_view() + response = self._request_trashed_document_delete_get_view() self.assertEqual(response.status_code, 200) self.assertEqual( DeletedDocument.objects.count(), trashed_document_count ) - def _request_document_delete_post_view(self): - return self.post( - viewname='documents:document_delete', kwargs={ - 'pk': self.test_document.pk - } - ) - def test_document_delete_post_view_no_permissions(self): self.test_document.delete() self.assertEqual(Document.objects.count(), 0) self.assertEqual(DeletedDocument.objects.count(), 1) - response = self._request_document_delete_post_view() + response = self._request_trashed_document_delete_post_view() self.assertEqual(response.status_code, 404) self.assertEqual(Document.objects.count(), 0) @@ -192,19 +199,16 @@ class TrashedDocumentTestCase(GenericDocumentViewTestCase): obj=self.test_document, permission=permission_document_delete ) - response = self._request_document_delete_post_view() + response = self._request_trashed_document_delete_post_view() self.assertEqual(response.status_code, 302) self.assertEqual(DeletedDocument.objects.count(), 0) self.assertEqual(Document.objects.count(), 0) - def _request_document_list_deleted_view(self): - return self.get(viewname='documents:document_list_deleted') - def test_deleted_document_list_view_no_permissions(self): self.test_document.delete() - response = self._request_document_list_deleted_view() + response = self._request_trashed_document_list_view() self.assertNotContains( response=response, text=self.test_document.label, status_code=200 ) @@ -216,7 +220,7 @@ class TrashedDocumentTestCase(GenericDocumentViewTestCase): obj=self.test_document, permission=permission_document_view ) - response = self._request_document_list_deleted_view() + response = self._request_trashed_document_list_view() self.assertContains( response=response, text=self.test_document.label, status_code=200 ) diff --git a/mayan/apps/documents/urls.py b/mayan/apps/documents/urls.py index 84aec25094..a0e25cc670 100644 --- a/mayan/apps/documents/urls.py +++ b/mayan/apps/documents/urls.py @@ -4,21 +4,24 @@ from django.conf.urls import url from .api_views import ( APITrashedDocumentListView, APIDeletedDocumentRestoreView, - APIDeletedDocumentView, APIDocumentDownloadView, APIDocumentView, - APIDocumentListView, APIDocumentVersionDownloadView, + APIDeletedDocumentView, APIDocumentDownloadView, APIDocumentPageListView, + APIDocumentView, APIDocumentListView, APIDocumentVersionDownloadView, APIDocumentPageImageView, APIDocumentPageView, APIDocumentTypeDocumentListView, APIDocumentTypeListView, APIDocumentTypeView, APIDocumentVersionsListView, APIDocumentVersionPageListView, APIDocumentVersionView, - APIRecentDocumentListView + APIRecentDocumentListView, + APIDocumentVersionPageView, + APIDocumentVersionPageImageView ) from .views.document_views import ( DocumentDocumentTypeEditView, DocumentDownloadFormView, DocumentDownloadView, DocumentDuplicatesListView, DocumentEditView, DocumentListView, DocumentPreviewView, DocumentPrint, - DocumentTransformationsClearView, DocumentTransformationsCloneView, - DocumentUpdatePageCountView, DocumentView, DuplicatedDocumentListView, - RecentAccessDocumentListView, RecentAddedDocumentListView + DocumentPagesResetView, DocumentTransformationsClearView, + DocumentTransformationsCloneView, DocumentView, + DuplicatedDocumentListView, RecentAccessDocumentListView, + RecentAddedDocumentListView ) from .views.document_page_views import ( DocumentPageDisable, DocumentPageEnable, DocumentPageListView, @@ -30,7 +33,8 @@ from .views.document_page_views import ( ) from .views.document_version_views import ( DocumentVersionDownloadFormView, DocumentVersionDownloadView, - DocumentVersionListView, DocumentVersionRevertView, DocumentVersionView, + DocumentVersionListView, DocumentVersionRevertView, + DocumentVersionUpdatePageCountView, DocumentVersionView, ) from .views.document_type_views import ( DocumentTypeCreateView, DocumentTypeDeleteView, @@ -172,14 +176,14 @@ urlpatterns_documents = [ name='document_print' ), url( - regex=r'^documents/(?P\d+)/reset_page_count/$', - view=DocumentUpdatePageCountView.as_view(), - name='document_update_page_count' + regex=r'^documents/(?P\d+)/pages/reset/$', + view=DocumentPagesResetView.as_view(), + name='document_pages_reset' ), url( - regex=r'^documents/multiple/reset_page_count/$', - view=DocumentUpdatePageCountView.as_view(), - name='document_multiple_update_page_count' + regex=r'^documents/multiple/pages/reset/$', + view=DocumentPagesResetView.as_view(), + name='document_multiple_pages_reset' ), url( regex=r'^documents/(?P\d+)/download/form/$', @@ -305,6 +309,16 @@ urlpatterns_document_versions = [ view=DocumentVersionDownloadView.as_view(), name='document_version_download' ), + url( + regex=r'^documents/versions/(?P\d+)/pages/update/$', + view=DocumentVersionUpdatePageCountView.as_view(), + name='document_version_page_count_update' + ), + url( + regex=r'^documents/versions/multiple/pages/update/$', + view=DocumentVersionUpdatePageCountView.as_view(), + name='document_version_multiple_page_count_update' + ), url( regex=r'^documents/versions/(?P\d+)/revert/$', view=DocumentVersionRevertView.as_view(), @@ -405,6 +419,11 @@ api_urls = [ view=APIDocumentVersionPageListView.as_view(), name='documentversion-page-list' ), + url( + regex=r'^documents/(?P[0-9]+)/pages/$', + view=APIDocumentPageListView.as_view(), + name='document-page-list' + ), url( regex=r'^documents/(?P[0-9]+)/versions/(?P[0-9]+)/download/$', view=APIDocumentVersionDownloadView.as_view(), @@ -416,12 +435,20 @@ api_urls = [ ), url( regex=r'^documents/(?P[0-9]+)/versions/(?P[0-9]+)/pages/(?P[0-9]+)$', + view=APIDocumentVersionPageView.as_view(), name='documentversionpage-detail' + ), + url( + regex=r'^documents/(?P[0-9]+)/pages/(?P[0-9]+)$', view=APIDocumentPageView.as_view(), name='documentpage-detail' ), url( - regex=r'^documents/(?P[0-9]+)/versions/(?P[0-9]+)/pages/(?P[0-9]+)/image/$', + regex=r'^documents/(?P[0-9]+)/pages/(?P[0-9]+)/image/$', view=APIDocumentPageImageView.as_view(), name='documentpage-image' ), + url( + regex=r'^documents/(?P[0-9]+)/versions/(?P[0-9]+)/pages/(?P[0-9]+)/image/$', + view=APIDocumentVersionPageImageView.as_view(), name='documentversionpage-image' + ), url( regex=r'^trashed_documents/$', view=APITrashedDocumentListView.as_view(), name='trasheddocument-list' diff --git a/mayan/apps/documents/views/document_page_views.py b/mayan/apps/documents/views/document_page_views.py index f05767d627..46fcacd9e7 100644 --- a/mayan/apps/documents/views/document_page_views.py +++ b/mayan/apps/documents/views/document_page_views.py @@ -20,8 +20,8 @@ from mayan.apps.converter.literals import DEFAULT_ROTATION, DEFAULT_ZOOM_LEVEL from ..forms import DocumentPageForm from ..icons import icon_document_pages -from ..links import link_document_update_page_count -from ..models import Document, DocumentPage +from ..links import link_document_pages_reset +from ..models import Document, DocumentPage, DocumentVersionPage from ..permissions import permission_document_edit, permission_document_view from ..settings import ( setting_rotation_step, setting_zoom_percent_step, setting_zoom_max_level, @@ -50,13 +50,13 @@ class DocumentPageListView(ExternalObjectMixin, SingleObjectListView): 'hide_object': True, 'list_as_items': True, 'no_results_icon': icon_document_pages, - 'no_results_main_link': link_document_update_page_count.resolve( + 'no_results_main_link': link_document_pages_reset.resolve( request=self.request, resolved_object=self.external_object ), 'no_results_text': _( 'This could mean that the document is of a format that is ' - 'not supported, that it is corrupted or that the upload ' - 'process was interrupted. Use the document page recalculation ' + 'not supported, that it is corrupted, or that the upload ' + 'process was interrupted. Use the document page reset ' 'action to attempt to introspect the page count again.' ), 'no_results_title': _('No document pages available'), diff --git a/mayan/apps/documents/views/document_version_views.py b/mayan/apps/documents/views/document_version_views.py index 4c53bbff36..8a2c3cd1ed 100644 --- a/mayan/apps/documents/views/document_version_views.py +++ b/mayan/apps/documents/views/document_version_views.py @@ -3,10 +3,11 @@ from __future__ import absolute_import, unicode_literals import logging from django.contrib import messages -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import ugettext_lazy as _, ungettext from mayan.apps.common.generics import ( - ConfirmView, SingleObjectDetailView, SingleObjectListView + ConfirmView, MultipleObjectConfirmActionView, SingleObjectDetailView, + SingleObjectListView ) from mayan.apps.common.mixins import ExternalObjectMixin @@ -14,9 +15,10 @@ from ..events import event_document_view from ..forms import DocumentVersionDownloadForm, DocumentVersionPreviewForm from ..models import Document, DocumentVersion from ..permissions import ( - permission_document_download, permission_document_version_revert, - permission_document_version_view + permission_document_download, permission_document_tools, + permission_document_version_revert, permission_document_version_view ) +from ..tasks import task_update_page_count from .document_views import DocumentDownloadFormView, DocumentDownloadView @@ -142,6 +144,45 @@ class DocumentVersionRevertView(ExternalObjectMixin, ConfirmView): ) +class DocumentVersionUpdatePageCountView(MultipleObjectConfirmActionView): + model = DocumentVersion + object_permission = permission_document_tools + success_message = _( + '%(count)d document version queued for page count recalculation' + ) + success_message_plural = _( + '%(count)d documents version queued for page count recalculation' + ) + + def get_extra_context(self): + queryset = self.object_list + + result = { + 'title': ungettext( + singular='Recalculate the page count of the selected document version?', + plural='Recalculate the page count of the selected document versions?', + number=queryset.count() + ) + } + + if queryset.count() == 1: + result.update( + { + 'object': queryset.first(), + 'title': _( + 'Recalculate the page count of the document version: %s?' + ) % queryset.first() + } + ) + + return result + + def object_action(self, form, instance): + task_update_page_count.apply_async( + kwargs={'version_id': instance.pk} + ) + + class DocumentVersionView(SingleObjectDetailView): form_class = DocumentVersionPreviewForm model = DocumentVersion diff --git a/mayan/apps/documents/views/document_views.py b/mayan/apps/documents/views/document_views.py index 566077c99c..dab008a75c 100644 --- a/mayan/apps/documents/views/document_views.py +++ b/mayan/apps/documents/views/document_views.py @@ -44,14 +44,14 @@ from ..permissions import ( from ..settings import ( setting_print_width, setting_print_height, setting_recent_added_count ) -from ..tasks import task_update_page_count +from ..tasks import task_document_pages_reset from ..utils import parse_range __all__ = ( 'DocumentListView', 'DocumentDocumentTypeEditView', 'DocumentDuplicatesListView', 'DocumentEditView', 'DocumentPreviewView', 'DocumentView', 'DocumentDownloadFormView', 'DocumentDownloadView', - 'DocumentUpdatePageCountView', 'DocumentTransformationsClearView', + 'DocumentPagesResetView', 'DocumentTransformationsClearView', 'DocumentTransformationsCloneView', 'DocumentPrint', 'DuplicatedDocumentListView', 'RecentAccessDocumentListView', 'RecentAddedDocumentListView' @@ -418,6 +418,52 @@ class DocumentPreviewView(SingleObjectDetailView): } +class DocumentPagesResetView(MultipleObjectConfirmActionView): + model = Document + object_permission = permission_document_tools + success_message = _('%(count)d document queued for pages reset') + success_message_plural = _('%(count)d documents queued for pages reset') + + def get_extra_context(self): + queryset = self.object_list + + result = { + 'title': ungettext( + singular='Reset the pages of the selected document?', + plural='Reset the pages of the selected documents?', + number=queryset.count() + ) + } + + if queryset.count() == 1: + result.update( + { + 'object': queryset.first(), + 'title': _( + 'Reset the pages of the document: %s?' + ) % queryset.first() + } + ) + + return result + + def object_action(self, form, instance): + latest_version = instance.latest_version + if latest_version: + task_document_pages_reset.apply_async( + kwargs={'document_id': instance.pk} + ) + else: + messages.error( + self.request, _( + 'Document "%(document)s" is empty. Upload at least one ' + 'document version before attempting to reset the pages. ' + ) % { + 'document': instance, + } + ) + + class DocumentView(SingleObjectDetailView): form_class = DocumentPropertiesForm model = Document @@ -436,57 +482,6 @@ class DocumentView(SingleObjectDetailView): } -class DocumentUpdatePageCountView(MultipleObjectConfirmActionView): - model = Document - object_permission = permission_document_tools - success_message = _( - '%(count)d document queued for page count recalculation' - ) - success_message_plural = _( - '%(count)d documents queued for page count recalculation' - ) - - def get_extra_context(self): - queryset = self.object_list - - result = { - 'title': ungettext( - singular='Recalculate the page count of the selected document?', - plural='Recalculate the page count of the selected documents?', - number=queryset.count() - ) - } - - if queryset.count() == 1: - result.update( - { - 'object': queryset.first(), - 'title': _( - 'Recalculate the page count of the document: %s?' - ) % queryset.first() - } - ) - - return result - - def object_action(self, form, instance): - latest_version = instance.latest_version - if latest_version: - task_update_page_count.apply_async( - kwargs={'version_id': latest_version.pk} - ) - else: - messages.error( - self.request, _( - 'Document "%(document)s" is empty. Upload at least one ' - 'document version before attempting to detect the ' - 'page count.' - ) % { - 'document': instance, - } - ) - - class DocumentTransformationsClearView(MultipleObjectConfirmActionView): model = Document object_permission = permission_transformation_delete diff --git a/mayan/apps/dynamic_search/classes.py b/mayan/apps/dynamic_search/classes.py index 2476f07a15..1630bcc7fb 100644 --- a/mayan/apps/dynamic_search/classes.py +++ b/mayan/apps/dynamic_search/classes.py @@ -184,7 +184,14 @@ class SearchModel(object): query_string=query_string, global_and_search=global_and_search ) - queryset = self.get_queryset().filter(search_query.query).distinct() + try: + queryset = self.get_queryset().filter(search_query.query).distinct() + except Exception: + logger.error( + 'Error filtering model %s with queryset: %s', self.model, + search_query.query + ) + raise if self.permission: queryset = AccessControlList.objects.restrict_queryset( diff --git a/mayan/apps/file_caching/migrations/0003_auto_20191008_1510.py b/mayan/apps/file_caching/migrations/0003_auto_20191008_1510.py new file mode 100644 index 0000000000..14382e0479 --- /dev/null +++ b/mayan/apps/file_caching/migrations/0003_auto_20191008_1510.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.22 on 2019-10-08 15:10 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('file_caching', '0002_auto_20190729_0236'), + ] + + operations = [ + migrations.AlterField( + model_name='cache', + name='name', + field=models.CharField(db_index=True, help_text='Internal name of the cache.', max_length=128, unique=True, verbose_name='Name'), + ), + ] diff --git a/mayan/apps/file_metadata/apps.py b/mayan/apps/file_metadata/apps.py index a9830d467d..675ebe3365 100644 --- a/mayan/apps/file_metadata/apps.py +++ b/mayan/apps/file_metadata/apps.py @@ -12,7 +12,9 @@ from mayan.apps.common.menus import ( menu_tools ) from mayan.apps.document_indexing.handlers import handler_index_document -from mayan.apps.documents.search import document_page_search, document_search +from mayan.apps.documents.search import ( + document_page_search, document_search, document_version_page_search +) from mayan.apps.documents.signals import post_version_upload from mayan.apps.events.classes import ModelEventType from mayan.apps.navigation.classes import SourceColumn @@ -156,11 +158,19 @@ class FileMetadataApp(MayanAppConfig): label=_('File metadata value') ) - document_page_search.add_model_field( + #document_page_search.add_model_field( + # field='document__document_version__file_metadata_drivers__entries__key', + # label=_('File metadata key') + #) + #document_page_search.add_model_field( + # field='document__document_version__file_metadata_drivers__entries__value', + # label=_('File metadata value') + #) + document_version_page_search.add_model_field( field='document_version__file_metadata_drivers__entries__key', label=_('File metadata key') ) - document_page_search.add_model_field( + document_version_page_search.add_model_field( field='document_version__file_metadata_drivers__entries__value', label=_('File metadata value') ) diff --git a/mayan/apps/metadata/apps.py b/mayan/apps/metadata/apps.py index 44fd6bb546..b0224e9ceb 100644 --- a/mayan/apps/metadata/apps.py +++ b/mayan/apps/metadata/apps.py @@ -16,7 +16,9 @@ from mayan.apps.common.menus import ( menu_facet, menu_list_facet, menu_multi_item, menu_object, menu_secondary, menu_setup ) -from mayan.apps.documents.search import document_page_search, document_search +from mayan.apps.documents.search import ( + document_page_search, document_search, document_version_page_search +) from mayan.apps.documents.signals import post_document_type_change from mayan.apps.events.classes import ModelEventType from mayan.apps.events.links import ( @@ -76,7 +78,7 @@ class MetadataApp(MayanAppConfig): app_label='documents', model_name='Document' ) DocumentPageResult = apps.get_model( - app_label='documents', model_name='DocumentPageResult' + app_label='documents', model_name='DocumentVersionPageResult' ) DocumentType = apps.get_model( @@ -188,10 +190,18 @@ class MetadataApp(MayanAppConfig): ) document_page_search.add_model_field( - field='document_version__document__metadata__metadata_type__name', + field='document__metadata__metadata_type__name', label=_('Metadata type') ) document_page_search.add_model_field( + field='document__metadata__value', + label=_('Metadata value') + ) + document_version_page_search.add_model_field( + field='document_version__document__metadata__metadata_type__name', + label=_('Metadata type') + ) + document_version_page_search.add_model_field( field='document_version__document__metadata__value', label=_('Metadata value') ) diff --git a/mayan/apps/ocr/admin.py b/mayan/apps/ocr/admin.py index 865481d602..9a927c5260 100644 --- a/mayan/apps/ocr/admin.py +++ b/mayan/apps/ocr/admin.py @@ -3,13 +3,15 @@ from __future__ import unicode_literals from django.contrib import admin from .models import ( - DocumentPageOCRContent, DocumentTypeSettings, DocumentVersionOCRError + DocumentTypeSettings, DocumentVersionPageOCRContent, + DocumentVersionOCRError ) -@admin.register(DocumentPageOCRContent) -class DocumentPageOCRContentAdmin(admin.ModelAdmin): - list_display = ('document_page',) +@admin.register(DocumentVersionPageOCRContent) +class DocumentVersionPageOCRContentAdmin(admin.ModelAdmin): + pass + #list_display = ('document_page',) @admin.register(DocumentTypeSettings) diff --git a/mayan/apps/ocr/api_views.py b/mayan/apps/ocr/api_views.py index 0dea74beb3..20a4fe833e 100644 --- a/mayan/apps/ocr/api_views.py +++ b/mayan/apps/ocr/api_views.py @@ -8,7 +8,7 @@ from rest_framework.response import Response from mayan.apps.documents.models import Document, DocumentVersion from mayan.apps.rest_api.permissions import MayanPermission -from .models import DocumentPageOCRContent +from .models import DocumentVersionPageOCRContent from .permissions import permission_ocr_content_view, permission_ocr_document from .serializers import DocumentPageOCRContentSerializer @@ -90,8 +90,8 @@ class APIDocumentPageOCRContentView(generics.RetrieveAPIView): try: ocr_content = instance.ocr_content - except DocumentPageOCRContent.DoesNotExist: - ocr_content = DocumentPageOCRContent.objects.none() + except DocumentVersionPageOCRContent.DoesNotExist: + ocr_content = DocumentVersionPageOCRContent.objects.none() serializer = self.get_serializer(ocr_content) return Response(serializer.data) diff --git a/mayan/apps/ocr/apps.py b/mayan/apps/ocr/apps.py index 33b4f1893b..772ec287fa 100644 --- a/mayan/apps/ocr/apps.py +++ b/mayan/apps/ocr/apps.py @@ -12,7 +12,9 @@ from mayan.apps.common.classes import ModelField from mayan.apps.common.menus import ( menu_facet, menu_list_facet, menu_multi_item, menu_secondary, menu_tools ) -from mayan.apps.documents.search import document_search, document_page_search +from mayan.apps.documents.search import ( + document_search, document_page_search, document_version_page_search +) from mayan.apps.documents.signals import post_version_upload from mayan.apps.events.classes import ModelEventType from mayan.apps.navigation.classes import SourceColumn @@ -32,17 +34,19 @@ from .links import ( link_document_ocr_content_delete_multiple, link_document_ocr_download, link_document_ocr_errors_list, link_document_submit, link_document_submit_multiple, link_document_type_ocr_settings, - link_document_type_submit, link_entry_list + link_document_type_submit, link_document_version_page_ocr_content, + link_entry_list ) from .methods import ( - method_document_ocr_submit, method_document_version_ocr_submit + method_document_ocr_submit, method_document_page_get_ocr_content, + method_document_version_ocr_submit ) from .permissions import ( permission_document_type_ocr_setup, permission_ocr_document, permission_ocr_content_view ) from .signals import post_document_version_ocr -from .utils import get_document_ocr_content +from .utils import get_document_ocr_content, get_document_version_ocr_content logger = logging.getLogger(__name__) @@ -73,6 +77,9 @@ class OCRApp(MayanAppConfig): DocumentVersion = apps.get_model( app_label='documents', model_name='DocumentVersion' ) + DocumentVersionPage = apps.get_model( + app_label='documents', model_name='DocumentVersionPage' + ) DocumentVersionOCRError = self.get_model( model_name='DocumentVersionOCRError' @@ -81,8 +88,11 @@ class OCRApp(MayanAppConfig): Document.add_to_class( name='submit_for_ocr', value=method_document_ocr_submit ) + DocumentPage.add_to_class( + name='get_ocr_content', value=method_document_page_get_ocr_content + ) DocumentVersion.add_to_class( - name='ocr_content', value=get_document_ocr_content + name='ocr_content', value=get_document_version_ocr_content ) DocumentVersion.add_to_class( name='submit_for_ocr', value=method_document_version_ocr_submit @@ -97,7 +107,7 @@ class OCRApp(MayanAppConfig): ) ModelField( - model=Document, name='versions__version_pages__ocr_content__content' + model=Document, name='versions__pages__ocr_content__content' ) ModelPermission.register( @@ -128,12 +138,14 @@ class OCRApp(MayanAppConfig): ) document_search.add_model_field( - field='versions__version_pages__ocr_content__content', label=_('OCR') + field='versions__pages__ocr_content__content', label=_('OCR') ) - - document_page_search.add_model_field( + document_version_page_search.add_model_field( field='ocr_content__content', label=_('OCR') ) + #document_page_search.add_model_field( + # field='ocr_content__content', label=_('OCR') + #) menu_facet.bind_links( links=(link_document_ocr_content,), sources=(Document,) @@ -141,6 +153,10 @@ class OCRApp(MayanAppConfig): menu_list_facet.bind_links( links=(link_document_page_ocr_content,), sources=(DocumentPage,) ) + menu_list_facet.bind_links( + links=(link_document_version_page_ocr_content,), + sources=(DocumentVersionPage,) + ) menu_list_facet.bind_links( links=(link_document_type_ocr_settings,), sources=(DocumentType,) ) diff --git a/mayan/apps/ocr/forms.py b/mayan/apps/ocr/forms.py index 6951861ecf..5379bce4ae 100644 --- a/mayan/apps/ocr/forms.py +++ b/mayan/apps/ocr/forms.py @@ -8,7 +8,7 @@ from django.utils.translation import ugettext_lazy as _, ugettext from mayan.apps.common.widgets import TextAreaDiv -from .models import DocumentPageOCRContent +from .models import DocumentVersionPageOCRContent class DocumentPageOCRContentForm(forms.Form): @@ -28,15 +28,26 @@ class DocumentPageOCRContentForm(forms.Form): content = '' self.fields['contents'].initial = '' - try: - page_content = page.ocr_content.content - except DocumentPageOCRContent.DoesNotExist: - pass - else: - content = conditional_escape(force_text(page_content)) + content = conditional_escape( + force_text(self.get_instance_ocr_content(instance=page)) + ) self.fields['contents'].initial = mark_safe(content) + def get_instance_ocr_content(self, instance): + try: + return instance.content_object.ocr_content.content + except DocumentVersionPageOCRContent.DoesNotExist: + return '' + + +class DocumentVersionPageOCRContentForm(DocumentPageOCRContentForm): + def get_instance_ocr_content(self, instance): + try: + return instance.ocr_content.content + except (AttributeError, DocumentVersionPageOCRContent.DoesNotExist): + return '' + class DocumentOCRContentForm(forms.Form): """ @@ -54,19 +65,15 @@ class DocumentOCRContentForm(forms.Form): ) def __init__(self, *args, **kwargs): - self.document = kwargs.pop('instance', None) + document = kwargs.pop('instance', None) super(DocumentOCRContentForm, self).__init__(*args, **kwargs) content = [] self.fields['contents'].initial = '' - try: - document_pages = self.document.pages.all() - except AttributeError: - document_pages = [] - for page in document_pages: + for document_page in document.pages.all(): try: - page_content = page.ocr_content.content - except DocumentPageOCRContent.DoesNotExist: + page_content = document_page.content_object.ocr_content.content + except (AttributeError, DocumentVersionPageOCRContent.DoesNotExist): pass else: content.append(conditional_escape(force_text(page_content))) @@ -74,7 +81,7 @@ class DocumentOCRContentForm(forms.Form): '\n\n\n
- %s -
\n\n\n' % ( ugettext( 'Page %(page_number)d' - ) % {'page_number': page.page_number} + ) % {'page_number': document_page.page_number} ) ) diff --git a/mayan/apps/ocr/icons.py b/mayan/apps/ocr/icons.py index f6e1ce3141..f3f7984992 100644 --- a/mayan/apps/ocr/icons.py +++ b/mayan/apps/ocr/icons.py @@ -19,7 +19,7 @@ icon_document_ocr_errors_list = Icon( icon_document_type_ocr_settings = Icon( driver_name='fontawesome', symbol='font' ) -icon_document_type_submit = Icon(driver_name='fontawesome', symbol='font') -icon_entry_list = Icon(driver_name='fontawesome', symbol='font') - icon_document_submit = icon_document_multiple_submit +icon_document_type_submit = Icon(driver_name='fontawesome', symbol='font') +icon_document_version_page_ocr_content = Icon(driver_name='fontawesome', symbol='font') +icon_entry_list = Icon(driver_name='fontawesome', symbol='font') diff --git a/mayan/apps/ocr/links.py b/mayan/apps/ocr/links.py index 343bf26d18..68284d26ef 100644 --- a/mayan/apps/ocr/links.py +++ b/mayan/apps/ocr/links.py @@ -58,10 +58,11 @@ link_document_type_submit = Link( permissions=(permission_ocr_document,), text=_('OCR documents per type'), view='ocr:document_type_submit' ) -link_entry_list = Link( - icon_class_path='mayan.apps.ocr.icons.icon_entry_list', - permissions=(permission_ocr_document,), text=_('OCR errors'), - view='ocr:entry_list' +link_document_version_page_ocr_content = Link( + args='resolved_object.id', + icon_class_path='mayan.apps.ocr.icons.icon_document_version_page_ocr_content', + permissions=(permission_ocr_content_view,), text=_('OCR'), + view='ocr:document_version_page_ocr_content', ) link_document_ocr_errors_list = Link( args='resolved_object.id', @@ -75,3 +76,8 @@ link_document_ocr_download = Link( permissions=(permission_ocr_content_view,), text=_('Download OCR text'), view='ocr:document_ocr_download' ) +link_entry_list = Link( + icon_class_path='mayan.apps.ocr.icons.icon_entry_list', + permissions=(permission_ocr_document,), text=_('OCR errors'), + view='ocr:entry_list' +) diff --git a/mayan/apps/ocr/managers.py b/mayan/apps/ocr/managers.py index ee8e1bd95a..869c454001 100644 --- a/mayan/apps/ocr/managers.py +++ b/mayan/apps/ocr/managers.py @@ -9,7 +9,9 @@ from django.conf import settings from django.db import models, transaction from mayan.apps.documents.literals import DOCUMENT_IMAGE_TASK_TIMEOUT -from mayan.apps.documents.tasks import task_generate_document_page_image +from mayan.apps.documents.tasks import ( + task_generate_document_version_page_image +) from .events import ( event_ocr_document_content_deleted, event_ocr_document_version_finish @@ -20,47 +22,53 @@ from .signals import post_document_version_ocr logger = logging.getLogger(__name__) -class DocumentPageOCRContentManager(models.Manager): +class DocumentVesionPageOCRContentManager(models.Manager): def delete_content_for(self, document, user=None): with transaction.atomic(): for document_page in document.pages.all(): - self.filter(document_page=document_page).delete() + self.filter( + document_version_page=document_page.content_object + ).delete() event_ocr_document_content_deleted.commit( actor=user, target=document ) - def process_document_page(self, document_page): + def process_document_version_page(self, document_version_page): logger.info( 'Processing page: %d of document version: %s', - document_page.page_number, document_page.document_version + document_version_page.page_number, + document_version_page.document_version ) - DocumentPageOCRContent = apps.get_model( - app_label='ocr', model_name='DocumentPageOCRContent' + DocumentVersionPageOCRContent = apps.get_model( + app_label='ocr', model_name='DocumentVersionPageOCRContent' ) - task = task_generate_document_page_image.apply_async( + task = task_generate_document_version_page_image.apply_async( kwargs=dict( - document_page_id=document_page.pk + document_version_page_id=document_version_page.pk ) ) - cache_filename = task.get(timeout=DOCUMENT_IMAGE_TASK_TIMEOUT, disable_sync_subtasks=False) + cache_filename = task.get( + timeout=DOCUMENT_IMAGE_TASK_TIMEOUT, disable_sync_subtasks=False + ) - with document_page.cache_partition.get_file(filename=cache_filename).open() as file_object: - document_page_content, created = DocumentPageOCRContent.objects.get_or_create( - document_page=document_page + with document_version_page.cache_partition.get_file(filename=cache_filename).open() as file_object: + document_version_page_content, created = DocumentVersionPageOCRContent.objects.get_or_create( + document_version_page=document_version_page ) - document_page_content.content = ocr_backend.execute( + document_version_page_content.content = ocr_backend.execute( file_object=file_object, - language=document_page.document.language + language=document_version_page.document.language ) - document_page_content.save() + document_version_page_content.save() logger.info( 'Finished processing page: %d of document version: %s', - document_page.page_number, document_page.document_version + document_version_page.page_number, + document_version_page.document_version ) def process_document_version(self, document_version): @@ -68,8 +76,10 @@ class DocumentPageOCRContentManager(models.Manager): logger.debug('document version: %d', document_version.pk) try: - for document_page in document_version.pages.all(): - self.process_document_page(document_page=document_page) + for document_version_page in document_version.pages.all(): + self.process_document_version_page( + document_version_page=document_version_page + ) except Exception as exception: logger.error( 'OCR error for document version: %d; %s', document_version.pk, diff --git a/mayan/apps/ocr/methods.py b/mayan/apps/ocr/methods.py index b1c11a8265..567fa4b79a 100644 --- a/mayan/apps/ocr/methods.py +++ b/mayan/apps/ocr/methods.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals from datetime import timedelta +from django.apps import apps from django.utils.timezone import now from mayan.apps.common.settings import settings_db_sync_task_delay @@ -17,6 +18,17 @@ def method_document_ocr_submit(self): latest_version.submit_for_ocr() +def method_document_page_get_ocr_content(self): + DocumentVersionPageOCRContent = apps.get_model( + app_label='ocr', model_name='DocumentVersionPageOCRContent' + ) + + try: + return self.content_object.ocr_content.content + except (AttributeError, DocumentVersionPageOCRContent.DoesNotExist): + return None + + def method_document_version_ocr_submit(self): event_ocr_document_version_submit.commit( action_object=self.document, target=self diff --git a/mayan/apps/ocr/migrations/0009_rename_page_content.py b/mayan/apps/ocr/migrations/0009_rename_page_content.py new file mode 100644 index 0000000000..f98ee91251 --- /dev/null +++ b/mayan/apps/ocr/migrations/0009_rename_page_content.py @@ -0,0 +1,42 @@ +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + dependencies = [ + ('ocr', '0008_auto_20180917_0646'), + ('documents', '0052_rename_document_page'), + ] + + operations = [ + migrations.RenameModel( + 'DocumentPageOCRContent', 'DocumentVersionPageOCRContent' + ), + migrations.AlterField( + model_name='documentversionpageocrcontent', + name='document_page', + field=models.OneToOneField( + on_delete=django.db.models.deletion.CASCADE, + #name='document_version_page', + related_name='ocr_content', + to='documents.DocumentVersionPage', + verbose_name='Document version page' + ), + ), + migrations.RenameField( + model_name='documentversionpageocrcontent', + old_name='document_page', + new_name='document_version_page', + ), + migrations.AlterModelOptions( + name='documentversionpageocrcontent', + options={ + 'verbose_name': 'Document version page OCR content', + 'verbose_name_plural': 'Document version pages OCR contents' + }, + ), + ] + + diff --git a/mayan/apps/ocr/models.py b/mayan/apps/ocr/models.py index 9f42843a7e..adaf142a0c 100644 --- a/mayan/apps/ocr/models.py +++ b/mayan/apps/ocr/models.py @@ -4,10 +4,12 @@ from django.db import models from django.utils.encoding import force_text, python_2_unicode_compatible from django.utils.translation import ugettext_lazy as _ -from mayan.apps.documents.models import DocumentPage, DocumentType, DocumentVersion +from mayan.apps.documents.models import ( + DocumentPage, DocumentType, DocumentVersion, DocumentVersionPage +) from .managers import ( - DocumentPageOCRContentManager, DocumentTypeSettingsManager + DocumentVesionPageOCRContentManager, DocumentTypeSettingsManager ) @@ -36,13 +38,13 @@ class DocumentTypeSettings(models.Model): @python_2_unicode_compatible -class DocumentPageOCRContent(models.Model): +class DocumentVersionPageOCRContent(models.Model): """ This model stores the OCR results for a document page. """ - document_page = models.OneToOneField( + document_version_page = models.OneToOneField( on_delete=models.CASCADE, related_name='ocr_content', - to=DocumentPage, verbose_name=_('Document page') + to=DocumentVersionPage, verbose_name=_('Document version page') ) content = models.TextField( blank=True, help_text=_( @@ -50,11 +52,11 @@ class DocumentPageOCRContent(models.Model): ), verbose_name=_('Content') ) - objects = DocumentPageOCRContentManager() + objects = DocumentVesionPageOCRContentManager() class Meta: - verbose_name = _('Document page OCR content') - verbose_name_plural = _('Document pages OCR contents') + verbose_name = _('Document version page OCR content') + verbose_name_plural = _('Document version pages OCR contents') def __str__(self): return force_text(self.document_page) diff --git a/mayan/apps/ocr/serializers.py b/mayan/apps/ocr/serializers.py index 3d9c06c18d..780e9a48a6 100644 --- a/mayan/apps/ocr/serializers.py +++ b/mayan/apps/ocr/serializers.py @@ -2,10 +2,10 @@ from __future__ import unicode_literals from rest_framework import serializers -from .models import DocumentPageOCRContent +from .models import DocumentVersionPageOCRContent class DocumentPageOCRContentSerializer(serializers.ModelSerializer): class Meta: fields = ('content',) - model = DocumentPageOCRContent + model = DocumentVersionPageOCRContent diff --git a/mayan/apps/ocr/tasks.py b/mayan/apps/ocr/tasks.py index a6d9c82efb..4ba1f5268f 100644 --- a/mayan/apps/ocr/tasks.py +++ b/mayan/apps/ocr/tasks.py @@ -19,8 +19,8 @@ def task_do_ocr(self, document_version_pk): DocumentVersion = apps.get_model( app_label='documents', model_name='DocumentVersion' ) - DocumentPageOCRContent = apps.get_model( - app_label='ocr', model_name='DocumentPageOCRContent' + DocumentVersionPageOCRContent = apps.get_model( + app_label='ocr', model_name='DocumentVersionPageOCRContent' ) lock_id = 'task_do_ocr_doc_version-%d' % document_version_pk @@ -39,7 +39,7 @@ def task_do_ocr(self, document_version_pk): 'Starting document OCR for document version: %s', document_version ) - DocumentPageOCRContent.objects.process_document_version( + DocumentVersionPageOCRContent.objects.process_document_version( document_version=document_version ) except OperationalError as exception: diff --git a/mayan/apps/ocr/tests/test_api.py b/mayan/apps/ocr/tests/test_api.py index e847550141..578e8a94b8 100644 --- a/mayan/apps/ocr/tests/test_api.py +++ b/mayan/apps/ocr/tests/test_api.py @@ -12,13 +12,34 @@ from ..permissions import ( from .literals import TEST_DOCUMENT_CONTENT -class OCRAPITestCase(DocumentTestMixin, BaseAPITestCase): +class OCRAPIViewTestMixin(object): def _request_document_ocr_submit_view(self): return self.post( viewname='rest_api:document-ocr-submit-view', kwargs={'pk': self.test_document.pk} ) + def _request_document_version_ocr_submit_view(self): + return self.post( + viewname='rest_api:document-version-ocr-submit-view', kwargs={ + 'document_pk': self.test_document.pk, + 'version_pk': self.test_document.latest_version.pk + } + ) + + def _request_document_version_page_content_view(self): + return self.get( + viewname='rest_api:document-page-ocr-content-view', kwargs={ + 'document_pk': self.test_document.pk, + 'version_pk': self.test_document.latest_version.pk, + 'page_pk': self.test_document.latest_version.pages.first().pk, + } + ) + + +class OCRAPIViewTestCase( + OCRAPIViewTestMixin, DocumentTestMixin, BaseAPITestCase +): def test_submit_document_no_access(self): response = self._request_document_ocr_submit_view() self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) @@ -35,15 +56,9 @@ class OCRAPITestCase(DocumentTestMixin, BaseAPITestCase): self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) self.assertTrue( - hasattr(self.test_document.pages.first(), 'ocr_content') - ) - - def _request_document_version_ocr_submit_view(self): - return self.post( - viewname='rest_api:document-version-ocr-submit-view', kwargs={ - 'document_pk': self.test_document.pk, - 'version_pk': self.test_document.latest_version.pk - } + hasattr( + self.test_document.pages.first().content_object, 'ocr_content' + ) ) def test_submit_document_version_no_access(self): @@ -62,20 +77,11 @@ class OCRAPITestCase(DocumentTestMixin, BaseAPITestCase): self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) self.assertTrue( - hasattr(self.test_document.pages.first(), 'ocr_content') - ) - - def _request_document_page_content_view(self): - return self.get( - viewname='rest_api:document-page-ocr-content-view', kwargs={ - 'document_pk': self.test_document.pk, - 'version_pk': self.test_document.latest_version.pk, - 'page_pk': self.test_document.latest_version.pages.first().pk, - } + hasattr(self.test_document_version.pages.first(), 'ocr_content') ) def test_get_document_version_page_content_no_access(self): - response = self._request_document_page_content_view() + response = self._request_document_version_page_content_view() self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) def test_get_document_version_page_content_with_access(self): @@ -83,7 +89,7 @@ class OCRAPITestCase(DocumentTestMixin, BaseAPITestCase): self.grant_access( permission=permission_ocr_content_view, obj=self.test_document ) - response = self._request_document_page_content_view() + response = self._request_document_version_page_content_view() self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertTrue( TEST_DOCUMENT_CONTENT in response.data['content'] diff --git a/mayan/apps/ocr/tests/test_events.py b/mayan/apps/ocr/tests/test_events.py index 4a4318ec64..c51cee8fad 100644 --- a/mayan/apps/ocr/tests/test_events.py +++ b/mayan/apps/ocr/tests/test_events.py @@ -8,13 +8,13 @@ from ..events import ( event_ocr_document_content_deleted, event_ocr_document_version_submit, event_ocr_document_version_finish ) -from ..models import DocumentPageOCRContent +from ..models import DocumentVersionPageOCRContent class OCREventsTestCase(GenericDocumentTestCase): def test_document_content_deleted_event(self): Action.objects.all().delete() - DocumentPageOCRContent.objects.delete_content_for( + DocumentVersionPageOCRContent.objects.delete_content_for( document=self.test_document ) diff --git a/mayan/apps/ocr/tests/test_models.py b/mayan/apps/ocr/tests/test_models.py index cd4992d0d4..fe31995990 100644 --- a/mayan/apps/ocr/tests/test_models.py +++ b/mayan/apps/ocr/tests/test_models.py @@ -19,7 +19,7 @@ class DocumentOCRTestCase(DocumentTestMixin, BaseTestCase): _skip_file_descriptor_test = True def test_ocr_language_backends_end(self): - content = self.test_document.pages.first().ocr_content.content + content = self.test_document.pages.first().content_object.ocr_content.content self.assertTrue(TEST_DOCUMENT_CONTENT in content) @@ -40,7 +40,7 @@ class GermanOCRSupportTestCase(DocumentTestMixin, BaseTestCase): ) def test_ocr_language_backends_end(self): - content = self.test_document.pages.first().ocr_content.content + content = self.test_document.pages.first().content_object.ocr_content.content self.assertTrue( TEST_DOCUMENT_CONTENT_DEU_1 in content diff --git a/mayan/apps/ocr/tests/test_views.py b/mayan/apps/ocr/tests/test_views.py index 6ed278c54a..03a7ec7baf 100644 --- a/mayan/apps/ocr/tests/test_views.py +++ b/mayan/apps/ocr/tests/test_views.py @@ -2,7 +2,7 @@ from __future__ import unicode_literals from mayan.apps.documents.tests.base import GenericDocumentViewTestCase -from ..models import DocumentPageOCRContent +from ..models import DocumentVersionPageOCRContent from ..permissions import ( permission_ocr_content_view, permission_ocr_document, permission_document_type_ocr_setup @@ -27,10 +27,10 @@ class OCRViewTestMixin(object): } ) - def _request_document_page_content_view(self): + def _request_document_version_page_content_view(self): return self.get( - viewname='ocr:document_page_ocr_content', kwargs={ - 'pk': self.test_document.pages.first().pk + viewname='ocr:document_version_page_ocr_content', kwargs={ + 'pk': self.test_document_version.pages.first().pk } ) @@ -86,8 +86,8 @@ class OCRViewsTestCase(OCRViewTestMixin, GenericDocumentViewTestCase): self.assertEqual(response.status_code, 404) self.assertTrue( - DocumentPageOCRContent.objects.filter( - document_page=self.test_document.pages.first() + DocumentVersionPageOCRContent.objects.filter( + document_version_page=self.test_document.pages.first().content_object ).exists() ) @@ -101,28 +101,11 @@ class OCRViewsTestCase(OCRViewTestMixin, GenericDocumentViewTestCase): self.assertEqual(response.status_code, 302) self.assertFalse( - DocumentPageOCRContent.objects.filter( - document_page=self.test_document.pages.first() + DocumentVersionPageOCRContent.objects.filter( + document_version_page=self.test_document.pages.first().content_object ).exists() ) - def test_document_page_content_view_no_permissions(self): - self.test_document.submit_for_ocr() - - response = self._request_document_page_content_view() - self.assertEqual(response.status_code, 404) - - def test_document_page_content_view_with_access(self): - self.test_document.submit_for_ocr() - self.grant_access( - obj=self.test_document, permission=permission_ocr_content_view - ) - - response = self._request_document_page_content_view() - self.assertContains( - response=response, text=TEST_DOCUMENT_CONTENT, status_code=200 - ) - def test_document_submit_view_no_permission(self): response = self._request_document_submit_view() self.assertEqual(response.status_code, 404) @@ -188,6 +171,23 @@ class OCRViewsTestCase(OCRViewTestMixin, GenericDocumentViewTestCase): ), ) + def test_document_version_page_content_view_no_permissions(self): + self.test_document.submit_for_ocr() + + response = self._request_document_version_page_content_view() + self.assertEqual(response.status_code, 404) + + def test_document_version_page_content_view_with_access(self): + self.test_document.submit_for_ocr() + self.grant_access( + obj=self.test_document, permission=permission_ocr_content_view + ) + + response = self._request_document_version_page_content_view() + self.assertContains( + response=response, text=TEST_DOCUMENT_CONTENT, status_code=200 + ) + class DocumentTypeOCRViewTestMixin(object): def _request_document_type_ocr_settings_view(self): diff --git a/mayan/apps/ocr/urls.py b/mayan/apps/ocr/urls.py index 3d2d9b0b25..a8b9d9791e 100644 --- a/mayan/apps/ocr/urls.py +++ b/mayan/apps/ocr/urls.py @@ -8,9 +8,10 @@ from .api_views import ( ) from .views import ( DocumentOCRContentDeleteView, DocumentOCRContentView, - DocumentOCRDownloadView, - DocumentOCRErrorsListView, DocumentPageOCRContentView, DocumentSubmitView, - DocumentTypeSettingsEditView, DocumentTypeSubmitView, EntryListView + DocumentOCRDownloadView, DocumentOCRErrorsListView, + DocumentPageOCRContentView, DocumentSubmitView, + DocumentTypeSettingsEditView, DocumentTypeSubmitView, + DocumentVersionPageOCRContentView, EntryListView ) urlpatterns = [ @@ -50,6 +51,11 @@ urlpatterns = [ view=DocumentPageOCRContentView.as_view(), name='document_page_ocr_content' ), + url( + regex=r'^documents/versions/pages/(?P\d+)/content/$', + view=DocumentVersionPageOCRContentView.as_view(), + name='document_version_page_ocr_content' + ), url( regex=r'^document_types/submit/$', view=DocumentTypeSubmitView.as_view(), name='document_type_submit' diff --git a/mayan/apps/ocr/utils.py b/mayan/apps/ocr/utils.py index 0ac5de5fb3..a192613999 100644 --- a/mayan/apps/ocr/utils.py +++ b/mayan/apps/ocr/utils.py @@ -4,15 +4,29 @@ from django.apps import apps from django.utils.encoding import force_text -def get_document_ocr_content(document): - DocumentPageOCRContent = apps.get_model( - app_label='ocr', model_name='DocumentPageOCRContent' +def get_document_version_ocr_content(document_version): + DocumentVersionPageOCRContent = apps.get_model( + app_label='ocr', model_name='DocumentVersionPageOCRContent' ) - for page in document.pages.all(): + for document_version_page in document_version.pages.all(): try: - page_content = page.ocr_content.content - except DocumentPageOCRContent.DoesNotExist: + page_content = document_version_page.ocr_content.content + except DocumentVersionPageOCRContent.DoesNotExist: + pass + else: + yield force_text(page_content) + + +def get_document_ocr_content(document): + DocumentVersionPageOCRContent = apps.get_model( + app_label='ocr', model_name='DocumentVersionPageOCRContent' + ) + + for document_page in document.pages.all(): + try: + page_content = document_page.content_object.ocr_content.content + except (AttributeError, DocumentVersionPageOCRContent.DoesNotExist): pass else: yield force_text(page_content) diff --git a/mayan/apps/ocr/views.py b/mayan/apps/ocr/views.py index 41b978388e..21e7c00fb0 100644 --- a/mayan/apps/ocr/views.py +++ b/mayan/apps/ocr/views.py @@ -12,10 +12,15 @@ from mayan.apps.common.generics import ( ) from mayan.apps.common.mixins import ExternalObjectMixin from mayan.apps.documents.forms import DocumentTypeFilteredSelectForm -from mayan.apps.documents.models import Document, DocumentPage, DocumentType +from mayan.apps.documents.models import ( + Document, DocumentPage, DocumentType, DocumentVersionPage +) -from .forms import DocumentPageOCRContentForm, DocumentOCRContentForm -from .models import DocumentPageOCRContent, DocumentVersionOCRError +from .forms import ( + DocumentPageOCRContentForm, DocumentOCRContentForm, + DocumentVersionPageOCRContentForm +) +from .models import DocumentVersionPageOCRContent, DocumentVersionOCRError from .permissions import ( permission_ocr_content_view, permission_ocr_document, permission_document_type_ocr_setup @@ -46,7 +51,7 @@ class DocumentOCRContentDeleteView(MultipleObjectConfirmActionView): return result def object_action(self, form, instance): - DocumentPageOCRContent.objects.delete_content_for( + DocumentVersionPageOCRContent.objects.delete_content_for( document=instance, user=self.request.user ) @@ -94,6 +99,30 @@ class DocumentPageOCRContentView(SingleObjectDetailView): } +class DocumentVersionPageOCRContentView(SingleObjectDetailView): + form_class = DocumentVersionPageOCRContentForm + model = DocumentVersionPage + object_permission = permission_ocr_content_view + + def dispatch(self, request, *args, **kwargs): + result = super(DocumentVersionPageOCRContentView, self).dispatch( + request, *args, **kwargs + ) + self.get_object().document.add_as_recent_document_for_user( + user=request.user + ) + return result + + def get_extra_context(self): + return { + 'hide_labels': True, + 'object': self.get_object(), + 'title': _( + 'OCR result for document version page: %s' + ) % self.get_object(), + } + + class DocumentSubmitView(MultipleObjectConfirmActionView): model = Document object_permission = permission_ocr_document diff --git a/mayan/apps/sources/apps.py b/mayan/apps/sources/apps.py index 7139e5a846..d948a54fa7 100644 --- a/mayan/apps/sources/apps.py +++ b/mayan/apps/sources/apps.py @@ -21,9 +21,10 @@ from .handlers import ( handler_create_default_document_source, handler_initialize_periodic_tasks ) from .links import ( - link_document_create_multiple, link_setup_sources, - link_setup_source_check_now, link_setup_source_create_imap_email, - link_setup_source_create_pop3_email, link_setup_source_create_sane_scanner, + link_document_create_multiple, link_document_pages_append, + link_setup_sources, link_setup_source_check_now, + link_setup_source_create_imap_email, link_setup_source_create_pop3_email, + link_setup_source_create_sane_scanner, link_setup_source_create_watch_folder, link_setup_source_create_webform, link_setup_source_create_staging_folder, link_setup_source_delete, link_setup_source_edit, link_setup_source_logs, link_staging_file_delete, @@ -145,10 +146,16 @@ class SourcesApp(MayanAppConfig): menu_secondary.bind_links( links=(link_document_version_upload,), sources=( - 'documents:document_version_list', 'documents:upload_version', + 'documents:document_version_list', 'sources:upload_version', 'documents:document_version_revert' ) ) + menu_secondary.bind_links( + links=(link_document_pages_append,), + sources=( + 'documents:document_pages', 'sources:document_pages_append' + ) + ) post_upgrade.connect( receiver=handler_initialize_periodic_tasks, diff --git a/mayan/apps/sources/forms.py b/mayan/apps/sources/forms.py index a6431f71e5..e4eed9de64 100644 --- a/mayan/apps/sources/forms.py +++ b/mayan/apps/sources/forms.py @@ -23,14 +23,25 @@ class NewDocumentForm(DocumentForm): class NewVersionForm(forms.Form): - def __init__(self, *args, **kwargs): - super(NewVersionForm, self).__init__(*args, **kwargs) + comment = forms.CharField( + help_text=_('An optional comment to explain the upload.'), + label=_('Comment'), required=False, + widget=forms.widgets.Textarea(attrs={'rows': 4}), + ) - self.fields['comment'] = forms.CharField( - label=_('Comment'), - required=False, - widget=forms.widgets.Textarea(attrs={'rows': 4}), - ) + append_pages = forms.BooleanField( + help_text=_( + 'If selected, the pages of the file uploaded will be appended ' + 'to the existing document pages. Otherwise the pages of the ' + 'upload will replace the existing pages of the document.' + ), label=_('Append pages?'), required=False, + ) + + def __init__(self, *args, **kwargs): + hide_append_pages = kwargs.pop('hide_append_pages', False) + super(NewVersionForm, self).__init__(*args, **kwargs) + if hide_append_pages: + self.fields['append_pages'].widget = forms.widgets.HiddenInput() class UploadBaseForm(forms.Form): diff --git a/mayan/apps/sources/icons.py b/mayan/apps/sources/icons.py index 1b0a6c8c84..a0a59fcded 100644 --- a/mayan/apps/sources/icons.py +++ b/mayan/apps/sources/icons.py @@ -9,6 +9,10 @@ icon_document_version_upload = Icon( driver_name='fontawesome', symbol='upload' ) icon_log = Icon(driver_name='fontawesome', symbol='exclamation-triangle') +icon_document_pages_append = Icon( + driver_name='fontawesome-dual', primary_symbol='copy', + secondary_symbol='plus' +) icon_setup_sources = Icon(driver_name='fontawesome', symbol='upload') icon_setup_source_check_now = Icon(driver_name='fontawesome', symbol='check') icon_setup_source_delete = Icon(driver_name='fontawesome', symbol='times') diff --git a/mayan/apps/sources/links.py b/mayan/apps/sources/links.py index b1793547f5..da5d3300d6 100644 --- a/mayan/apps/sources/links.py +++ b/mayan/apps/sources/links.py @@ -113,6 +113,13 @@ link_staging_file_delete = Link( permissions=(permission_document_new_version, permission_document_create), tags='dangerous', text=_('Delete'), view='sources:staging_file_delete', ) +link_document_pages_append = Link( + args='resolved_object.pk', + icon_class_path='mayan.apps.sources.icons.icon_document_pages_append', + permissions=(permission_document_new_version,), + text=_('Append pages'), + view='sources:document_pages_append' +) link_document_version_upload = Link( args='resolved_object.pk', condition=document_new_version_not_blocked, icon_class_path='mayan.apps.sources.icons.icon_document_version_upload', diff --git a/mayan/apps/sources/urls.py b/mayan/apps/sources/urls.py index fdc7adc330..173891f5f5 100644 --- a/mayan/apps/sources/urls.py +++ b/mayan/apps/sources/urls.py @@ -7,9 +7,10 @@ from .api_views import ( APIStagingSourceListView, APIStagingSourceView ) from .views import ( - SetupSourceCheckView, SetupSourceCreateView, SetupSourceDeleteView, - SetupSourceEditView, SetupSourceListView, SourceLogListView, - StagingFileDeleteView, UploadInteractiveVersionView, UploadInteractiveView + DocumentPagesAppendView, SetupSourceCheckView, SetupSourceCreateView, + SetupSourceDeleteView, SetupSourceEditView, SetupSourceListView, + SourceLogListView, StagingFileDeleteView, UploadInteractiveVersionView, + UploadInteractiveView ) from .wizards import DocumentCreateWizard @@ -41,6 +42,14 @@ urlpatterns = [ regex=r'^documents/(?P\d+)/versions/upload/interactive/$', view=UploadInteractiveVersionView.as_view(), name='upload_version' ), + url( + regex=r'^documents/(?P\d+)/pages/append/interactive/(?P\d+)/$', + view=DocumentPagesAppendView.as_view(), name='document_pages_append' + ), + url( + regex=r'^documents/(?P\d+)/pages/append/interactive/$', + view=DocumentPagesAppendView.as_view(), name='document_pages_append' + ), # Setup views diff --git a/mayan/apps/sources/views.py b/mayan/apps/sources/views.py index be55047b7d..3f26a650c2 100644 --- a/mayan/apps/sources/views.py +++ b/mayan/apps/sources/views.py @@ -362,7 +362,6 @@ class UploadInteractiveView(UploadBaseView): class UploadInteractiveVersionView(UploadBaseView): def dispatch(self, request, *args, **kwargs): - self.subtemplates_list = [] self.document = get_object_or_404( @@ -417,12 +416,17 @@ class UploadInteractiveVersionView(UploadBaseView): else: user_id = None - task_upload_new_version.apply_async(kwargs=dict( - shared_uploaded_file_id=shared_uploaded_file.pk, - document_id=self.document.pk, - user_id=user_id, - comment=forms['document_form'].cleaned_data.get('comment') - )) + task_upload_new_version.apply_async( + kwargs=dict( + append_pages=forms['document_form'].cleaned_data.get( + 'append_pages', False + ), + shared_uploaded_file_id=shared_uploaded_file.pk, + document_id=self.document.pk, + user_id=user_id, + comment=forms['document_form'].cleaned_data.get('comment') + ) + ) messages.success( message=_( @@ -448,13 +452,6 @@ class UploadInteractiveVersionView(UploadBaseView): files=kwargs.get('files', None), ) - def create_document_form_form(self, **kwargs): - return self.get_form_classes()['document_form']( - prefix=kwargs['prefix'], - data=kwargs.get('data', None), - files=kwargs.get('files', None), - ) - def get_form_classes(self): return { 'document_form': NewVersionForm, @@ -467,8 +464,33 @@ class UploadInteractiveVersionView(UploadBaseView): ).get_context_data(**kwargs) context['object'] = self.document context['title'] = _( - 'Upload a new version from source: %s' - ) % self.source.label + 'Upload a new version for document "%(document)s" from source: %(source)s' + ) % {'document': self.document, 'source': self.source.label} + + context['submit_label'] = _('Submit') + + return context + + +class DocumentPagesAppendView(UploadInteractiveVersionView): + def get_document_form_initial(self): + return { + 'append_pages': True, + } + + def get_form_extra_kwargs(self, form_name): + if form_name == 'document_form': + return { + 'hide_append_pages': True + } + + def get_context_data(self, **kwargs): + context = super( + DocumentPagesAppendView, self + ).get_context_data(**kwargs) + context['title'] = _( + 'Append pages to document "%(document)s" from source: %(source)s' + ) % {'document': self.document, 'source': self.source.label} return context diff --git a/mayan/apps/tags/apps.py b/mayan/apps/tags/apps.py index c64811558a..2828e06049 100644 --- a/mayan/apps/tags/apps.py +++ b/mayan/apps/tags/apps.py @@ -13,7 +13,9 @@ from mayan.apps.common.menus import ( menu_facet, menu_list_facet, menu_main, menu_multi_item, menu_object, menu_secondary ) -from mayan.apps.documents.search import document_page_search, document_search +from mayan.apps.documents.search import ( + document_page_search, document_search, document_version_page_search +) from mayan.apps.events.classes import ModelEventType from mayan.apps.events.links import ( link_events_for_object, link_object_event_types_user_subcriptions_list, @@ -62,7 +64,7 @@ class TagsApp(MayanAppConfig): ) DocumentPageResult = apps.get_model( - app_label='documents', model_name='DocumentPageResult' + app_label='documents', model_name='DocumentVersionPageResult' ) DocumentTag = self.get_model(model_name='DocumentTag') @@ -133,9 +135,12 @@ class TagsApp(MayanAppConfig): ) document_page_search.add_model_field( - field='document_version__document__tags__label', label=_('Tags') + field='document__tags__label', label=_('Tags') ) document_search.add_model_field(field='tags__label', label=_('Tags')) + document_version_page_search.add_model_field( + field='document_version__document__tags__label', label=_('Tags') + ) menu_facet.bind_links( links=(link_document_tag_list,), sources=(Document,) diff --git a/requirements/base.txt b/requirements/base.txt index 337001f564..87dd80811c 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,9 +1,7 @@ Pillow==6.0.0 PyPDF2==1.26.0 PyYAML==5.1.1 -celery==4.3.0 django-activity-stream==0.7.0 -django-celery-beat==1.5.0 django-colorful==1.3 django-cors-headers==2.5.2 django-downloadview==1.9 diff --git a/setup.py b/setup.py index 5b4c273ef0..a8ab7ec567 100644 --- a/setup.py +++ b/setup.py @@ -60,9 +60,7 @@ django==1.11.24 Pillow==6.0.0 PyPDF2==1.26.0 PyYAML==5.1.1 -celery==4.3.0 django-activity-stream==0.7.0 -django-celery-beat==1.5.0 django-colorful==1.3 django-cors-headers==2.5.2 django-downloadview==1.9