diff --git a/mayan/apps/document_parsing/apps.py b/mayan/apps/document_parsing/apps.py index ea4355c5a8..71cd9930ac 100644 --- a/mayan/apps/document_parsing/apps.py +++ b/mayan/apps/document_parsing/apps.py @@ -63,7 +63,7 @@ class DocumentParsingApp(MayanAppConfig): app_label='documents', model_name='Document' ) DocumentPage = apps.get_model( - app_label='documents', model_name='DocumentPage' + app_label='documents', model_name='DocumentVersionPage' ) DocumentType = apps.get_model( app_label='documents', model_name='DocumentType' @@ -100,9 +100,9 @@ class DocumentParsingApp(MayanAppConfig): ) ) - ModelField( - model=Document, name='versions__version_pages__content__content' - ) + #ModelField( + # model=Document, name='versions__pages__content__content' + #) ModelPermission.register( model=Document, permissions=( @@ -133,7 +133,7 @@ class DocumentParsingApp(MayanAppConfig): ) document_search.add_model_field( - field='versions__version_pages__content__content', label=_('Content') + field='versions__pages__content__content', label=_('Content') ) document_page_search.add_model_field( diff --git a/mayan/apps/document_parsing/models.py b/mayan/apps/document_parsing/models.py index 5e9b52320e..8d70eacd5b 100644 --- a/mayan/apps/document_parsing/models.py +++ b/mayan/apps/document_parsing/models.py @@ -17,8 +17,8 @@ class DocumentPageContent(models.Model): This model store's the parsed content of a document page. """ document_page = models.OneToOneField( - on_delete=models.CASCADE, related_name='content', to=DocumentPage, - verbose_name=_('Document page') + on_delete=models.CASCADE, related_name='content', + to=DocumentPage, verbose_name=_('Document page') ) content = models.TextField( blank=True, help_text=_( diff --git a/mayan/apps/document_parsing/views.py b/mayan/apps/document_parsing/views.py index de9cb3ef0b..2fde8527d0 100644 --- a/mayan/apps/document_parsing/views.py +++ b/mayan/apps/document_parsing/views.py @@ -12,7 +12,9 @@ from mayan.apps.common.generics import ( ) from mayan.apps.common.mixins import ExternalObjectMixin from mayan.apps.documents.forms import DocumentTypeFilteredSelectForm -from mayan.apps.documents.models import Document, DocumentPage, DocumentType +from mayan.apps.documents.models import ( + Document, DocumentType, DocumentVersionPage +) from .forms import DocumentContentForm, DocumentPageContentForm from .models import DocumentPageContent, DocumentVersionParseError @@ -87,7 +89,7 @@ class DocumentContentDownloadView(SingleObjectDownloadView): class DocumentPageContentView(SingleObjectDetailView): form_class = DocumentPageContentForm - model = DocumentPage + model = DocumentVersionPage object_permission = permission_content_view def dispatch(self, request, *args, **kwargs): diff --git a/mayan/apps/documents/admin.py b/mayan/apps/documents/admin.py index 62d30c0e1f..59b6004f31 100644 --- a/mayan/apps/documents/admin.py +++ b/mayan/apps/documents/admin.py @@ -3,18 +3,12 @@ from __future__ import unicode_literals from django.contrib import admin from .models import ( - DeletedDocument, Document, DocumentPage, DocumentType, - DocumentTypeFilename, DocumentVersion, DuplicatedDocument, RecentDocument + DeletedDocument, Document, DocumentType, DocumentTypeFilename, + DocumentVersion, DocumentVersionPage, DuplicatedDocument, + RecentDocument ) -class DocumentPageInline(admin.StackedInline): - model = DocumentPage - extra = 1 - classes = ('collapse-open',) - allow_add = True - - class DocumentTypeFilenameInline(admin.StackedInline): model = DocumentTypeFilename extra = 1 @@ -29,6 +23,13 @@ class DocumentVersionInline(admin.StackedInline): allow_add = True +class DocumentVersionPageInline(admin.StackedInline): + model = DocumentVersionPage + extra = 1 + classes = ('collapse-open',) + allow_add = True + + @admin.register(DeletedDocument) class DeletedDocumentAdmin(admin.ModelAdmin): date_hierarchy = 'deleted_date_time' diff --git a/mayan/apps/documents/api_views.py b/mayan/apps/documents/api_views.py index 4d17038722..7f0d6d0d46 100644 --- a/mayan/apps/documents/api_views.py +++ b/mayan/apps/documents/api_views.py @@ -33,7 +33,8 @@ from .serializers import ( DocumentTypeSerializer, DocumentVersionSerializer, NewDocumentSerializer, NewDocumentVersionSerializer, RecentDocumentSerializer, WritableDocumentSerializer, - WritableDocumentTypeSerializer, WritableDocumentVersionSerializer + WritableDocumentTypeSerializer, WritableDocumentVersionSerializer, + DocumentVersionPageSerializer ) from .settings import settings_document_page_image_cache_time from .tasks import task_generate_document_page_image @@ -168,13 +169,13 @@ class APIDocumentPageImageView(generics.RetrieveAPIView): ) return document - def get_document_version(self): - return get_object_or_404( - self.get_document().versions.all(), pk=self.kwargs['version_pk'] - ) + #def get_document_version(self): + # return get_object_or_404( + # self.get_document().versions.all(), pk=self.kwargs['version_pk'] + # ) def get_queryset(self): - return self.get_document_version().pages_all.all() + return self.get_document().pages_all.all() def get_serializer(self, *args, **kwargs): return None @@ -221,6 +222,95 @@ class APIDocumentPageImageView(generics.RetrieveAPIView): return response +class APIDocumentVersionPageImageView(generics.RetrieveAPIView): + """ + get: Returns an image representation of the selected document version page. + """ + lookup_url_kwarg = 'page_pk' + + def get_document(self): + if self.request.method == 'GET': + permission_required = permission_document_view + else: + permission_required = permission_document_edit + + document = get_object_or_404(Document.passthrough, pk=self.kwargs['pk']) + + AccessControlList.objects.check_access( + obj=document, permissions=(permission_required,), + user=self.request.user + ) + return document + + def get_document_version(self): + return get_object_or_404( + self.get_document().versions.all(), pk=self.kwargs['version_pk'] + ) + + def get_queryset(self): + return self.get_document_version().pages_all.all() + + def get_serializer(self, *args, **kwargs): + return None + + def get_serializer_class(self): + return None + + @cache_control(private=True) + def retrieve(self, request, *args, **kwargs): + width = request.GET.get('width') + height = request.GET.get('height') + zoom = request.GET.get('zoom') + + if zoom: + zoom = int(zoom) + + rotation = request.GET.get('rotation') + + if rotation: + rotation = int(rotation) + + maximum_layer_order = request.GET.get('maximum_layer_order') + if maximum_layer_order: + maximum_layer_order = int(maximum_layer_order) + + task = task_generate_document_version_page_image.apply_async( + kwargs=dict( + document_version_page_id=self.get_object().pk, width=width, + height=height, zoom=zoom, rotation=rotation, + maximum_layer_order=maximum_layer_order, + user_id=request.user.pk + ) + ) + + cache_filename = task.get(timeout=DOCUMENT_IMAGE_TASK_TIMEOUT) + cache_file = self.get_object().cache_partition.get_file(filename=cache_filename) + with cache_file.open() as file_object: + response = HttpResponse(file_object.read(), content_type='image') + if '_hash' in request.GET: + patch_cache_control( + response=response, + max_age=settings_document_page_image_cache_time.value + ) + return response + + +class APIDocumentPageListView(generics.ListAPIView): + serializer_class = DocumentPageSerializer + + def get_document(self): + document = get_object_or_404(Document, pk=self.kwargs['pk']) + + AccessControlList.objects.check_access( + obj=document, permissions=(permission_document_view,), + user=self.request.user + ) + return document + + def get_queryset(self): + return self.get_document().pages.all() + + class APIDocumentPageView(generics.RetrieveUpdateAPIView): """ get: Returns the selected document page details. @@ -230,6 +320,38 @@ class APIDocumentPageView(generics.RetrieveUpdateAPIView): lookup_url_kwarg = 'page_pk' serializer_class = DocumentPageSerializer + def get_document(self): + if self.request.method == 'GET': + permission_required = permission_document_view + else: + permission_required = permission_document_edit + + document = get_object_or_404(Document, pk=self.kwargs['pk']) + + AccessControlList.objects.check_access( + obj=document, permissions=(permission_required,), + user=self.request.user + ) + return document + + #def get_document_version(self): + # return get_object_or_404( + # self.get_document().versions.all(), pk=self.kwargs['version_pk'] + # ) + + def get_queryset(self): + return self.get_document().pages.all() + + +class APIDocumentVersionPageView(generics.RetrieveUpdateAPIView): + """ + get: Returns the selected document verion page details. + patch: Edit the selected document version page. + put: Edit the selected document version page. + """ + lookup_url_kwarg = 'page_pk' + serializer_class = DocumentVersionPageSerializer + def get_document(self): if self.request.method == 'GET': permission_required = permission_document_view @@ -289,8 +411,7 @@ class APIDocumentTypeView(generics.RetrieveUpdateDestroyAPIView): 'GET': (permission_document_type_view,), 'PUT': (permission_document_type_edit,), 'PATCH': (permission_document_type_edit,), - 'DELETE': (permission_document_type_delete,) - } + 'DELETE': (permission_document_type_delete,) } permission_classes = (MayanPermission,) queryset = DocumentType.objects.all() diff --git a/mayan/apps/documents/apps.py b/mayan/apps/documents/apps.py index 09e85928c7..738395c805 100644 --- a/mayan/apps/documents/apps.py +++ b/mayan/apps/documents/apps.py @@ -120,8 +120,8 @@ class DocumentsApp(MayanAppConfig): DeletedDocument = self.get_model(model_name='DeletedDocument') Document = self.get_model(model_name='Document') - DocumentPage = self.get_model(model_name='DocumentPage') - DocumentPageResult = self.get_model(model_name='DocumentPageResult') + DocumentPage = self.get_model(model_name='DocumentVersionPage') + DocumentPageResult = self.get_model(model_name='DocumentVersionPageResult') DocumentType = self.get_model(model_name='DocumentType') DocumentTypeFilename = self.get_model(model_name='DocumentTypeFilename') DocumentVersion = self.get_model(model_name='DocumentVersion') diff --git a/mayan/apps/documents/dashboard_widgets.py b/mayan/apps/documents/dashboard_widgets.py index 2ca9dd1b89..221e2cfbbd 100644 --- a/mayan/apps/documents/dashboard_widgets.py +++ b/mayan/apps/documents/dashboard_widgets.py @@ -32,12 +32,12 @@ class DashboardWidgetDocumentPagesTotal(DashboardWidgetNumeric): AccessControlList = apps.get_model( app_label='acls', model_name='AccessControlList' ) - DocumentPage = apps.get_model( - app_label='documents', model_name='DocumentPage' + DocumentVersionPage = apps.get_model( + app_label='documents', model_name='DocumentVersionPage' ) self.count = AccessControlList.objects.restrict_queryset( permission=permission_document_view, user=request.user, - queryset=DocumentPage.objects.all() + queryset=DocumentVersionPage.objects.all() ).count() return super(DashboardWidgetDocumentPagesTotal, self).render(request) diff --git a/mayan/apps/documents/literals.py b/mayan/apps/documents/literals.py index e29ed17e8d..f5b4587b07 100644 --- a/mayan/apps/documents/literals.py +++ b/mayan/apps/documents/literals.py @@ -42,3 +42,5 @@ PAGE_RANGE_RANGE = 'range' PAGE_RANGE_CHOICES = ( (PAGE_RANGE_ALL, _('All pages')), (PAGE_RANGE_RANGE, _('Page range')) ) + +RETRY_DELAY_DOCUMENT_RESET_PAGES = 30 diff --git a/mayan/apps/documents/managers.py b/mayan/apps/documents/managers.py index 6c9dd92640..7ec57862ef 100644 --- a/mayan/apps/documents/managers.py +++ b/mayan/apps/documents/managers.py @@ -28,15 +28,15 @@ class DocumentManager(models.Manager): class DocumentPageManager(models.Manager): def get_by_natural_key(self, page_number, document_version_natural_key): - DocumentVersion = apps.get_model( - app_label='documents', model_name='DocumentVersion' + Document = apps.get_model( + app_label='documents', model_name='Document' ) try: - document_version = DocumentVersion.objects.get_by_natural_key(*document_version_natural_key) - except DocumentVersion.DoesNotExist: + document = Document.objects.get_by_natural_key(*document_version_natural_key) + except Document.DoesNotExist: raise self.model.DoesNotExist - return self.get(document_version__pk=document_version.pk, page_number=page_number) + return self.get(document__pk=document.pk, page_number=page_number) def get_queryset(self): return models.QuerySet( @@ -124,6 +124,24 @@ class DocumentVersionManager(models.Manager): return self.get(document__pk=document.pk, checksum=checksum) +class DocumentVersionPageManager(models.Manager): + def get_by_natural_key(self, page_number, document_version_natural_key): + DocumentVersion = apps.get_model( + app_label='documents', model_name='DocumentVersion' + ) + try: + document_version = DocumentVersion.objects.get_by_natural_key(*document_version_natural_key) + except DocumentVersion.DoesNotExist: + raise self.model.DoesNotExist + + return self.get(document_version__pk=document_version.pk, page_number=page_number) + + def get_queryset(self): + return models.QuerySet( + model=self.model, using=self._db + ).filter(enabled=True) + + class DuplicatedDocumentManager(models.Manager): def clean_empty_duplicate_lists(self): self.filter(documents=None).delete() diff --git a/mayan/apps/documents/migrations/0052_auto_20191007_1921.py b/mayan/apps/documents/migrations/0052_auto_20191007_1921.py new file mode 100644 index 0000000000..5cfcc3040d --- /dev/null +++ b/mayan/apps/documents/migrations/0052_auto_20191007_1921.py @@ -0,0 +1,27 @@ +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('documents', '0051_documentpage_enabled'), + ] + + operations = [ + migrations.DeleteModel( + name='DocumentPageResult', + ), + migrations.RenameModel('DocumentPage', 'DocumentVersionPage'), + migrations.AlterField( + model_name='documentversionpage', + name='document_version', + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name='version_pages', to='documents.DocumentVersion', + verbose_name='Document version' + ), + ), + ] diff --git a/mayan/apps/documents/migrations/0053_auto_20191007_1922.py b/mayan/apps/documents/migrations/0053_auto_20191007_1922.py new file mode 100644 index 0000000000..ee4175bd28 --- /dev/null +++ b/mayan/apps/documents/migrations/0053_auto_20191007_1922.py @@ -0,0 +1,27 @@ +from __future__ import unicode_literals + +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ('documents', '0052_auto_20191007_1921'), + ('ocr', '0008_auto_20180917_0646'), + ('document_parsing', '0004_auto_20180917_0645'), + ] + + operations = [ + migrations.CreateModel( + name='DocumentPageResult', + fields=[ + ], + options={ + 'verbose_name': 'Document version page', + 'verbose_name_plural': 'Document version pages', + 'ordering': ('document_version__document', 'page_number'), + 'proxy': True, + 'indexes': [], + }, + bases=('documents.documentversionpage',), + ), + ] diff --git a/mayan/apps/documents/migrations/0054_auto_20191008_1522.py b/mayan/apps/documents/migrations/0054_auto_20191008_1522.py new file mode 100644 index 0000000000..4bfca889db --- /dev/null +++ b/mayan/apps/documents/migrations/0054_auto_20191008_1522.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.22 on 2019-10-08 15:22 +from __future__ import unicode_literals + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('documents', '0053_auto_20191007_1922'), + ] + + operations = [ + migrations.DeleteModel( + name='DocumentPageResult', + ), + migrations.CreateModel( + name='DocumentVersionPageResult', + fields=[ + ], + options={ + 'verbose_name': 'Document version page', + 'verbose_name_plural': 'Document version pages', + 'ordering': ('document_version__document', 'page_number'), + 'proxy': True, + 'indexes': [], + }, + bases=('documents.documentversionpage',), + ), + ] diff --git a/mayan/apps/documents/migrations/0055_auto_20191008_2116.py b/mayan/apps/documents/migrations/0055_auto_20191008_2116.py new file mode 100644 index 0000000000..5f6ad39fa8 --- /dev/null +++ b/mayan/apps/documents/migrations/0055_auto_20191008_2116.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.22 on 2019-10-08 21:16 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('contenttypes', '0002_remove_content_type_name'), + ('documents', '0054_auto_20191008_1522'), + ] + + operations = [ + migrations.CreateModel( + name='DocumentPage', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('enabled', models.BooleanField(default=True, verbose_name='Enabled')), + ('page_number', models.PositiveIntegerField(blank=True, db_index=True, null=True, verbose_name='Page number')), + ('object_id', models.PositiveIntegerField()), + ('content_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType')), + ('document', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='pages', to='documents.Document', verbose_name='Document')), + ], + options={ + 'verbose_name': 'Document page', + 'verbose_name_plural': 'Document pages', + 'ordering': ('page_number',), + }, + ), + migrations.AlterModelOptions( + name='documentversionpage', + options={'ordering': ('page_number',), 'verbose_name': 'Document version page', 'verbose_name_plural': 'Document version pages'}, + ), + migrations.AlterField( + model_name='documentversionpage', + name='document_version', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='pages', to='documents.DocumentVersion', verbose_name='Document version'), + ), + migrations.AlterUniqueTogether( + name='documentpage', + unique_together=set([('document', 'page_number')]), + ), + ] diff --git a/mayan/apps/documents/models/__init__.py b/mayan/apps/documents/models/__init__.py index bb4b55b82a..69063382c8 100644 --- a/mayan/apps/documents/models/__init__.py +++ b/mayan/apps/documents/models/__init__.py @@ -2,4 +2,5 @@ from .document_models import * # NOQA from .document_page_models import * # NOQA from .document_type_models import * # NOQA from .document_version_models import * # NOQA +from .document_version_page_models import * # NOQA from .misc_models import * # NOQA diff --git a/mayan/apps/documents/models/document_models.py b/mayan/apps/documents/models/document_models.py index 8b6fef2400..651254166c 100644 --- a/mayan/apps/documents/models/document_models.py +++ b/mayan/apps/documents/models/document_models.py @@ -5,9 +5,10 @@ import uuid from django.apps import apps from django.core.files import File -from django.db import models +from django.db import models, transaction from django.urls import reverse from django.utils.encoding import python_2_unicode_compatible +from django.utils.functional import cached_property from django.utils.timezone import now from django.utils.translation import ugettext, ugettext_lazy as _ @@ -15,6 +16,7 @@ from ..events import ( event_document_create, event_document_properties_edit, event_document_type_change, ) +from ..literals import DOCUMENT_IMAGES_CACHE_NAME from ..managers import DocumentManager, PassthroughManager, TrashCanManager from ..settings import setting_language from ..signals import post_document_type_change @@ -102,6 +104,18 @@ class Document(models.Model): ) return RecentDocument.objects.add_document_for_user(user, self) + @cached_property + def cache(self): + Cache = apps.get_model(app_label='file_caching', model_name='Cache') + return Cache.objects.get(name=DOCUMENT_IMAGES_CACHE_NAME) + + @cached_property + def cache_partition(self): + partition, created = self.cache.partitions.get_or_create( + name='document-{}'.format(self.uuid) + ) + return partition + def delete(self, *args, **kwargs): to_trash = kwargs.pop('to_trash', True) @@ -165,6 +179,22 @@ class Document(models.Model): """ return self.latest_version.open(*args, **kwargs) + def reset_pages(self): + with transaction.atomic(): + for page in self.pages.all(): + page.delete() + + self.latest_version.update_page_count() + + for version_page in self.latest_version.pages.all(): + document_page = self.pages.create( + #content_type = models.ForeignKey( + # on_delete=models.CASCADE, to=ContentType + #) + #object_id = models.PositiveIntegerField() + content_object = version_page + ) + def restore(self): self.in_trash = False self.save() @@ -234,28 +264,31 @@ class Document(models.Model): @property def page_count(self): - return self.latest_version.page_count + return self.pages.count() + #return self.latest_version.page_count @property def pages_all(self): - try: - return self.latest_version.pages_all - except AttributeError: - # Document has no version yet - DocumentPage = apps.get_model( - app_label='documents', model_name='DocumentPage' - ) + return self.pages.all() + #try: + # return self.latest_version.pages_all + #except AttributeError: + # # Document has no version yet + # DocumentPage = apps.get_model( + # app_label='documents', model_name='DocumentPage' + # ) - return DocumentPage.objects.none() + # return DocumentPage.objects.none() @property def pages(self): - try: - return self.latest_version.pages - except AttributeError: - # Document has no version yet - DocumentPage = apps.get_model( - app_label='documents', model_name='DocumentPage' - ) + return self.pages.all() + #try: + # return self.latest_version.pages + #except AttributeError: + # # Document has no version yet + # DocumentPage = apps.get_model( + # app_label='documents', model_name='DocumentVersionPage' + # ) - return DocumentPage.objects.none() + # return DocumentPage.objects.none() diff --git a/mayan/apps/documents/models/document_page_models.py b/mayan/apps/documents/models/document_page_models.py index 00be66ab4b..87cac77f7a 100644 --- a/mayan/apps/documents/models/document_page_models.py +++ b/mayan/apps/documents/models/document_page_models.py @@ -4,14 +4,16 @@ import logging from furl import furl +from django.contrib.contenttypes.fields import GenericForeignKey +from django.contrib.contenttypes.models import ContentType from django.db import models +from django.db.models import Max from django.urls import reverse from django.utils.encoding import force_text, python_2_unicode_compatible from django.utils.functional import cached_property from django.utils.translation import ugettext_lazy as _ from mayan.apps.converter.literals import DEFAULT_ZOOM_LEVEL, DEFAULT_ROTATION - from mayan.apps.converter.models import LayerTransformation from mayan.apps.converter.transformations import ( BaseTransformation, TransformationResize, TransformationRotate, @@ -26,25 +28,32 @@ from ..settings import ( setting_zoom_min_level ) -from .document_version_models import DocumentVersion +from .document_models import Document +#from .document_version_page_models import DocumentVersionPage -__all__ = ('DocumentPage', 'DocumentPageResult') +__all__ = ('DocumentPage',)# 'DocumentPageResult') logger = logging.getLogger(__name__) @python_2_unicode_compatible class DocumentPage(models.Model): """ - Model that describes a document version page + Model that describes a document page """ - document_version = models.ForeignKey( - on_delete=models.CASCADE, related_name='version_pages', to=DocumentVersion, - verbose_name=_('Document version') + document = models.ForeignKey( + on_delete=models.CASCADE, related_name='pages', to=Document, + verbose_name=_('Document') ) enabled = models.BooleanField(default=True, verbose_name=_('Enabled')) page_number = models.PositiveIntegerField( - db_index=True, default=1, editable=False, - verbose_name=_('Page number') + db_index=True, blank=True, null=True, verbose_name=_('Page number') + ) + content_type = models.ForeignKey( + on_delete=models.CASCADE, to=ContentType + ) + object_id = models.PositiveIntegerField() + content_object = GenericForeignKey( + ct_field='content_type', fk_field='object_id' ) objects = DocumentPageManager() @@ -52,6 +61,7 @@ class DocumentPage(models.Model): class Meta: ordering = ('page_number',) + unique_together = ('document', 'page_number') verbose_name = _('Document page') verbose_name_plural = _('Document pages') @@ -60,7 +70,7 @@ class DocumentPage(models.Model): @cached_property def cache_partition(self): - partition, created = self.document_version.cache.partitions.get_or_create( + partition, created = self.document.cache.partitions.get_or_create( name=self.uuid ) return partition @@ -69,19 +79,19 @@ class DocumentPage(models.Model): self.cache_partition.delete() super(DocumentPage, self).delete(*args, **kwargs) - def detect_orientation(self): - with self.document_version.open() as file_object: - converter = get_converter_class()( - file_object=file_object, - mime_type=self.document_version.mimetype - ) - return converter.detect_orientation( - page_number=self.page_number - ) + #def detect_orientation(self): + # with self.document_version.open() as file_object: + # converter = get_converter_class()( + # file_object=file_object, + # mime_type=self.document_version.mimetype + # ) + # return converter.detect_orientation( + # page_number=self.page_number + # ) - @property - def document(self): - return self.document_version.document + #@property + #def document(self): + # return self.document_version.document def generate_image(self, user=None, **kwargs): transformation_list = self.get_combined_transformation_list(user=user, **kwargs) @@ -90,7 +100,7 @@ class DocumentPage(models.Model): # Check is transformed image is available logger.debug('transformations cache filename: %s', combined_cache_filename) - if not setting_disable_transformed_image_cache.value and self.cache_partition.get_file(filename=combined_cache_filename): + if self.cache_partition.get_file(filename=combined_cache_filename): logger.debug( 'transformations cache file "%s" found', combined_cache_filename ) @@ -128,7 +138,8 @@ class DocumentPage(models.Model): final_url.args = kwargs final_url.path = reverse( viewname='rest_api:documentpage-image', kwargs={ - 'pk': self.document.pk, 'version_pk': self.document_version.pk, + 'pk': self.document.pk, + #'version_pk': self.document_version.pk, 'page_pk': self.pk } ) @@ -190,12 +201,12 @@ class DocumentPage(models.Model): return transformation_list def get_image(self, transformations=None): - cache_filename = 'base_image' + cache_filename = 'document_page' logger.debug('Page cache filename: %s', cache_filename) cache_file = self.cache_partition.get_file(filename=cache_filename) - if not setting_disable_base_image_cache.value and cache_file: + if cache_file: logger.debug('Page cache file "%s" found', cache_filename) with cache_file.open() as file_object: @@ -216,7 +227,12 @@ class DocumentPage(models.Model): logger.debug('Page cache file "%s" not found', cache_filename) try: - with self.document_version.get_intermediate_file() as file_object: + #with self.document_version.get_intermediate_file() as file_object: + #Render or get cached document version page + self.content_object.get_image() + cache_filename = 'base_image' + cache_file = self.content_object.cache_partition.get_file(filename=cache_filename) + with cache_file.open() as file_object: converter = get_converter_class()( file_object=file_object ) @@ -241,28 +257,39 @@ class DocumentPage(models.Model): ) raise + def get_label(self): + return _( + 'Page %(page_number)d out of %(total_pages)d of %(document)s' + ) % { + 'document': force_text(self.document), + 'page_number': self.page_number, + 'total_pages': self.document.pages_all.count() + } + get_label.short_description = _('Label') + @property def is_in_trash(self): return self.document.is_in_trash - def get_label(self): - return _( - 'Page %(page_num)d out of %(total_pages)d of %(document)s' - ) % { - 'document': force_text(self.document), - 'page_num': self.page_number, - 'total_pages': self.document_version.pages_all.count() - } - get_label.short_description = _('Label') - def natural_key(self): - return (self.page_number, self.document_version.natural_key()) - natural_key.dependencies = ['documents.DocumentVersion'] + return (self.page_number, self.document.natural_key()) + natural_key.dependencies = ['documents.Document'] + + def save(self, *args, **kwargs): + if not self.page_number: + last_page_number = DocumentPage.objects.filter( + document=self.document + ).aggregate(Max('page_number'))['page_number__max'] + if last_page_number is not None: + self.page_number = last_page_number + 1 + else: + self.page_number = 1 + super(DocumentPage, self).save(*args, **kwargs) @property def siblings(self): return DocumentPage.objects.filter( - document_version=self.document_version + document=self.document ) @property @@ -271,12 +298,12 @@ class DocumentPage(models.Model): Make cache UUID a mix of version ID and page ID to avoid using stale images """ - return '{}-{}'.format(self.document_version.uuid, self.pk) + return '{}-{}'.format(self.document.uuid, self.pk) -class DocumentPageResult(DocumentPage): - class Meta: - ordering = ('document_version__document', 'page_number') - proxy = True - verbose_name = _('Document page') - verbose_name_plural = _('Document pages') +#class DocumentVersionPageResult(DocumentVersionPage): +# class Meta: +# ordering = ('document_version__document', 'page_number') +# proxy = True +# verbose_name = _('Document version page') +# verbose_name_plural = _('Document version pages') diff --git a/mayan/apps/documents/models/document_version_models.py b/mayan/apps/documents/models/document_version_models.py index 1c24fbfeda..7a398fecb1 100644 --- a/mayan/apps/documents/models/document_version_models.py +++ b/mayan/apps/documents/models/document_version_models.py @@ -246,16 +246,16 @@ class DocumentVersion(models.Model): return result - @property - def pages_all(self): - DocumentPage = apps.get_model( - app_label='documents', model_name='DocumentPage' - ) - return DocumentPage.passthrough.filter(document_version=self) + #@property + #def pages_all(self): + # DocumentPage = apps.get_model( + # app_label='documents', model_name='DocumentVersionPage' + # ) + # return DocumentPage.passthrough.filter(document_version=self) - @property - def pages(self): - return self.version_pages.all() + #@property + #def pages(self): + # return self.pages.all() @property def page_count(self): @@ -410,7 +410,7 @@ class DocumentVersion(models.Model): pass else: DocumentPage = apps.get_model( - app_label='documents', model_name='DocumentPage' + app_label='documents', model_name='DocumentVersionPage' ) with transaction.atomic(): diff --git a/mayan/apps/documents/models/document_version_page_models.py b/mayan/apps/documents/models/document_version_page_models.py new file mode 100644 index 0000000000..724de125b7 --- /dev/null +++ b/mayan/apps/documents/models/document_version_page_models.py @@ -0,0 +1,284 @@ +from __future__ import absolute_import, unicode_literals + +import logging + +from furl import furl + +from django.db import models +from django.urls import reverse +from django.utils.encoding import force_text, python_2_unicode_compatible +from django.utils.functional import cached_property +from django.utils.translation import ugettext_lazy as _ + +from mayan.apps.converter.literals import DEFAULT_ZOOM_LEVEL, DEFAULT_ROTATION + +from mayan.apps.converter.models import LayerTransformation +from mayan.apps.converter.transformations import ( + BaseTransformation, TransformationResize, TransformationRotate, + TransformationZoom +) +from mayan.apps.converter.utils import get_converter_class + +from ..managers import DocumentVersionPageManager +from ..settings import ( + setting_disable_base_image_cache, setting_disable_transformed_image_cache, + setting_display_width, setting_display_height, setting_zoom_max_level, + setting_zoom_min_level +) + +from .document_version_models import DocumentVersion + +__all__ = ('DocumentVersionPage', 'DocumentVersionPageResult') +logger = logging.getLogger(__name__) + + +@python_2_unicode_compatible +class DocumentVersionPage(models.Model): + """ + Model that describes a document version page + """ + document_version = models.ForeignKey( + on_delete=models.CASCADE, related_name='pages', to=DocumentVersion, + verbose_name=_('Document version') + ) + enabled = models.BooleanField(default=True, verbose_name=_('Enabled')) + page_number = models.PositiveIntegerField( + db_index=True, default=1, editable=False, + verbose_name=_('Page number') + ) + + objects = DocumentVersionPageManager() + #passthrough = models.Manager() + + class Meta: + ordering = ('page_number',) + verbose_name = _('Document version page') + verbose_name_plural = _('Document version pages') + + def __str__(self): + return self.get_label() + + @cached_property + def cache_partition(self): + partition, created = self.document_version.cache.partitions.get_or_create( + name=self.uuid + ) + return partition + + def delete(self, *args, **kwargs): + self.cache_partition.delete() + super(DocumentVersionPage, self).delete(*args, **kwargs) + + #def detect_orientation(self): + # with self.document_version.open() as file_object: + # converter = get_converter_class()( + # file_object=file_object, + # mime_type=self.document_version.mimetype + # ) + # return converter.detect_orientation( + # page_number=self.page_number + # ) + + @property + def document(self): + return self.document_version.document + + def generate_image(self, user=None, **kwargs): + transformation_list = self.get_combined_transformation_list(user=user, **kwargs) + combined_cache_filename = BaseTransformation.combine(transformation_list) + + # Check is transformed image is available + logger.debug('transformations cache filename: %s', combined_cache_filename) + + if self.cache_partition.get_file(filename=combined_cache_filename): + logger.debug( + 'transformations cache file "%s" found', combined_cache_filename + ) + else: + logger.debug( + 'transformations cache file "%s" not found', combined_cache_filename + ) + image = self.get_image(transformations=transformation_list) + with self.cache_partition.create_file(filename=combined_cache_filename) as file_object: + file_object.write(image.getvalue()) + + return combined_cache_filename + + #def get_absolute_url(self): + # return reverse( + # viewname='documents:document_version_page_view', kwargs={ + # 'pk': self.pk + # } + # ) + + def get_api_image_url(self, *args, **kwargs): + """ + Create an unique URL combining: + - the page's image URL + - the interactive argument + - a hash from the server side and interactive transformations + The purpose of this unique URL is to allow client side caching + if document page images. + """ + transformations_hash = BaseTransformation.combine( + self.get_combined_transformation_list(*args, **kwargs) + ) + + kwargs.pop('transformations', None) + + final_url = furl() + final_url.args = kwargs + final_url.path = reverse( + viewname='rest_api:documentversionpage-image', kwargs={ + 'pk': self.document.pk, 'version_pk': self.document_version.pk, + 'page_pk': self.pk + } + ) + final_url.args['_hash'] = transformations_hash + + return final_url.tostr() + + def get_combined_transformation_list(self, user=None, *args, **kwargs): + """ + Return a list of transformation containing the server side + document page transformation as well as tranformations created + from the arguments as transient interactive transformation. + """ + # Convert arguments into transformations + transformations = kwargs.get('transformations', []) + + # Set sensible defaults if the argument is not specified or if the + # argument is None + width = kwargs.get('width', setting_display_width.value) or setting_display_width.value + height = kwargs.get('height', setting_display_height.value) or setting_display_height.value + rotation = kwargs.get('rotation', DEFAULT_ROTATION) or DEFAULT_ROTATION + zoom_level = kwargs.get('zoom', DEFAULT_ZOOM_LEVEL) or DEFAULT_ZOOM_LEVEL + + if zoom_level < setting_zoom_min_level.value: + zoom_level = setting_zoom_min_level.value + + if zoom_level > setting_zoom_max_level.value: + zoom_level = setting_zoom_max_level.value + + # Generate transformation hash + transformation_list = [] + + maximum_layer_order = kwargs.get('maximum_layer_order', None) + + # Stored transformations first + for stored_transformation in LayerTransformation.objects.get_for_object( + self, maximum_layer_order=maximum_layer_order, as_classes=True, + user=user + ): + transformation_list.append(stored_transformation) + + # Interactive transformations second + for transformation in transformations: + transformation_list.append(transformation) + + if rotation: + transformation_list.append( + TransformationRotate(degrees=rotation) + ) + + if width: + transformation_list.append( + TransformationResize(width=width, height=height) + ) + + if zoom_level: + transformation_list.append(TransformationZoom(percent=zoom_level)) + + return transformation_list + + def get_image(self, transformations=None): + cache_filename = 'base_image' + logger.debug('Page cache filename: %s', cache_filename) + + cache_file = self.cache_partition.get_file(filename=cache_filename) + + if cache_file: + logger.debug('Page cache file "%s" found', cache_filename) + + with cache_file.open() as file_object: + converter = get_converter_class()( + file_object=file_object + ) + + converter.seek_page(page_number=0) + + # This code is also repeated below to allow using a context + # manager with cache_file.open and close it automatically. + # Apply runtime transformations + for transformation in transformations or []: + converter.transform(transformation=transformation) + + return converter.get_page() + else: + logger.debug('Page cache file "%s" not found', cache_filename) + + try: + with self.document_version.get_intermediate_file() as file_object: + converter = get_converter_class()( + file_object=file_object + ) + converter.seek_page(page_number=self.page_number - 1) + + page_image = converter.get_page() + + # Since open "wb+" doesn't create files, create it explicitly + with self.cache_partition.create_file(filename=cache_filename) as file_object: + file_object.write(page_image.getvalue()) + + # Apply runtime transformations + for transformation in transformations or []: + converter.transform(transformation=transformation) + + return converter.get_page() + except Exception as exception: + # Cleanup in case of error + logger.error( + 'Error creating page cache file "%s"; %s', + cache_filename, exception + ) + raise + + #@property + #def is_in_trash(self): + # return self.document.is_in_trash + + def get_label(self): + return _( + 'Version page %(page_number)d out of %(total_pages)d of %(document)s' + ) % { + 'document': force_text(self.document), + 'page_number': self.page_number, + 'total_pages': self.document_version.pages.count() + } + get_label.short_description = _('Label') + + def natural_key(self): + return (self.page_number, self.document_version.natural_key()) + natural_key.dependencies = ['documents.DocumentVersion'] + + #@property + #def siblings(self): + # return DocumentVersionPage.objects.filter( + # document_version=self.document_version + # ) + + @property + def uuid(self): + """ + Make cache UUID a mix of version ID and page ID to avoid using stale + images + """ + return '{}-{}'.format(self.document_version.uuid, self.pk) + + +class DocumentVersionPageResult(DocumentVersionPage): + class Meta: + ordering = ('document_version__document', 'page_number') + proxy = True + verbose_name = _('Document version page') + verbose_name_plural = _('Document version pages') diff --git a/mayan/apps/documents/queues.py b/mayan/apps/documents/queues.py index 5ac0fa5f01..a98b0e757f 100644 --- a/mayan/apps/documents/queues.py +++ b/mayan/apps/documents/queues.py @@ -30,6 +30,10 @@ queue_converter.add_task_type( dotted_path='mayan.apps.documents.tasks.task_generate_document_page_image', label=_('Generate document page image') ) +queue_converter.add_task_type( + dotted_path='mayan.apps.documents.tasks.task_generate_document_version_page_image', + label=_('Generate document version page image') +) queue_documents.add_task_type( dotted_path='mayan.apps.documents.tasks.task_delete_document', @@ -66,6 +70,10 @@ queue_tools.add_task_type( label=_('Duplicated document scan') ) +queue_uploads.add_task_type( + dotted_path='mayan.apps.documents.tasks.task_document_reset_pages', + label=_('Reset document pages') +) queue_uploads.add_task_type( dotted_path='mayan.apps.documents.tasks.task_update_page_count', label=_('Update document page count') diff --git a/mayan/apps/documents/search.py b/mayan/apps/documents/search.py index de03a36dd0..20044afadd 100644 --- a/mayan/apps/documents/search.py +++ b/mayan/apps/documents/search.py @@ -20,7 +20,7 @@ def transformation_format_uuid(term_string): def get_queryset_page_search_queryset(): # Ignore documents in trash can DocumentPage = apps.get_model( - app_label='documents', model_name='DocumentPage' + app_label='documents', model_name='DocumentVersionPage' ) return DocumentPage.objects.filter(document_version__document__in_trash=False) @@ -49,7 +49,7 @@ document_search.add_model_field( document_page_search = SearchModel( app_label='documents', list_mode=LIST_MODE_CHOICE_ITEM, - model_name='DocumentPage', permission=permission_document_view, + model_name='DocumentVersionPage', permission=permission_document_view, queryset=get_queryset_page_search_queryset, serializer_path='mayan.apps.documents.serializers.DocumentPageSerializer' ) diff --git a/mayan/apps/documents/serializers.py b/mayan/apps/documents/serializers.py index 9632a546cb..5a6c172f7a 100644 --- a/mayan/apps/documents/serializers.py +++ b/mayan/apps/documents/serializers.py @@ -8,42 +8,40 @@ from rest_framework.reverse import reverse from mayan.apps.common.models import SharedUploadedFile from .models import ( - Document, DocumentVersion, DocumentPage, DocumentType, - DocumentTypeFilename, RecentDocument + Document, DocumentPage, DocumentType, DocumentTypeFilename, + DocumentVersion, DocumentVersionPage, RecentDocument ) from .settings import setting_language from .tasks import task_upload_new_version class DocumentPageSerializer(serializers.HyperlinkedModelSerializer): - document_version_url = serializers.SerializerMethodField() + document_url = serializers.SerializerMethodField() image_url = serializers.SerializerMethodField() url = serializers.SerializerMethodField() class Meta: - fields = ('document_version_url', 'image_url', 'page_number', 'url') + fields = ('document_url', 'image_url', 'page_number', 'url') model = DocumentPage - def get_document_version_url(self, instance): + def get_document_url(self, instance): return reverse( - viewname='rest_api:documentversion-detail', args=( - instance.document.pk, instance.document_version.pk, + viewname='rest_api:document-detail', args=( + instance.document.pk, ), request=self.context['request'], format=self.context['format'] ) def get_image_url(self, instance): return reverse( viewname='rest_api:documentpage-image', args=( - instance.document.pk, instance.document_version.pk, - instance.pk, + instance.document.pk, instance.pk, ), request=self.context['request'], format=self.context['format'] ) def get_url(self, instance): return reverse( viewname='rest_api:documentpage-detail', args=( - instance.document.pk, instance.document_version.pk, - instance.pk, + instance.document.pk, instance.pk, ), request=self.context['request'], format=self.context['format'] ) @@ -97,6 +95,39 @@ class WritableDocumentTypeSerializer(serializers.ModelSerializer): return obj.documents.count() +class DocumentVersionPageSerializer(serializers.HyperlinkedModelSerializer): + document_version_url = serializers.SerializerMethodField() + image_url = serializers.SerializerMethodField() + url = serializers.SerializerMethodField() + + class Meta: + fields = ('document_version_url', 'image_url', 'page_number', 'url') + model = DocumentVersionPage + + def get_document_version_url(self, instance): + return reverse( + viewname='rest_api:documentversion-detail', args=( + instance.document.pk, instance.document_version.pk, + ), request=self.context['request'], format=self.context['format'] + ) + + def get_image_url(self, instance): + return reverse( + viewname='rest_api:documentversionpage-image', args=( + instance.document.pk, instance.document_version.pk, + instance.pk, + ), request=self.context['request'], format=self.context['format'] + ) + + def get_url(self, instance): + return reverse( + viewname='rest_api:documentversionpage-detail', args=( + instance.document.pk, instance.document_version.pk, + instance.pk, + ), request=self.context['request'], format=self.context['format'] + ) + + class DocumentVersionSerializer(serializers.HyperlinkedModelSerializer): document_url = serializers.SerializerMethodField() download_url = serializers.SerializerMethodField() diff --git a/mayan/apps/documents/statistics.py b/mayan/apps/documents/statistics.py index 2f1059e7d1..94d91be994 100644 --- a/mayan/apps/documents/statistics.py +++ b/mayan/apps/documents/statistics.py @@ -41,7 +41,7 @@ def new_documents_per_month(): def new_document_pages_per_month(): DocumentPage = apps.get_model( - app_label='documents', model_name='DocumentPage' + app_label='documents', model_name='DocumentVersionPage' ) qss = qsstats.QuerySetStats( @@ -106,7 +106,7 @@ def new_document_pages_this_month(user=None): app_label='acls', model_name='AccessControlList' ) DocumentPage = apps.get_model( - app_label='documents', model_name='DocumentPage' + app_label='documents', model_name='DocumentVersionPage' ) queryset = DocumentPage.objects.all() @@ -195,7 +195,7 @@ def total_document_version_per_month(): def total_document_page_per_month(): DocumentPage = apps.get_model( - app_label='documents', model_name='DocumentPage' + app_label='documents', model_name='DocumentVersionPage' ) qss = qsstats.QuerySetStats( diff --git a/mayan/apps/documents/tasks.py b/mayan/apps/documents/tasks.py index 34bfad3616..f62a015538 100644 --- a/mayan/apps/documents/tasks.py +++ b/mayan/apps/documents/tasks.py @@ -9,7 +9,8 @@ from django.db import OperationalError from mayan.celery import app from .literals import ( - UPDATE_PAGE_COUNT_RETRY_DELAY, UPLOAD_NEW_VERSION_RETRY_DELAY + RETRY_DELAY_DOCUMENT_RESET_PAGES, UPDATE_PAGE_COUNT_RETRY_DELAY, + UPLOAD_NEW_VERSION_RETRY_DELAY ) logger = logging.getLogger(__name__) @@ -64,6 +65,25 @@ def task_delete_stubs(): logger.info(msg='Finshed') +@app.task(bind=True, default_retry_delay=RETRY_DELAY_DOCUMENT_RESET_PAGES, ignore_result=True) +def task_document_reset_pages(self, document_id): + Document = apps.get_model( + app_label='documents', model_name='Document' + ) + + document = Document.objects.get(pk=document_id) + + try: + document.reset_pages() + except OperationalError as exception: + logger.warning( + 'Operational error during attempt to reset pages for ' + 'document: %s; %s. Retrying.', document, + exception + ) + raise self.retry(exc=exception) + + @app.task() def task_generate_document_page_image(document_page_id, user_id=None, **kwargs): DocumentPage = apps.get_model( @@ -80,6 +100,22 @@ def task_generate_document_page_image(document_page_id, user_id=None, **kwargs): return document_page.generate_image(user=user, **kwargs) +@app.task() +def task_generate_document_version_page_image(document_version_page_id, user_id=None, **kwargs): + DocumentVersionPage = apps.get_model( + app_label='documents', model_name='DocumentVersionPage' + ) + User = get_user_model() + + if user_id: + user = User.objects.get(pk=user_id) + else: + user = None + + document_version_page = DocumentVersionPage.objects.get(pk=document_version_page_id) + return document_version_page.generate_image(user=user, **kwargs) + + @app.task(ignore_result=True) def task_scan_duplicates_all(): DuplicatedDocument = apps.get_model( diff --git a/mayan/apps/documents/urls.py b/mayan/apps/documents/urls.py index 84aec25094..ed5219f230 100644 --- a/mayan/apps/documents/urls.py +++ b/mayan/apps/documents/urls.py @@ -4,13 +4,15 @@ from django.conf.urls import url from .api_views import ( APITrashedDocumentListView, APIDeletedDocumentRestoreView, - APIDeletedDocumentView, APIDocumentDownloadView, APIDocumentView, - APIDocumentListView, APIDocumentVersionDownloadView, + APIDeletedDocumentView, APIDocumentDownloadView, APIDocumentPageListView, + APIDocumentView, APIDocumentListView, APIDocumentVersionDownloadView, APIDocumentPageImageView, APIDocumentPageView, APIDocumentTypeDocumentListView, APIDocumentTypeListView, APIDocumentTypeView, APIDocumentVersionsListView, APIDocumentVersionPageListView, APIDocumentVersionView, - APIRecentDocumentListView + APIRecentDocumentListView, + APIDocumentVersionPageView, + APIDocumentVersionPageImageView ) from .views.document_views import ( DocumentDocumentTypeEditView, DocumentDownloadFormView, @@ -405,6 +407,11 @@ api_urls = [ view=APIDocumentVersionPageListView.as_view(), name='documentversion-page-list' ), + url( + regex=r'^documents/(?P[0-9]+)/pages/$', + view=APIDocumentPageListView.as_view(), + name='document-page-list' + ), url( regex=r'^documents/(?P[0-9]+)/versions/(?P[0-9]+)/download/$', view=APIDocumentVersionDownloadView.as_view(), @@ -416,12 +423,20 @@ api_urls = [ ), url( regex=r'^documents/(?P[0-9]+)/versions/(?P[0-9]+)/pages/(?P[0-9]+)$', + view=APIDocumentVersionPageView.as_view(), name='documentversionpage-detail' + ), + url( + regex=r'^documents/(?P[0-9]+)/pages/(?P[0-9]+)$', view=APIDocumentPageView.as_view(), name='documentpage-detail' ), url( - regex=r'^documents/(?P[0-9]+)/versions/(?P[0-9]+)/pages/(?P[0-9]+)/image/$', + regex=r'^documents/(?P[0-9]+)/pages/(?P[0-9]+)/image/$', view=APIDocumentPageImageView.as_view(), name='documentpage-image' ), + url( + regex=r'^documents/(?P[0-9]+)/versions/(?P[0-9]+)/pages/(?P[0-9]+)/image/$', + view=APIDocumentVersionPageImageView.as_view(), name='documentversionpage-image' + ), url( regex=r'^trashed_documents/$', view=APITrashedDocumentListView.as_view(), name='trasheddocument-list' diff --git a/mayan/apps/documents/views/document_page_views.py b/mayan/apps/documents/views/document_page_views.py index f05767d627..614cba4585 100644 --- a/mayan/apps/documents/views/document_page_views.py +++ b/mayan/apps/documents/views/document_page_views.py @@ -21,7 +21,7 @@ from mayan.apps.converter.literals import DEFAULT_ROTATION, DEFAULT_ZOOM_LEVEL from ..forms import DocumentPageForm from ..icons import icon_document_pages from ..links import link_document_update_page_count -from ..models import Document, DocumentPage +from ..models import Document, DocumentVersionPage from ..permissions import permission_document_edit, permission_document_view from ..settings import ( setting_rotation_step, setting_zoom_percent_step, setting_zoom_max_level, @@ -69,7 +69,7 @@ class DocumentPageListView(ExternalObjectMixin, SingleObjectListView): class DocumentPageNavigationBase(ExternalObjectMixin, RedirectView): - external_object_class = DocumentPage + external_object_class = DocumentVersionPage external_object_permission = permission_document_view external_object_pk_url_kwarg = 'pk' @@ -164,7 +164,7 @@ class DocumentPageNavigationPrevious(DocumentPageNavigationBase): class DocumentPageView(ExternalObjectMixin, SimpleView): - external_object_class = DocumentPage + external_object_class = DocumentVersionPage external_object_permission = permission_document_view external_object_pk_url_kwarg = 'pk' template_name = 'appearance/generic_form.html' @@ -204,7 +204,7 @@ class DocumentPageViewResetView(RedirectView): class DocumentPageInteractiveTransformation(ExternalObjectMixin, RedirectView): - external_object_class = DocumentPage + external_object_class = DocumentVersionPage external_object_permission = permission_document_view external_object_pk_url_kwarg = 'pk' @@ -289,7 +289,7 @@ class DocumentPageDisable(MultipleObjectConfirmActionView): return result def get_source_queryset(self): - return DocumentPage.passthrough.all() + return DocumentVersionPage.passthrough.all() def object_action(self, form, instance): instance.enabled = False @@ -319,7 +319,7 @@ class DocumentPageEnable(MultipleObjectConfirmActionView): return result def get_source_queryset(self): - return DocumentPage.passthrough.all() + return DocumentVersionPage.passthrough.all() def object_action(self, form, instance): instance.enabled = True diff --git a/mayan/apps/file_caching/migrations/0003_auto_20191008_1510.py b/mayan/apps/file_caching/migrations/0003_auto_20191008_1510.py new file mode 100644 index 0000000000..14382e0479 --- /dev/null +++ b/mayan/apps/file_caching/migrations/0003_auto_20191008_1510.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.22 on 2019-10-08 15:10 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('file_caching', '0002_auto_20190729_0236'), + ] + + operations = [ + migrations.AlterField( + model_name='cache', + name='name', + field=models.CharField(db_index=True, help_text='Internal name of the cache.', max_length=128, unique=True, verbose_name='Name'), + ), + ] diff --git a/mayan/apps/metadata/apps.py b/mayan/apps/metadata/apps.py index 44fd6bb546..5856484193 100644 --- a/mayan/apps/metadata/apps.py +++ b/mayan/apps/metadata/apps.py @@ -76,7 +76,7 @@ class MetadataApp(MayanAppConfig): app_label='documents', model_name='Document' ) DocumentPageResult = apps.get_model( - app_label='documents', model_name='DocumentPageResult' + app_label='documents', model_name='DocumentVersionPageResult' ) DocumentType = apps.get_model( diff --git a/mayan/apps/ocr/apps.py b/mayan/apps/ocr/apps.py index 33b4f1893b..8ec9be9800 100644 --- a/mayan/apps/ocr/apps.py +++ b/mayan/apps/ocr/apps.py @@ -62,7 +62,7 @@ class OCRApp(MayanAppConfig): app_label='documents', model_name='Document' ) DocumentPage = apps.get_model( - app_label='documents', model_name='DocumentPage' + app_label='documents', model_name='DocumentVersionPage' ) DocumentType = apps.get_model( app_label='documents', model_name='DocumentType' @@ -96,9 +96,9 @@ class OCRApp(MayanAppConfig): ) ) - ModelField( - model=Document, name='versions__version_pages__ocr_content__content' - ) + #ModelField( + # model=Document, name='versions__pages__ocr_content__content' + #) ModelPermission.register( model=Document, permissions=( diff --git a/mayan/apps/ocr/models.py b/mayan/apps/ocr/models.py index 9f42843a7e..8402d2079c 100644 --- a/mayan/apps/ocr/models.py +++ b/mayan/apps/ocr/models.py @@ -4,7 +4,9 @@ from django.db import models from django.utils.encoding import force_text, python_2_unicode_compatible from django.utils.translation import ugettext_lazy as _ -from mayan.apps.documents.models import DocumentPage, DocumentType, DocumentVersion +from mayan.apps.documents.models import ( + DocumentPage, DocumentType, DocumentVersion +) from .managers import ( DocumentPageOCRContentManager, DocumentTypeSettingsManager diff --git a/mayan/apps/ocr/tasks.py b/mayan/apps/ocr/tasks.py index a6d9c82efb..4ba1f5268f 100644 --- a/mayan/apps/ocr/tasks.py +++ b/mayan/apps/ocr/tasks.py @@ -19,8 +19,8 @@ def task_do_ocr(self, document_version_pk): DocumentVersion = apps.get_model( app_label='documents', model_name='DocumentVersion' ) - DocumentPageOCRContent = apps.get_model( - app_label='ocr', model_name='DocumentPageOCRContent' + DocumentVersionPageOCRContent = apps.get_model( + app_label='ocr', model_name='DocumentVersionPageOCRContent' ) lock_id = 'task_do_ocr_doc_version-%d' % document_version_pk @@ -39,7 +39,7 @@ def task_do_ocr(self, document_version_pk): 'Starting document OCR for document version: %s', document_version ) - DocumentPageOCRContent.objects.process_document_version( + DocumentVersionPageOCRContent.objects.process_document_version( document_version=document_version ) except OperationalError as exception: diff --git a/mayan/apps/ocr/tests/test_api.py b/mayan/apps/ocr/tests/test_api.py index e847550141..95c03e288a 100644 --- a/mayan/apps/ocr/tests/test_api.py +++ b/mayan/apps/ocr/tests/test_api.py @@ -65,7 +65,7 @@ class OCRAPITestCase(DocumentTestMixin, BaseAPITestCase): hasattr(self.test_document.pages.first(), 'ocr_content') ) - def _request_document_page_content_view(self): + def _request_document_version_page_content_view(self): return self.get( viewname='rest_api:document-page-ocr-content-view', kwargs={ 'document_pk': self.test_document.pk, @@ -75,7 +75,7 @@ class OCRAPITestCase(DocumentTestMixin, BaseAPITestCase): ) def test_get_document_version_page_content_no_access(self): - response = self._request_document_page_content_view() + response = self._request_document_version_page_content_view() self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) def test_get_document_version_page_content_with_access(self): @@ -83,7 +83,7 @@ class OCRAPITestCase(DocumentTestMixin, BaseAPITestCase): self.grant_access( permission=permission_ocr_content_view, obj=self.test_document ) - response = self._request_document_page_content_view() + response = self._request_document_version_page_content_view() self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertTrue( TEST_DOCUMENT_CONTENT in response.data['content'] diff --git a/mayan/apps/ocr/tests/test_events.py b/mayan/apps/ocr/tests/test_events.py index 4a4318ec64..c51cee8fad 100644 --- a/mayan/apps/ocr/tests/test_events.py +++ b/mayan/apps/ocr/tests/test_events.py @@ -8,13 +8,13 @@ from ..events import ( event_ocr_document_content_deleted, event_ocr_document_version_submit, event_ocr_document_version_finish ) -from ..models import DocumentPageOCRContent +from ..models import DocumentVersionPageOCRContent class OCREventsTestCase(GenericDocumentTestCase): def test_document_content_deleted_event(self): Action.objects.all().delete() - DocumentPageOCRContent.objects.delete_content_for( + DocumentVersionPageOCRContent.objects.delete_content_for( document=self.test_document ) diff --git a/mayan/apps/ocr/tests/test_views.py b/mayan/apps/ocr/tests/test_views.py index 6ed278c54a..d7759fbfc1 100644 --- a/mayan/apps/ocr/tests/test_views.py +++ b/mayan/apps/ocr/tests/test_views.py @@ -2,7 +2,7 @@ from __future__ import unicode_literals from mayan.apps.documents.tests.base import GenericDocumentViewTestCase -from ..models import DocumentPageOCRContent +from ..models import DocumentVersionPageOCRContent from ..permissions import ( permission_ocr_content_view, permission_ocr_document, permission_document_type_ocr_setup @@ -27,9 +27,9 @@ class OCRViewTestMixin(object): } ) - def _request_document_page_content_view(self): + def _request_document_version_page_content_view(self): return self.get( - viewname='ocr:document_page_ocr_content', kwargs={ + viewname='ocr:document_version_page_ocr_content', kwargs={ 'pk': self.test_document.pages.first().pk } ) @@ -86,8 +86,8 @@ class OCRViewsTestCase(OCRViewTestMixin, GenericDocumentViewTestCase): self.assertEqual(response.status_code, 404) self.assertTrue( - DocumentPageOCRContent.objects.filter( - document_page=self.test_document.pages.first() + DocumentVersionPageOCRContent.objects.filter( + document_version_page=self.test_document.pages.first() ).exists() ) @@ -101,28 +101,11 @@ class OCRViewsTestCase(OCRViewTestMixin, GenericDocumentViewTestCase): self.assertEqual(response.status_code, 302) self.assertFalse( - DocumentPageOCRContent.objects.filter( - document_page=self.test_document.pages.first() + DocumentVersionPageOCRContent.objects.filter( + document_version_page=self.test_document.pages.first() ).exists() ) - def test_document_page_content_view_no_permissions(self): - self.test_document.submit_for_ocr() - - response = self._request_document_page_content_view() - self.assertEqual(response.status_code, 404) - - def test_document_page_content_view_with_access(self): - self.test_document.submit_for_ocr() - self.grant_access( - obj=self.test_document, permission=permission_ocr_content_view - ) - - response = self._request_document_page_content_view() - self.assertContains( - response=response, text=TEST_DOCUMENT_CONTENT, status_code=200 - ) - def test_document_submit_view_no_permission(self): response = self._request_document_submit_view() self.assertEqual(response.status_code, 404) @@ -188,6 +171,23 @@ class OCRViewsTestCase(OCRViewTestMixin, GenericDocumentViewTestCase): ), ) + def test_document_version_page_content_view_no_permissions(self): + self.test_document.submit_for_ocr() + + response = self._request_document_version_page_content_view() + self.assertEqual(response.status_code, 404) + + def test_document_version_page_content_view_with_access(self): + self.test_document.submit_for_ocr() + self.grant_access( + obj=self.test_document, permission=permission_ocr_content_view + ) + + response = self._request_document_version_page_content_view() + self.assertContains( + response=response, text=TEST_DOCUMENT_CONTENT, status_code=200 + ) + class DocumentTypeOCRViewTestMixin(object): def _request_document_type_ocr_settings_view(self): diff --git a/mayan/apps/ocr/views.py b/mayan/apps/ocr/views.py index 41b978388e..1524b3d006 100644 --- a/mayan/apps/ocr/views.py +++ b/mayan/apps/ocr/views.py @@ -12,7 +12,9 @@ from mayan.apps.common.generics import ( ) from mayan.apps.common.mixins import ExternalObjectMixin from mayan.apps.documents.forms import DocumentTypeFilteredSelectForm -from mayan.apps.documents.models import Document, DocumentPage, DocumentType +from mayan.apps.documents.models import ( + Document, DocumentType, DocumentVersionPage +) from .forms import DocumentPageOCRContentForm, DocumentOCRContentForm from .models import DocumentPageOCRContent, DocumentVersionOCRError @@ -74,7 +76,7 @@ class DocumentOCRContentView(SingleObjectDetailView): class DocumentPageOCRContentView(SingleObjectDetailView): form_class = DocumentPageOCRContentForm - model = DocumentPage + model = DocumentVersionPage object_permission = permission_ocr_content_view def dispatch(self, request, *args, **kwargs): diff --git a/mayan/apps/redactions/apps.py b/mayan/apps/redactions/apps.py index a7460b817d..732262545f 100644 --- a/mayan/apps/redactions/apps.py +++ b/mayan/apps/redactions/apps.py @@ -28,7 +28,7 @@ class RedactionsApp(MayanAppConfig): super(RedactionsApp, self).ready() DocumentPage = apps.get_model( - app_label='documents', model_name='DocumentPage' + app_label='documents', model_name='DocumentVersionPage' ) link_redaction_list = link_transformation_list.copy( diff --git a/mayan/apps/tags/apps.py b/mayan/apps/tags/apps.py index c64811558a..f3d36c1611 100644 --- a/mayan/apps/tags/apps.py +++ b/mayan/apps/tags/apps.py @@ -62,7 +62,7 @@ class TagsApp(MayanAppConfig): ) DocumentPageResult = apps.get_model( - app_label='documents', model_name='DocumentPageResult' + app_label='documents', model_name='DocumentVersionPageResult' ) DocumentTag = self.get_model(model_name='DocumentTag')