From 57e7722f597dd42661e69ad2216a5b55ac345c0c Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Mon, 2 Apr 2018 01:45:30 -0400 Subject: [PATCH] Don't show documents with 0 duplicates in the duplicated document list. Also clean up the duplicated document model after a document is deleted. Fix queue name typo. Signed-off-by: Roberto Rosario --- HISTORY.rst | 2 + docs/releases/3.0.rst | 12 +++++ mayan/apps/documents/apps.py | 32 ++++++++----- mayan/apps/documents/handlers.py | 6 ++- mayan/apps/documents/managers.py | 15 ++++++ mayan/apps/documents/queues.py | 25 ++++++---- mayan/apps/documents/tasks.py | 8 ++++ mayan/apps/documents/tests/base.py | 47 +++++++++++++++--- mayan/apps/documents/tests/test_models.py | 50 ++++++++++---------- mayan/apps/documents/views/document_views.py | 6 +-- 10 files changed, 144 insertions(+), 59 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index f2154f6caf..34a867cea0 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -101,6 +101,8 @@ - Add the "to=" keyword argument to all ForeignKey, ManayToMany and OneToOne Fields. - Add Makefile target to check the format of the README.rst file. - Mark the feature to detect and fix the orientatin of PDF as experimental. +- Don't show documents with 0 duplicates in the duplicated document list. +- Clean up the duplicated document model after a document is deleted. 2.7.3 (2017-09-11) ================== diff --git a/docs/releases/3.0.rst b/docs/releases/3.0.rst index c2738f8a93..a231a24cd4 100644 --- a/docs/releases/3.0.rst +++ b/docs/releases/3.0.rst @@ -260,6 +260,18 @@ project. MERCs 1 and 2 have been approved. MERC-1 outlines the MERC process itself and MERC-2 documents the way API tests are to be written for Mayan EDMS. +Duplicated documents +-------------------- +The duplicated documents system has been improved to also better detect when +the duplicate of a primary document has been move to the trash. In this +instance the duplicate count of the primary document would be zero and will +cause the primary document to not show in the duplicated document list view. + +If the duplicated document is deleted from the trash the system now will launch +a background clean up task to permanently delete the empty primary document's +duplicate document entry from the database. + + Other changes worth mentioning ------------------------------ - Add Makefile target to check the format of the README.rst file. diff --git a/mayan/apps/documents/apps.py b/mayan/apps/documents/apps.py index 4f68dd4217..79fff3962f 100644 --- a/mayan/apps/documents/apps.py +++ b/mayan/apps/documents/apps.py @@ -4,6 +4,7 @@ from datetime import timedelta from kombu import Exchange, Queue +from django.db.models.signals import post_delete from django.utils.translation import ugettext_lazy as _ from acls import ModelPermission @@ -46,7 +47,8 @@ from .events import ( event_document_view ) from .handlers import ( - create_default_document_type, handler_scan_duplicates_for + create_default_document_type, handler_remove_empty_duplicates_lists, + handler_scan_duplicates_for, ) from .links import ( link_clear_image_cache, link_document_clear_transformations, @@ -359,29 +361,32 @@ class DocumentsApp(MayanAppConfig): 'documents.tasks.task_check_trash_periods': { 'queue': 'documents_periodic' }, - 'documents.tasks.task_delete_stubs': { - 'queue': 'documents_periodic' + 'documents.tasks.task_clean_empty_duplicate_lists': { + 'queue': 'documents' }, 'documents.tasks.task_clear_image_cache': { 'queue': 'tools' }, + 'documents.tasks.task_delete_document': { + 'queue': 'documents' + }, + 'documents.tasks.task_delete_stubs': { + 'queue': 'documents_periodic' + }, 'documents.tasks.task_generate_document_page_image': { 'queue': 'converter' }, - 'documents.tasks.task_update_page_count': { - 'queue': 'uploads' - }, - 'documents.tasks.task_upload_new_version': { - 'queue': 'uploads' - }, 'documents.tasks.task_scan_duplicates_all': { 'queue': 'tools' }, 'documents.tasks.task_scan_duplicates_for': { 'queue': 'uploads' }, - 'documents.tasks.task_delete_document': { - 'queue': 'documents' + 'documents.tasks.task_update_page_count': { + 'queue': 'uploads' + }, + 'documents.tasks.task_upload_new_version': { + 'queue': 'uploads' }, } ) @@ -576,6 +581,11 @@ class DocumentsApp(MayanAppConfig): minute='0' ) + post_delete.connect( + dispatch_uid='handler_remove_empty_duplicates_lists', + receiver=handler_remove_empty_duplicates_lists, + sender=Document, + ) post_initial_setup.connect( create_default_document_type, dispatch_uid='create_default_document_type' diff --git a/mayan/apps/documents/handlers.py b/mayan/apps/documents/handlers.py index c0137f20fd..62fa444a0e 100644 --- a/mayan/apps/documents/handlers.py +++ b/mayan/apps/documents/handlers.py @@ -4,7 +4,7 @@ from django.apps import apps from .literals import DEFAULT_DOCUMENT_TYPE_LABEL from .signals import post_initial_document_type -from .tasks import task_scan_duplicates_for +from .tasks import task_clean_empty_duplicate_lists, task_scan_duplicates_for def create_default_document_type(sender, **kwargs): @@ -25,3 +25,7 @@ def handler_scan_duplicates_for(sender, instance, **kwargs): task_scan_duplicates_for.apply_async( kwargs={'document_id': instance.document.pk} ) + + +def handler_remove_empty_duplicates_lists(sender, **kwargs): + task_clean_empty_duplicate_lists.apply_async() diff --git a/mayan/apps/documents/managers.py b/mayan/apps/documents/managers.py index 652f32d134..a783670af0 100644 --- a/mayan/apps/documents/managers.py +++ b/mayan/apps/documents/managers.py @@ -100,6 +100,19 @@ class DocumentTypeManager(models.Manager): class DuplicatedDocumentManager(models.Manager): + def clean_empty_duplicate_lists(self): + self.filter(documents=None).delete() + + def get_duplicated_documents(self): + Document = apps.get_model( + app_label='documents', model_name='Document' + ) + return Document.objects.filter( + pk__in=self.filter(documents__isnull=False).values_list( + 'document_id', flat=True + ) + ) + def scan(self): """ Find duplicates by iterating over all documents and then @@ -135,6 +148,8 @@ class DuplicatedDocumentManager(models.Manager): if duplicates.exists(): instance, created = self.get_or_create(document=document) instance.documents.add(*duplicates) + else: + self.filter(document=document).delete() if scan_children: for document in duplicates: diff --git a/mayan/apps/documents/queues.py b/mayan/apps/documents/queues.py index a544b20ab0..20c9a43659 100644 --- a/mayan/apps/documents/queues.py +++ b/mayan/apps/documents/queues.py @@ -14,10 +14,24 @@ queue_documents_periodic = CeleryQueue( queue_uploads = CeleryQueue( name='uploads', label=_('Uploads') ) -queue_uploads = CeleryQueue( +queue_documents = CeleryQueue( name='documents', label=_('Documents') ) +queue_converter.add_task_type( + name='documents.tasks.task_generate_document_page_image', + label=_('Generate document page image') +) + +queue_documents.add_task_type( + name='documents.tasks.task_delete_document', + label=_('Delete a document') +) +queue_documents.add_task_type( + name='documents.tasks.task_clean_empty_duplicate_lists', + label=_('Clean empty duplicate lists') +) + queue_documents_periodic.add_task_type( name='documents.tasks.task_check_delete_periods', label=_('Check document type delete periods') @@ -36,11 +50,6 @@ queue_tools.add_task_type( label=_('Clear image cache') ) -queue_converter.add_task_type( - name='documents.tasks.task_generate_document_page_image', - label=_('Generate document page image') -) - queue_uploads.add_task_type( name='documents.tasks.task_update_page_count', label=_('Update document page count') @@ -49,7 +58,3 @@ queue_uploads.add_task_type( name='documents.tasks.task_upload_new_version', label=_('Upload new document version') ) -queue_uploads.add_task_type( - name='documents.tasks.task_delete_document', - label=_('Delete a document') -) diff --git a/mayan/apps/documents/tasks.py b/mayan/apps/documents/tasks.py index cd4c0a17c5..c4a9256aa9 100644 --- a/mayan/apps/documents/tasks.py +++ b/mayan/apps/documents/tasks.py @@ -15,6 +15,14 @@ from .literals import ( logger = logging.getLogger(__name__) +@app.task(ignore_result=True) +def task_clean_empty_duplicate_lists(): + DuplicatedDocument = apps.get_model( + app_label='documents', model_name='DuplicatedDocument' + ) + DuplicatedDocument.objects.clean_empty_duplicate_lists() + + @app.task(ignore_result=True) def task_check_delete_periods(): DocumentType = apps.get_model( diff --git a/mayan/apps/documents/tests/base.py b/mayan/apps/documents/tests/base.py index 1079fb9221..9dc5c89f6f 100644 --- a/mayan/apps/documents/tests/base.py +++ b/mayan/apps/documents/tests/base.py @@ -2,9 +2,12 @@ from __future__ import unicode_literals +import os + +from django.conf import settings from django.test import override_settings -from common.tests import GenericViewTestCase +from common.tests import BaseTestCase, GenericViewTestCase from ..models import DocumentType @@ -14,21 +17,53 @@ from .literals import ( ) +@override_settings(OCR_AUTO_OCR=False) +class GenericDocumentTestCase(BaseTestCase): + test_document_filename = TEST_SMALL_DOCUMENT_FILENAME + + def upload_document(self): + with open(self.test_document_path) as file_object: + document = self.document_type.new_document( + file_object=file_object, label=self.test_document_filename + ) + return document + + def setUp(self): + super(GenericDocumentTestCase, self).setUp() + self.test_document_path = os.path.join( + settings.BASE_DIR, 'apps', 'documents', 'tests', 'contrib', + 'sample_documents', self.test_document_filename + ) + + self.document_type = DocumentType.objects.create( + label=TEST_DOCUMENT_TYPE_LABEL + ) + + self.document = self.upload_document() + + def tearDown(self): + self.document_type.delete() + super(GenericDocumentTestCase, self).tearDown() + + @override_settings(OCR_AUTO_OCR=False) class GenericDocumentViewTestCase(GenericViewTestCase): test_document_filename = TEST_SMALL_DOCUMENT_FILENAME test_document_path = TEST_SMALL_DOCUMENT_PATH + def upload_document(self): + with open(self.test_document_path) as file_object: + document = self.document_type.new_document( + file_object=file_object, label=self.test_document_filename + ) + return document + def setUp(self): super(GenericDocumentViewTestCase, self).setUp() self.document_type = DocumentType.objects.create( label=TEST_DOCUMENT_TYPE_LABEL ) - - with open(self.test_document_path) as file_object: - self.document = self.document_type.new_document( - file_object=file_object, label=self.test_document_filename - ) + self.document = self.upload_document() def tearDown(self): if self.document_type.pk: diff --git a/mayan/apps/documents/tests/test_models.py b/mayan/apps/documents/tests/test_models.py index 6ac64d303e..d2c6d644ee 100644 --- a/mayan/apps/documents/tests/test_models.py +++ b/mayan/apps/documents/tests/test_models.py @@ -10,8 +10,11 @@ from django.test import override_settings from common.tests import BaseTestCase from ..literals import STUB_EXPIRATION_INTERVAL -from ..models import DeletedDocument, Document, DocumentType +from ..models import ( + DeletedDocument, Document, DocumentType, DuplicatedDocument +) +from .base import GenericDocumentTestCase from .literals import ( TEST_DOCUMENT_TYPE_LABEL, TEST_DOCUMENT_PATH, TEST_MULTI_PAGE_TIFF_PATH, TEST_PDF_INDIRECT_ROTATE_PATH, TEST_OFFICE_DOCUMENT_PATH, @@ -19,31 +22,6 @@ from .literals import ( ) -@override_settings(OCR_AUTO_OCR=False) -class GenericDocumentTestCase(BaseTestCase): - test_document_filename = TEST_SMALL_DOCUMENT_FILENAME - - def setUp(self): - super(GenericDocumentTestCase, self).setUp() - self.test_document_path = os.path.join( - settings.BASE_DIR, 'apps', 'documents', 'tests', 'contrib', - 'sample_documents', self.test_document_filename - ) - - self.document_type = DocumentType.objects.create( - label=TEST_DOCUMENT_TYPE_LABEL - ) - - with open(self.test_document_path) as file_object: - self.document = self.document_type.new_document( - file_object=file_object, label=self.test_document_filename - ) - - def tearDown(self): - self.document_type.delete() - super(GenericDocumentTestCase, self).tearDown() - - @override_settings(OCR_AUTO_OCR=False) class DocumentTestCase(BaseTestCase): def setUp(self): @@ -302,3 +280,23 @@ class DocumentManagerTestCase(BaseTestCase): Document.objects.delete_stubs() self.assertEqual(Document.objects.count(), 0) + + +class DuplicatedDocumentsTestCase(GenericDocumentTestCase): + def test_duplicates_after_delete(self): + document_2 = self.upload_document() + document_2.delete() + document_2.delete() + + self.assertEqual(DuplicatedDocument.objects.filter(document=self.document).count(), 0) + + def test_duplicates_after_trash(self): + document_2 = self.upload_document() + document_2.delete() + + self.assertFalse(document_2 in DuplicatedDocument.objects.get(document=self.document).documents.all()) + + def test_duplicate_scan(self): + document_2 = self.upload_document() + + self.assertTrue(document_2 in DuplicatedDocument.objects.get(document=self.document).documents.all()) diff --git a/mayan/apps/documents/views/document_views.py b/mayan/apps/documents/views/document_views.py index 92203a799f..dd3fcebc00 100644 --- a/mayan/apps/documents/views/document_views.py +++ b/mayan/apps/documents/views/document_views.py @@ -787,11 +787,7 @@ class DocumentPrint(FormView): class DuplicatedDocumentListView(DocumentListView): def get_document_queryset(self): - return Document.objects.filter( - pk__in=DuplicatedDocument.objects.values_list( - 'document_id', flat=True - ) - ) + return DuplicatedDocument.objects.get_duplicated_documents() def get_extra_context(self): context = super(DuplicatedDocumentListView, self).get_extra_context()