Don't show documents with 0 duplicates in the duplicated document list.

Also clean up the duplicated document model after a document is deleted.
Fix queue name typo.

Signed-off-by: Roberto Rosario <roberto.rosario.gonzalez@gmail.com>

HISTORY.rst

@@ -101,6 +101,8 @@
 - Add the "to=" keyword argument to all ForeignKey, ManayToMany and OneToOne Fields.
 - Add Makefile target to check the format of the README.rst file.
 - Mark the feature to detect and fix the orientatin of PDF as experimental.
+- Don't show documents with 0 duplicates in the duplicated document list.
+- Clean up the duplicated document model after a document is deleted.
 
 2.7.3 (2017-09-11)
 ==================

docs/releases/3.0.rst

@@ -260,6 +260,18 @@ project. MERCs 1 and 2 have been approved. MERC-1 outlines the MERC process
 itself and MERC-2 documents the way API tests are to be written for Mayan EDMS.
 
 
+Duplicated documents
+--------------------
+The duplicated documents system has been improved to also better detect when
+the duplicate of a primary document has been move to the trash. In this
+instance the duplicate count of the primary document would be zero and will
+cause the primary document to not show in the duplicated document list view.
+
+If the duplicated document is deleted from the trash the system now will launch
+a background clean up task to permanently delete the empty primary document's
+duplicate document entry from the database.
+
+
 Other changes worth mentioning
 ------------------------------
 - Add Makefile target to check the format of the README.rst file.

mayan/apps/documents/apps.py

@@ -4,6 +4,7 @@ from datetime import timedelta
 
 from kombu import Exchange, Queue
 
+from django.db.models.signals import post_delete
 from django.utils.translation import ugettext_lazy as _
 
 from acls import ModelPermission
@@ -46,7 +47,8 @@ from .events import (
     event_document_view
 )
 from .handlers import (
-    create_default_document_type, handler_scan_duplicates_for
+    create_default_document_type, handler_remove_empty_duplicates_lists,
+    handler_scan_duplicates_for,
 )
 from .links import (
     link_clear_image_cache, link_document_clear_transformations,
@@ -359,29 +361,32 @@ class DocumentsApp(MayanAppConfig):
                 'documents.tasks.task_check_trash_periods': {
                     'queue': 'documents_periodic'
                 },
-                'documents.tasks.task_delete_stubs': {
+                'documents.tasks.task_clean_empty_duplicate_lists': {
-                    'queue': 'documents_periodic'
+                    'queue': 'documents'
                 },
                 'documents.tasks.task_clear_image_cache': {
                     'queue': 'tools'
                 },
+                'documents.tasks.task_delete_document': {
+                    'queue': 'documents'
+                },
+                'documents.tasks.task_delete_stubs': {
+                    'queue': 'documents_periodic'
+                },
                 'documents.tasks.task_generate_document_page_image': {
                     'queue': 'converter'
                 },
-                'documents.tasks.task_update_page_count': {
-                    'queue': 'uploads'
-                },
-                'documents.tasks.task_upload_new_version': {
-                    'queue': 'uploads'
-                },
                 'documents.tasks.task_scan_duplicates_all': {
                     'queue': 'tools'
                 },
                 'documents.tasks.task_scan_duplicates_for': {
                     'queue': 'uploads'
                 },
-                'documents.tasks.task_delete_document': {
+                'documents.tasks.task_update_page_count': {
-                    'queue': 'documents'
+                    'queue': 'uploads'
+                },
+                'documents.tasks.task_upload_new_version': {
+                    'queue': 'uploads'
                 },
             }
         )
@@ -576,6 +581,11 @@ class DocumentsApp(MayanAppConfig):
             minute='0'
         )
 
+        post_delete.connect(
+            dispatch_uid='handler_remove_empty_duplicates_lists',
+            receiver=handler_remove_empty_duplicates_lists,
+            sender=Document,
+        )
         post_initial_setup.connect(
             create_default_document_type,
             dispatch_uid='create_default_document_type'

mayan/apps/documents/handlers.py

@@ -4,7 +4,7 @@ from django.apps import apps
 
 from .literals import DEFAULT_DOCUMENT_TYPE_LABEL
 from .signals import post_initial_document_type
-from .tasks import task_scan_duplicates_for
+from .tasks import task_clean_empty_duplicate_lists, task_scan_duplicates_for
 
 
 def create_default_document_type(sender, **kwargs):
@@ -25,3 +25,7 @@ def handler_scan_duplicates_for(sender, instance, **kwargs):
     task_scan_duplicates_for.apply_async(
         kwargs={'document_id': instance.document.pk}
     )
+
+
+def handler_remove_empty_duplicates_lists(sender, **kwargs):
+    task_clean_empty_duplicate_lists.apply_async()

mayan/apps/documents/managers.py

@@ -100,6 +100,19 @@ class DocumentTypeManager(models.Manager):
 
 
 class DuplicatedDocumentManager(models.Manager):
+    def clean_empty_duplicate_lists(self):
+        self.filter(documents=None).delete()
+
+    def get_duplicated_documents(self):
+        Document = apps.get_model(
+            app_label='documents', model_name='Document'
+        )
+        return Document.objects.filter(
+            pk__in=self.filter(documents__isnull=False).values_list(
+                'document_id', flat=True
+            )
+        )
+
     def scan(self):
         """
         Find duplicates by iterating over all documents and then
@@ -135,6 +148,8 @@ class DuplicatedDocumentManager(models.Manager):
         if duplicates.exists():
             instance, created = self.get_or_create(document=document)
             instance.documents.add(*duplicates)
+        else:
+            self.filter(document=document).delete()
 
         if scan_children:
             for document in duplicates:

mayan/apps/documents/queues.py

@@ -14,10 +14,24 @@ queue_documents_periodic = CeleryQueue(
 queue_uploads = CeleryQueue(
     name='uploads', label=_('Uploads')
 )
-queue_uploads = CeleryQueue(
+queue_documents = CeleryQueue(
     name='documents', label=_('Documents')
 )
 
+queue_converter.add_task_type(
+    name='documents.tasks.task_generate_document_page_image',
+    label=_('Generate document page image')
+)
+
+queue_documents.add_task_type(
+    name='documents.tasks.task_delete_document',
+    label=_('Delete a document')
+)
+queue_documents.add_task_type(
+    name='documents.tasks.task_clean_empty_duplicate_lists',
+    label=_('Clean empty duplicate lists')
+)
+
 queue_documents_periodic.add_task_type(
     name='documents.tasks.task_check_delete_periods',
     label=_('Check document type delete periods')
@@ -36,11 +50,6 @@ queue_tools.add_task_type(
     label=_('Clear image cache')
 )
 
-queue_converter.add_task_type(
-    name='documents.tasks.task_generate_document_page_image',
-    label=_('Generate document page image')
-)
-
 queue_uploads.add_task_type(
     name='documents.tasks.task_update_page_count',
     label=_('Update document page count')
@@ -49,7 +58,3 @@ queue_uploads.add_task_type(
     name='documents.tasks.task_upload_new_version',
     label=_('Upload new document version')
 )
-queue_uploads.add_task_type(
-    name='documents.tasks.task_delete_document',
-    label=_('Delete a document')
-)

mayan/apps/documents/tasks.py

@@ -15,6 +15,14 @@ from .literals import (
 logger = logging.getLogger(__name__)
 
 
+@app.task(ignore_result=True)
+def task_clean_empty_duplicate_lists():
+    DuplicatedDocument = apps.get_model(
+        app_label='documents', model_name='DuplicatedDocument'
+    )
+    DuplicatedDocument.objects.clean_empty_duplicate_lists()
+
+
 @app.task(ignore_result=True)
 def task_check_delete_periods():
     DocumentType = apps.get_model(

mayan/apps/documents/tests/base.py

@@ -2,9 +2,12 @@
 
 from __future__ import unicode_literals
 
+import os
+
+from django.conf import settings
 from django.test import override_settings
 
-from common.tests import GenericViewTestCase
+from common.tests import BaseTestCase, GenericViewTestCase
 
 from ..models import DocumentType
 
@@ -14,21 +17,53 @@ from .literals import (
 )
 
 
+@override_settings(OCR_AUTO_OCR=False)
+class GenericDocumentTestCase(BaseTestCase):
+    test_document_filename = TEST_SMALL_DOCUMENT_FILENAME
+
+    def upload_document(self):
+        with open(self.test_document_path) as file_object:
+            document = self.document_type.new_document(
+                file_object=file_object, label=self.test_document_filename
+            )
+        return document
+
+    def setUp(self):
+        super(GenericDocumentTestCase, self).setUp()
+        self.test_document_path = os.path.join(
+            settings.BASE_DIR, 'apps', 'documents', 'tests', 'contrib',
+            'sample_documents', self.test_document_filename
+        )
+
+        self.document_type = DocumentType.objects.create(
+            label=TEST_DOCUMENT_TYPE_LABEL
+        )
+
+        self.document = self.upload_document()
+
+    def tearDown(self):
+        self.document_type.delete()
+        super(GenericDocumentTestCase, self).tearDown()
+
+
 @override_settings(OCR_AUTO_OCR=False)
 class GenericDocumentViewTestCase(GenericViewTestCase):
     test_document_filename = TEST_SMALL_DOCUMENT_FILENAME
     test_document_path = TEST_SMALL_DOCUMENT_PATH
 
+    def upload_document(self):
+        with open(self.test_document_path) as file_object:
+            document = self.document_type.new_document(
+                file_object=file_object, label=self.test_document_filename
+            )
+        return document
+
     def setUp(self):
         super(GenericDocumentViewTestCase, self).setUp()
         self.document_type = DocumentType.objects.create(
             label=TEST_DOCUMENT_TYPE_LABEL
         )
-
+        self.document = self.upload_document()
-        with open(self.test_document_path) as file_object:
-            self.document = self.document_type.new_document(
-                file_object=file_object, label=self.test_document_filename
-            )
 
     def tearDown(self):
         if self.document_type.pk:

mayan/apps/documents/tests/test_models.py

@@ -10,8 +10,11 @@ from django.test import override_settings
 from common.tests import BaseTestCase
 
 from ..literals import STUB_EXPIRATION_INTERVAL
-from ..models import DeletedDocument, Document, DocumentType
+from ..models import (
+    DeletedDocument, Document, DocumentType, DuplicatedDocument
+)
 
+from .base import GenericDocumentTestCase
 from .literals import (
     TEST_DOCUMENT_TYPE_LABEL, TEST_DOCUMENT_PATH, TEST_MULTI_PAGE_TIFF_PATH,
     TEST_PDF_INDIRECT_ROTATE_PATH, TEST_OFFICE_DOCUMENT_PATH,
@@ -19,31 +22,6 @@ from .literals import (
 )
 
 
-@override_settings(OCR_AUTO_OCR=False)
-class GenericDocumentTestCase(BaseTestCase):
-    test_document_filename = TEST_SMALL_DOCUMENT_FILENAME
-
-    def setUp(self):
-        super(GenericDocumentTestCase, self).setUp()
-        self.test_document_path = os.path.join(
-            settings.BASE_DIR, 'apps', 'documents', 'tests', 'contrib',
-            'sample_documents', self.test_document_filename
-        )
-
-        self.document_type = DocumentType.objects.create(
-            label=TEST_DOCUMENT_TYPE_LABEL
-        )
-
-        with open(self.test_document_path) as file_object:
-            self.document = self.document_type.new_document(
-                file_object=file_object, label=self.test_document_filename
-            )
-
-    def tearDown(self):
-        self.document_type.delete()
-        super(GenericDocumentTestCase, self).tearDown()
-
-
 @override_settings(OCR_AUTO_OCR=False)
 class DocumentTestCase(BaseTestCase):
     def setUp(self):
@@ -302,3 +280,23 @@ class DocumentManagerTestCase(BaseTestCase):
         Document.objects.delete_stubs()
 
         self.assertEqual(Document.objects.count(), 0)
+
+
+class DuplicatedDocumentsTestCase(GenericDocumentTestCase):
+    def test_duplicates_after_delete(self):
+        document_2 = self.upload_document()
+        document_2.delete()
+        document_2.delete()
+
+        self.assertEqual(DuplicatedDocument.objects.filter(document=self.document).count(), 0)
+
+    def test_duplicates_after_trash(self):
+        document_2 = self.upload_document()
+        document_2.delete()
+
+        self.assertFalse(document_2 in DuplicatedDocument.objects.get(document=self.document).documents.all())
+
+    def test_duplicate_scan(self):
+        document_2 = self.upload_document()
+
+        self.assertTrue(document_2 in DuplicatedDocument.objects.get(document=self.document).documents.all())

mayan/apps/documents/views/document_views.py

@@ -787,11 +787,7 @@ class DocumentPrint(FormView):
 
 class DuplicatedDocumentListView(DocumentListView):
     def get_document_queryset(self):
-        return Document.objects.filter(
+        return DuplicatedDocument.objects.get_duplicated_documents()
-            pk__in=DuplicatedDocument.objects.values_list(
-                'document_id', flat=True
-            )
-        )
 
     def get_extra_context(self):
         context = super(DuplicatedDocumentListView, self).get_extra_context()