matthias/mayan-edms
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Don't show documents with 0 duplicates in the duplicated document list.

Browse Source 
Also clean up the duplicated document model after a document is deleted.
Fix queue name typo.

Signed-off-by: Roberto Rosario <roberto.rosario.gonzalez@gmail.com>
This commit is contained in:
Roberto Rosario
2018-04-02 01:45:30 -04:00
parent 010d56bda9
commit 57e7722f59
10 changed files with 144 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
HISTORY.rst
View File

@@ -101,6 +101,8 @@
- Add the "to=" keyword argument to all ForeignKey, ManayToMany and OneToOne Fields.
- Add Makefile target to check the format of the README.rst file.
- Mark the feature to detect and fix the orientatin of PDF as experimental.
- Don't show documents with 0 duplicates in the duplicated document list.
- Clean up the duplicated document model after a document is deleted.
2.7.3 (2017-09-11)
==================

12
docs/releases/3.0.rst
View File

@@ -260,6 +260,18 @@ project. MERCs 1 and 2 have been approved. MERC-1 outlines the MERC process
itself and MERC-2 documents the way API tests are to be written for Mayan EDMS.
Duplicated documents
--------------------
The duplicated documents system has been improved to also better detect when
the duplicate of a primary document has been move to the trash. In this
instance the duplicate count of the primary document would be zero and will
cause the primary document to not show in the duplicated document list view.
If the duplicated document is deleted from the trash the system now will launch
a background clean up task to permanently delete the empty primary document's
duplicate document entry from the database.
Other changes worth mentioning
------------------------------
- Add Makefile target to check the format of the README.rst file.

32
mayan/apps/documents/apps.py
View File

@@ -4,6 +4,7 @@ from datetime import timedelta
from kombu import Exchange, Queue
from django.db.models.signals import post_delete
from django.utils.translation import ugettext_lazy as _
from acls import ModelPermission
@@ -46,7 +47,8 @@ from .events import (
event_document_view
)
from .handlers import (
create_default_document_type, handler_scan_duplicates_for
create_default_document_type, handler_remove_empty_duplicates_lists,
handler_scan_duplicates_for,
)
from .links import (
link_clear_image_cache, link_document_clear_transformations,
@@ -359,29 +361,32 @@ class DocumentsApp(MayanAppConfig):
'documents.tasks.task_check_trash_periods': {
'queue': 'documents_periodic'
},
'documents.tasks.task_delete_stubs': {
'queue': 'documents_periodic'
'documents.tasks.task_clean_empty_duplicate_lists': {
'queue': 'documents'
},
'documents.tasks.task_clear_image_cache': {
'queue': 'tools'
},
'documents.tasks.task_delete_document': {
'queue': 'documents'
},
'documents.tasks.task_delete_stubs': {
'queue': 'documents_periodic'
},
'documents.tasks.task_generate_document_page_image': {
'queue': 'converter'
},
'documents.tasks.task_update_page_count': {
'queue': 'uploads'
},
'documents.tasks.task_upload_new_version': {
'queue': 'uploads'
},
'documents.tasks.task_scan_duplicates_all': {
'queue': 'tools'
},
'documents.tasks.task_scan_duplicates_for': {
'queue': 'uploads'
},
'documents.tasks.task_delete_document': {
'queue': 'documents'
'documents.tasks.task_update_page_count': {
'queue': 'uploads'
},
'documents.tasks.task_upload_new_version': {
'queue': 'uploads'
},
}
)
@@ -576,6 +581,11 @@ class DocumentsApp(MayanAppConfig):
minute='0'
)
post_delete.connect(
dispatch_uid='handler_remove_empty_duplicates_lists',
receiver=handler_remove_empty_duplicates_lists,
sender=Document,
)
post_initial_setup.connect(
create_default_document_type,
dispatch_uid='create_default_document_type'

6
mayan/apps/documents/handlers.py
View File

@@ -4,7 +4,7 @@ from django.apps import apps
from .literals import DEFAULT_DOCUMENT_TYPE_LABEL
from .signals import post_initial_document_type
from .tasks import task_scan_duplicates_for
from .tasks import task_clean_empty_duplicate_lists, task_scan_duplicates_for
def create_default_document_type(sender, **kwargs):
@@ -25,3 +25,7 @@ def handler_scan_duplicates_for(sender, instance, **kwargs):
task_scan_duplicates_for.apply_async(
kwargs={'document_id': instance.document.pk}
)
def handler_remove_empty_duplicates_lists(sender, **kwargs):
task_clean_empty_duplicate_lists.apply_async()

15
mayan/apps/documents/managers.py
View File

@@ -100,6 +100,19 @@ class DocumentTypeManager(models.Manager):
class DuplicatedDocumentManager(models.Manager):
def clean_empty_duplicate_lists(self):
self.filter(documents=None).delete()
def get_duplicated_documents(self):
Document = apps.get_model(
app_label='documents', model_name='Document'
)
return Document.objects.filter(
pk__in=self.filter(documents__isnull=False).values_list(
'document_id', flat=True
)
)
def scan(self):
"""
Find duplicates by iterating over all documents and then
@@ -135,6 +148,8 @@ class DuplicatedDocumentManager(models.Manager):
if duplicates.exists():
instance, created = self.get_or_create(document=document)
instance.documents.add(*duplicates)
else:
self.filter(document=document).delete()
if scan_children:
for document in duplicates:

25
mayan/apps/documents/queues.py
View File

@@ -14,10 +14,24 @@ queue_documents_periodic = CeleryQueue(
queue_uploads = CeleryQueue(
name='uploads', label=_('Uploads')
)
queue_uploads = CeleryQueue(
queue_documents = CeleryQueue(
name='documents', label=_('Documents')
)
queue_converter.add_task_type(
name='documents.tasks.task_generate_document_page_image',
label=_('Generate document page image')
)
queue_documents.add_task_type(
name='documents.tasks.task_delete_document',
label=_('Delete a document')
)
queue_documents.add_task_type(
name='documents.tasks.task_clean_empty_duplicate_lists',
label=_('Clean empty duplicate lists')
)
queue_documents_periodic.add_task_type(
name='documents.tasks.task_check_delete_periods',
label=_('Check document type delete periods')
@@ -36,11 +50,6 @@ queue_tools.add_task_type(
label=_('Clear image cache')
)
queue_converter.add_task_type(
name='documents.tasks.task_generate_document_page_image',
label=_('Generate document page image')
)
queue_uploads.add_task_type(
name='documents.tasks.task_update_page_count',
label=_('Update document page count')
@@ -49,7 +58,3 @@ queue_uploads.add_task_type(
name='documents.tasks.task_upload_new_version',
label=_('Upload new document version')
)
queue_uploads.add_task_type(
name='documents.tasks.task_delete_document',
label=_('Delete a document')
)

8
mayan/apps/documents/tasks.py
View File

@@ -15,6 +15,14 @@ from .literals import (
logger = logging.getLogger(__name__)
@app.task(ignore_result=True)
def task_clean_empty_duplicate_lists():
DuplicatedDocument = apps.get_model(
app_label='documents', model_name='DuplicatedDocument'
)
DuplicatedDocument.objects.clean_empty_duplicate_lists()
@app.task(ignore_result=True)
def task_check_delete_periods():
DocumentType = apps.get_model(

47
mayan/apps/documents/tests/base.py
View File

@@ -2,9 +2,12 @@
from __future__ import unicode_literals
import os
from django.conf import settings
from django.test import override_settings
from common.tests import GenericViewTestCase
from common.tests import BaseTestCase, GenericViewTestCase
from ..models import DocumentType
@@ -14,21 +17,53 @@ from .literals import (
)
@override_settings(OCR_AUTO_OCR=False)
class GenericDocumentTestCase(BaseTestCase):
test_document_filename = TEST_SMALL_DOCUMENT_FILENAME
def upload_document(self):
with open(self.test_document_path) as file_object:
document = self.document_type.new_document(
file_object=file_object, label=self.test_document_filename
)
return document
def setUp(self):
super(GenericDocumentTestCase, self).setUp()
self.test_document_path = os.path.join(
settings.BASE_DIR, 'apps', 'documents', 'tests', 'contrib',
'sample_documents', self.test_document_filename
)
self.document_type = DocumentType.objects.create(
label=TEST_DOCUMENT_TYPE_LABEL
)
self.document = self.upload_document()
def tearDown(self):
self.document_type.delete()
super(GenericDocumentTestCase, self).tearDown()
@override_settings(OCR_AUTO_OCR=False)
class GenericDocumentViewTestCase(GenericViewTestCase):
test_document_filename = TEST_SMALL_DOCUMENT_FILENAME
test_document_path = TEST_SMALL_DOCUMENT_PATH
def upload_document(self):
with open(self.test_document_path) as file_object:
document = self.document_type.new_document(
file_object=file_object, label=self.test_document_filename
)
return document
def setUp(self):
super(GenericDocumentViewTestCase, self).setUp()
self.document_type = DocumentType.objects.create(
label=TEST_DOCUMENT_TYPE_LABEL
)
with open(self.test_document_path) as file_object:
self.document = self.document_type.new_document(
file_object=file_object, label=self.test_document_filename
)
self.document = self.upload_document()
def tearDown(self):
if self.document_type.pk:

50
mayan/apps/documents/tests/test_models.py
View File

@@ -10,8 +10,11 @@ from django.test import override_settings
from common.tests import BaseTestCase
from ..literals import STUB_EXPIRATION_INTERVAL
from ..models import DeletedDocument, Document, DocumentType
from ..models import (
DeletedDocument, Document, DocumentType, DuplicatedDocument
)
from .base import GenericDocumentTestCase
from .literals import (
TEST_DOCUMENT_TYPE_LABEL, TEST_DOCUMENT_PATH, TEST_MULTI_PAGE_TIFF_PATH,
TEST_PDF_INDIRECT_ROTATE_PATH, TEST_OFFICE_DOCUMENT_PATH,
@@ -19,31 +22,6 @@ from .literals import (
)
@override_settings(OCR_AUTO_OCR=False)
class GenericDocumentTestCase(BaseTestCase):
test_document_filename = TEST_SMALL_DOCUMENT_FILENAME
def setUp(self):
super(GenericDocumentTestCase, self).setUp()
self.test_document_path = os.path.join(
settings.BASE_DIR, 'apps', 'documents', 'tests', 'contrib',
'sample_documents', self.test_document_filename
)
self.document_type = DocumentType.objects.create(
label=TEST_DOCUMENT_TYPE_LABEL
)
with open(self.test_document_path) as file_object:
self.document = self.document_type.new_document(
file_object=file_object, label=self.test_document_filename
)
def tearDown(self):
self.document_type.delete()
super(GenericDocumentTestCase, self).tearDown()
@override_settings(OCR_AUTO_OCR=False)
class DocumentTestCase(BaseTestCase):
def setUp(self):
@@ -302,3 +280,23 @@ class DocumentManagerTestCase(BaseTestCase):
Document.objects.delete_stubs()
self.assertEqual(Document.objects.count(), 0)
class DuplicatedDocumentsTestCase(GenericDocumentTestCase):
def test_duplicates_after_delete(self):
document_2 = self.upload_document()
document_2.delete()
document_2.delete()
self.assertEqual(DuplicatedDocument.objects.filter(document=self.document).count(), 0)
def test_duplicates_after_trash(self):
document_2 = self.upload_document()
document_2.delete()
self.assertFalse(document_2 in DuplicatedDocument.objects.get(document=self.document).documents.all())
def test_duplicate_scan(self):
document_2 = self.upload_document()
self.assertTrue(document_2 in DuplicatedDocument.objects.get(document=self.document).documents.all())

6
mayan/apps/documents/views/document_views.py
View File

@@ -787,11 +787,7 @@ class DocumentPrint(FormView):
class DuplicatedDocumentListView(DocumentListView):
def get_document_queryset(self):
return Document.objects.filter(
pk__in=DuplicatedDocument.objects.values_list(
'document_id', flat=True
)
)
return DuplicatedDocument.objects.get_duplicated_documents()
def get_extra_context(self):
context = super(DuplicatedDocumentListView, self).get_extra_context()