Merge remote-tracking branch 'origin/master' into development

Signed-off-by: Roberto Rosario <roberto.rosario.gonzalez@gmail.com>

mayan/apps/documents/admin.py

@@ -4,7 +4,7 @@ from django.contrib import admin
 
 from .models import (
     DeletedDocument, Document, DocumentPage, DocumentType,
-    DocumentTypeFilename, DocumentVersion, RecentDocument
+    DocumentTypeFilename, DocumentVersion, DuplicatedDocument, RecentDocument
 )
 
 
@@ -55,6 +55,13 @@ class DocumentTypeAdmin(admin.ModelAdmin):
     )
 
 
+@admin.register(DuplicatedDocument)
+class DuplicatedDocumentAdmin(admin.ModelAdmin):
+    list_display = (
+        'document', 'datetime_added'
+    )
+
+
 @admin.register(RecentDocument)
 class RecentDocumentAdmin(admin.ModelAdmin):
     date_hierarchy = 'datetime_accessed'

mayan/apps/documents/apps.py

@@ -4,7 +4,7 @@ from datetime import timedelta
 
 from kombu import Exchange, Queue
 
-from django.core.urlresolvers import reverse_lazy
+from django.urls import reverse_lazy
 from django.utils.translation import ugettext_lazy as _
 
 from acls import ModelPermission
@@ -31,11 +31,13 @@ from rest_api.classes import APIEndPoint, APIResource
 from rest_api.fields import DynamicSerializerField
 from statistics.classes import StatisticNamespace, CharJSLine
 
-from .handlers import create_default_document_type
+from .handlers import (
+    create_default_document_type, handler_scan_duplicates_for
+)
 from .links import (
     link_clear_image_cache, link_document_clear_transformations,
     link_document_clone_transformations, link_document_delete,
-    link_document_document_type_edit,
+    link_document_document_type_edit, link_document_duplicates_list,
     link_document_multiple_document_type_edit, link_document_download,
     link_document_edit, link_document_list, link_document_list_deleted,
     link_document_list_recent, link_document_multiple_delete,
@@ -55,7 +57,8 @@ from .links import (
     link_document_type_filename_list, link_document_type_list,
     link_document_type_setup, link_document_update_page_count,
     link_document_version_download, link_document_version_list,
-    link_document_version_revert, link_trash_can_empty
+    link_document_version_revert, link_duplicated_document_list,
+    link_duplicated_document_scan, link_trash_can_empty
 )
 from .literals import (
     CHECK_DELETE_PERIOD_INTERVAL, CHECK_TRASH_PERIOD_INTERVAL,
@@ -67,12 +70,15 @@ from .permissions import (
     permission_document_download, permission_document_edit,
     permission_document_new_version, permission_document_print,
     permission_document_properties_edit, permission_document_restore,
-    permission_document_trash, permission_document_version_revert,
+    permission_document_trash, permission_document_type_delete,
+    permission_document_type_edit, permission_document_type_view,
+    permission_document_version_revert, permission_document_version_view,
     permission_document_view
 )
 from .queues import *  # NOQA
 # Just import to initialize the search models
 from .search import document_search, document_page_search  # NOQA
+from .signals import post_version_upload
 from .statistics import (
     new_documents_per_month, new_document_pages_per_month,
     new_document_pages_this_month, new_documents_this_month,
@@ -103,6 +109,7 @@ class DocumentsApp(MayanAppConfig):
         DocumentType = self.get_model('DocumentType')
         DocumentTypeFilename = self.get_model('DocumentTypeFilename')
         DocumentVersion = self.get_model('DocumentVersion')
+        DuplicatedDocument = self.get_model('DuplicatedDocument')
 
         DynamicSerializerField.add_serializer(
             klass=Document,
@@ -171,7 +178,8 @@ class DocumentsApp(MayanAppConfig):
                 permission_document_edit, permission_document_new_version,
                 permission_document_print, permission_document_properties_edit,
                 permission_document_restore, permission_document_trash,
-                permission_document_version_revert, permission_document_view,
+                permission_document_version_revert,
+                permission_document_version_view, permission_document_view,
                 permission_events_view, permission_transformation_create,
                 permission_transformation_delete,
                 permission_transformation_edit, permission_transformation_view,
@@ -179,7 +187,10 @@ class DocumentsApp(MayanAppConfig):
         )
 
         ModelPermission.register(
-            model=DocumentType, permissions=(permission_document_create,)
+            model=DocumentType, permissions=(
+                permission_document_create, permission_document_type_delete,
+                permission_document_type_edit, permission_document_type_view
+            )
         )
 
         ModelPermission.register_proxy(
@@ -189,14 +200,15 @@ class DocumentsApp(MayanAppConfig):
         ModelPermission.register_inheritance(
             model=Document, related='document_type',
         )
-
-        ModelPermission.register_inheritance(
-            model=DocumentVersion, related='document',
-        )
-
         ModelPermission.register_inheritance(
             model=DocumentPage, related='document',
         )
+        ModelPermission.register_inheritance(
+            model=DocumentTypeFilename, related='document_type',
+        )
+        ModelPermission.register_inheritance(
+            model=DocumentVersion, related='document',
+        )
 
         # Document and document page thumbnail widget
         document_thumbnail_widget = DocumentThumbnailWidget()
@@ -274,6 +286,16 @@ class DocumentsApp(MayanAppConfig):
             source=DocumentVersion, label=_('Comment'),
             attribute='comment'
         )
+        SourceColumn(
+            source=DuplicatedDocument, label=_('Thumbnail'),
+            func=lambda context: document_thumbnail_widget.render(
+                instance=context['object'].document
+            )
+        )
+        SourceColumn(
+            source=DuplicatedDocument, label=_('Duplicates'),
+            func=lambda context: context['object'].documents.count()
+        )
 
         app.conf.CELERYBEAT_SCHEDULE.update(
             {
@@ -305,6 +327,9 @@ class DocumentsApp(MayanAppConfig):
                     routing_key='documents_periodic', delivery_mode=1
                 ),
                 Queue('uploads', Exchange('uploads'), routing_key='uploads'),
+                Queue(
+                    'documents', Exchange('documents'), routing_key='documents'
+                ),
             )
         )
 
@@ -331,20 +356,31 @@ class DocumentsApp(MayanAppConfig):
                 'documents.tasks.task_upload_new_version': {
                     'queue': 'uploads'
                 },
+                'documents.tasks.task_scan_duplicates_all': {
+                    'queue': 'tools'
+                },
+                'documents.tasks.task_scan_duplicates_for': {
+                    'queue': 'uploads'
+                },
+                'documents.tasks.task_delete_document': {
+                    'queue': 'documents'
+                },
             }
         )
 
         menu_documents.bind_links(
             links=(
                 link_document_list_recent, link_document_list,
-                link_document_list_deleted
+                link_document_list_deleted, link_duplicated_document_list
             )
         )
 
         menu_main.bind_links(links=(menu_documents,), position=0)
 
         menu_setup.bind_links(links=(link_document_type_setup,))
-        menu_tools.bind_links(links=(link_clear_image_cache,))
+        menu_tools.bind_links(
+            links=(link_clear_image_cache, link_duplicated_document_scan)
+        )
 
         # Document type links
         menu_object.bind_links(
@@ -387,7 +423,7 @@ class DocumentsApp(MayanAppConfig):
                 link_document_print, link_document_trash,
                 link_document_download, link_document_clear_transformations,
                 link_document_clone_transformations,
-                link_document_update_page_count
+                link_document_update_page_count,
             ), sources=(Document,)
         )
         menu_object.bind_links(
@@ -396,7 +432,10 @@ class DocumentsApp(MayanAppConfig):
         )
 
         # Document facet links
-        menu_facet.bind_links(links=(link_acl_list,), sources=(Document,))
+        menu_facet.bind_links(
+            links=(link_document_duplicates_list, link_acl_list,),
+            sources=(Document,)
+        )
         menu_facet.bind_links(
             links=(link_document_preview,), sources=(Document,), position=0
         )
@@ -502,6 +541,10 @@ class DocumentsApp(MayanAppConfig):
             create_default_document_type,
             dispatch_uid='create_default_document_type'
         )
+        post_version_upload.connect(
+            handler_scan_duplicates_for,
+            dispatch_uid='handler_scan_duplicates_for',
+        )
 
         registry.register(DeletedDocument)
         registry.register(Document)

mayan/apps/documents/handlers.py

@@ -4,6 +4,7 @@ from django.apps import apps
 
 from .literals import DEFAULT_DOCUMENT_TYPE_LABEL
 from .signals import post_initial_document_type
+from .tasks import task_scan_duplicates_for
 
 
 def create_default_document_type(sender, **kwargs):
@@ -18,3 +19,9 @@ def create_default_document_type(sender, **kwargs):
         post_initial_document_type.send(
             sender=DocumentType, instance=document_type
         )
+
+
+def handler_scan_duplicates_for(sender, instance, **kwargs):
+    task_scan_duplicates_for.apply_async(
+        kwargs={'document_id': instance.document.pk}
+    )

mayan/apps/documents/links.py

@@ -285,3 +285,16 @@ link_document_type_setup = Link(
     icon='fa fa-file', permissions=(permission_document_type_view,),
     text=_('Document types'), view='documents:document_type_list'
 )
+link_duplicated_document_list = Link(
+    icon='fa fa-clone', text=_('Duplicated documents'),
+    view='documents:duplicated_document_list'
+)
+link_document_duplicates_list = Link(
+    args='resolved_object.id', icon='fa fa-clone',
+    permissions=(permission_document_view,), text=_('Duplicates'),
+    view='documents:document_duplicates_list',
+)
+link_duplicated_document_scan = Link(
+    icon='fa fa-clone', text=_('Duplicated document scan'),
+    view='documents:duplicated_document_scan'
+)

mayan/apps/documents/managers.py

@@ -5,6 +5,7 @@ import logging
 
 from django.apps import apps
 from django.db import models
+from django.db.models import F, Max
 from django.utils.timezone import now
 
 from .literals import STUB_EXPIRATION_INTERVAL
@@ -98,13 +99,52 @@ class DocumentTypeManager(models.Manager):
         return self.get(label=label)
 
 
+class DuplicatedDocumentManager(models.Manager):
+    def scan(self):
+        """
+        Find duplicates by iterating over all documents and then
+        find matching latest version checksums
+        """
+        Document = apps.get_model(
+            app_label='documents', model_name='Document'
+        )
+
+        for document in Document.objects.all():
+            self.scan_for(document=document, scan_children=False)
+
+    def scan_for(self, document, scan_children=True):
+        """
+        Find duplicates by matching latest version checksums
+        """
+        Document = apps.get_model(
+            app_label='documents', model_name='Document'
+        )
+
+        # Get the documents whose latest version matches the checksum
+        # of the current document and exclude the current document
+        duplicates = Document.objects.annotate(
+            max_timestamp=Max('versions__timestamp')
+        ).filter(
+            versions__timestamp=F('max_timestamp'),
+            versions__checksum=document.checksum
+        ).exclude(pk=document.pk)
+
+        if duplicates.exists():
+            instance, created = self.get_or_create(document=document)
+            instance.documents.add(*duplicates)
+
+        if scan_children:
+            for document in duplicates:
+                self.scan_for(document=document, scan_children=False)
+
+
 class PassthroughManager(models.Manager):
     pass
 
 
 class RecentDocumentManager(models.Manager):
     def add_document_for_user(self, user, document):
-        if user.is_authenticated():
+        if user.is_authenticated:
             new_recent, created = self.model.objects.get_or_create(
                 user=user, document=document
             )
@@ -120,7 +160,7 @@ class RecentDocumentManager(models.Manager):
     def get_for_user(self, user):
         document_model = apps.get_model('documents', 'document')
 
-        if user.is_authenticated():
+        if user.is_authenticated:
             return document_model.objects.filter(
                 recentdocument__user=user
             ).order_by('-recentdocument__datetime_accessed')

mayan/apps/documents/migrations/0038_auto_20170705_2008.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Generated by Django 1.10.7 on 2017-07-05 19:42
+# Generated by Django 1.10.7 on 2017-07-05 20:08
 from __future__ import unicode_literals
 
 from django.db import migrations, models
@@ -15,9 +15,9 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='documentversion',
             name='checksum',
-            field=models.TextField(
-                blank=True, db_index=True, editable=False, null=True,
-                verbose_name='Checksum'
+            field=models.CharField(
+                blank=True, db_index=True, editable=False, max_length=64,
+                null=True, verbose_name='Checksum'
             ),
         ),
     ]

mayan/apps/documents/migrations/0039_duplicateddocument.py

@@ -0,0 +1,50 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.10.7 on 2017-07-06 03:30
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('documents', '0038_auto_20170705_2008'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='DuplicatedDocument',
+            fields=[
+                (
+                    'id', models.AutoField(
+                        auto_created=True, primary_key=True, serialize=False,
+                        verbose_name='ID'
+                    )
+                ),
+                (
+                    'datetime_added', models.DateTimeField(
+                        auto_now_add=True, db_index=True,
+                        verbose_name='Added'
+                    )
+                ),
+                (
+                    'document', models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name='duplicates', to='documents.Document',
+                        verbose_name='Document'
+                    )
+                ),
+                (
+                    'documents', models.ManyToManyField(
+                        to='documents.Document',
+                        verbose_name='Duplicated documents'
+                    )
+                ),
+            ],
+            options={
+                'verbose_name': 'Duplicated document',
+                'verbose_name_plural': 'Duplicated documents',
+            },
+        ),
+    ]

mayan/apps/documents/migrations/0040_auto_20170725_1111.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.10.7 on 2017-07-25 11:11
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('documents', '0039_duplicateddocument'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='documentversion',
+            options={'ordering': ('timestamp',), 'verbose_name': 'Document version', 'verbose_name_plural': 'Document version'},
+        ),
+    ]

mayan/apps/documents/models.py

@@ -6,8 +6,8 @@ import uuid
 
 from django.conf import settings
 from django.core.files import File
-from django.core.urlresolvers import reverse
 from django.db import models, transaction
+from django.urls import reverse
 from django.utils.encoding import force_text, python_2_unicode_compatible
 from django.utils.timezone import now
 from django.utils.translation import ugettext, ugettext_lazy as _
@@ -30,12 +30,13 @@ from .events import (
 )
 from .literals import DEFAULT_DELETE_PERIOD, DEFAULT_DELETE_TIME_UNIT
 from .managers import (
-    DocumentManager, DocumentTypeManager, PassthroughManager,
-    RecentDocumentManager, TrashCanManager
+    DocumentManager, DocumentTypeManager, DuplicatedDocumentManager,
+    PassthroughManager, RecentDocumentManager, TrashCanManager
 )
 from .permissions import permission_document_view
 from .runtime import cache_storage_backend, storage_backend
 from .settings import (
+    setting_disable_base_image_cache, setting_disable_transformed_image_cache,
     setting_display_size, setting_language, setting_zoom_max_level,
     setting_zoom_min_level
 )
@@ -149,7 +150,7 @@ class Document(models.Model):
 
     uuid = models.UUIDField(default=uuid.uuid4, editable=False)
     document_type = models.ForeignKey(
-        DocumentType, related_name='documents',
+        DocumentType, on_delete=models.CASCADE, related_name='documents',
         verbose_name=_('Document type')
     )
     label = models.CharField(
@@ -362,7 +363,8 @@ class DocumentVersion(models.Model):
         cls._post_save_hooks[order] = func
 
     document = models.ForeignKey(
-        Document, related_name='versions', verbose_name=_('Document')
+        Document, on_delete=models.CASCADE, related_name='versions',
+        verbose_name=_('Document')
     )
     timestamp = models.DateTimeField(
         auto_now_add=True, db_index=True, verbose_name=_('Timestamp')
@@ -382,11 +384,16 @@ class DocumentVersion(models.Model):
     encoding = models.CharField(
         blank=True, editable=False, max_length=64, null=True
     )
-    checksum = models.TextField(
-        blank=True, db_index=True, editable=False, null=True,
+    checksum = models.CharField(
+        blank=True, db_index=True, editable=False, max_length=64, null=True,
         verbose_name=_('Checksum')
     )
 
+    class Meta:
+        ordering = ('timestamp',)
+        verbose_name = _('Document version')
+        verbose_name_plural = _('Document version')
+
     def __str__(self):
         return '{0} - {1}'.format(self.document, self.timestamp)
 
@@ -448,17 +455,13 @@ class DocumentVersion(models.Model):
                 event_document_new_version.commit(
                     actor=user, target=self.document
                 )
-                post_version_upload.send(sender=self.__class__, instance=self)
+                post_version_upload.send(sender=DocumentVersion, instance=self)
 
                 if tuple(self.document.versions.all()) == (self,):
                     post_document_created.send(
-                        sender=self.document.__class__, instance=self.document
+                        sender=Document, instance=self.document
                     )
 
-    class Meta:
-        verbose_name = _('Document version')
-        verbose_name_plural = _('Document version')
-
     @property
     def cache_filename(self):
         return 'document-version-{}'.format(self.uuid)
@@ -647,7 +650,7 @@ class DocumentTypeFilename(models.Model):
     quick rename functionality
     """
     document_type = models.ForeignKey(
-        DocumentType, related_name='filenames',
+        DocumentType, on_delete=models.CASCADE, related_name='filenames',
         verbose_name=_('Document type')
     )
     filename = models.CharField(
@@ -671,7 +674,7 @@ class DocumentPage(models.Model):
     Model that describes a document version page
     """
     document_version = models.ForeignKey(
-        DocumentVersion, related_name='pages',
+        DocumentVersion, on_delete=models.CASCADE, related_name='pages',
         verbose_name=_('Document version')
     )
     page_number = models.PositiveIntegerField(
@@ -768,11 +771,14 @@ class DocumentPage(models.Model):
         # Check is transformed image is available
         logger.debug('transformations cache filename: %s', cache_filename)
 
-        if cache_storage_backend.exists(cache_filename):
+        if not setting_disable_transformed_image_cache.value and cache_storage_backend.exists(cache_filename):
             logger.debug(
                 'transformations cache file "%s" found', cache_filename
             )
         else:
+            logger.debug(
+                'transformations cache file "%s" not found', cache_filename
+            )
             image = self.get_image(transformations=transformation_list)
             with cache_storage_backend.open(cache_filename, 'wb+') as file_object:
                 file_object.write(image.getvalue())
@@ -785,7 +791,7 @@ class DocumentPage(models.Model):
         cache_filename = self.cache_filename
         logger.debug('Page cache filename: %s', cache_filename)
 
-        if cache_storage_backend.exists(cache_filename):
+        if not setting_disable_base_image_cache.value and cache_storage_backend.exists(cache_filename):
             logger.debug('Page cache file "%s" found', cache_filename)
             converter = converter_class(
                 file_object=cache_storage_backend.open(cache_filename)
@@ -841,7 +847,7 @@ class DocumentPage(models.Model):
 
 class DocumentPageCachedImage(models.Model):
     document_page = models.ForeignKey(
-        DocumentPage, related_name='cached_images',
+        DocumentPage, on_delete=models.CASCADE, related_name='cached_images',
         verbose_name=_('Document page')
     )
     filename = models.CharField(max_length=128, verbose_name=_('Filename'))
@@ -870,10 +876,12 @@ class RecentDocument(models.Model):
     a given user
     """
     user = models.ForeignKey(
-        settings.AUTH_USER_MODEL, db_index=True, editable=False, verbose_name=_('User')
+        settings.AUTH_USER_MODEL, db_index=True, editable=False,
+        on_delete=models.CASCADE, verbose_name=_('User')
     )
     document = models.ForeignKey(
-        Document, editable=False, verbose_name=_('Document')
+        Document, editable=False, on_delete=models.CASCADE,
+        verbose_name=_('Document')
     )
     datetime_accessed = models.DateTimeField(
         auto_now=True, db_index=True, verbose_name=_('Accessed')
@@ -892,3 +900,26 @@ class RecentDocument(models.Model):
         ordering = ('-datetime_accessed',)
         verbose_name = _('Recent document')
         verbose_name_plural = _('Recent documents')
+
+
+@python_2_unicode_compatible
+class DuplicatedDocument(models.Model):
+    document = models.ForeignKey(
+        Document, on_delete=models.CASCADE, related_name='duplicates',
+        verbose_name=_('Document')
+    )
+    documents = models.ManyToManyField(
+        Document, verbose_name=_('Duplicated documents')
+    )
+    datetime_added = models.DateTimeField(
+        auto_now_add=True, db_index=True, verbose_name=_('Added')
+    )
+
+    objects = DuplicatedDocumentManager()
+
+    def __str__(self):
+        return force_text(self.document)
+
+    class Meta:
+        verbose_name = _('Duplicated document')
+        verbose_name_plural = _('Duplicated documents')

mayan/apps/documents/permissions.py

@@ -51,8 +51,9 @@ permission_empty_trash = namespace.add_permission(
     name='document_empty_trash', label=_('Empty trash')
 )
 
+# TODO: rename 'document_setup' to 'document_types' on the next major version
 setup_namespace = PermissionNamespace(
-    'documents_setup', label=_('Documents setup')
+    'documents_setup', label=_('Document types')
 )
 permission_document_type_create = setup_namespace.add_permission(
     name='document_type_create', label=_('Create document types')

mayan/apps/documents/queues.py

@@ -14,6 +14,9 @@ queue_documents_periodic = CeleryQueue(
 queue_uploads = CeleryQueue(
     name='uploads', label=_('Uploads')
 )
+queue_uploads = CeleryQueue(
+    name='documents', label=_('Documents')
+)
 
 queue_documents_periodic.add_task_type(
     name='documents.tasks.task_check_delete_periods',
@@ -46,3 +49,7 @@ queue_uploads.add_task_type(
     name='documents.tasks.task_upload_new_version',
     label=_('Upload new document version')
 )
+queue_uploads.add_task_type(
+    name='documents.tasks.task_delete_document',
+    label=_('Delete a document')
+)

mayan/apps/documents/settings.py

@@ -21,7 +21,7 @@ setting_print_size = namespace.add_setting(
     global_name='DOCUMENTS_PRINT_SIZE', default='3600'
 )
 setting_thumbnail_size = namespace.add_setting(
-    global_name='DOCUMENTS_THUMBNAIL_SIZE', default='50x50'
+    global_name='DOCUMENTS_THUMBNAIL_SIZE', default='800'
 )
 setting_recent_count = namespace.add_setting(
     global_name='DOCUMENTS_RECENT_COUNT', default=40,
@@ -73,3 +73,18 @@ setting_language_choices = namespace.add_setting(
     global_name='DOCUMENTS_LANGUAGE_CHOICES', default=LANGUAGE_CHOICES,
     help_text=_('List of supported document languages.')
 )
+setting_disable_base_image_cache = namespace.add_setting(
+    global_name='DOCUMENTS_DISABLE_BASE_IMAGE_CACHE', default=False,
+    help_text=_(
+        'Disables the first cache tier which stores high resolution, '
+        'non transformed versions of documents\'s pages.'
+    )
+)
+setting_disable_transformed_image_cache = namespace.add_setting(
+    global_name='DOCUMENTS_DISABLE_TRANSFORMED_IMAGE_CACHE', default=False,
+    help_text=_(
+        'Disables the second cache tier which stores medium to low '
+        'resolution, transformed (rotated, zoomed, etc) versions '
+        'of documents\' pages.'
+    )
+)

mayan/apps/documents/tasks.py

@@ -44,6 +44,18 @@ def task_clear_image_cache():
     logger.info('Finished document cache invalidation')
 
 
+@app.task(ignore_result=True)
+def task_delete_document(deleted_document_id):
+    DeletedDocument = apps.get_model(
+        app_label='documents', model_name='DeletedDocument'
+    )
+
+    logger.debug('Executing')
+    deleted_document = DeletedDocument.objects.get(pk=deleted_document_id)
+    deleted_document.delete()
+    logger.debug('Finshed')
+
+
 @app.task(ignore_result=True)
 def task_delete_stubs():
     Document = apps.get_model(
@@ -66,6 +78,29 @@ def task_generate_document_page_image(document_page_id, *args, **kwargs):
     return document_page.generate_image(*args, **kwargs)
 
 
+@app.task(ignore_result=True)
+def task_scan_duplicates_all():
+    DuplicatedDocument = apps.get_model(
+        app_label='documents', model_name='DuplicatedDocument'
+    )
+
+    DuplicatedDocument.objects.scan()
+
+
+@app.task(ignore_result=True)
+def task_scan_duplicates_for(document_id):
+    Document = apps.get_model(
+        app_label='documents', model_name='Document'
+    )
+    DuplicatedDocument = apps.get_model(
+        app_label='documents', model_name='DuplicatedDocument'
+    )
+
+    document = Document.objects.get(pk=document_id)
+
+    DuplicatedDocument.objects.scan_for(document=document)
+
+
 @app.task(bind=True, default_retry_delay=UPDATE_PAGE_COUNT_RETRY_DELAY, ignore_result=True)
 def task_update_page_count(self, version_id):
     DocumentVersion = apps.get_model(

mayan/apps/documents/templates/documents/document_print.html

@@ -4,6 +4,6 @@
 
 {% block content_plain %}
     {% for page in pages %}
-        <img src="{% url 'rest_api:documentpage-image' page.id %}?size={{ size }}" />
+        <img src="{% url 'rest_api:documentpage-image' page.document.pk page.document.latest_version.pk page.pk %}?size={{ size }}" />
     {% endfor %}
 {% endblock %}

mayan/apps/documents/tests/literals.py

@@ -9,14 +9,15 @@ __all__ = (
     'TEST_COMPRESSED_DOCUMENTS_FILENAME', 'TEST_COMPRESSED_DOCUMENT_PATH',
     'TEST_DEU_DOCUMENT_FILENAME', 'TEST_DEU_DOCUMENT_PATH',
     'TEST_DOCUMENT_PATH', 'TEST_HYBRID_DOCUMENT_PATH',
-    'TEST_DOCUMENT_FILENAME', 'TEST_DOCUMENT_TYPE',
+    'TEST_DOCUMENT_FILENAME', 'TEST_DOCUMENT_TYPE_LABEL',
+    'TEST_DOCUMENT_TYPE_2_LABEL', 'TEST_DOCUMENT_TYPE_LABEL_EDITED',
     'TEST_DOCUMENT_DESCRIPTION', 'TEST_HYBRID_DOCUMENT',
     'TEST_MULTI_PAGE_TIFF', 'TEST_MULTI_PAGE_TIFF_PATH',
     'TEST_NON_ASCII_COMPRESSED_DOCUMENT_FILENAME',
     'TEST_NON_ASCII_COMPRESSED_DOCUMENT_PATH',
     'TEST_NON_ASCII_DOCUMENT_FILENAME', 'TEST_NON_ASCII_DOCUMENT_PATH',
-    'TEST_SMALL_DOCUMENT_FILENAME', 'TEST_SMALL_DOCUMENT_PATH',
-    'TEST_DOCUMENT_VERSION_COMMENT_EDITED',
+    'TEST_SMALL_DOCUMENT_CHECKSUM', 'TEST_SMALL_DOCUMENT_FILENAME',
+    'TEST_SMALL_DOCUMENT_PATH', 'TEST_DOCUMENT_VERSION_COMMENT_EDITED',
 )
 
 # Filenames
@@ -25,8 +26,11 @@ TEST_DEU_DOCUMENT_FILENAME = 'deu_website.png'
 TEST_DOCUMENT_DESCRIPTION = 'test description'
 TEST_DOCUMENT_DESCRIPTION_EDITED = 'test document description edited'
 TEST_DOCUMENT_FILENAME = 'mayan_11_1.pdf'
-TEST_DOCUMENT_TYPE = 'test_document_type'
-TEST_DOCUMENT_TYPE_2 = 'test document type 2'
+TEST_DOCUMENT_TYPE_LABEL = 'test_document_type'
+TEST_DOCUMENT_TYPE_2_LABEL = 'test document type 2'
+TEST_DOCUMENT_TYPE_LABEL_EDITED = 'test document type edited label'
+TEST_DOCUMENT_TYPE_QUICK_LABEL = 'test quick label'
+TEST_DOCUMENT_TYPE_QUICK_LABEL_EDITED = 'test quick label edited'
 TEST_DOCUMENT_VERSION_COMMENT_EDITED = 'test document version comment edited'
 TEST_HYBRID_DOCUMENT = 'hybrid_text_and_image.pdf'
 TEST_MULTI_PAGE_TIFF = 'multi_page.tiff'
@@ -36,6 +40,8 @@ TEST_OFFICE_DOCUMENT = 'simple_2_page_document.doc'
 TEST_SMALL_DOCUMENT_FILENAME = 'title_page.png'
 TEST_SMALL_DOCUMENT_CHECKSUM = 'efa10e6cc21f83078aaa94d5cbe51de67b51af706143b\
 afc7fd6d4c02124879a'
+TEST_TRANSFORMATION_NAME = 'rotate'
+TEST_TRANSFORMATION_ARGUMENT = 'degrees: 180'
 
 # File paths
 TEST_COMPRESSED_DOCUMENT_PATH = os.path.join(
@@ -50,7 +56,6 @@ TEST_DOCUMENT_PATH = os.path.join(
     settings.BASE_DIR, 'apps', 'documents', 'tests', 'contrib',
     'sample_documents', TEST_DOCUMENT_FILENAME
 )
-TEST_DOCUMENT_TYPE_QUICK_LABEL = 'test quick label'
 TEST_HYBRID_DOCUMENT_PATH = os.path.join(
     settings.BASE_DIR, 'apps', 'documents', 'tests', 'contrib',
     'sample_documents', TEST_HYBRID_DOCUMENT

mayan/apps/documents/tests/test_api.py

@@ -7,8 +7,8 @@ import time
 from json import loads
 
 from django.contrib.auth import get_user_model
-from django.core.urlresolvers import reverse
 from django.test import override_settings
+from django.urls import reverse
 from django.utils.encoding import force_text
 
 from django_downloadview import assert_download_response
@@ -21,9 +21,9 @@ from user_management.tests.literals import (
 
 from .literals import (
     TEST_DOCUMENT_DESCRIPTION_EDITED, TEST_DOCUMENT_FILENAME,
-    TEST_DOCUMENT_PATH, TEST_DOCUMENT_TYPE,
-    TEST_DOCUMENT_VERSION_COMMENT_EDITED, TEST_SMALL_DOCUMENT_FILENAME,
-    TEST_SMALL_DOCUMENT_PATH
+    TEST_DOCUMENT_PATH, TEST_DOCUMENT_TYPE_LABEL,
+    TEST_DOCUMENT_TYPE_LABEL_EDITED, TEST_DOCUMENT_VERSION_COMMENT_EDITED,
+    TEST_SMALL_DOCUMENT_FILENAME, TEST_SMALL_DOCUMENT_PATH
 )
 from ..models import Document, DocumentType
 
@@ -49,40 +49,46 @@ class DocumentTypeAPITestCase(BaseAPITestCase):
 
         response = self.client.post(
             reverse('rest_api:documenttype-list'), data={
-                'label': TEST_DOCUMENT_TYPE
+                'label': TEST_DOCUMENT_TYPE_LABEL
             }
         )
 
         self.assertEqual(response.status_code, 201)
         self.assertEqual(DocumentType.objects.all().count(), 1)
         self.assertEqual(
-            DocumentType.objects.all().first().label, TEST_DOCUMENT_TYPE
+            DocumentType.objects.all().first().label, TEST_DOCUMENT_TYPE_LABEL
         )
 
     def test_document_type_edit_via_put(self):
-        document_type = DocumentType.objects.create(label=TEST_DOCUMENT_TYPE)
+        document_type = DocumentType.objects.create(
+            label=TEST_DOCUMENT_TYPE_LABEL
+        )
 
         self.client.put(
             reverse('rest_api:documenttype-detail', args=(document_type.pk,)),
-            {'label': TEST_DOCUMENT_TYPE + 'edited'}
+            {'label': TEST_DOCUMENT_TYPE_LABEL_EDITED}
         )
 
         document_type = DocumentType.objects.get(pk=document_type.pk)
-        self.assertEqual(document_type.label, TEST_DOCUMENT_TYPE + 'edited')
+        self.assertEqual(document_type.label, TEST_DOCUMENT_TYPE_LABEL_EDITED)
 
     def test_document_type_edit_via_patch(self):
-        document_type = DocumentType.objects.create(label=TEST_DOCUMENT_TYPE)
+        document_type = DocumentType.objects.create(
+            label=TEST_DOCUMENT_TYPE_LABEL
+        )
 
         self.client.patch(
             reverse('rest_api:documenttype-detail', args=(document_type.pk,)),
-            {'label': TEST_DOCUMENT_TYPE + 'edited'}
+            {'label': TEST_DOCUMENT_TYPE_LABEL_EDITED}
         )
 
         document_type = DocumentType.objects.get(pk=document_type.pk)
-        self.assertEqual(document_type.label, TEST_DOCUMENT_TYPE + 'edited')
+        self.assertEqual(document_type.label, TEST_DOCUMENT_TYPE_LABEL_EDITED)
 
     def test_document_type_delete(self):
-        document_type = DocumentType.objects.create(label=TEST_DOCUMENT_TYPE)
+        document_type = DocumentType.objects.create(
+            label=TEST_DOCUMENT_TYPE_LABEL
+        )
 
         self.client.delete(
             reverse('rest_api:documenttype-detail', args=(document_type.pk,))
@@ -105,7 +111,7 @@ class DocumentAPITestCase(BaseAPITestCase):
         )
 
         self.document_type = DocumentType.objects.create(
-            label=TEST_DOCUMENT_TYPE
+            label=TEST_DOCUMENT_TYPE_LABEL
         )
 
     def tearDown(self):
@@ -225,9 +231,9 @@ class DocumentAPITestCase(BaseAPITestCase):
         with open(TEST_DOCUMENT_PATH) as file_object:
             document.new_version(file_object=file_object)
 
-        self.assertEqual(document.versions.count(), 2)
+        document.refresh_from_db()
 
-        last_version = document.versions.last()
+        self.assertEqual(document.versions.count(), 2)
 
         response = self.client.get(
             reverse(
@@ -235,8 +241,10 @@ class DocumentAPITestCase(BaseAPITestCase):
                 args=(document.pk,)
             )
         )
+
         self.assertEqual(
-            response.data['results'][1]['checksum'], last_version.checksum
+            response.data['results'][1]['checksum'],
+            document.latest_version.checksum
         )
 
     def test_document_download(self):
@@ -361,7 +369,7 @@ class TrashedDocumentAPITestCase(BaseAPITestCase):
         )
 
         self.document_type = DocumentType.objects.create(
-            label=TEST_DOCUMENT_TYPE
+            label=TEST_DOCUMENT_TYPE_LABEL
         )
 
     def tearDown(self):
@@ -430,6 +438,3 @@ class TrashedDocumentAPITestCase(BaseAPITestCase):
 
         self.assertEqual(Document.trash.count(), 0)
         self.assertEqual(Document.objects.count(), 1)
-
-    # TODO: def test_document_set_document_type(self):
-    #    pass

mayan/apps/documents/tests/test_links.py

@@ -4,7 +4,7 @@ from __future__ import unicode_literals
 
 import time
 
-from django.core.urlresolvers import reverse
+from django.urls import reverse
 
 from acls.models import AccessControlList
 from user_management.tests.literals import (

mayan/apps/documents/tests/test_models.py

@@ -10,7 +10,7 @@ from ..literals import STUB_EXPIRATION_INTERVAL
 from ..models import DeletedDocument, Document, DocumentType
 
 from .literals import (
-    TEST_DOCUMENT_TYPE, TEST_DOCUMENT_PATH, TEST_MULTI_PAGE_TIFF_PATH,
+    TEST_DOCUMENT_TYPE_LABEL, TEST_DOCUMENT_PATH, TEST_MULTI_PAGE_TIFF_PATH,
     TEST_OFFICE_DOCUMENT_PATH, TEST_SMALL_DOCUMENT_PATH
 )
 
@@ -21,7 +21,7 @@ class GenericDocumentTestCase(BaseTestCase):
         super(GenericDocumentTestCase, self).setUp()
 
         self.document_type = DocumentType.objects.create(
-            label=TEST_DOCUMENT_TYPE
+            label=TEST_DOCUMENT_TYPE_LABEL
         )
 
         with open(TEST_SMALL_DOCUMENT_PATH) as file_object:
@@ -40,7 +40,7 @@ class DocumentTestCase(BaseTestCase):
         super(DocumentTestCase, self).setUp()
 
         self.document_type = DocumentType.objects.create(
-            label=TEST_DOCUMENT_TYPE
+            label=TEST_DOCUMENT_TYPE_LABEL
         )
 
         with open(TEST_DOCUMENT_PATH) as file_object:
@@ -53,7 +53,7 @@ class DocumentTestCase(BaseTestCase):
         super(DocumentTestCase, self).tearDown()
 
     def test_document_creation(self):
-        self.assertEqual(self.document_type.label, TEST_DOCUMENT_TYPE)
+        self.assertEqual(self.document_type.label, TEST_DOCUMENT_TYPE_LABEL)
 
         self.assertEqual(self.document.exists(), True)
         self.assertEqual(self.document.size, 272213)
@@ -162,7 +162,7 @@ class OfficeDocumentTestCase(BaseTestCase):
         super(OfficeDocumentTestCase, self).setUp()
 
         self.document_type = DocumentType.objects.create(
-            label=TEST_DOCUMENT_TYPE
+            label=TEST_DOCUMENT_TYPE_LABEL
         )
 
         with open(TEST_OFFICE_DOCUMENT_PATH) as file_object:
@@ -191,7 +191,7 @@ class MultiPageTiffTestCase(BaseTestCase):
     def setUp(self):
         super(MultiPageTiffTestCase, self).setUp()
         self.document_type = DocumentType.objects.create(
-            label=TEST_DOCUMENT_TYPE
+            label=TEST_DOCUMENT_TYPE_LABEL
         )
 
         with open(TEST_MULTI_PAGE_TIFF_PATH) as file_object:
@@ -253,7 +253,7 @@ class DocumentManagerTestCase(BaseTestCase):
     def setUp(self):
         super(DocumentManagerTestCase, self).setUp()
         self.document_type = DocumentType.objects.create(
-            label=TEST_DOCUMENT_TYPE
+            label=TEST_DOCUMENT_TYPE_LABEL
         )
 
     def tearDown(self):

mayan/apps/documents/urls.py

@@ -16,8 +16,8 @@ from .views import (
     ClearImageCacheView, DeletedDocumentDeleteView,
     DeletedDocumentDeleteManyView, DeletedDocumentListView,
     DocumentDocumentTypeEditView, DocumentDownloadFormView,
-    DocumentDownloadView, DocumentEditView, DocumentListView,
-    DocumentPageListView, DocumentPageNavigationFirst,
+    DocumentDownloadView, DocumentDuplicatesListView, DocumentEditView,
+    DocumentListView, DocumentPageListView, DocumentPageNavigationFirst,
     DocumentPageNavigationLast, DocumentPageNavigationNext,
     DocumentPageNavigationPrevious, DocumentPageRotateLeftView,
     DocumentPageRotateRightView, DocumentPageView, DocumentPageViewResetView,
@@ -31,7 +31,8 @@ from .views import (
     DocumentTypeListView, DocumentTypeEditView, DocumentUpdatePageCountView,
     DocumentVersionDownloadFormView, DocumentVersionDownloadView,
     DocumentVersionListView, DocumentVersionRevertView, DocumentView,
-    EmptyTrashCanView, RecentDocumentListView
+    DuplicatedDocumentListView, EmptyTrashCanView, RecentDocumentListView,
+    ScanDuplicatedDocuments
 )
 
 
@@ -45,7 +46,11 @@ urlpatterns = [
         r'^list/deleted/$', DeletedDocumentListView.as_view(),
         name='document_list_deleted'
     ),
-
+    url(
+        r'^list/duplicated/$',
+        DuplicatedDocumentListView.as_view(),
+        name='duplicated_document_list'
+    ),
     url(
         r'^(?P<pk>\d+)/preview/$', DocumentPreviewView.as_view(),
         name='document_preview'
@@ -54,6 +59,10 @@ urlpatterns = [
         r'^(?P<pk>\d+)/properties/$', DocumentView.as_view(),
         name='document_properties'
     ),
+    url(
+        r'^(?P<pk>\d+)/duplicates/$', DocumentDuplicatesListView.as_view(),
+        name='document_duplicates_list'
+    ),
     url(
         r'^(?P<pk>\d+)/restore/$', DocumentRestoreView.as_view(),
         name='document_restore'
@@ -255,6 +264,14 @@ urlpatterns = [
         DocumentTypeFilenameCreateView.as_view(),
         name='document_type_filename_create'
     ),
+
+    # Tools
+
+    url(
+        r'^tools/documents/duplicated/scan/$',
+        ScanDuplicatedDocuments.as_view(),
+        name='duplicated_document_scan'
+    ),
 ]
 
 api_urls = [

mayan/apps/documents/views/document_page_views.py

@@ -5,8 +5,8 @@ import urlparse
 
 from django.conf import settings
 from django.contrib import messages
-from django.core.urlresolvers import reverse
-from django.shortcuts import get_object_or_404
+from django.shortcuts import get_object_or_404, resolve_url
+from django.urls import reverse
 from django.utils.http import urlencode
 from django.utils.translation import ugettext_lazy as _
 from django.views.generic import RedirectView
@@ -46,6 +46,7 @@ class DocumentPageListView(SingleObjectListView):
 
     def get_extra_context(self):
         return {
+            'list_as_items': True,
             'object': self.get_document(),
             'title': _('Pages for document: %s') % self.get_document(),
         }
@@ -70,7 +71,7 @@ class DocumentPageNavigationBase(RedirectView):
     def get_redirect_url(self, *args, **kwargs):
         parse_result = urlparse.urlparse(
             self.request.META.get(
-                'HTTP_REFERER', reverse(
+                'HTTP_REFERER', resolve_url(
                     settings.LOGIN_REDIRECT_URL
                 )
             )

mayan/apps/documents/views/document_type_views.py

@@ -2,8 +2,8 @@ from __future__ import absolute_import, unicode_literals
 
 import logging
 
-from django.core.urlresolvers import reverse, reverse_lazy
 from django.shortcuts import get_object_or_404
+from django.urls import reverse, reverse_lazy
 from django.utils.translation import ugettext_lazy as _
 
 from acls.models import AccessControlList
@@ -32,16 +32,19 @@ class DocumentTypeDocumentListView(DocumentListView):
         return self.get_document_type().documents.all()
 
     def get_extra_context(self):
-        return {
-            'hide_links': True,
-            'object': self.get_document_type(),
-            'title': _('Documents of type: %s') % self.get_document_type()
-        }
+        context = super(DocumentTypeDocumentListView, self).get_extra_context()
+        context.update(
+            {
+                'object': self.get_document_type(),
+                'title': _('Documents of type: %s') % self.get_document_type()
+            }
+        )
+        return context
 
 
 class DocumentTypeListView(SingleObjectListView):
     model = DocumentType
-    view_permission = permission_document_type_view
+    object_permission = permission_document_type_view
 
     def get_extra_context(self):
         return {
@@ -67,8 +70,8 @@ class DocumentTypeCreateView(SingleObjectCreateView):
 
 class DocumentTypeDeleteView(SingleObjectDeleteView):
     model = DocumentType
+    object_permission = permission_document_type_delete
     post_action_redirect = reverse_lazy('documents:document_type_list')
-    view_permission = permission_document_type_delete
 
     def get_extra_context(self):
         return {
@@ -84,8 +87,8 @@ class DocumentTypeEditView(SingleObjectEditView):
         'delete_time_unit'
     )
     model = DocumentType
+    object_permission = permission_document_type_edit
     post_action_redirect = reverse_lazy('documents:document_type_list')
-    view_permission = permission_document_type_edit
 
     def get_extra_context(self):
         return {
@@ -126,7 +129,7 @@ class DocumentTypeFilenameCreateView(SingleObjectCreateView):
 class DocumentTypeFilenameEditView(SingleObjectEditView):
     fields = ('enabled', 'filename',)
     model = DocumentTypeFilename
-    view_permission = permission_document_type_edit
+    object_permission = permission_document_type_edit
 
     def get_extra_context(self):
         document_type_filename = self.get_object()
@@ -153,7 +156,7 @@ class DocumentTypeFilenameEditView(SingleObjectEditView):
 
 class DocumentTypeFilenameDeleteView(SingleObjectDeleteView):
     model = DocumentTypeFilename
-    view_permission = permission_document_type_edit
+    object_permission = permission_document_type_edit
 
     def get_extra_context(self):
         return {
@@ -178,7 +181,7 @@ class DocumentTypeFilenameDeleteView(SingleObjectDeleteView):
 
 class DocumentTypeFilenameListView(SingleObjectListView):
     model = DocumentType
-    view_permission = permission_document_type_view
+    object_permission = permission_document_type_view
 
     def get_document_type(self):
         return get_object_or_404(DocumentType, pk=self.kwargs['pk'])

mayan/apps/documents/views/document_version_views.py

@@ -12,7 +12,7 @@ from common.generics import ConfirmView, SingleObjectListView
 from ..models import Document, DocumentVersion
 from ..permissions import (
     permission_document_download, permission_document_version_revert,
-    permission_document_version_view, permission_document_view
+    permission_document_version_view
 )
 
 from .document_views import DocumentDownloadFormView, DocumentDownloadView

mayan/apps/documents/views/document_views.py

@@ -4,9 +4,9 @@ import logging
 
 from django.contrib import messages
 from django.core.exceptions import PermissionDenied
-from django.core.urlresolvers import reverse, reverse_lazy
 from django.http import HttpResponseRedirect
 from django.shortcuts import get_object_or_404
+from django.urls import reverse, reverse_lazy
 from django.utils.encoding import force_text
 from django.utils.http import urlencode
 from django.utils.translation import ugettext_lazy as _, ungettext
@@ -19,6 +19,7 @@ from common.generics import (
     SingleObjectDownloadView, SingleObjectEditView, SingleObjectListView
 )
 from common.mixins import MultipleInstanceActionMixin
+from common.utils import encapsulate
 from converter.models import Transformation
 from converter.permissions import (
     permission_transformation_delete, permission_transformation_edit
@@ -31,7 +32,9 @@ from ..forms import (
     DocumentTypeSelectForm,
 )
 from ..literals import PAGE_RANGE_RANGE, DEFAULT_ZIP_FILENAME
-from ..models import DeletedDocument, Document, RecentDocument
+from ..models import (
+    DeletedDocument, Document, DuplicatedDocument, RecentDocument
+)
 from ..permissions import (
     permission_document_delete, permission_document_download,
     permission_document_print, permission_document_properties_edit,
@@ -40,42 +43,47 @@ from ..permissions import (
     permission_empty_trash
 )
 from ..settings import setting_print_size
-from ..tasks import task_update_page_count
+from ..tasks import task_delete_document, task_update_page_count
 from ..utils import parse_range
 
 logger = logging.getLogger(__name__)
 
 
 class DocumentListView(SingleObjectListView):
-    extra_context = {
-        'hide_links': True,
-        'title': _('All documents'),
-    }
-
     object_permission = permission_document_view
 
     def get_document_queryset(self):
         return Document.objects.defer('description', 'uuid', 'date_added', 'language', 'in_trash', 'deleted_date_time').all()
 
+    def get_extra_context(self):
+        return {
+            'hide_links': True,
+            'list_as_items': True,
+            'title': _('All documents'),
+        }
+
     def get_queryset(self):
-        self.queryset = self.get_document_queryset().filter(is_stub=False)
-        return super(DocumentListView, self).get_queryset()
+        return self.get_document_queryset().filter(is_stub=False)
 
 
 class DeletedDocumentListView(DocumentListView):
     object_permission = None
 
-    extra_context = {
-        'hide_link': True,
-        'title': _('Documents in trash'),
-    }
-
     def get_document_queryset(self):
         return AccessControlList.objects.filter_by_access(
             permission_document_view, self.request.user,
             queryset=DeletedDocument.trash.all()
         )
 
+    def get_extra_context(self):
+        context = super(DeletedDocumentListView, self).get_extra_context()
+        context.update(
+            {
+                'title': _('Documents in trash'),
+            }
+        )
+        return context
+
 
 class DeletedDocumentDeleteView(ConfirmView):
     extra_context = {
@@ -92,7 +100,9 @@ class DeletedDocumentDeleteView(ConfirmView):
             obj=source_document
         )
 
-        instance.delete()
+        task_delete_document.apply_async(
+            kwargs={'deleted_document_id': instance.pk}
+        )
 
     def view_action(self):
         instance = get_object_or_404(DeletedDocument, pk=self.kwargs['pk'])
@@ -167,6 +177,39 @@ class DocumentDocumentTypeEditView(MultipleObjectFormActionView):
         )
 
 
+class DocumentDuplicatesListView(DocumentListView):
+    def dispatch(self, request, *args, **kwargs):
+        AccessControlList.objects.check_access(
+            permissions=permission_document_view, user=self.request.user,
+            obj=self.get_document()
+        )
+
+        return super(
+            DocumentDuplicatesListView, self
+        ).dispatch(request, *args, **kwargs)
+
+    def get_document(self):
+        return get_object_or_404(Document, pk=self.kwargs['pk'])
+
+    def get_extra_context(self):
+        context = super(DocumentDuplicatesListView, self).get_extra_context()
+        context.update(
+            {
+                'object': self.get_document(),
+                'title': _('Duplicates for document: %s') % self.get_document(),
+            }
+        )
+        return context
+
+    def get_queryset(self):
+        try:
+            return DuplicatedDocument.objects.get(
+                document=self.get_document()
+            ).documents.all()
+        except DuplicatedDocument.DoesNotExist:
+            return Document.objects.none()
+
+
 class DocumentEditView(SingleObjectEditView):
     form_class = DocumentForm
     model = Document
@@ -331,20 +374,26 @@ class EmptyTrashCanView(ConfirmView):
 
     def view_action(self):
         for deleted_document in DeletedDocument.objects.all():
-            deleted_document.delete()
+            task_delete_document.apply_async(
+                kwargs={'deleted_document_id': deleted_document.pk}
+            )
 
         messages.success(self.request, _('Trash emptied successfully'))
 
 
 class RecentDocumentListView(DocumentListView):
-    extra_context = {
-        'hide_links': True,
-        'title': _('Recent documents'),
-    }
-
     def get_document_queryset(self):
         return RecentDocument.objects.get_for_user(self.request.user)
 
+    def get_extra_context(self):
+        context = super(RecentDocumentListView, self).get_extra_context()
+        context.update(
+            {
+                'title': _('Recent documents'),
+            }
+        )
+        return context
+
 
 class DocumentDownloadFormView(FormView):
     form_class = DocumentDownloadForm
@@ -367,7 +416,7 @@ class DocumentDownloadFormView(FormView):
     def get_extra_context(self):
         subtemplates_list = [
             {
-                'name': 'appearance/generic_list_subtemplate.html',
+                'name': 'appearance/generic_list_items_subtemplate.html',
                 'context': {
                     'object_list': self.queryset,
                     'hide_link': True,
@@ -719,6 +768,34 @@ class DocumentPrint(FormView):
 
     def get_template_names(self):
         if self.request.method == 'POST':
-            return ['documents/document_print.html']
+            return ('documents/document_print.html',)
         else:
-            return [self.template_name]
+            return (self.template_name,)
+
+
+class DuplicatedDocumentListView(DocumentListView):
+    def get_document_queryset(self):
+        return Document.objects.filter(
+            pk__in=DuplicatedDocument.objects.values_list(
+                'document_id', flat=True
+            )
+        )
+
+    def get_extra_context(self):
+        context = super(DuplicatedDocumentListView, self).get_extra_context()
+        context.update(
+            {
+                'extra_columns': (
+                    {
+                        'name': _('Duplicates'),
+                        'attribute': encapsulate(
+                            lambda document: DuplicatedDocument.objects.get(
+                                document=document
+                            ).documents.count()
+                        )
+                    },
+                ),
+                'title': _('Duplicated documents')
+            }
+        )
+        return context

mayan/apps/documents/views/misc_views.py

@@ -8,7 +8,7 @@ from django.utils.translation import ugettext_lazy as _
 from common.generics import ConfirmView
 
 from ..permissions import permission_document_tools
-from ..tasks import task_clear_image_cache
+from ..tasks import task_clear_image_cache, task_scan_duplicates_all
 
 logger = logging.getLogger(__name__)
 
@@ -24,3 +24,16 @@ class ClearImageCacheView(ConfirmView):
         messages.success(
             self.request, _('Document cache clearing queued successfully.')
         )
+
+
+class ScanDuplicatedDocuments(ConfirmView):
+    extra_context = {
+        'title': _('Scan for duplicated documents?')
+    }
+    view_permission = permission_document_tools
+
+    def view_action(self):
+        task_scan_duplicates_all.apply_async()
+        messages.success(
+            self.request, _('Duplicated document scan queued successfully.')
+        )

mayan/apps/documents/widgets.py

@@ -1,13 +1,15 @@
 from __future__ import unicode_literals
 
 from django import forms
-from django.core.urlresolvers import reverse
+from django.urls import reverse
 from django.utils.encoding import force_text
 from django.utils.html import strip_tags
 from django.utils.http import urlencode
 from django.utils.safestring import mark_safe
 from django.utils.translation import ugettext, ugettext_lazy as _
 
+from common.utils import index_or_default
+
 from .settings import (
     setting_display_size, setting_preview_size, setting_thumbnail_size
 )
@@ -99,6 +101,8 @@ class InstanceImageWidget(object):
     preview_query_dict = {}
     image_class = 'lazy-load'
     title = None
+    width = None
+    height = None
 
     # Click view
     def get_click_view_kwargs(self, instance):
@@ -221,10 +225,16 @@ class InstanceImageWidget(object):
                 )
 
             result.append(
-                '<i class="spinner fa fa-spinner fa-pulse fa-3x fa-fw"></i> '
-                '<img class="thin_border {image_class}" '
+                '<div class="spinner-container text-primary" style="height: {height}px;">'
+                '<span class="spinner-icon fa-stack fa-lg">'
+                '<i class="fa fa-file-o fa-stack-2x"></i>'
+                '<i class="fa fa-clock-o fa-stack-1x"></i>'
+                '</span>'
+                '</div>'
+                '<img class="thin_border {image_class} pull-left" style="width: {width};"'
                 'data-url="{preview_full_url}" src="#" '
-                'alt="{alt_text}" /> '.format(
+                '/> '.format(
+                    width=self.width or '100%', height=self.height or '150',
                     image_class=self.image_class,
                     preview_full_url=self.get_preview_view_url(instance=instance),
                     alt_text=self.alt_text
@@ -253,6 +263,8 @@ class BaseDocumentThumbnailWidget(InstanceImageWidget):
     preview_view_query_dict = {
         'size': setting_thumbnail_size.value
     }
+    width = setting_thumbnail_size.value.split('x')[0]
+    height = 150
 
     def get_destination_url(self, instance):
         return instance.get_absolute_url()
@@ -265,9 +277,16 @@ class CarouselDocumentPageThumbnailWidget(BaseDocumentThumbnailWidget):
     preview_view_query_dict = {
         'size': setting_display_size.value
     }
+    width = setting_preview_size.value.split('x')[0]
+    height = index_or_default(
+        instance=setting_preview_size.value.split('x'),
+        index=1, default=setting_preview_size.value.split('x')[0]
+    )
 
 
 class DocumentThumbnailWidget(BaseDocumentThumbnailWidget):
+    width = '100%'
+
     def get_click_view_kwargs(self, instance):
         return {
             'pk': instance.pk,
@@ -308,6 +327,8 @@ class DocumentThumbnailWidget(BaseDocumentThumbnailWidget):
 
 
 class DocumentPageThumbnailWidget(BaseDocumentThumbnailWidget):
+    width = '100%'
+
     def get_title(self, instance):
         return force_text(instance)