Caching: Initial experitmental cache model

signed-off-by: Roberto Rosario <rosarior@t60.lan>

mayan/apps/common/migrations/0012_auto_20181130_2348.py

@@ -0,0 +1,55 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.16 on 2018-11-30 23:48
+from __future__ import unicode_literals
+
+import common.models
+import django.core.files.storage
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('contenttypes', '0002_remove_content_type_name'),
+        ('common', '0011_auto_20180429_0758'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Cache',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=128, verbose_name='Name')),
+                ('label', models.CharField(max_length=128, verbose_name='Label')),
+                ('maximum_size', models.PositiveIntegerField(verbose_name='Maximum size')),
+                ('object_id', models.PositiveIntegerField()),
+                ('storage_instance_path', models.CharField(max_length=255, verbose_name='Storage instance path')),
+                ('content_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType')),
+            ],
+            options={
+                'verbose_name': 'Cache',
+                'verbose_name_plural': 'Caches',
+            },
+        ),
+        migrations.CreateModel(
+            name='CacheFile',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('datetime', models.DateTimeField(auto_now_add=True, db_index=True, verbose_name='Date time')),
+                ('filename', models.CharField(max_length=128, verbose_name='Filename')),
+                ('file_size', models.PositiveIntegerField(db_index=True, default=0, verbose_name='File size')),
+                ('cache', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='files', to='common.Cache', verbose_name='Cache')),
+            ],
+            options={
+                'get_latest_by': 'datetime',
+                'verbose_name': 'Cache file',
+                'verbose_name_plural': 'Cache files',
+            },
+        ),
+        migrations.AlterField(
+            model_name='shareduploadedfile',
+            name='file',
+            field=models.FileField(storage=django.core.files.storage.FileSystemStorage(location='/usr/local/development/mayan-edms/mayan/media/shared_files'), upload_to=common.models.upload_to, verbose_name='File'),
+        ),
+    ]

mayan/apps/common/models.py

@@ -8,7 +8,10 @@ from django.conf import settings
 from django.contrib.contenttypes.fields import GenericForeignKey
 from django.contrib.contenttypes.models import ContentType
 from django.db import models
+from django.db.models import Sum
 from django.utils.encoding import force_text, python_2_unicode_compatible
+from django.utils.functional import cached_property
+from django.utils.module_loading import import_string
 from django.utils.translation import ugettext_lazy as _
 
 from .managers import ErrorLogEntryManager, UserLocaleProfileManager
@@ -19,6 +22,67 @@ def upload_to(instance, filename):
     return 'shared-file-{}'.format(uuid.uuid4().hex)
 
 
+@python_2_unicode_compatible
+class Cache(models.Model):
+    name = models.CharField(max_length=128, verbose_name=_('Name'))
+    label = models.CharField(max_length=128, verbose_name=_('Label'))
+    maximum_size = models.PositiveIntegerField(verbose_name=_('Maximum size'))
+    content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE)
+    object_id = models.PositiveIntegerField()
+    content_object = GenericForeignKey('content_type', 'object_id')
+    storage_instance_path = models.CharField(
+        max_length=255, verbose_name=_('Storage instance path')
+    )
+
+    class Meta:
+        verbose_name = _('Cache')
+        verbose_name_plural = _('Caches')
+
+    def __str__(self):
+        return self.label
+
+    def get_total_size(self):
+        return self.files.aggregate(
+            file_size__sum=Sum('file_size')
+        )['file_size__sum']
+
+    def prune(self):
+        while self.get_total_size() > self.maximum_size:
+            self.files.earliest().delete()
+
+    @cached_property
+    def storage_instance(self):
+        return import_string(self.storage_instance_path)
+
+
+class CacheFile(models.Model):
+    cache = models.ForeignKey(
+        on_delete=models.CASCADE, related_name='files',
+        to=Cache, verbose_name=_('Cache')
+    )
+    datetime = models.DateTimeField(
+        auto_now_add=True, db_index=True, verbose_name=_('Date time')
+    )
+    filename = models.CharField(max_length=128, verbose_name=_('Filename'))
+    file_size = models.PositiveIntegerField(
+        db_index=True, default=0, verbose_name=_('File size')
+    )
+
+    class Meta:
+        get_latest_by = 'datetime'
+        verbose_name = _('Cache file')
+        verbose_name_plural = _('Cache files')
+
+    def delete(self, *args, **kwargs):
+        self.cache.storage_instance.delete(self.filename)
+        return super(CacheFile, self).delete(*args, **kwargs)
+
+    def save(self, *args, **kwargs):
+        self.cache.prune()
+        self.file_size = self.cache.storage_instance.size(self.filename)
+        return super(CacheFile, self).save(*args, **kwargs)
+
+
 class ErrorLogEntry(models.Model):
     """
     Class to store an error log for any object. Uses generic foreign keys to

mayan/apps/documents/apps.py

@@ -4,7 +4,7 @@ from datetime import timedelta
 
 from kombu import Exchange, Queue
 
-from django.db.models.signals import post_delete
+from django.db.models.signals import post_delete, post_migrate
 from django.utils.translation import ugettext_lazy as _
 
 from acls import ModelPermission
@@ -48,8 +48,8 @@ from .events import (
     event_document_view
 )
 from .handlers import (
-    create_default_document_type, handler_remove_empty_duplicates_lists,
-    handler_scan_duplicates_for,
+    create_default_document_type, handler_create_document_cache,
+    handler_remove_empty_duplicates_lists, handler_scan_duplicates_for,
 )
 from .links import (
     link_clear_image_cache, link_document_clear_transformations,
@@ -591,6 +591,10 @@ class DocumentsApp(MayanAppConfig):
             create_default_document_type,
             dispatch_uid='create_default_document_type'
         )
+        post_migrate.connect(
+            dispatch_uid='documents_handler_create_document_cache',
+            receiver=handler_create_document_cache,
+        )
         post_version_upload.connect(
             handler_scan_duplicates_for,
             dispatch_uid='handler_scan_duplicates_for',

mayan/apps/documents/handlers.py

@@ -1,6 +1,7 @@
 from __future__ import unicode_literals
 
 from django.apps import apps
+from django.utils.translation import ugettext_lazy as _
 
 from .literals import DEFAULT_DOCUMENT_TYPE_LABEL
 from .signals import post_initial_document_type
@@ -21,6 +22,14 @@ def create_default_document_type(sender, **kwargs):
         )
 
 
+def handler_create_document_cache(sender, **kwargs):
+    Cache = apps.get_model(app_label='common', model_name='Cache')
+    Cache.objects.get_or_create(
+        name='document_images', label=_('Document images'),
+        storage_instance_path='documents.storages.storage_documentimagecache'
+    )
+
+
 def handler_scan_duplicates_for(sender, instance, **kwargs):
     task_scan_duplicates_for.apply_async(
         kwargs={'document_id': instance.document.pk}

mayan/apps/documents/literals.py

@@ -9,6 +9,7 @@ CHECK_TRASH_PERIOD_INTERVAL = 60
 DELETE_STALE_STUBS_INTERVAL = 60 * 10  # 10 minutes
 DEFAULT_DELETE_PERIOD = 30
 DEFAULT_DELETE_TIME_UNIT = TIME_DELTA_UNIT_DAYS
+DEFAULT_DOCUMENTS_CACHE_MAXIMUM_SIZE = 50 * 2 ** 20  # 50 Megabytes
 DEFAULT_LANGUAGE = 'eng'
 DEFAULT_LANGUAGE_CODES = (
     'ilo', 'run', 'uig', 'hin', 'pan', 'pnb', 'wuu', 'msa', 'kxd', 'ind',

mayan/apps/documents/settings.py

@@ -7,7 +7,9 @@ from django.utils.translation import ugettext_lazy as _
 
 from smart_settings import Namespace
 
-from .literals import DEFAULT_LANGUAGE, DEFAULT_LANGUAGE_CODES
+from .literals import (
+    DEFAULT_DOCUMENTS_CACHE_MAXIMUM_SIZE, DEFAULT_LANGUAGE, DEFAULT_LANGUAGE_CODES
+)
 
 namespace = Namespace(name='documents', label=_('Documents'))
 
@@ -25,6 +27,15 @@ setting_documentimagecache_storage_arguments = namespace.add_setting(
         'Arguments to pass to the DOCUMENT_CACHE_STORAGE_BACKEND.'
     )
 )
+setting_document_cache_maximum_size = namespace.add_setting(
+    global_name='DOCUMENTS_CACHE_MAXIMUM_SIZE',
+    default=DEFAULT_DOCUMENTS_CACHE_MAXIMUM_SIZE,
+    help_text=_(
+        'The threshold at which the DOCUMENT_CACHE_STORAGE_BACKEND will start '
+        'deleting the oldest document image cache files. Specify the size in '
+        'bytes.'
+    )
+)
 setting_disable_base_image_cache = namespace.add_setting(
     global_name='DOCUMENTS_DISABLE_BASE_IMAGE_CACHE', default=False,
     help_text=_(