From a414b8df92fa23fee58e7169c36830f19c144e6b Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Fri, 30 Nov 2018 19:48:26 -0400 Subject: [PATCH] Caching: Initial experitmental cache model signed-off-by: Roberto Rosario --- .../migrations/0012_auto_20181130_2348.py | 55 ++++++++++++++++ mayan/apps/common/models.py | 64 +++++++++++++++++++ mayan/apps/documents/apps.py | 10 ++- mayan/apps/documents/handlers.py | 9 +++ mayan/apps/documents/literals.py | 1 + mayan/apps/documents/settings.py | 13 +++- 6 files changed, 148 insertions(+), 4 deletions(-) create mode 100644 mayan/apps/common/migrations/0012_auto_20181130_2348.py diff --git a/mayan/apps/common/migrations/0012_auto_20181130_2348.py b/mayan/apps/common/migrations/0012_auto_20181130_2348.py new file mode 100644 index 0000000000..e3bee0c4a9 --- /dev/null +++ b/mayan/apps/common/migrations/0012_auto_20181130_2348.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.16 on 2018-11-30 23:48 +from __future__ import unicode_literals + +import common.models +import django.core.files.storage +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('contenttypes', '0002_remove_content_type_name'), + ('common', '0011_auto_20180429_0758'), + ] + + operations = [ + migrations.CreateModel( + name='Cache', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=128, verbose_name='Name')), + ('label', models.CharField(max_length=128, verbose_name='Label')), + ('maximum_size', models.PositiveIntegerField(verbose_name='Maximum size')), + ('object_id', models.PositiveIntegerField()), + ('storage_instance_path', models.CharField(max_length=255, verbose_name='Storage instance path')), + ('content_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType')), + ], + options={ + 'verbose_name': 'Cache', + 'verbose_name_plural': 'Caches', + }, + ), + migrations.CreateModel( + name='CacheFile', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('datetime', models.DateTimeField(auto_now_add=True, db_index=True, verbose_name='Date time')), + ('filename', models.CharField(max_length=128, verbose_name='Filename')), + ('file_size', models.PositiveIntegerField(db_index=True, default=0, verbose_name='File size')), + ('cache', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='files', to='common.Cache', verbose_name='Cache')), + ], + options={ + 'get_latest_by': 'datetime', + 'verbose_name': 'Cache file', + 'verbose_name_plural': 'Cache files', + }, + ), + migrations.AlterField( + model_name='shareduploadedfile', + name='file', + field=models.FileField(storage=django.core.files.storage.FileSystemStorage(location='/usr/local/development/mayan-edms/mayan/media/shared_files'), upload_to=common.models.upload_to, verbose_name='File'), + ), + ] diff --git a/mayan/apps/common/models.py b/mayan/apps/common/models.py index 41f736c700..6e535202fe 100644 --- a/mayan/apps/common/models.py +++ b/mayan/apps/common/models.py @@ -8,7 +8,10 @@ from django.conf import settings from django.contrib.contenttypes.fields import GenericForeignKey from django.contrib.contenttypes.models import ContentType from django.db import models +from django.db.models import Sum from django.utils.encoding import force_text, python_2_unicode_compatible +from django.utils.functional import cached_property +from django.utils.module_loading import import_string from django.utils.translation import ugettext_lazy as _ from .managers import ErrorLogEntryManager, UserLocaleProfileManager @@ -19,6 +22,67 @@ def upload_to(instance, filename): return 'shared-file-{}'.format(uuid.uuid4().hex) +@python_2_unicode_compatible +class Cache(models.Model): + name = models.CharField(max_length=128, verbose_name=_('Name')) + label = models.CharField(max_length=128, verbose_name=_('Label')) + maximum_size = models.PositiveIntegerField(verbose_name=_('Maximum size')) + content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE) + object_id = models.PositiveIntegerField() + content_object = GenericForeignKey('content_type', 'object_id') + storage_instance_path = models.CharField( + max_length=255, verbose_name=_('Storage instance path') + ) + + class Meta: + verbose_name = _('Cache') + verbose_name_plural = _('Caches') + + def __str__(self): + return self.label + + def get_total_size(self): + return self.files.aggregate( + file_size__sum=Sum('file_size') + )['file_size__sum'] + + def prune(self): + while self.get_total_size() > self.maximum_size: + self.files.earliest().delete() + + @cached_property + def storage_instance(self): + return import_string(self.storage_instance_path) + + +class CacheFile(models.Model): + cache = models.ForeignKey( + on_delete=models.CASCADE, related_name='files', + to=Cache, verbose_name=_('Cache') + ) + datetime = models.DateTimeField( + auto_now_add=True, db_index=True, verbose_name=_('Date time') + ) + filename = models.CharField(max_length=128, verbose_name=_('Filename')) + file_size = models.PositiveIntegerField( + db_index=True, default=0, verbose_name=_('File size') + ) + + class Meta: + get_latest_by = 'datetime' + verbose_name = _('Cache file') + verbose_name_plural = _('Cache files') + + def delete(self, *args, **kwargs): + self.cache.storage_instance.delete(self.filename) + return super(CacheFile, self).delete(*args, **kwargs) + + def save(self, *args, **kwargs): + self.cache.prune() + self.file_size = self.cache.storage_instance.size(self.filename) + return super(CacheFile, self).save(*args, **kwargs) + + class ErrorLogEntry(models.Model): """ Class to store an error log for any object. Uses generic foreign keys to diff --git a/mayan/apps/documents/apps.py b/mayan/apps/documents/apps.py index 04763c42cb..8aa9b8a1c2 100644 --- a/mayan/apps/documents/apps.py +++ b/mayan/apps/documents/apps.py @@ -4,7 +4,7 @@ from datetime import timedelta from kombu import Exchange, Queue -from django.db.models.signals import post_delete +from django.db.models.signals import post_delete, post_migrate from django.utils.translation import ugettext_lazy as _ from acls import ModelPermission @@ -48,8 +48,8 @@ from .events import ( event_document_view ) from .handlers import ( - create_default_document_type, handler_remove_empty_duplicates_lists, - handler_scan_duplicates_for, + create_default_document_type, handler_create_document_cache, + handler_remove_empty_duplicates_lists, handler_scan_duplicates_for, ) from .links import ( link_clear_image_cache, link_document_clear_transformations, @@ -591,6 +591,10 @@ class DocumentsApp(MayanAppConfig): create_default_document_type, dispatch_uid='create_default_document_type' ) + post_migrate.connect( + dispatch_uid='documents_handler_create_document_cache', + receiver=handler_create_document_cache, + ) post_version_upload.connect( handler_scan_duplicates_for, dispatch_uid='handler_scan_duplicates_for', diff --git a/mayan/apps/documents/handlers.py b/mayan/apps/documents/handlers.py index 62fa444a0e..c59e666a60 100644 --- a/mayan/apps/documents/handlers.py +++ b/mayan/apps/documents/handlers.py @@ -1,6 +1,7 @@ from __future__ import unicode_literals from django.apps import apps +from django.utils.translation import ugettext_lazy as _ from .literals import DEFAULT_DOCUMENT_TYPE_LABEL from .signals import post_initial_document_type @@ -21,6 +22,14 @@ def create_default_document_type(sender, **kwargs): ) +def handler_create_document_cache(sender, **kwargs): + Cache = apps.get_model(app_label='common', model_name='Cache') + Cache.objects.get_or_create( + name='document_images', label=_('Document images'), + storage_instance_path='documents.storages.storage_documentimagecache' + ) + + def handler_scan_duplicates_for(sender, instance, **kwargs): task_scan_duplicates_for.apply_async( kwargs={'document_id': instance.document.pk} diff --git a/mayan/apps/documents/literals.py b/mayan/apps/documents/literals.py index 47777469aa..8a6da28df8 100644 --- a/mayan/apps/documents/literals.py +++ b/mayan/apps/documents/literals.py @@ -9,6 +9,7 @@ CHECK_TRASH_PERIOD_INTERVAL = 60 DELETE_STALE_STUBS_INTERVAL = 60 * 10 # 10 minutes DEFAULT_DELETE_PERIOD = 30 DEFAULT_DELETE_TIME_UNIT = TIME_DELTA_UNIT_DAYS +DEFAULT_DOCUMENTS_CACHE_MAXIMUM_SIZE = 50 * 2 ** 20 # 50 Megabytes DEFAULT_LANGUAGE = 'eng' DEFAULT_LANGUAGE_CODES = ( 'ilo', 'run', 'uig', 'hin', 'pan', 'pnb', 'wuu', 'msa', 'kxd', 'ind', diff --git a/mayan/apps/documents/settings.py b/mayan/apps/documents/settings.py index 334a3abd6a..111bdc5d98 100644 --- a/mayan/apps/documents/settings.py +++ b/mayan/apps/documents/settings.py @@ -7,7 +7,9 @@ from django.utils.translation import ugettext_lazy as _ from smart_settings import Namespace -from .literals import DEFAULT_LANGUAGE, DEFAULT_LANGUAGE_CODES +from .literals import ( + DEFAULT_DOCUMENTS_CACHE_MAXIMUM_SIZE, DEFAULT_LANGUAGE, DEFAULT_LANGUAGE_CODES +) namespace = Namespace(name='documents', label=_('Documents')) @@ -25,6 +27,15 @@ setting_documentimagecache_storage_arguments = namespace.add_setting( 'Arguments to pass to the DOCUMENT_CACHE_STORAGE_BACKEND.' ) ) +setting_document_cache_maximum_size = namespace.add_setting( + global_name='DOCUMENTS_CACHE_MAXIMUM_SIZE', + default=DEFAULT_DOCUMENTS_CACHE_MAXIMUM_SIZE, + help_text=_( + 'The threshold at which the DOCUMENT_CACHE_STORAGE_BACKEND will start ' + 'deleting the oldest document image cache files. Specify the size in ' + 'bytes.' + ) +) setting_disable_base_image_cache = namespace.add_setting( global_name='DOCUMENTS_DISABLE_BASE_IMAGE_CACHE', default=False, help_text=_(