From 713977ed4623e14d37794591b6b86bdbffb2080d Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Mon, 13 Jul 2015 01:06:40 -0400 Subject: [PATCH] Document image and intermediate file caching now has it's own storage backend. --- .gitignore | 2 +- mayan/apps/documents/links.py | 2 +- mayan/apps/documents/literals.py | 1 + mayan/apps/documents/models.py | 33 ++++++++++++++++---------------- mayan/apps/documents/runtime.py | 3 ++- mayan/apps/documents/settings.py | 2 +- mayan/apps/documents/storage.py | 18 +++++++++++++++++ 7 files changed, 41 insertions(+), 20 deletions(-) create mode 100644 mayan/apps/documents/storage.py diff --git a/.gitignore b/.gitignore index dad5458e19..0f9c54dc3c 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,7 @@ settings_local.py /celerybeat-schedule document_storage/ /misc/mayan.geany -image_cache/ +mayan/media/document_cache/ build/ _build/ gpg_home/ diff --git a/mayan/apps/documents/links.py b/mayan/apps/documents/links.py index bb12cf2706..bd626b3995 100644 --- a/mayan/apps/documents/links.py +++ b/mayan/apps/documents/links.py @@ -71,7 +71,7 @@ link_document_list_deleted = Link(icon='fa fa-trash', text=_('Trash'), view='doc link_clear_image_cache = Link( icon='fa fa-file-image-o', description=_('Clear the graphics representations used to speed up the documents\' display and interactive transformations results.'), - permissions=[permission_document_tools], text=_('Clear image cache'), + permissions=[permission_document_tools], text=_('Clear document cache'), view='documents:document_clear_image_cache' ) link_trash_can_empty = Link(permissions=[permission_empty_trash], text=_('Empty trash'), view='documents:trash_can_empty') diff --git a/mayan/apps/documents/literals.py b/mayan/apps/documents/literals.py index 9eb99dfb14..2e9b5ed480 100644 --- a/mayan/apps/documents/literals.py +++ b/mayan/apps/documents/literals.py @@ -1,5 +1,6 @@ from __future__ import unicode_literals +CACHE_PATH = 'document_cache/' CHECK_DELETE_PERIOD_INTERVAL = 60 CHECK_TRASH_PERIOD_INTERVAL = 60 DEFAULT_DELETE_PERIOD = 30 diff --git a/mayan/apps/documents/models.py b/mayan/apps/documents/models.py index 994a5f27f7..36391a4955 100644 --- a/mayan/apps/documents/models.py +++ b/mayan/apps/documents/models.py @@ -16,7 +16,6 @@ from django.utils.translation import ugettext_lazy as _ from common.literals import TIME_DELTA_UNIT_CHOICES from common.models import SharedUploadedFile from common.settings import setting_temporary_directory -from common.utils import fs_cleanup from converter import ( converter_class, TransformationResize, TransformationRotate, TransformationZoom @@ -35,10 +34,10 @@ from .managers import ( DocumentManager, DocumentTypeManager, PassthroughManager, RecentDocumentManager, TrashCanManager ) -from .runtime import storage_backend +from .runtime import cache_storage_backend, storage_backend from .settings import ( - setting_cache_path, setting_display_size, setting_language, - setting_language_choices, setting_zoom_max_level, setting_zoom_min_level + setting_display_size, setting_language, setting_language_choices, + setting_zoom_max_level, setting_zoom_min_level ) from .signals import ( post_document_created, post_document_type_change, post_version_upload @@ -340,6 +339,7 @@ class DocumentVersion(models.Model): post_document_created.send(sender=self.document.__class__, instance=self.document) def invalidate_cache(self): + cache_storage_backend.delete(self.cache_filename) for page in self.pages.all(): page.invalidate_cache() @@ -472,16 +472,16 @@ class DocumentVersion(models.Model): @property def cache_filename(self): - return os.path.join(setting_cache_path.value, 'document-version-{}'.format(self.uuid)) + return 'document-version-{}'.format(self.uuid) def get_intermidiate_file(self): cache_filename = self.cache_filename logger.debug('Intermidiate filename: %s', cache_filename) - if os.path.exists(cache_filename): + if cache_storage_backend.exists(cache_filename): logger.debug('Intermidiate file "%s" found.', cache_filename) - return open(cache_filename) + return cache_storage_backend.open(cache_filename) else: logger.debug('Intermidiate file "%s" not found.', cache_filename) @@ -489,17 +489,17 @@ class DocumentVersion(models.Model): converter = converter_class(file_object=self.open()) pdf_file_object = converter.to_pdf() - with open(cache_filename, 'wb+') as file_object: + with cache_storage_backend.open(cache_filename, 'wb+') as file_object: for chunk in pdf_file_object: file_object.write(chunk) - return open(cache_filename) + return cache_storage_backend.open(cache_filename) except InvalidOfficeFormat: return self.open() except Exception as exception: # Cleanup in case of error logger.error('Error creating intermediate file "%s"; %s.', cache_filename, exception) - fs_cleanup(cache_filename) + cache_storage_backend.delete(cache_filename) raise @@ -560,7 +560,7 @@ class DocumentPage(models.Model): return self.document_version.document def invalidate_cache(self): - fs_cleanup(self.cache_filename) + cache_storage_backend.delete(self.cache_filename) @property def uuid(self): @@ -572,7 +572,7 @@ class DocumentPage(models.Model): @property def cache_filename(self): - return os.path.join(setting_cache_path.value, 'page-cache-{}'.format(self.uuid)) + return 'page-cache-{}'.format(self.uuid) def get_image(self, *args, **kwargs): as_base64 = kwargs.pop('as_base64', False) @@ -592,9 +592,9 @@ class DocumentPage(models.Model): cache_filename = self.cache_filename logger.debug('Page cache filename: %s', cache_filename) - if os.path.exists(cache_filename): + if cache_storage_backend.exists(cache_filename): logger.debug('Page cache file "%s" found', cache_filename) - converter = converter_class(file_object=open(cache_filename)) + converter = converter_class(file_object=cache_storage_backend.open(cache_filename)) converter.seek(0) else: @@ -605,12 +605,13 @@ class DocumentPage(models.Model): converter.seek(page_number=self.page_number - 1) page_image = converter.get_page() - with open(cache_filename, 'wb+') as file_object: + + with cache_storage_backend.open(cache_filename, 'wb+') as file_object: file_object.write(page_image.getvalue()) except Exception as exception: # Cleanup in case of error logger.error('Error creating page cache file "%s"; %s', cache_filename, exception) - fs_cleanup(cache_filename) + cache_storage_backend.delete(cache_filename) raise # Stored transformations diff --git a/mayan/apps/documents/runtime.py b/mayan/apps/documents/runtime.py index 7d5bdfc4ca..b36c95e8c0 100644 --- a/mayan/apps/documents/runtime.py +++ b/mayan/apps/documents/runtime.py @@ -1,5 +1,6 @@ from django.utils.module_loading import import_string -from .settings import setting_storage_backend +from .settings import setting_cache_storage_backend, setting_storage_backend storage_backend = import_string(setting_storage_backend.value)() +cache_storage_backend = import_string(setting_cache_storage_backend.value)() diff --git a/mayan/apps/documents/settings.py b/mayan/apps/documents/settings.py index c08242e287..63cf47537e 100644 --- a/mayan/apps/documents/settings.py +++ b/mayan/apps/documents/settings.py @@ -24,6 +24,6 @@ setting_zoom_percent_step = namespace.add_setting(global_name='DOCUMENTS_ZOOM_PE setting_zoom_max_level = namespace.add_setting(global_name='DOCUMENTS_ZOOM_MAX_LEVEL', default=300, help_text=_('Maximum amount in percent (%) to allow user to zoom in a document page interactively.')) setting_zoom_min_level = namespace.add_setting(global_name='DOCUMENTS_ZOOM_MIN_LEVEL', default=25, help_text=_('Minimum amount in percent (%) to allow user to zoom out a document page interactively.')) setting_rotation_step = namespace.add_setting(global_name='DOCUMENTS_ROTATION_STEP', default=90, help_text=_('Amount in degrees to rotate a document page per user interaction.')) -setting_cache_path = namespace.add_setting(global_name='DOCUMENTS_CACHE_PATH', default=os.path.join(settings.MEDIA_ROOT, 'image_cache'), is_path=True) +setting_cache_storage_backend = namespace.add_setting(global_name='DOCUMENTS_CACHE_STORAGE_BACKEND', default='documents.storage.LocalCacheFileStorage') setting_language = namespace.add_setting(global_name='DOCUMENTS_LANGUAGE', default='eng', help_text=_('Default documents language (in ISO639-2 format).')) setting_language_choices = namespace.add_setting(global_name='DOCUMENTS_LANGUAGE_CHOICES', default=LANGUAGE_CHOICES, help_text=_('List of supported document languages.')) diff --git a/mayan/apps/documents/storage.py b/mayan/apps/documents/storage.py new file mode 100644 index 0000000000..eaf26fde39 --- /dev/null +++ b/mayan/apps/documents/storage.py @@ -0,0 +1,18 @@ +from __future__ import unicode_literals + +import os + +from django.conf import settings +from django.core.files.storage import FileSystemStorage + +from .literals import CACHE_PATH + + +class LocalCacheFileStorage(FileSystemStorage): + """Simple wrapper for the stock Django FileSystemStorage class""" + + def __init__(self, *args, **kwargs): + super(LocalCacheFileStorage, self).__init__(*args, **kwargs) + self.location = os.path.join(settings.MEDIA_ROOT, CACHE_PATH) + if not os.path.exists(os.path.dirname(self.location)): + os.makedirs(os.path.dirname(self.location))