From 424d6a5af98c94ce2090425c14a3b93dbb1fb79d Mon Sep 17 00:00:00 2001 From: Michael Price Date: Wed, 21 Mar 2018 19:56:53 -0400 Subject: [PATCH] Standardize the way storages are used. All apps that use storage now define their storages in the .storages modules instead of the .runtime module. The storage.backends.filebasedstorage.FileBasedStorage has been remove, instead Django's default storage is used and each app is responsible of specifying their default path. Signed-off-by: Michael Price --- HISTORY.rst | 8 +++-- mayan/apps/common/models.py | 4 +-- mayan/apps/common/runtime.py | 5 --- mayan/apps/common/settings.py | 6 +++- mayan/apps/common/storages.py | 17 ++++++++++ mayan/apps/document_signatures/models.py | 2 +- mayan/apps/document_signatures/settings.py | 4 +-- .../{runtime.py => storages.py} | 0 mayan/apps/documents/api_views.py | 4 +-- mayan/apps/documents/literals.py | 1 - mayan/apps/documents/models.py | 32 +++++++++---------- mayan/apps/documents/settings.py | 12 +++---- mayan/apps/documents/storage.py | 18 ----------- .../documents/{runtime.py => storages.py} | 10 +++--- mayan/apps/ocr/managers.py | 4 +-- .../apps/storage/backends/filebasedstorage.py | 20 ------------ mayan/apps/storage/backends/literals.py | 3 -- 17 files changed, 63 insertions(+), 87 deletions(-) delete mode 100644 mayan/apps/common/runtime.py create mode 100644 mayan/apps/common/storages.py rename mayan/apps/document_signatures/{runtime.py => storages.py} (100%) delete mode 100644 mayan/apps/documents/storage.py rename mayan/apps/documents/{runtime.py => storages.py} (55%) delete mode 100644 mayan/apps/storage/backends/filebasedstorage.py delete mode 100644 mayan/apps/storage/backends/literals.py diff --git a/HISTORY.rst b/HISTORY.rst index 71723a5e9a..a8fc5237be 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -117,9 +117,6 @@ - Support passing arguments to the document, document cache and document signatures storage backends. New settings: DOCUMENTS_STORAGE_BACKEND_ARGUMENTS, DOCUMENTS_CACHE_STORAGE_BACKEND_ARGUMENTS, SIGNATURES_STORAGE_BACKEND_ARGUMENTS -- Support passing arguments to the document, document cache and document signatures - storage backends. New settings: DOCUMENTS_STORAGE_BACKEND_ARGUMENTS, - DOCUMENTS_CACHE_STORAGE_BACKEND_ARGUMENTS, SIGNATURES_STORAGE_BACKEND_ARGUMENTS - Remove the setting STORAGE_FILESTORAGE_LOCATION. Document storage location for the storage.backend.filebasedstorage.FileBasedStorage backdend must now passed via the DOCUMENTS_STORAGE_BACKEND_ARGUMENTS, @@ -129,6 +126,11 @@ DOCUMENTS_STORAGE_BACKEND_ARGUMENTS = '{ location: }' If no path is specified the backend will default to 'mayan/media/document_storage'. +- Standardize the way storages are used. All apps that use storage now define + their storages in the .storages modules instead of the .runtime module. + The storage.backends.filebasedstorage.FileBasedStorage has been remove, + instead Django's default storage is used and each app is responsible + of specifying their default path. 2.7.3 (2017-09-11) diff --git a/mayan/apps/common/models.py b/mayan/apps/common/models.py index 5d75ff363b..e76a54a85e 100644 --- a/mayan/apps/common/models.py +++ b/mayan/apps/common/models.py @@ -12,7 +12,7 @@ from django.utils.encoding import force_text, python_2_unicode_compatible from django.utils.translation import ugettext_lazy as _ from .managers import ErrorLogEntryManager -from .runtime import shared_storage_backend +from .storages import sharedupload_storage def upload_to(instance, filename): @@ -47,7 +47,7 @@ class ErrorLogEntry(models.Model): @python_2_unicode_compatible class SharedUploadedFile(models.Model): file = models.FileField( - storage=shared_storage_backend, upload_to=upload_to, + storage=sharedupload_storage, upload_to=upload_to, verbose_name=_('File') ) filename = models.CharField(max_length=255, verbose_name=_('Filename')) diff --git a/mayan/apps/common/runtime.py b/mayan/apps/common/runtime.py deleted file mode 100644 index 59621c1341..0000000000 --- a/mayan/apps/common/runtime.py +++ /dev/null @@ -1,5 +0,0 @@ -from django.utils.module_loading import import_string - -from .settings import setting_shared_storage - -shared_storage_backend = import_string(setting_shared_storage.value)() diff --git a/mayan/apps/common/settings.py b/mayan/apps/common/settings.py index 99c4e121c6..e8a8d45abb 100644 --- a/mayan/apps/common/settings.py +++ b/mayan/apps/common/settings.py @@ -38,9 +38,13 @@ setting_paginate_by = namespace.add_setting( ) setting_shared_storage = namespace.add_setting( global_name='COMMON_SHARED_STORAGE', - default='storage.backends.filebasedstorage.FileBasedStorage', + default='django.core.files.storage.FileSystemStorage', help_text=_('A storage backend that all workers can use to share files.') ) +setting_shared_storage_arguments = namespace.add_setting( + global_name='COMMON_SHARED_STORAGE_ARGUMENTS', + default='{location: mayan/media/shared_files}', +) setting_temporary_directory = namespace.add_setting( global_name='COMMON_TEMPORARY_DIRECTORY', default=tempfile.gettempdir(), help_text=_( diff --git a/mayan/apps/common/storages.py b/mayan/apps/common/storages.py new file mode 100644 index 0000000000..87e04b1b64 --- /dev/null +++ b/mayan/apps/common/storages.py @@ -0,0 +1,17 @@ +from __future__ import unicode_literals + +import yaml + +from django.utils.module_loading import import_string + +from .settings import ( + setting_shared_storage, setting_shared_storage_arguments +) + +sharedupload_storage = import_string( + dotted_path=setting_shared_storage.value +)( + **yaml.safe_load( + setting_shared_storage_arguments.value or '{}' + ) +) diff --git a/mayan/apps/document_signatures/models.py b/mayan/apps/document_signatures/models.py index 17d1b6bb9d..6366b23789 100644 --- a/mayan/apps/document_signatures/models.py +++ b/mayan/apps/document_signatures/models.py @@ -15,7 +15,7 @@ from django_gpg.models import Key from documents.models import DocumentVersion from .managers import EmbeddedSignatureManager -from .runtime import storage_backend +from .storages import storage_backend logger = logging.getLogger(__name__) diff --git a/mayan/apps/document_signatures/settings.py b/mayan/apps/document_signatures/settings.py index edaf2d2a58..962b00ee2c 100644 --- a/mayan/apps/document_signatures/settings.py +++ b/mayan/apps/document_signatures/settings.py @@ -7,9 +7,9 @@ from smart_settings import Namespace namespace = Namespace(name='signatures', label=_('Document signatures')) setting_storage_backend = namespace.add_setting( global_name='SIGNATURES_STORAGE_BACKEND', - default='storage.backends.filebasedstorage.FileBasedStorage' + default='django.core.files.storage.FileSystemStorage' ) setting_storage_backend_arguments = namespace.add_setting( global_name='SIGNATURES_STORAGE_BACKEND_ARGUMENTS', - default='' + default='{location: mayan/media/document_storage}' ) diff --git a/mayan/apps/document_signatures/runtime.py b/mayan/apps/document_signatures/storages.py similarity index 100% rename from mayan/apps/document_signatures/runtime.py rename to mayan/apps/document_signatures/storages.py diff --git a/mayan/apps/documents/api_views.py b/mayan/apps/documents/api_views.py index efb387b18f..5d79656fb7 100644 --- a/mayan/apps/documents/api_views.py +++ b/mayan/apps/documents/api_views.py @@ -27,7 +27,6 @@ from .permissions import ( permission_document_type_view, permission_document_version_revert, permission_document_version_view ) -from .runtime import cache_storage_backend from .serializers import ( DeletedDocumentSerializer, DocumentPageSerializer, DocumentSerializer, DocumentTypeSerializer, DocumentVersionSerializer, @@ -35,6 +34,7 @@ from .serializers import ( RecentDocumentSerializer, WritableDocumentSerializer, WritableDocumentTypeSerializer, WritableDocumentVersionSerializer ) +from .storages import documentimagecache_storage from .tasks import task_generate_document_page_image logger = logging.getLogger(__name__) @@ -288,7 +288,7 @@ class APIDocumentPageImageView(generics.RetrieveAPIView): ) cache_filename = task.get(timeout=DOCUMENT_IMAGE_TASK_TIMEOUT) - with cache_storage_backend.open(cache_filename) as file_object: + with documentimagecache_storage.open(cache_filename) as file_object: return HttpResponse(file_object.read(), content_type='image') diff --git a/mayan/apps/documents/literals.py b/mayan/apps/documents/literals.py index bc58ba9096..77e27106a6 100644 --- a/mayan/apps/documents/literals.py +++ b/mayan/apps/documents/literals.py @@ -4,7 +4,6 @@ from django.utils.translation import ugettext_lazy as _ from common.literals import TIME_DELTA_UNIT_DAYS -CACHE_PATH = 'document_cache/' CHECK_DELETE_PERIOD_INTERVAL = 60 CHECK_TRASH_PERIOD_INTERVAL = 60 DELETE_STALE_STUBS_INTERVAL = 60 * 10 # 10 minutes diff --git a/mayan/apps/documents/models.py b/mayan/apps/documents/models.py index f02dd43908..6e055e46eb 100644 --- a/mayan/apps/documents/models.py +++ b/mayan/apps/documents/models.py @@ -36,7 +36,6 @@ from .managers import ( PassthroughManager, RecentDocumentManager, TrashCanManager ) from .permissions import permission_document_view -from .runtime import cache_storage_backend, storage_backend from .settings import ( setting_disable_base_image_cache, setting_disable_transformed_image_cache, setting_display_width, setting_display_height, setting_fix_orientation, @@ -45,6 +44,7 @@ from .settings import ( from .signals import ( post_document_created, post_document_type_change, post_version_upload ) +from .storages import documentversion_storage, documentimagecache_storage logger = logging.getLogger(__name__) @@ -393,7 +393,7 @@ class DocumentVersion(models.Model): # File related fields file = models.FileField( - storage=storage_backend, upload_to=UUID_FUNCTION, + storage=documentversion_storage, upload_to=UUID_FUNCTION, verbose_name=_('File') ) mimetype = models.CharField( @@ -459,10 +459,10 @@ class DocumentVersion(models.Model): cache_filename = self.cache_filename logger.debug('Intermidiate filename: %s', cache_filename) - if cache_storage_backend.exists(cache_filename): + if documentimagecache_storage.exists(cache_filename): logger.debug('Intermidiate file "%s" found.', cache_filename) - return cache_storage_backend.open(cache_filename) + return documentimagecache_storage.open(cache_filename) else: logger.debug('Intermidiate file "%s" not found.', cache_filename) @@ -470,11 +470,11 @@ class DocumentVersion(models.Model): converter = converter_class(file_object=self.open()) pdf_file_object = converter.to_pdf() - with cache_storage_backend.open(cache_filename, 'wb+') as file_object: + with documentimagecache_storage.open(cache_filename, 'wb+') as file_object: for chunk in pdf_file_object: file_object.write(chunk) - return cache_storage_backend.open(cache_filename) + return documentimagecache_storage.open(cache_filename) except InvalidOfficeFormat: return self.open() except Exception as exception: @@ -483,7 +483,7 @@ class DocumentVersion(models.Model): 'Error creating intermediate file "%s"; %s.', cache_filename, exception ) - cache_storage_backend.delete(cache_filename) + documentimagecache_storage.delete(cache_filename) raise def get_rendered_string(self, preserve_extension=False): @@ -503,7 +503,7 @@ class DocumentVersion(models.Model): ) def invalidate_cache(self): - cache_storage_backend.delete(self.cache_filename) + documentimagecache_storage.delete(self.cache_filename) for page in self.pages.all(): page.invalidate_cache() @@ -810,7 +810,7 @@ class DocumentPage(models.Model): # Check is transformed image is available logger.debug('transformations cache filename: %s', cache_filename) - if not setting_disable_transformed_image_cache.value and cache_storage_backend.exists(cache_filename): + if not setting_disable_transformed_image_cache.value and documentimagecache_storage.exists(cache_filename): logger.debug( 'transformations cache file "%s" found', cache_filename ) @@ -819,7 +819,7 @@ class DocumentPage(models.Model): 'transformations cache file "%s" not found', cache_filename ) image = self.get_image(transformations=transformation_list) - with cache_storage_backend.open(cache_filename, 'wb+') as file_object: + with documentimagecache_storage.open(cache_filename, 'wb+') as file_object: file_object.write(image.getvalue()) self.cached_images.create(filename=cache_filename) @@ -840,10 +840,10 @@ class DocumentPage(models.Model): cache_filename = self.cache_filename logger.debug('Page cache filename: %s', cache_filename) - if not setting_disable_base_image_cache.value and cache_storage_backend.exists(cache_filename): + if not setting_disable_base_image_cache.value and documentimagecache_storage.exists(cache_filename): logger.debug('Page cache file "%s" found', cache_filename) converter = converter_class( - file_object=cache_storage_backend.open(cache_filename) + file_object=documentimagecache_storage.open(cache_filename) ) converter.seek(0) @@ -858,7 +858,7 @@ class DocumentPage(models.Model): page_image = converter.get_page() - with cache_storage_backend.open(cache_filename, 'wb+') as file_object: + with documentimagecache_storage.open(cache_filename, 'wb+') as file_object: file_object.write(page_image.getvalue()) except Exception as exception: # Cleanup in case of error @@ -866,7 +866,7 @@ class DocumentPage(models.Model): 'Error creating page cache file "%s"; %s', cache_filename, exception ) - cache_storage_backend.delete(cache_filename) + documentimagecache_storage.delete(cache_filename) raise for transformation in transformations: @@ -875,7 +875,7 @@ class DocumentPage(models.Model): return converter.get_page() def invalidate_cache(self): - cache_storage_backend.delete(self.cache_filename) + documentimagecache_storage.delete(self.cache_filename) for cached_image in self.cached_images.all(): cached_image.delete() @@ -906,7 +906,7 @@ class DocumentPageCachedImage(models.Model): verbose_name_plural = _('Document page cached images') def delete(self, *args, **kwargs): - cache_storage_backend.delete(self.filename) + documentimagecache_storage.delete(self.filename) return super(DocumentPageCachedImage, self).delete(*args, **kwargs) diff --git a/mayan/apps/documents/settings.py b/mayan/apps/documents/settings.py index 5579ecda27..81ef69ce13 100644 --- a/mayan/apps/documents/settings.py +++ b/mayan/apps/documents/settings.py @@ -45,11 +45,11 @@ setting_recent_count = namespace.add_setting( ) setting_storage_backend = namespace.add_setting( global_name='DOCUMENTS_STORAGE_BACKEND', - default='storage.backends.filebasedstorage.FileBasedStorage' + default='django.core.files.storage.FileSystemStorage' ) setting_storage_backend_arguments = namespace.add_setting( global_name='DOCUMENTS_STORAGE_BACKEND_ARGUMENTS', - default='' + default='{location: mayan/media/document_storage}' ) setting_zoom_percent_step = namespace.add_setting( global_name='DOCUMENTS_ZOOM_PERCENT_STEP', default=25, @@ -78,13 +78,13 @@ setting_rotation_step = namespace.add_setting( 'Amount in degrees to rotate a document page per user interaction.' ) ) -setting_cache_storage_backend = namespace.add_setting( +setting_documentimagecache_storage = namespace.add_setting( global_name='DOCUMENTS_CACHE_STORAGE_BACKEND', - default='documents.storage.LocalCacheFileStorage' + default='django.core.files.storage.FileSystemStorage' ) -setting_cache_storage_backend_arguments = namespace.add_setting( +setting_documentimagecache_storage_arguments = namespace.add_setting( global_name='DOCUMENTS_CACHE_STORAGE_BACKEND_ARGUMENTS', - default='' + default='{location: mayan/media/document_cache}' ) setting_language = namespace.add_setting( global_name='DOCUMENTS_LANGUAGE', default='eng', diff --git a/mayan/apps/documents/storage.py b/mayan/apps/documents/storage.py deleted file mode 100644 index eaf26fde39..0000000000 --- a/mayan/apps/documents/storage.py +++ /dev/null @@ -1,18 +0,0 @@ -from __future__ import unicode_literals - -import os - -from django.conf import settings -from django.core.files.storage import FileSystemStorage - -from .literals import CACHE_PATH - - -class LocalCacheFileStorage(FileSystemStorage): - """Simple wrapper for the stock Django FileSystemStorage class""" - - def __init__(self, *args, **kwargs): - super(LocalCacheFileStorage, self).__init__(*args, **kwargs) - self.location = os.path.join(settings.MEDIA_ROOT, CACHE_PATH) - if not os.path.exists(os.path.dirname(self.location)): - os.makedirs(os.path.dirname(self.location)) diff --git a/mayan/apps/documents/runtime.py b/mayan/apps/documents/storages.py similarity index 55% rename from mayan/apps/documents/runtime.py rename to mayan/apps/documents/storages.py index dd58e1d0bf..f7c9a4aeb8 100644 --- a/mayan/apps/documents/runtime.py +++ b/mayan/apps/documents/storages.py @@ -5,11 +5,11 @@ import yaml from django.utils.module_loading import import_string from .settings import ( - setting_cache_storage_backend, setting_cache_storage_backend_arguments, + setting_documentimagecache_storage, setting_documentimagecache_storage_arguments, setting_storage_backend, setting_storage_backend_arguments ) -storage_backend = import_string( +documentversion_storage = import_string( dotted_path=setting_storage_backend.value )( **yaml.safe_load( @@ -17,10 +17,10 @@ storage_backend = import_string( ) ) -cache_storage_backend = import_string( - dotted_path=setting_cache_storage_backend.value +documentimagecache_storage = import_string( + dotted_path=setting_documentimagecache_storage.value )( **yaml.safe_load( - setting_cache_storage_backend_arguments.value or '{}' + setting_documentimagecache_storage_arguments.value or '{}' ) ) diff --git a/mayan/apps/ocr/managers.py b/mayan/apps/ocr/managers.py index be26e0257e..55dfd203c5 100644 --- a/mayan/apps/ocr/managers.py +++ b/mayan/apps/ocr/managers.py @@ -8,7 +8,7 @@ from django.apps import apps from django.conf import settings from django.db import models -from documents.runtime import cache_storage_backend +from documents.storages import documentimagecache_storage from .events import event_ocr_document_version_finish from .runtime import ocr_backend @@ -31,7 +31,7 @@ class DocumentPageOCRContentManager(models.Manager): # TODO: Call task and wait cache_filename = document_page.generate_image() - with cache_storage_backend.open(cache_filename) as file_object: + with documentimagecache_storage.open(cache_filename) as file_object: document_page_content, created = DocumentPageOCRContent.objects.get_or_create( document_page=document_page ) diff --git a/mayan/apps/storage/backends/filebasedstorage.py b/mayan/apps/storage/backends/filebasedstorage.py deleted file mode 100644 index dfd3757ba9..0000000000 --- a/mayan/apps/storage/backends/filebasedstorage.py +++ /dev/null @@ -1,20 +0,0 @@ -from __future__ import unicode_literals - -import os - -from django.core.files.storage import FileSystemStorage -from django.conf import settings - -from .literals import DEFAULT_PATH - - -class FileBasedStorage(FileSystemStorage): - """Simple wrapper for the stock Django FileSystemStorage class""" - - separator = os.path.sep - - def __init__(self, *args, **kwargs): - self.location = kwargs.pop( - 'location', os.path.join(settings.MEDIA_ROOT, DEFAULT_PATH) - ) - super(FileBasedStorage, self).__init__(*args, **kwargs) diff --git a/mayan/apps/storage/backends/literals.py b/mayan/apps/storage/backends/literals.py deleted file mode 100644 index 74d1d49b1b..0000000000 --- a/mayan/apps/storage/backends/literals.py +++ /dev/null @@ -1,3 +0,0 @@ -from __future__ import unicode_literals - -DEFAULT_PATH = 'document_storage'