diff --git a/HISTORY.rst b/HISTORY.rst index 71723a5e9a..a8fc5237be 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -117,9 +117,6 @@ - Support passing arguments to the document, document cache and document signatures storage backends. New settings: DOCUMENTS_STORAGE_BACKEND_ARGUMENTS, DOCUMENTS_CACHE_STORAGE_BACKEND_ARGUMENTS, SIGNATURES_STORAGE_BACKEND_ARGUMENTS -- Support passing arguments to the document, document cache and document signatures - storage backends. New settings: DOCUMENTS_STORAGE_BACKEND_ARGUMENTS, - DOCUMENTS_CACHE_STORAGE_BACKEND_ARGUMENTS, SIGNATURES_STORAGE_BACKEND_ARGUMENTS - Remove the setting STORAGE_FILESTORAGE_LOCATION. Document storage location for the storage.backend.filebasedstorage.FileBasedStorage backdend must now passed via the DOCUMENTS_STORAGE_BACKEND_ARGUMENTS, @@ -129,6 +126,11 @@ DOCUMENTS_STORAGE_BACKEND_ARGUMENTS = '{ location: }' If no path is specified the backend will default to 'mayan/media/document_storage'. +- Standardize the way storages are used. All apps that use storage now define + their storages in the .storages modules instead of the .runtime module. + The storage.backends.filebasedstorage.FileBasedStorage has been remove, + instead Django's default storage is used and each app is responsible + of specifying their default path. 2.7.3 (2017-09-11) diff --git a/mayan/apps/common/models.py b/mayan/apps/common/models.py index 5d75ff363b..e76a54a85e 100644 --- a/mayan/apps/common/models.py +++ b/mayan/apps/common/models.py @@ -12,7 +12,7 @@ from django.utils.encoding import force_text, python_2_unicode_compatible from django.utils.translation import ugettext_lazy as _ from .managers import ErrorLogEntryManager -from .runtime import shared_storage_backend +from .storages import sharedupload_storage def upload_to(instance, filename): @@ -47,7 +47,7 @@ class ErrorLogEntry(models.Model): @python_2_unicode_compatible class SharedUploadedFile(models.Model): file = models.FileField( - storage=shared_storage_backend, upload_to=upload_to, + storage=sharedupload_storage, upload_to=upload_to, verbose_name=_('File') ) filename = models.CharField(max_length=255, verbose_name=_('Filename')) diff --git a/mayan/apps/common/runtime.py b/mayan/apps/common/runtime.py deleted file mode 100644 index 59621c1341..0000000000 --- a/mayan/apps/common/runtime.py +++ /dev/null @@ -1,5 +0,0 @@ -from django.utils.module_loading import import_string - -from .settings import setting_shared_storage - -shared_storage_backend = import_string(setting_shared_storage.value)() diff --git a/mayan/apps/common/settings.py b/mayan/apps/common/settings.py index 99c4e121c6..e8a8d45abb 100644 --- a/mayan/apps/common/settings.py +++ b/mayan/apps/common/settings.py @@ -38,9 +38,13 @@ setting_paginate_by = namespace.add_setting( ) setting_shared_storage = namespace.add_setting( global_name='COMMON_SHARED_STORAGE', - default='storage.backends.filebasedstorage.FileBasedStorage', + default='django.core.files.storage.FileSystemStorage', help_text=_('A storage backend that all workers can use to share files.') ) +setting_shared_storage_arguments = namespace.add_setting( + global_name='COMMON_SHARED_STORAGE_ARGUMENTS', + default='{location: mayan/media/shared_files}', +) setting_temporary_directory = namespace.add_setting( global_name='COMMON_TEMPORARY_DIRECTORY', default=tempfile.gettempdir(), help_text=_( diff --git a/mayan/apps/common/storages.py b/mayan/apps/common/storages.py new file mode 100644 index 0000000000..87e04b1b64 --- /dev/null +++ b/mayan/apps/common/storages.py @@ -0,0 +1,17 @@ +from __future__ import unicode_literals + +import yaml + +from django.utils.module_loading import import_string + +from .settings import ( + setting_shared_storage, setting_shared_storage_arguments +) + +sharedupload_storage = import_string( + dotted_path=setting_shared_storage.value +)( + **yaml.safe_load( + setting_shared_storage_arguments.value or '{}' + ) +) diff --git a/mayan/apps/document_signatures/models.py b/mayan/apps/document_signatures/models.py index 17d1b6bb9d..6366b23789 100644 --- a/mayan/apps/document_signatures/models.py +++ b/mayan/apps/document_signatures/models.py @@ -15,7 +15,7 @@ from django_gpg.models import Key from documents.models import DocumentVersion from .managers import EmbeddedSignatureManager -from .runtime import storage_backend +from .storages import storage_backend logger = logging.getLogger(__name__) diff --git a/mayan/apps/document_signatures/settings.py b/mayan/apps/document_signatures/settings.py index edaf2d2a58..962b00ee2c 100644 --- a/mayan/apps/document_signatures/settings.py +++ b/mayan/apps/document_signatures/settings.py @@ -7,9 +7,9 @@ from smart_settings import Namespace namespace = Namespace(name='signatures', label=_('Document signatures')) setting_storage_backend = namespace.add_setting( global_name='SIGNATURES_STORAGE_BACKEND', - default='storage.backends.filebasedstorage.FileBasedStorage' + default='django.core.files.storage.FileSystemStorage' ) setting_storage_backend_arguments = namespace.add_setting( global_name='SIGNATURES_STORAGE_BACKEND_ARGUMENTS', - default='' + default='{location: mayan/media/document_storage}' ) diff --git a/mayan/apps/document_signatures/runtime.py b/mayan/apps/document_signatures/storages.py similarity index 100% rename from mayan/apps/document_signatures/runtime.py rename to mayan/apps/document_signatures/storages.py diff --git a/mayan/apps/documents/api_views.py b/mayan/apps/documents/api_views.py index efb387b18f..5d79656fb7 100644 --- a/mayan/apps/documents/api_views.py +++ b/mayan/apps/documents/api_views.py @@ -27,7 +27,6 @@ from .permissions import ( permission_document_type_view, permission_document_version_revert, permission_document_version_view ) -from .runtime import cache_storage_backend from .serializers import ( DeletedDocumentSerializer, DocumentPageSerializer, DocumentSerializer, DocumentTypeSerializer, DocumentVersionSerializer, @@ -35,6 +34,7 @@ from .serializers import ( RecentDocumentSerializer, WritableDocumentSerializer, WritableDocumentTypeSerializer, WritableDocumentVersionSerializer ) +from .storages import documentimagecache_storage from .tasks import task_generate_document_page_image logger = logging.getLogger(__name__) @@ -288,7 +288,7 @@ class APIDocumentPageImageView(generics.RetrieveAPIView): ) cache_filename = task.get(timeout=DOCUMENT_IMAGE_TASK_TIMEOUT) - with cache_storage_backend.open(cache_filename) as file_object: + with documentimagecache_storage.open(cache_filename) as file_object: return HttpResponse(file_object.read(), content_type='image') diff --git a/mayan/apps/documents/literals.py b/mayan/apps/documents/literals.py index bc58ba9096..77e27106a6 100644 --- a/mayan/apps/documents/literals.py +++ b/mayan/apps/documents/literals.py @@ -4,7 +4,6 @@ from django.utils.translation import ugettext_lazy as _ from common.literals import TIME_DELTA_UNIT_DAYS -CACHE_PATH = 'document_cache/' CHECK_DELETE_PERIOD_INTERVAL = 60 CHECK_TRASH_PERIOD_INTERVAL = 60 DELETE_STALE_STUBS_INTERVAL = 60 * 10 # 10 minutes diff --git a/mayan/apps/documents/models.py b/mayan/apps/documents/models.py index f02dd43908..6e055e46eb 100644 --- a/mayan/apps/documents/models.py +++ b/mayan/apps/documents/models.py @@ -36,7 +36,6 @@ from .managers import ( PassthroughManager, RecentDocumentManager, TrashCanManager ) from .permissions import permission_document_view -from .runtime import cache_storage_backend, storage_backend from .settings import ( setting_disable_base_image_cache, setting_disable_transformed_image_cache, setting_display_width, setting_display_height, setting_fix_orientation, @@ -45,6 +44,7 @@ from .settings import ( from .signals import ( post_document_created, post_document_type_change, post_version_upload ) +from .storages import documentversion_storage, documentimagecache_storage logger = logging.getLogger(__name__) @@ -393,7 +393,7 @@ class DocumentVersion(models.Model): # File related fields file = models.FileField( - storage=storage_backend, upload_to=UUID_FUNCTION, + storage=documentversion_storage, upload_to=UUID_FUNCTION, verbose_name=_('File') ) mimetype = models.CharField( @@ -459,10 +459,10 @@ class DocumentVersion(models.Model): cache_filename = self.cache_filename logger.debug('Intermidiate filename: %s', cache_filename) - if cache_storage_backend.exists(cache_filename): + if documentimagecache_storage.exists(cache_filename): logger.debug('Intermidiate file "%s" found.', cache_filename) - return cache_storage_backend.open(cache_filename) + return documentimagecache_storage.open(cache_filename) else: logger.debug('Intermidiate file "%s" not found.', cache_filename) @@ -470,11 +470,11 @@ class DocumentVersion(models.Model): converter = converter_class(file_object=self.open()) pdf_file_object = converter.to_pdf() - with cache_storage_backend.open(cache_filename, 'wb+') as file_object: + with documentimagecache_storage.open(cache_filename, 'wb+') as file_object: for chunk in pdf_file_object: file_object.write(chunk) - return cache_storage_backend.open(cache_filename) + return documentimagecache_storage.open(cache_filename) except InvalidOfficeFormat: return self.open() except Exception as exception: @@ -483,7 +483,7 @@ class DocumentVersion(models.Model): 'Error creating intermediate file "%s"; %s.', cache_filename, exception ) - cache_storage_backend.delete(cache_filename) + documentimagecache_storage.delete(cache_filename) raise def get_rendered_string(self, preserve_extension=False): @@ -503,7 +503,7 @@ class DocumentVersion(models.Model): ) def invalidate_cache(self): - cache_storage_backend.delete(self.cache_filename) + documentimagecache_storage.delete(self.cache_filename) for page in self.pages.all(): page.invalidate_cache() @@ -810,7 +810,7 @@ class DocumentPage(models.Model): # Check is transformed image is available logger.debug('transformations cache filename: %s', cache_filename) - if not setting_disable_transformed_image_cache.value and cache_storage_backend.exists(cache_filename): + if not setting_disable_transformed_image_cache.value and documentimagecache_storage.exists(cache_filename): logger.debug( 'transformations cache file "%s" found', cache_filename ) @@ -819,7 +819,7 @@ class DocumentPage(models.Model): 'transformations cache file "%s" not found', cache_filename ) image = self.get_image(transformations=transformation_list) - with cache_storage_backend.open(cache_filename, 'wb+') as file_object: + with documentimagecache_storage.open(cache_filename, 'wb+') as file_object: file_object.write(image.getvalue()) self.cached_images.create(filename=cache_filename) @@ -840,10 +840,10 @@ class DocumentPage(models.Model): cache_filename = self.cache_filename logger.debug('Page cache filename: %s', cache_filename) - if not setting_disable_base_image_cache.value and cache_storage_backend.exists(cache_filename): + if not setting_disable_base_image_cache.value and documentimagecache_storage.exists(cache_filename): logger.debug('Page cache file "%s" found', cache_filename) converter = converter_class( - file_object=cache_storage_backend.open(cache_filename) + file_object=documentimagecache_storage.open(cache_filename) ) converter.seek(0) @@ -858,7 +858,7 @@ class DocumentPage(models.Model): page_image = converter.get_page() - with cache_storage_backend.open(cache_filename, 'wb+') as file_object: + with documentimagecache_storage.open(cache_filename, 'wb+') as file_object: file_object.write(page_image.getvalue()) except Exception as exception: # Cleanup in case of error @@ -866,7 +866,7 @@ class DocumentPage(models.Model): 'Error creating page cache file "%s"; %s', cache_filename, exception ) - cache_storage_backend.delete(cache_filename) + documentimagecache_storage.delete(cache_filename) raise for transformation in transformations: @@ -875,7 +875,7 @@ class DocumentPage(models.Model): return converter.get_page() def invalidate_cache(self): - cache_storage_backend.delete(self.cache_filename) + documentimagecache_storage.delete(self.cache_filename) for cached_image in self.cached_images.all(): cached_image.delete() @@ -906,7 +906,7 @@ class DocumentPageCachedImage(models.Model): verbose_name_plural = _('Document page cached images') def delete(self, *args, **kwargs): - cache_storage_backend.delete(self.filename) + documentimagecache_storage.delete(self.filename) return super(DocumentPageCachedImage, self).delete(*args, **kwargs) diff --git a/mayan/apps/documents/settings.py b/mayan/apps/documents/settings.py index 5579ecda27..81ef69ce13 100644 --- a/mayan/apps/documents/settings.py +++ b/mayan/apps/documents/settings.py @@ -45,11 +45,11 @@ setting_recent_count = namespace.add_setting( ) setting_storage_backend = namespace.add_setting( global_name='DOCUMENTS_STORAGE_BACKEND', - default='storage.backends.filebasedstorage.FileBasedStorage' + default='django.core.files.storage.FileSystemStorage' ) setting_storage_backend_arguments = namespace.add_setting( global_name='DOCUMENTS_STORAGE_BACKEND_ARGUMENTS', - default='' + default='{location: mayan/media/document_storage}' ) setting_zoom_percent_step = namespace.add_setting( global_name='DOCUMENTS_ZOOM_PERCENT_STEP', default=25, @@ -78,13 +78,13 @@ setting_rotation_step = namespace.add_setting( 'Amount in degrees to rotate a document page per user interaction.' ) ) -setting_cache_storage_backend = namespace.add_setting( +setting_documentimagecache_storage = namespace.add_setting( global_name='DOCUMENTS_CACHE_STORAGE_BACKEND', - default='documents.storage.LocalCacheFileStorage' + default='django.core.files.storage.FileSystemStorage' ) -setting_cache_storage_backend_arguments = namespace.add_setting( +setting_documentimagecache_storage_arguments = namespace.add_setting( global_name='DOCUMENTS_CACHE_STORAGE_BACKEND_ARGUMENTS', - default='' + default='{location: mayan/media/document_cache}' ) setting_language = namespace.add_setting( global_name='DOCUMENTS_LANGUAGE', default='eng', diff --git a/mayan/apps/documents/storage.py b/mayan/apps/documents/storage.py deleted file mode 100644 index eaf26fde39..0000000000 --- a/mayan/apps/documents/storage.py +++ /dev/null @@ -1,18 +0,0 @@ -from __future__ import unicode_literals - -import os - -from django.conf import settings -from django.core.files.storage import FileSystemStorage - -from .literals import CACHE_PATH - - -class LocalCacheFileStorage(FileSystemStorage): - """Simple wrapper for the stock Django FileSystemStorage class""" - - def __init__(self, *args, **kwargs): - super(LocalCacheFileStorage, self).__init__(*args, **kwargs) - self.location = os.path.join(settings.MEDIA_ROOT, CACHE_PATH) - if not os.path.exists(os.path.dirname(self.location)): - os.makedirs(os.path.dirname(self.location)) diff --git a/mayan/apps/documents/runtime.py b/mayan/apps/documents/storages.py similarity index 55% rename from mayan/apps/documents/runtime.py rename to mayan/apps/documents/storages.py index dd58e1d0bf..f7c9a4aeb8 100644 --- a/mayan/apps/documents/runtime.py +++ b/mayan/apps/documents/storages.py @@ -5,11 +5,11 @@ import yaml from django.utils.module_loading import import_string from .settings import ( - setting_cache_storage_backend, setting_cache_storage_backend_arguments, + setting_documentimagecache_storage, setting_documentimagecache_storage_arguments, setting_storage_backend, setting_storage_backend_arguments ) -storage_backend = import_string( +documentversion_storage = import_string( dotted_path=setting_storage_backend.value )( **yaml.safe_load( @@ -17,10 +17,10 @@ storage_backend = import_string( ) ) -cache_storage_backend = import_string( - dotted_path=setting_cache_storage_backend.value +documentimagecache_storage = import_string( + dotted_path=setting_documentimagecache_storage.value )( **yaml.safe_load( - setting_cache_storage_backend_arguments.value or '{}' + setting_documentimagecache_storage_arguments.value or '{}' ) ) diff --git a/mayan/apps/ocr/managers.py b/mayan/apps/ocr/managers.py index be26e0257e..55dfd203c5 100644 --- a/mayan/apps/ocr/managers.py +++ b/mayan/apps/ocr/managers.py @@ -8,7 +8,7 @@ from django.apps import apps from django.conf import settings from django.db import models -from documents.runtime import cache_storage_backend +from documents.storages import documentimagecache_storage from .events import event_ocr_document_version_finish from .runtime import ocr_backend @@ -31,7 +31,7 @@ class DocumentPageOCRContentManager(models.Manager): # TODO: Call task and wait cache_filename = document_page.generate_image() - with cache_storage_backend.open(cache_filename) as file_object: + with documentimagecache_storage.open(cache_filename) as file_object: document_page_content, created = DocumentPageOCRContent.objects.get_or_create( document_page=document_page ) diff --git a/mayan/apps/storage/backends/filebasedstorage.py b/mayan/apps/storage/backends/filebasedstorage.py deleted file mode 100644 index dfd3757ba9..0000000000 --- a/mayan/apps/storage/backends/filebasedstorage.py +++ /dev/null @@ -1,20 +0,0 @@ -from __future__ import unicode_literals - -import os - -from django.core.files.storage import FileSystemStorage -from django.conf import settings - -from .literals import DEFAULT_PATH - - -class FileBasedStorage(FileSystemStorage): - """Simple wrapper for the stock Django FileSystemStorage class""" - - separator = os.path.sep - - def __init__(self, *args, **kwargs): - self.location = kwargs.pop( - 'location', os.path.join(settings.MEDIA_ROOT, DEFAULT_PATH) - ) - super(FileBasedStorage, self).__init__(*args, **kwargs) diff --git a/mayan/apps/storage/backends/literals.py b/mayan/apps/storage/backends/literals.py deleted file mode 100644 index 74d1d49b1b..0000000000 --- a/mayan/apps/storage/backends/literals.py +++ /dev/null @@ -1,3 +0,0 @@ -from __future__ import unicode_literals - -DEFAULT_PATH = 'document_storage'