diff --git a/HISTORY.rst b/HISTORY.rst index 315942b3e3..a8fc5237be 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -114,7 +114,24 @@ revoke permissions for the selected role. - Only show the new document link if the user has access to create documents of at least one document type. GitLab Issue #302. Thanks to kg @kgraves. -- Remove the data filters feature. +- Support passing arguments to the document, document cache and document signatures + storage backends. New settings: DOCUMENTS_STORAGE_BACKEND_ARGUMENTS, + DOCUMENTS_CACHE_STORAGE_BACKEND_ARGUMENTS, SIGNATURES_STORAGE_BACKEND_ARGUMENTS +- Remove the setting STORAGE_FILESTORAGE_LOCATION. Document storage + location for the storage.backend.filebasedstorage.FileBasedStorage + backdend must now passed via the DOCUMENTS_STORAGE_BACKEND_ARGUMENTS, + DOCUMENTS_CACHE_STORAGE_BACKEND_ARGUMENTS, or + SIGNATURES_STORAGE_BACKEND_ARGUMENTS if the backend is used to documents, + the document image cache and/or document signatures. Use + DOCUMENTS_STORAGE_BACKEND_ARGUMENTS = '{ location: }' + If no path is specified the backend will default to + 'mayan/media/document_storage'. +- Standardize the way storages are used. All apps that use storage now define + their storages in the .storages modules instead of the .runtime module. + The storage.backends.filebasedstorage.FileBasedStorage has been remove, + instead Django's default storage is used and each app is responsible + of specifying their default path. + 2.7.3 (2017-09-11) ================== diff --git a/mayan/apps/common/models.py b/mayan/apps/common/models.py index 5d75ff363b..e76a54a85e 100644 --- a/mayan/apps/common/models.py +++ b/mayan/apps/common/models.py @@ -12,7 +12,7 @@ from django.utils.encoding import force_text, python_2_unicode_compatible from django.utils.translation import ugettext_lazy as _ from .managers import ErrorLogEntryManager -from .runtime import shared_storage_backend +from .storages import sharedupload_storage def upload_to(instance, filename): @@ -47,7 +47,7 @@ class ErrorLogEntry(models.Model): @python_2_unicode_compatible class SharedUploadedFile(models.Model): file = models.FileField( - storage=shared_storage_backend, upload_to=upload_to, + storage=sharedupload_storage, upload_to=upload_to, verbose_name=_('File') ) filename = models.CharField(max_length=255, verbose_name=_('Filename')) diff --git a/mayan/apps/common/runtime.py b/mayan/apps/common/runtime.py deleted file mode 100644 index 59621c1341..0000000000 --- a/mayan/apps/common/runtime.py +++ /dev/null @@ -1,5 +0,0 @@ -from django.utils.module_loading import import_string - -from .settings import setting_shared_storage - -shared_storage_backend = import_string(setting_shared_storage.value)() diff --git a/mayan/apps/common/settings.py b/mayan/apps/common/settings.py index 99c4e121c6..e8a8d45abb 100644 --- a/mayan/apps/common/settings.py +++ b/mayan/apps/common/settings.py @@ -38,9 +38,13 @@ setting_paginate_by = namespace.add_setting( ) setting_shared_storage = namespace.add_setting( global_name='COMMON_SHARED_STORAGE', - default='storage.backends.filebasedstorage.FileBasedStorage', + default='django.core.files.storage.FileSystemStorage', help_text=_('A storage backend that all workers can use to share files.') ) +setting_shared_storage_arguments = namespace.add_setting( + global_name='COMMON_SHARED_STORAGE_ARGUMENTS', + default='{location: mayan/media/shared_files}', +) setting_temporary_directory = namespace.add_setting( global_name='COMMON_TEMPORARY_DIRECTORY', default=tempfile.gettempdir(), help_text=_( diff --git a/mayan/apps/common/storages.py b/mayan/apps/common/storages.py new file mode 100644 index 0000000000..87e04b1b64 --- /dev/null +++ b/mayan/apps/common/storages.py @@ -0,0 +1,17 @@ +from __future__ import unicode_literals + +import yaml + +from django.utils.module_loading import import_string + +from .settings import ( + setting_shared_storage, setting_shared_storage_arguments +) + +sharedupload_storage = import_string( + dotted_path=setting_shared_storage.value +)( + **yaml.safe_load( + setting_shared_storage_arguments.value or '{}' + ) +) diff --git a/mayan/apps/document_signatures/models.py b/mayan/apps/document_signatures/models.py index 17d1b6bb9d..6366b23789 100644 --- a/mayan/apps/document_signatures/models.py +++ b/mayan/apps/document_signatures/models.py @@ -15,7 +15,7 @@ from django_gpg.models import Key from documents.models import DocumentVersion from .managers import EmbeddedSignatureManager -from .runtime import storage_backend +from .storages import storage_backend logger = logging.getLogger(__name__) diff --git a/mayan/apps/document_signatures/runtime.py b/mayan/apps/document_signatures/runtime.py deleted file mode 100644 index 7d5bdfc4ca..0000000000 --- a/mayan/apps/document_signatures/runtime.py +++ /dev/null @@ -1,5 +0,0 @@ -from django.utils.module_loading import import_string - -from .settings import setting_storage_backend - -storage_backend = import_string(setting_storage_backend.value)() diff --git a/mayan/apps/document_signatures/settings.py b/mayan/apps/document_signatures/settings.py index 012b357694..962b00ee2c 100644 --- a/mayan/apps/document_signatures/settings.py +++ b/mayan/apps/document_signatures/settings.py @@ -7,5 +7,9 @@ from smart_settings import Namespace namespace = Namespace(name='signatures', label=_('Document signatures')) setting_storage_backend = namespace.add_setting( global_name='SIGNATURES_STORAGE_BACKEND', - default='storage.backends.filebasedstorage.FileBasedStorage' + default='django.core.files.storage.FileSystemStorage' +) +setting_storage_backend_arguments = namespace.add_setting( + global_name='SIGNATURES_STORAGE_BACKEND_ARGUMENTS', + default='{location: mayan/media/document_storage}' ) diff --git a/mayan/apps/document_signatures/storages.py b/mayan/apps/document_signatures/storages.py new file mode 100644 index 0000000000..52000433f0 --- /dev/null +++ b/mayan/apps/document_signatures/storages.py @@ -0,0 +1,17 @@ +from __future__ import unicode_literals + +import yaml + +from django.utils.module_loading import import_string + +from .settings import ( + setting_storage_backend, setting_storage_backend_arguments +) + +storage_backend = import_string( + dotted_path=setting_storage_backend.value +)( + **yaml.safe_load( + setting_storage_backend_arguments.value or '{}' + ) +) diff --git a/mayan/apps/documents/api_views.py b/mayan/apps/documents/api_views.py index efb387b18f..5d79656fb7 100644 --- a/mayan/apps/documents/api_views.py +++ b/mayan/apps/documents/api_views.py @@ -27,7 +27,6 @@ from .permissions import ( permission_document_type_view, permission_document_version_revert, permission_document_version_view ) -from .runtime import cache_storage_backend from .serializers import ( DeletedDocumentSerializer, DocumentPageSerializer, DocumentSerializer, DocumentTypeSerializer, DocumentVersionSerializer, @@ -35,6 +34,7 @@ from .serializers import ( RecentDocumentSerializer, WritableDocumentSerializer, WritableDocumentTypeSerializer, WritableDocumentVersionSerializer ) +from .storages import documentimagecache_storage from .tasks import task_generate_document_page_image logger = logging.getLogger(__name__) @@ -288,7 +288,7 @@ class APIDocumentPageImageView(generics.RetrieveAPIView): ) cache_filename = task.get(timeout=DOCUMENT_IMAGE_TASK_TIMEOUT) - with cache_storage_backend.open(cache_filename) as file_object: + with documentimagecache_storage.open(cache_filename) as file_object: return HttpResponse(file_object.read(), content_type='image') diff --git a/mayan/apps/documents/literals.py b/mayan/apps/documents/literals.py index bc58ba9096..77e27106a6 100644 --- a/mayan/apps/documents/literals.py +++ b/mayan/apps/documents/literals.py @@ -4,7 +4,6 @@ from django.utils.translation import ugettext_lazy as _ from common.literals import TIME_DELTA_UNIT_DAYS -CACHE_PATH = 'document_cache/' CHECK_DELETE_PERIOD_INTERVAL = 60 CHECK_TRASH_PERIOD_INTERVAL = 60 DELETE_STALE_STUBS_INTERVAL = 60 * 10 # 10 minutes diff --git a/mayan/apps/documents/models.py b/mayan/apps/documents/models.py index f02dd43908..6e055e46eb 100644 --- a/mayan/apps/documents/models.py +++ b/mayan/apps/documents/models.py @@ -36,7 +36,6 @@ from .managers import ( PassthroughManager, RecentDocumentManager, TrashCanManager ) from .permissions import permission_document_view -from .runtime import cache_storage_backend, storage_backend from .settings import ( setting_disable_base_image_cache, setting_disable_transformed_image_cache, setting_display_width, setting_display_height, setting_fix_orientation, @@ -45,6 +44,7 @@ from .settings import ( from .signals import ( post_document_created, post_document_type_change, post_version_upload ) +from .storages import documentversion_storage, documentimagecache_storage logger = logging.getLogger(__name__) @@ -393,7 +393,7 @@ class DocumentVersion(models.Model): # File related fields file = models.FileField( - storage=storage_backend, upload_to=UUID_FUNCTION, + storage=documentversion_storage, upload_to=UUID_FUNCTION, verbose_name=_('File') ) mimetype = models.CharField( @@ -459,10 +459,10 @@ class DocumentVersion(models.Model): cache_filename = self.cache_filename logger.debug('Intermidiate filename: %s', cache_filename) - if cache_storage_backend.exists(cache_filename): + if documentimagecache_storage.exists(cache_filename): logger.debug('Intermidiate file "%s" found.', cache_filename) - return cache_storage_backend.open(cache_filename) + return documentimagecache_storage.open(cache_filename) else: logger.debug('Intermidiate file "%s" not found.', cache_filename) @@ -470,11 +470,11 @@ class DocumentVersion(models.Model): converter = converter_class(file_object=self.open()) pdf_file_object = converter.to_pdf() - with cache_storage_backend.open(cache_filename, 'wb+') as file_object: + with documentimagecache_storage.open(cache_filename, 'wb+') as file_object: for chunk in pdf_file_object: file_object.write(chunk) - return cache_storage_backend.open(cache_filename) + return documentimagecache_storage.open(cache_filename) except InvalidOfficeFormat: return self.open() except Exception as exception: @@ -483,7 +483,7 @@ class DocumentVersion(models.Model): 'Error creating intermediate file "%s"; %s.', cache_filename, exception ) - cache_storage_backend.delete(cache_filename) + documentimagecache_storage.delete(cache_filename) raise def get_rendered_string(self, preserve_extension=False): @@ -503,7 +503,7 @@ class DocumentVersion(models.Model): ) def invalidate_cache(self): - cache_storage_backend.delete(self.cache_filename) + documentimagecache_storage.delete(self.cache_filename) for page in self.pages.all(): page.invalidate_cache() @@ -810,7 +810,7 @@ class DocumentPage(models.Model): # Check is transformed image is available logger.debug('transformations cache filename: %s', cache_filename) - if not setting_disable_transformed_image_cache.value and cache_storage_backend.exists(cache_filename): + if not setting_disable_transformed_image_cache.value and documentimagecache_storage.exists(cache_filename): logger.debug( 'transformations cache file "%s" found', cache_filename ) @@ -819,7 +819,7 @@ class DocumentPage(models.Model): 'transformations cache file "%s" not found', cache_filename ) image = self.get_image(transformations=transformation_list) - with cache_storage_backend.open(cache_filename, 'wb+') as file_object: + with documentimagecache_storage.open(cache_filename, 'wb+') as file_object: file_object.write(image.getvalue()) self.cached_images.create(filename=cache_filename) @@ -840,10 +840,10 @@ class DocumentPage(models.Model): cache_filename = self.cache_filename logger.debug('Page cache filename: %s', cache_filename) - if not setting_disable_base_image_cache.value and cache_storage_backend.exists(cache_filename): + if not setting_disable_base_image_cache.value and documentimagecache_storage.exists(cache_filename): logger.debug('Page cache file "%s" found', cache_filename) converter = converter_class( - file_object=cache_storage_backend.open(cache_filename) + file_object=documentimagecache_storage.open(cache_filename) ) converter.seek(0) @@ -858,7 +858,7 @@ class DocumentPage(models.Model): page_image = converter.get_page() - with cache_storage_backend.open(cache_filename, 'wb+') as file_object: + with documentimagecache_storage.open(cache_filename, 'wb+') as file_object: file_object.write(page_image.getvalue()) except Exception as exception: # Cleanup in case of error @@ -866,7 +866,7 @@ class DocumentPage(models.Model): 'Error creating page cache file "%s"; %s', cache_filename, exception ) - cache_storage_backend.delete(cache_filename) + documentimagecache_storage.delete(cache_filename) raise for transformation in transformations: @@ -875,7 +875,7 @@ class DocumentPage(models.Model): return converter.get_page() def invalidate_cache(self): - cache_storage_backend.delete(self.cache_filename) + documentimagecache_storage.delete(self.cache_filename) for cached_image in self.cached_images.all(): cached_image.delete() @@ -906,7 +906,7 @@ class DocumentPageCachedImage(models.Model): verbose_name_plural = _('Document page cached images') def delete(self, *args, **kwargs): - cache_storage_backend.delete(self.filename) + documentimagecache_storage.delete(self.filename) return super(DocumentPageCachedImage, self).delete(*args, **kwargs) diff --git a/mayan/apps/documents/runtime.py b/mayan/apps/documents/runtime.py deleted file mode 100644 index b36c95e8c0..0000000000 --- a/mayan/apps/documents/runtime.py +++ /dev/null @@ -1,6 +0,0 @@ -from django.utils.module_loading import import_string - -from .settings import setting_cache_storage_backend, setting_storage_backend - -storage_backend = import_string(setting_storage_backend.value)() -cache_storage_backend = import_string(setting_cache_storage_backend.value)() diff --git a/mayan/apps/documents/settings.py b/mayan/apps/documents/settings.py index 21eb910b3c..81ef69ce13 100644 --- a/mayan/apps/documents/settings.py +++ b/mayan/apps/documents/settings.py @@ -45,7 +45,11 @@ setting_recent_count = namespace.add_setting( ) setting_storage_backend = namespace.add_setting( global_name='DOCUMENTS_STORAGE_BACKEND', - default='storage.backends.filebasedstorage.FileBasedStorage' + default='django.core.files.storage.FileSystemStorage' +) +setting_storage_backend_arguments = namespace.add_setting( + global_name='DOCUMENTS_STORAGE_BACKEND_ARGUMENTS', + default='{location: mayan/media/document_storage}' ) setting_zoom_percent_step = namespace.add_setting( global_name='DOCUMENTS_ZOOM_PERCENT_STEP', default=25, @@ -74,9 +78,13 @@ setting_rotation_step = namespace.add_setting( 'Amount in degrees to rotate a document page per user interaction.' ) ) -setting_cache_storage_backend = namespace.add_setting( +setting_documentimagecache_storage = namespace.add_setting( global_name='DOCUMENTS_CACHE_STORAGE_BACKEND', - default='documents.storage.LocalCacheFileStorage' + default='django.core.files.storage.FileSystemStorage' +) +setting_documentimagecache_storage_arguments = namespace.add_setting( + global_name='DOCUMENTS_CACHE_STORAGE_BACKEND_ARGUMENTS', + default='{location: mayan/media/document_cache}' ) setting_language = namespace.add_setting( global_name='DOCUMENTS_LANGUAGE', default='eng', diff --git a/mayan/apps/documents/storage.py b/mayan/apps/documents/storage.py deleted file mode 100644 index eaf26fde39..0000000000 --- a/mayan/apps/documents/storage.py +++ /dev/null @@ -1,18 +0,0 @@ -from __future__ import unicode_literals - -import os - -from django.conf import settings -from django.core.files.storage import FileSystemStorage - -from .literals import CACHE_PATH - - -class LocalCacheFileStorage(FileSystemStorage): - """Simple wrapper for the stock Django FileSystemStorage class""" - - def __init__(self, *args, **kwargs): - super(LocalCacheFileStorage, self).__init__(*args, **kwargs) - self.location = os.path.join(settings.MEDIA_ROOT, CACHE_PATH) - if not os.path.exists(os.path.dirname(self.location)): - os.makedirs(os.path.dirname(self.location)) diff --git a/mayan/apps/documents/storages.py b/mayan/apps/documents/storages.py new file mode 100644 index 0000000000..f7c9a4aeb8 --- /dev/null +++ b/mayan/apps/documents/storages.py @@ -0,0 +1,26 @@ +from __future__ import unicode_literals + +import yaml + +from django.utils.module_loading import import_string + +from .settings import ( + setting_documentimagecache_storage, setting_documentimagecache_storage_arguments, + setting_storage_backend, setting_storage_backend_arguments +) + +documentversion_storage = import_string( + dotted_path=setting_storage_backend.value +)( + **yaml.safe_load( + setting_storage_backend_arguments.value or '{}' + ) +) + +documentimagecache_storage = import_string( + dotted_path=setting_documentimagecache_storage.value +)( + **yaml.safe_load( + setting_documentimagecache_storage_arguments.value or '{}' + ) +) diff --git a/mayan/apps/ocr/managers.py b/mayan/apps/ocr/managers.py index be26e0257e..55dfd203c5 100644 --- a/mayan/apps/ocr/managers.py +++ b/mayan/apps/ocr/managers.py @@ -8,7 +8,7 @@ from django.apps import apps from django.conf import settings from django.db import models -from documents.runtime import cache_storage_backend +from documents.storages import documentimagecache_storage from .events import event_ocr_document_version_finish from .runtime import ocr_backend @@ -31,7 +31,7 @@ class DocumentPageOCRContentManager(models.Manager): # TODO: Call task and wait cache_filename = document_page.generate_image() - with cache_storage_backend.open(cache_filename) as file_object: + with documentimagecache_storage.open(cache_filename) as file_object: document_page_content, created = DocumentPageOCRContent.objects.get_or_create( document_page=document_page ) diff --git a/mayan/apps/storage/backends/compressedstorage.py b/mayan/apps/storage/backends/compressedstorage.py index d76fa76294..008c575bc4 100644 --- a/mayan/apps/storage/backends/compressedstorage.py +++ b/mayan/apps/storage/backends/compressedstorage.py @@ -17,8 +17,6 @@ except ImportError: from django.core.files import File from django.core.files.storage import FileSystemStorage -from ..settings import FILESTORAGE_LOCATION - class CompressedStorage(FileSystemStorage): """Simple wrapper for the stock Django FileSystemStorage class""" @@ -26,8 +24,8 @@ class CompressedStorage(FileSystemStorage): separator = os.path.sep def __init__(self, *args, **kwargs): + self.location = kwargs.pop('location') super(CompressedStorage, self).__init__(*args, **kwargs) - self.location = FILESTORAGE_LOCATION def save(self, name, content): descriptor = StringIO() diff --git a/mayan/apps/storage/backends/filebasedstorage.py b/mayan/apps/storage/backends/filebasedstorage.py deleted file mode 100644 index e446f7401b..0000000000 --- a/mayan/apps/storage/backends/filebasedstorage.py +++ /dev/null @@ -1,17 +0,0 @@ -from __future__ import unicode_literals - -import os - -from django.core.files.storage import FileSystemStorage - -from ..settings import setting_filestorage_location - - -class FileBasedStorage(FileSystemStorage): - """Simple wrapper for the stock Django FileSystemStorage class""" - - separator = os.path.sep - - def __init__(self, *args, **kwargs): - super(FileBasedStorage, self).__init__(*args, **kwargs) - self.location = setting_filestorage_location.value diff --git a/mayan/apps/storage/settings.py b/mayan/apps/storage/settings.py deleted file mode 100644 index 12184f50bd..0000000000 --- a/mayan/apps/storage/settings.py +++ /dev/null @@ -1,14 +0,0 @@ -from __future__ import unicode_literals - -import os - -from django.conf import settings -from django.utils.translation import ugettext_lazy as _ - -from smart_settings import Namespace - -namespace = Namespace(name='storage', label=_('Storage')) -setting_filestorage_location = namespace.add_setting( - global_name='STORAGE_FILESTORAGE_LOCATION', - default=os.path.join(settings.MEDIA_ROOT, 'document_storage'), is_path=True -)