Standardize the way storages are used. All apps that use storage now define their storages in the .storages modules instead of the .runtime module. The storage.backends.filebasedstorage.FileBasedStorage has been remove, instead Django's default storage is used and each app is responsible of specifying their default path.

Signed-off-by: Michael Price <loneviking72@gmail.com>

HISTORY.rst

@@ -117,9 +117,6 @@
 - Support passing arguments to the document, document cache and document signatures
   storage backends. New settings: DOCUMENTS_STORAGE_BACKEND_ARGUMENTS,
   DOCUMENTS_CACHE_STORAGE_BACKEND_ARGUMENTS, SIGNATURES_STORAGE_BACKEND_ARGUMENTS
-- Support passing arguments to the document, document cache and document signatures
-  storage backends. New settings: DOCUMENTS_STORAGE_BACKEND_ARGUMENTS,
-  DOCUMENTS_CACHE_STORAGE_BACKEND_ARGUMENTS, SIGNATURES_STORAGE_BACKEND_ARGUMENTS
 - Remove the setting STORAGE_FILESTORAGE_LOCATION. Document storage
   location for the storage.backend.filebasedstorage.FileBasedStorage
   backdend must now passed via the DOCUMENTS_STORAGE_BACKEND_ARGUMENTS,
@@ -129,6 +126,11 @@
   DOCUMENTS_STORAGE_BACKEND_ARGUMENTS = '{ location: <specific_path> }'
   If no path is specified the backend will default to
   'mayan/media/document_storage'.
+- Standardize the way storages are used. All apps that use storage now define
+  their storages in the .storages modules instead of the .runtime module.
+  The storage.backends.filebasedstorage.FileBasedStorage has been remove,
+  instead Django's default storage is used and each app is responsible
+  of specifying their default path.
 
 
 2.7.3 (2017-09-11)

mayan/apps/common/models.py

@@ -12,7 +12,7 @@ from django.utils.encoding import force_text, python_2_unicode_compatible
 from django.utils.translation import ugettext_lazy as _
 
 from .managers import ErrorLogEntryManager
-from .runtime import shared_storage_backend
+from .storages import sharedupload_storage
 
 
 def upload_to(instance, filename):
@@ -47,7 +47,7 @@ class ErrorLogEntry(models.Model):
 @python_2_unicode_compatible
 class SharedUploadedFile(models.Model):
     file = models.FileField(
-        storage=shared_storage_backend, upload_to=upload_to,
+        storage=sharedupload_storage, upload_to=upload_to,
         verbose_name=_('File')
     )
     filename = models.CharField(max_length=255, verbose_name=_('Filename'))

mayan/apps/common/runtime.py

@@ -1,5 +0,0 @@
-from django.utils.module_loading import import_string
-
-from .settings import setting_shared_storage
-
-shared_storage_backend = import_string(setting_shared_storage.value)()

mayan/apps/common/settings.py

@@ -38,9 +38,13 @@ setting_paginate_by = namespace.add_setting(
 )
 setting_shared_storage = namespace.add_setting(
     global_name='COMMON_SHARED_STORAGE',
-    default='storage.backends.filebasedstorage.FileBasedStorage',
+    default='django.core.files.storage.FileSystemStorage',
     help_text=_('A storage backend that all workers can use to share files.')
 )
+setting_shared_storage_arguments = namespace.add_setting(
+    global_name='COMMON_SHARED_STORAGE_ARGUMENTS',
+    default='{location: mayan/media/shared_files}',
+)
 setting_temporary_directory = namespace.add_setting(
     global_name='COMMON_TEMPORARY_DIRECTORY', default=tempfile.gettempdir(),
     help_text=_(

mayan/apps/common/storages.py

@@ -0,0 +1,17 @@
+from __future__ import unicode_literals
+
+import yaml
+
+from django.utils.module_loading import import_string
+
+from .settings import (
+    setting_shared_storage, setting_shared_storage_arguments
+)
+
+sharedupload_storage = import_string(
+    dotted_path=setting_shared_storage.value
+)(
+    **yaml.safe_load(
+        setting_shared_storage_arguments.value or '{}'
+    )
+)

mayan/apps/document_signatures/models.py

@@ -15,7 +15,7 @@ from django_gpg.models import Key
 from documents.models import DocumentVersion
 
 from .managers import EmbeddedSignatureManager
-from .runtime import storage_backend
+from .storages import storage_backend
 
 logger = logging.getLogger(__name__)
 

mayan/apps/document_signatures/settings.py

@@ -7,9 +7,9 @@ from smart_settings import Namespace
 namespace = Namespace(name='signatures', label=_('Document signatures'))
 setting_storage_backend = namespace.add_setting(
     global_name='SIGNATURES_STORAGE_BACKEND',
-    default='storage.backends.filebasedstorage.FileBasedStorage'
+    default='django.core.files.storage.FileSystemStorage'
 )
 setting_storage_backend_arguments = namespace.add_setting(
     global_name='SIGNATURES_STORAGE_BACKEND_ARGUMENTS',
-    default=''
+    default='{location: mayan/media/document_storage}'
 )

mayan/apps/documents/api_views.py

@@ -27,7 +27,6 @@ from .permissions import (
     permission_document_type_view, permission_document_version_revert,
     permission_document_version_view
 )
-from .runtime import cache_storage_backend
 from .serializers import (
     DeletedDocumentSerializer, DocumentPageSerializer, DocumentSerializer,
     DocumentTypeSerializer, DocumentVersionSerializer,
@@ -35,6 +34,7 @@ from .serializers import (
     RecentDocumentSerializer, WritableDocumentSerializer,
     WritableDocumentTypeSerializer, WritableDocumentVersionSerializer
 )
+from .storages import documentimagecache_storage
 from .tasks import task_generate_document_page_image
 
 logger = logging.getLogger(__name__)
@@ -288,7 +288,7 @@ class APIDocumentPageImageView(generics.RetrieveAPIView):
         )
 
         cache_filename = task.get(timeout=DOCUMENT_IMAGE_TASK_TIMEOUT)
-        with cache_storage_backend.open(cache_filename) as file_object:
+        with documentimagecache_storage.open(cache_filename) as file_object:
             return HttpResponse(file_object.read(), content_type='image')
 
 

mayan/apps/documents/literals.py

@@ -4,7 +4,6 @@ from django.utils.translation import ugettext_lazy as _
 
 from common.literals import TIME_DELTA_UNIT_DAYS
 
-CACHE_PATH = 'document_cache/'
 CHECK_DELETE_PERIOD_INTERVAL = 60
 CHECK_TRASH_PERIOD_INTERVAL = 60
 DELETE_STALE_STUBS_INTERVAL = 60 * 10  # 10 minutes

mayan/apps/documents/models.py

@@ -36,7 +36,6 @@ from .managers import (
     PassthroughManager, RecentDocumentManager, TrashCanManager
 )
 from .permissions import permission_document_view
-from .runtime import cache_storage_backend, storage_backend
 from .settings import (
     setting_disable_base_image_cache, setting_disable_transformed_image_cache,
     setting_display_width, setting_display_height, setting_fix_orientation,
@@ -45,6 +44,7 @@ from .settings import (
 from .signals import (
     post_document_created, post_document_type_change, post_version_upload
 )
+from .storages import documentversion_storage, documentimagecache_storage
 
 logger = logging.getLogger(__name__)
 
@@ -393,7 +393,7 @@ class DocumentVersion(models.Model):
 
     # File related fields
     file = models.FileField(
-        storage=storage_backend, upload_to=UUID_FUNCTION,
+        storage=documentversion_storage, upload_to=UUID_FUNCTION,
         verbose_name=_('File')
     )
     mimetype = models.CharField(
@@ -459,10 +459,10 @@ class DocumentVersion(models.Model):
         cache_filename = self.cache_filename
         logger.debug('Intermidiate filename: %s', cache_filename)
 
-        if cache_storage_backend.exists(cache_filename):
+        if documentimagecache_storage.exists(cache_filename):
             logger.debug('Intermidiate file "%s" found.', cache_filename)
 
-            return cache_storage_backend.open(cache_filename)
+            return documentimagecache_storage.open(cache_filename)
         else:
             logger.debug('Intermidiate file "%s" not found.', cache_filename)
 
@@ -470,11 +470,11 @@ class DocumentVersion(models.Model):
                 converter = converter_class(file_object=self.open())
                 pdf_file_object = converter.to_pdf()
 
-                with cache_storage_backend.open(cache_filename, 'wb+') as file_object:
+                with documentimagecache_storage.open(cache_filename, 'wb+') as file_object:
                     for chunk in pdf_file_object:
                         file_object.write(chunk)
 
-                return cache_storage_backend.open(cache_filename)
+                return documentimagecache_storage.open(cache_filename)
             except InvalidOfficeFormat:
                 return self.open()
             except Exception as exception:
@@ -483,7 +483,7 @@ class DocumentVersion(models.Model):
                     'Error creating intermediate file "%s"; %s.',
                     cache_filename, exception
                 )
-                cache_storage_backend.delete(cache_filename)
+                documentimagecache_storage.delete(cache_filename)
                 raise
 
     def get_rendered_string(self, preserve_extension=False):
@@ -503,7 +503,7 @@ class DocumentVersion(models.Model):
         )
 
     def invalidate_cache(self):
-        cache_storage_backend.delete(self.cache_filename)
+        documentimagecache_storage.delete(self.cache_filename)
         for page in self.pages.all():
             page.invalidate_cache()
 
@@ -810,7 +810,7 @@ class DocumentPage(models.Model):
         # Check is transformed image is available
         logger.debug('transformations cache filename: %s', cache_filename)
 
-        if not setting_disable_transformed_image_cache.value and cache_storage_backend.exists(cache_filename):
+        if not setting_disable_transformed_image_cache.value and documentimagecache_storage.exists(cache_filename):
             logger.debug(
                 'transformations cache file "%s" found', cache_filename
             )
@@ -819,7 +819,7 @@ class DocumentPage(models.Model):
                 'transformations cache file "%s" not found', cache_filename
             )
             image = self.get_image(transformations=transformation_list)
-            with cache_storage_backend.open(cache_filename, 'wb+') as file_object:
+            with documentimagecache_storage.open(cache_filename, 'wb+') as file_object:
                 file_object.write(image.getvalue())
 
             self.cached_images.create(filename=cache_filename)
@@ -840,10 +840,10 @@ class DocumentPage(models.Model):
         cache_filename = self.cache_filename
         logger.debug('Page cache filename: %s', cache_filename)
 
-        if not setting_disable_base_image_cache.value and cache_storage_backend.exists(cache_filename):
+        if not setting_disable_base_image_cache.value and documentimagecache_storage.exists(cache_filename):
             logger.debug('Page cache file "%s" found', cache_filename)
             converter = converter_class(
-                file_object=cache_storage_backend.open(cache_filename)
+                file_object=documentimagecache_storage.open(cache_filename)
             )
 
             converter.seek(0)
@@ -858,7 +858,7 @@ class DocumentPage(models.Model):
 
                 page_image = converter.get_page()
 
-                with cache_storage_backend.open(cache_filename, 'wb+') as file_object:
+                with documentimagecache_storage.open(cache_filename, 'wb+') as file_object:
                     file_object.write(page_image.getvalue())
             except Exception as exception:
                 # Cleanup in case of error
@@ -866,7 +866,7 @@ class DocumentPage(models.Model):
                     'Error creating page cache file "%s"; %s',
                     cache_filename, exception
                 )
-                cache_storage_backend.delete(cache_filename)
+                documentimagecache_storage.delete(cache_filename)
                 raise
 
         for transformation in transformations:
@@ -875,7 +875,7 @@ class DocumentPage(models.Model):
         return converter.get_page()
 
     def invalidate_cache(self):
-        cache_storage_backend.delete(self.cache_filename)
+        documentimagecache_storage.delete(self.cache_filename)
         for cached_image in self.cached_images.all():
             cached_image.delete()
 
@@ -906,7 +906,7 @@ class DocumentPageCachedImage(models.Model):
         verbose_name_plural = _('Document page cached images')
 
     def delete(self, *args, **kwargs):
-        cache_storage_backend.delete(self.filename)
+        documentimagecache_storage.delete(self.filename)
         return super(DocumentPageCachedImage, self).delete(*args, **kwargs)
 
 

mayan/apps/documents/settings.py

@@ -45,11 +45,11 @@ setting_recent_count = namespace.add_setting(
 )
 setting_storage_backend = namespace.add_setting(
     global_name='DOCUMENTS_STORAGE_BACKEND',
-    default='storage.backends.filebasedstorage.FileBasedStorage'
+    default='django.core.files.storage.FileSystemStorage'
 )
 setting_storage_backend_arguments = namespace.add_setting(
     global_name='DOCUMENTS_STORAGE_BACKEND_ARGUMENTS',
-    default=''
+    default='{location: mayan/media/document_storage}'
 )
 setting_zoom_percent_step = namespace.add_setting(
     global_name='DOCUMENTS_ZOOM_PERCENT_STEP', default=25,
@@ -78,13 +78,13 @@ setting_rotation_step = namespace.add_setting(
         'Amount in degrees to rotate a document page per user interaction.'
     )
 )
-setting_cache_storage_backend = namespace.add_setting(
+setting_documentimagecache_storage = namespace.add_setting(
     global_name='DOCUMENTS_CACHE_STORAGE_BACKEND',
-    default='documents.storage.LocalCacheFileStorage'
+    default='django.core.files.storage.FileSystemStorage'
 )
-setting_cache_storage_backend_arguments = namespace.add_setting(
+setting_documentimagecache_storage_arguments = namespace.add_setting(
     global_name='DOCUMENTS_CACHE_STORAGE_BACKEND_ARGUMENTS',
-    default=''
+    default='{location: mayan/media/document_cache}'
 )
 setting_language = namespace.add_setting(
     global_name='DOCUMENTS_LANGUAGE', default='eng',

mayan/apps/documents/storage.py

@@ -1,18 +0,0 @@
-from __future__ import unicode_literals
-
-import os
-
-from django.conf import settings
-from django.core.files.storage import FileSystemStorage
-
-from .literals import CACHE_PATH
-
-
-class LocalCacheFileStorage(FileSystemStorage):
-    """Simple wrapper for the stock Django FileSystemStorage class"""
-
-    def __init__(self, *args, **kwargs):
-        super(LocalCacheFileStorage, self).__init__(*args, **kwargs)
-        self.location = os.path.join(settings.MEDIA_ROOT, CACHE_PATH)
-        if not os.path.exists(os.path.dirname(self.location)):
-            os.makedirs(os.path.dirname(self.location))

mayan/apps/documents/storages.py

@@ -5,11 +5,11 @@ import yaml
 from django.utils.module_loading import import_string
 
 from .settings import (
-    setting_cache_storage_backend, setting_cache_storage_backend_arguments,
+    setting_documentimagecache_storage, setting_documentimagecache_storage_arguments,
     setting_storage_backend, setting_storage_backend_arguments
 )
 
-storage_backend = import_string(
+documentversion_storage = import_string(
     dotted_path=setting_storage_backend.value
 )(
     **yaml.safe_load(
@@ -17,10 +17,10 @@ storage_backend = import_string(
     )
 )
 
-cache_storage_backend = import_string(
+documentimagecache_storage = import_string(
-    dotted_path=setting_cache_storage_backend.value
+    dotted_path=setting_documentimagecache_storage.value
 )(
     **yaml.safe_load(
-        setting_cache_storage_backend_arguments.value or '{}'
+        setting_documentimagecache_storage_arguments.value or '{}'
     )
 )

mayan/apps/ocr/managers.py

@@ -8,7 +8,7 @@ from django.apps import apps
 from django.conf import settings
 from django.db import models
 
-from documents.runtime import cache_storage_backend
+from documents.storages import documentimagecache_storage
 
 from .events import event_ocr_document_version_finish
 from .runtime import ocr_backend
@@ -31,7 +31,7 @@ class DocumentPageOCRContentManager(models.Manager):
         # TODO: Call task and wait
         cache_filename = document_page.generate_image()
 
-        with cache_storage_backend.open(cache_filename) as file_object:
+        with documentimagecache_storage.open(cache_filename) as file_object:
             document_page_content, created = DocumentPageOCRContent.objects.get_or_create(
                 document_page=document_page
             )

mayan/apps/storage/backends/filebasedstorage.py

@@ -1,20 +0,0 @@
-from __future__ import unicode_literals
-
-import os
-
-from django.core.files.storage import FileSystemStorage
-from django.conf import settings
-
-from .literals import DEFAULT_PATH
-
-
-class FileBasedStorage(FileSystemStorage):
-    """Simple wrapper for the stock Django FileSystemStorage class"""
-
-    separator = os.path.sep
-
-    def __init__(self, *args, **kwargs):
-        self.location = kwargs.pop(
-            'location', os.path.join(settings.MEDIA_ROOT, DEFAULT_PATH)
-        )
-        super(FileBasedStorage, self).__init__(*args, **kwargs)

mayan/apps/storage/backends/literals.py

@@ -1,3 +0,0 @@
-from __future__ import unicode_literals
-
-DEFAULT_PATH = 'document_storage'