Merge remote-tracking branch 'origin/feature/storages' into merge-test

HISTORY.rst

@@ -114,7 +114,24 @@
   revoke permissions for the selected role.
 - Only show the new document link if the user has access to create documents of
   at least one document type. GitLab Issue #302. Thanks to kg @kgraves.
-- Remove the data filters feature.
+- Support passing arguments to the document, document cache and document signatures
+  storage backends. New settings: DOCUMENTS_STORAGE_BACKEND_ARGUMENTS,
+  DOCUMENTS_CACHE_STORAGE_BACKEND_ARGUMENTS, SIGNATURES_STORAGE_BACKEND_ARGUMENTS
+- Remove the setting STORAGE_FILESTORAGE_LOCATION. Document storage
+  location for the storage.backend.filebasedstorage.FileBasedStorage
+  backdend must now passed via the DOCUMENTS_STORAGE_BACKEND_ARGUMENTS,
+  DOCUMENTS_CACHE_STORAGE_BACKEND_ARGUMENTS, or
+  SIGNATURES_STORAGE_BACKEND_ARGUMENTS if the backend is used to documents,
+  the document image cache and/or document signatures. Use
+  DOCUMENTS_STORAGE_BACKEND_ARGUMENTS = '{ location: <specific_path> }'
+  If no path is specified the backend will default to
+  'mayan/media/document_storage'.
+- Standardize the way storages are used. All apps that use storage now define
+  their storages in the .storages modules instead of the .runtime module.
+  The storage.backends.filebasedstorage.FileBasedStorage has been remove,
+  instead Django's default storage is used and each app is responsible
+  of specifying their default path.
+
 
 2.7.3 (2017-09-11)
 ==================

mayan/apps/common/models.py

@@ -12,7 +12,7 @@ from django.utils.encoding import force_text, python_2_unicode_compatible
 from django.utils.translation import ugettext_lazy as _
 
 from .managers import ErrorLogEntryManager
-from .runtime import shared_storage_backend
+from .storages import sharedupload_storage
 
 
 def upload_to(instance, filename):
@@ -47,7 +47,7 @@ class ErrorLogEntry(models.Model):
 @python_2_unicode_compatible
 class SharedUploadedFile(models.Model):
     file = models.FileField(
-        storage=shared_storage_backend, upload_to=upload_to,
+        storage=sharedupload_storage, upload_to=upload_to,
         verbose_name=_('File')
     )
     filename = models.CharField(max_length=255, verbose_name=_('Filename'))

mayan/apps/common/runtime.py

@@ -1,5 +0,0 @@
-from django.utils.module_loading import import_string
-
-from .settings import setting_shared_storage
-
-shared_storage_backend = import_string(setting_shared_storage.value)()

mayan/apps/common/settings.py

@@ -38,9 +38,13 @@ setting_paginate_by = namespace.add_setting(
 )
 setting_shared_storage = namespace.add_setting(
     global_name='COMMON_SHARED_STORAGE',
-    default='storage.backends.filebasedstorage.FileBasedStorage',
+    default='django.core.files.storage.FileSystemStorage',
     help_text=_('A storage backend that all workers can use to share files.')
 )
+setting_shared_storage_arguments = namespace.add_setting(
+    global_name='COMMON_SHARED_STORAGE_ARGUMENTS',
+    default='{location: mayan/media/shared_files}',
+)
 setting_temporary_directory = namespace.add_setting(
     global_name='COMMON_TEMPORARY_DIRECTORY', default=tempfile.gettempdir(),
     help_text=_(

mayan/apps/common/storages.py

@@ -0,0 +1,17 @@
+from __future__ import unicode_literals
+
+import yaml
+
+from django.utils.module_loading import import_string
+
+from .settings import (
+    setting_shared_storage, setting_shared_storage_arguments
+)
+
+sharedupload_storage = import_string(
+    dotted_path=setting_shared_storage.value
+)(
+    **yaml.safe_load(
+        setting_shared_storage_arguments.value or '{}'
+    )
+)

mayan/apps/document_signatures/models.py

@@ -15,7 +15,7 @@ from django_gpg.models import Key
 from documents.models import DocumentVersion
 
 from .managers import EmbeddedSignatureManager
-from .runtime import storage_backend
+from .storages import storage_backend
 
 logger = logging.getLogger(__name__)
 

mayan/apps/document_signatures/runtime.py

@@ -1,5 +0,0 @@
-from django.utils.module_loading import import_string
-
-from .settings import setting_storage_backend
-
-storage_backend = import_string(setting_storage_backend.value)()

mayan/apps/document_signatures/settings.py

@@ -7,5 +7,9 @@ from smart_settings import Namespace
 namespace = Namespace(name='signatures', label=_('Document signatures'))
 setting_storage_backend = namespace.add_setting(
     global_name='SIGNATURES_STORAGE_BACKEND',
-    default='storage.backends.filebasedstorage.FileBasedStorage'
+    default='django.core.files.storage.FileSystemStorage'
+)
+setting_storage_backend_arguments = namespace.add_setting(
+    global_name='SIGNATURES_STORAGE_BACKEND_ARGUMENTS',
+    default='{location: mayan/media/document_storage}'
 )

mayan/apps/document_signatures/storages.py

@@ -0,0 +1,17 @@
+from __future__ import unicode_literals
+
+import yaml
+
+from django.utils.module_loading import import_string
+
+from .settings import (
+    setting_storage_backend, setting_storage_backend_arguments
+)
+
+storage_backend = import_string(
+    dotted_path=setting_storage_backend.value
+)(
+    **yaml.safe_load(
+        setting_storage_backend_arguments.value or '{}'
+    )
+)

mayan/apps/documents/api_views.py

@@ -27,7 +27,6 @@ from .permissions import (
     permission_document_type_view, permission_document_version_revert,
     permission_document_version_view
 )
-from .runtime import cache_storage_backend
 from .serializers import (
     DeletedDocumentSerializer, DocumentPageSerializer, DocumentSerializer,
     DocumentTypeSerializer, DocumentVersionSerializer,
@@ -35,6 +34,7 @@ from .serializers import (
     RecentDocumentSerializer, WritableDocumentSerializer,
     WritableDocumentTypeSerializer, WritableDocumentVersionSerializer
 )
+from .storages import documentimagecache_storage
 from .tasks import task_generate_document_page_image
 
 logger = logging.getLogger(__name__)
@@ -288,7 +288,7 @@ class APIDocumentPageImageView(generics.RetrieveAPIView):
         )
 
         cache_filename = task.get(timeout=DOCUMENT_IMAGE_TASK_TIMEOUT)
-        with cache_storage_backend.open(cache_filename) as file_object:
+        with documentimagecache_storage.open(cache_filename) as file_object:
             return HttpResponse(file_object.read(), content_type='image')
 
 

mayan/apps/documents/literals.py

@@ -4,7 +4,6 @@ from django.utils.translation import ugettext_lazy as _
 
 from common.literals import TIME_DELTA_UNIT_DAYS
 
-CACHE_PATH = 'document_cache/'
 CHECK_DELETE_PERIOD_INTERVAL = 60
 CHECK_TRASH_PERIOD_INTERVAL = 60
 DELETE_STALE_STUBS_INTERVAL = 60 * 10  # 10 minutes

mayan/apps/documents/models.py

@@ -36,7 +36,6 @@ from .managers import (
     PassthroughManager, RecentDocumentManager, TrashCanManager
 )
 from .permissions import permission_document_view
-from .runtime import cache_storage_backend, storage_backend
 from .settings import (
     setting_disable_base_image_cache, setting_disable_transformed_image_cache,
     setting_display_width, setting_display_height, setting_fix_orientation,
@@ -45,6 +44,7 @@ from .settings import (
 from .signals import (
     post_document_created, post_document_type_change, post_version_upload
 )
+from .storages import documentversion_storage, documentimagecache_storage
 
 logger = logging.getLogger(__name__)
 
@@ -393,7 +393,7 @@ class DocumentVersion(models.Model):
 
     # File related fields
     file = models.FileField(
-        storage=storage_backend, upload_to=UUID_FUNCTION,
+        storage=documentversion_storage, upload_to=UUID_FUNCTION,
         verbose_name=_('File')
     )
     mimetype = models.CharField(
@@ -459,10 +459,10 @@ class DocumentVersion(models.Model):
         cache_filename = self.cache_filename
         logger.debug('Intermidiate filename: %s', cache_filename)
 
-        if cache_storage_backend.exists(cache_filename):
+        if documentimagecache_storage.exists(cache_filename):
             logger.debug('Intermidiate file "%s" found.', cache_filename)
 
-            return cache_storage_backend.open(cache_filename)
+            return documentimagecache_storage.open(cache_filename)
         else:
             logger.debug('Intermidiate file "%s" not found.', cache_filename)
 
@@ -470,11 +470,11 @@ class DocumentVersion(models.Model):
                 converter = converter_class(file_object=self.open())
                 pdf_file_object = converter.to_pdf()
 
-                with cache_storage_backend.open(cache_filename, 'wb+') as file_object:
+                with documentimagecache_storage.open(cache_filename, 'wb+') as file_object:
                     for chunk in pdf_file_object:
                         file_object.write(chunk)
 
-                return cache_storage_backend.open(cache_filename)
+                return documentimagecache_storage.open(cache_filename)
             except InvalidOfficeFormat:
                 return self.open()
             except Exception as exception:
@@ -483,7 +483,7 @@ class DocumentVersion(models.Model):
                     'Error creating intermediate file "%s"; %s.',
                     cache_filename, exception
                 )
-                cache_storage_backend.delete(cache_filename)
+                documentimagecache_storage.delete(cache_filename)
                 raise
 
     def get_rendered_string(self, preserve_extension=False):
@@ -503,7 +503,7 @@ class DocumentVersion(models.Model):
         )
 
     def invalidate_cache(self):
-        cache_storage_backend.delete(self.cache_filename)
+        documentimagecache_storage.delete(self.cache_filename)
         for page in self.pages.all():
             page.invalidate_cache()
 
@@ -810,7 +810,7 @@ class DocumentPage(models.Model):
         # Check is transformed image is available
         logger.debug('transformations cache filename: %s', cache_filename)
 
-        if not setting_disable_transformed_image_cache.value and cache_storage_backend.exists(cache_filename):
+        if not setting_disable_transformed_image_cache.value and documentimagecache_storage.exists(cache_filename):
             logger.debug(
                 'transformations cache file "%s" found', cache_filename
             )
@@ -819,7 +819,7 @@ class DocumentPage(models.Model):
                 'transformations cache file "%s" not found', cache_filename
             )
             image = self.get_image(transformations=transformation_list)
-            with cache_storage_backend.open(cache_filename, 'wb+') as file_object:
+            with documentimagecache_storage.open(cache_filename, 'wb+') as file_object:
                 file_object.write(image.getvalue())
 
             self.cached_images.create(filename=cache_filename)
@@ -840,10 +840,10 @@ class DocumentPage(models.Model):
         cache_filename = self.cache_filename
         logger.debug('Page cache filename: %s', cache_filename)
 
-        if not setting_disable_base_image_cache.value and cache_storage_backend.exists(cache_filename):
+        if not setting_disable_base_image_cache.value and documentimagecache_storage.exists(cache_filename):
             logger.debug('Page cache file "%s" found', cache_filename)
             converter = converter_class(
-                file_object=cache_storage_backend.open(cache_filename)
+                file_object=documentimagecache_storage.open(cache_filename)
             )
 
             converter.seek(0)
@@ -858,7 +858,7 @@ class DocumentPage(models.Model):
 
                 page_image = converter.get_page()
 
-                with cache_storage_backend.open(cache_filename, 'wb+') as file_object:
+                with documentimagecache_storage.open(cache_filename, 'wb+') as file_object:
                     file_object.write(page_image.getvalue())
             except Exception as exception:
                 # Cleanup in case of error
@@ -866,7 +866,7 @@ class DocumentPage(models.Model):
                     'Error creating page cache file "%s"; %s',
                     cache_filename, exception
                 )
-                cache_storage_backend.delete(cache_filename)
+                documentimagecache_storage.delete(cache_filename)
                 raise
 
         for transformation in transformations:
@@ -875,7 +875,7 @@ class DocumentPage(models.Model):
         return converter.get_page()
 
     def invalidate_cache(self):
-        cache_storage_backend.delete(self.cache_filename)
+        documentimagecache_storage.delete(self.cache_filename)
         for cached_image in self.cached_images.all():
             cached_image.delete()
 
@@ -906,7 +906,7 @@ class DocumentPageCachedImage(models.Model):
         verbose_name_plural = _('Document page cached images')
 
     def delete(self, *args, **kwargs):
-        cache_storage_backend.delete(self.filename)
+        documentimagecache_storage.delete(self.filename)
         return super(DocumentPageCachedImage, self).delete(*args, **kwargs)
 
 

mayan/apps/documents/runtime.py

@@ -1,6 +0,0 @@
-from django.utils.module_loading import import_string
-
-from .settings import setting_cache_storage_backend, setting_storage_backend
-
-storage_backend = import_string(setting_storage_backend.value)()
-cache_storage_backend = import_string(setting_cache_storage_backend.value)()

mayan/apps/documents/settings.py

@@ -45,7 +45,11 @@ setting_recent_count = namespace.add_setting(
 )
 setting_storage_backend = namespace.add_setting(
     global_name='DOCUMENTS_STORAGE_BACKEND',
-    default='storage.backends.filebasedstorage.FileBasedStorage'
+    default='django.core.files.storage.FileSystemStorage'
+)
+setting_storage_backend_arguments = namespace.add_setting(
+    global_name='DOCUMENTS_STORAGE_BACKEND_ARGUMENTS',
+    default='{location: mayan/media/document_storage}'
 )
 setting_zoom_percent_step = namespace.add_setting(
     global_name='DOCUMENTS_ZOOM_PERCENT_STEP', default=25,
@@ -74,9 +78,13 @@ setting_rotation_step = namespace.add_setting(
         'Amount in degrees to rotate a document page per user interaction.'
     )
 )
-setting_cache_storage_backend = namespace.add_setting(
+setting_documentimagecache_storage = namespace.add_setting(
     global_name='DOCUMENTS_CACHE_STORAGE_BACKEND',
-    default='documents.storage.LocalCacheFileStorage'
+    default='django.core.files.storage.FileSystemStorage'
+)
+setting_documentimagecache_storage_arguments = namespace.add_setting(
+    global_name='DOCUMENTS_CACHE_STORAGE_BACKEND_ARGUMENTS',
+    default='{location: mayan/media/document_cache}'
 )
 setting_language = namespace.add_setting(
     global_name='DOCUMENTS_LANGUAGE', default='eng',

mayan/apps/documents/storage.py

@@ -1,18 +0,0 @@
-from __future__ import unicode_literals
-
-import os
-
-from django.conf import settings
-from django.core.files.storage import FileSystemStorage
-
-from .literals import CACHE_PATH
-
-
-class LocalCacheFileStorage(FileSystemStorage):
-    """Simple wrapper for the stock Django FileSystemStorage class"""
-
-    def __init__(self, *args, **kwargs):
-        super(LocalCacheFileStorage, self).__init__(*args, **kwargs)
-        self.location = os.path.join(settings.MEDIA_ROOT, CACHE_PATH)
-        if not os.path.exists(os.path.dirname(self.location)):
-            os.makedirs(os.path.dirname(self.location))

mayan/apps/documents/storages.py

@@ -0,0 +1,26 @@
+from __future__ import unicode_literals
+
+import yaml
+
+from django.utils.module_loading import import_string
+
+from .settings import (
+    setting_documentimagecache_storage, setting_documentimagecache_storage_arguments,
+    setting_storage_backend, setting_storage_backend_arguments
+)
+
+documentversion_storage = import_string(
+    dotted_path=setting_storage_backend.value
+)(
+    **yaml.safe_load(
+        setting_storage_backend_arguments.value or '{}'
+    )
+)
+
+documentimagecache_storage = import_string(
+    dotted_path=setting_documentimagecache_storage.value
+)(
+    **yaml.safe_load(
+        setting_documentimagecache_storage_arguments.value or '{}'
+    )
+)

mayan/apps/ocr/managers.py

@@ -8,7 +8,7 @@ from django.apps import apps
 from django.conf import settings
 from django.db import models
 
-from documents.runtime import cache_storage_backend
+from documents.storages import documentimagecache_storage
 
 from .events import event_ocr_document_version_finish
 from .runtime import ocr_backend
@@ -31,7 +31,7 @@ class DocumentPageOCRContentManager(models.Manager):
         # TODO: Call task and wait
         cache_filename = document_page.generate_image()
 
-        with cache_storage_backend.open(cache_filename) as file_object:
+        with documentimagecache_storage.open(cache_filename) as file_object:
             document_page_content, created = DocumentPageOCRContent.objects.get_or_create(
                 document_page=document_page
             )

mayan/apps/storage/backends/compressedstorage.py

@@ -17,8 +17,6 @@ except ImportError:
 from django.core.files import File
 from django.core.files.storage import FileSystemStorage
 
-from ..settings import FILESTORAGE_LOCATION
-
 
 class CompressedStorage(FileSystemStorage):
     """Simple wrapper for the stock Django FileSystemStorage class"""
@@ -26,8 +24,8 @@ class CompressedStorage(FileSystemStorage):
     separator = os.path.sep
 
     def __init__(self, *args, **kwargs):
+        self.location = kwargs.pop('location')
         super(CompressedStorage, self).__init__(*args, **kwargs)
-        self.location = FILESTORAGE_LOCATION
 
     def save(self, name, content):
         descriptor = StringIO()

mayan/apps/storage/backends/filebasedstorage.py

@@ -1,17 +0,0 @@
-from __future__ import unicode_literals
-
-import os
-
-from django.core.files.storage import FileSystemStorage
-
-from ..settings import setting_filestorage_location
-
-
-class FileBasedStorage(FileSystemStorage):
-    """Simple wrapper for the stock Django FileSystemStorage class"""
-
-    separator = os.path.sep
-
-    def __init__(self, *args, **kwargs):
-        super(FileBasedStorage, self).__init__(*args, **kwargs)
-        self.location = setting_filestorage_location.value

mayan/apps/storage/settings.py

@@ -1,14 +0,0 @@
-from __future__ import unicode_literals
-
-import os
-
-from django.conf import settings
-from django.utils.translation import ugettext_lazy as _
-
-from smart_settings import Namespace
-
-namespace = Namespace(name='storage', label=_('Storage'))
-setting_filestorage_location = namespace.add_setting(
-    global_name='STORAGE_FILESTORAGE_LOCATION',
-    default=os.path.join(settings.MEDIA_ROOT, 'document_storage'), is_path=True
-)