matthias/mayan-edms
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge remote-tracking branch 'origin/feature/storages' into merge-test

Browse Source
This commit is contained in:
Roberto Rosario
2018-04-03 02:43:24 -04:00
parent 271838888c 424d6a5af9
commit bb7caa3c10
20 changed files with 123 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mayan/apps/documents/api_views.py
+2 -2
View File
@@ -27,7 +27,6 @@ from .permissions import (
permission_document_type_view, permission_document_version_revert,
permission_document_version_view
)
from .runtime import cache_storage_backend
from .serializers import (
DeletedDocumentSerializer, DocumentPageSerializer, DocumentSerializer,
DocumentTypeSerializer, DocumentVersionSerializer,
@@ -35,6 +34,7 @@ from .serializers import (
RecentDocumentSerializer, WritableDocumentSerializer,
WritableDocumentTypeSerializer, WritableDocumentVersionSerializer
)
from .storages import documentimagecache_storage
from .tasks import task_generate_document_page_image
logger = logging.getLogger(__name__)
@@ -288,7 +288,7 @@ class APIDocumentPageImageView(generics.RetrieveAPIView):
)
cache_filename = task.get(timeout=DOCUMENT_IMAGE_TASK_TIMEOUT)
with cache_storage_backend.open(cache_filename) as file_object:
with documentimagecache_storage.open(cache_filename) as file_object:
return HttpResponse(file_object.read(), content_type='image')
mayan/apps/documents/literals.py
-1
View File
@@ -4,7 +4,6 @@ from django.utils.translation import ugettext_lazy as _
from common.literals import TIME_DELTA_UNIT_DAYS
CACHE_PATH = 'document_cache/'
CHECK_DELETE_PERIOD_INTERVAL = 60
CHECK_TRASH_PERIOD_INTERVAL = 60
DELETE_STALE_STUBS_INTERVAL = 60 * 10 # 10 minutes
mayan/apps/documents/models.py
+16 -16
View File
@@ -36,7 +36,6 @@ from .managers import (
PassthroughManager, RecentDocumentManager, TrashCanManager
)
from .permissions import permission_document_view
from .runtime import cache_storage_backend, storage_backend
from .settings import (
setting_disable_base_image_cache, setting_disable_transformed_image_cache,
setting_display_width, setting_display_height, setting_fix_orientation,
@@ -45,6 +44,7 @@ from .settings import (
from .signals import (
post_document_created, post_document_type_change, post_version_upload
)
from .storages import documentversion_storage, documentimagecache_storage
logger = logging.getLogger(__name__)
@@ -393,7 +393,7 @@ class DocumentVersion(models.Model):
# File related fields
file = models.FileField(
storage=storage_backend, upload_to=UUID_FUNCTION,
storage=documentversion_storage, upload_to=UUID_FUNCTION,
verbose_name=_('File')
)
mimetype = models.CharField(
@@ -459,10 +459,10 @@ class DocumentVersion(models.Model):
cache_filename = self.cache_filename
logger.debug('Intermidiate filename: %s', cache_filename)
if cache_storage_backend.exists(cache_filename):
if documentimagecache_storage.exists(cache_filename):
logger.debug('Intermidiate file "%s" found.', cache_filename)
return cache_storage_backend.open(cache_filename)
return documentimagecache_storage.open(cache_filename)
else:
logger.debug('Intermidiate file "%s" not found.', cache_filename)
@@ -470,11 +470,11 @@ class DocumentVersion(models.Model):
converter = converter_class(file_object=self.open())
pdf_file_object = converter.to_pdf()
with cache_storage_backend.open(cache_filename, 'wb+') as file_object:
with documentimagecache_storage.open(cache_filename, 'wb+') as file_object:
for chunk in pdf_file_object:
file_object.write(chunk)
return cache_storage_backend.open(cache_filename)
return documentimagecache_storage.open(cache_filename)
except InvalidOfficeFormat:
return self.open()
except Exception as exception:
@@ -483,7 +483,7 @@ class DocumentVersion(models.Model):
'Error creating intermediate file "%s"; %s.',
cache_filename, exception
)
cache_storage_backend.delete(cache_filename)
documentimagecache_storage.delete(cache_filename)
raise
def get_rendered_string(self, preserve_extension=False):
@@ -503,7 +503,7 @@ class DocumentVersion(models.Model):
)
def invalidate_cache(self):
cache_storage_backend.delete(self.cache_filename)
documentimagecache_storage.delete(self.cache_filename)
for page in self.pages.all():
page.invalidate_cache()
@@ -810,7 +810,7 @@ class DocumentPage(models.Model):
# Check is transformed image is available
logger.debug('transformations cache filename: %s', cache_filename)
if not setting_disable_transformed_image_cache.value and cache_storage_backend.exists(cache_filename):
if not setting_disable_transformed_image_cache.value and documentimagecache_storage.exists(cache_filename):
logger.debug(
'transformations cache file "%s" found', cache_filename
)
@@ -819,7 +819,7 @@ class DocumentPage(models.Model):
'transformations cache file "%s" not found', cache_filename
)
image = self.get_image(transformations=transformation_list)
with cache_storage_backend.open(cache_filename, 'wb+') as file_object:
with documentimagecache_storage.open(cache_filename, 'wb+') as file_object:
file_object.write(image.getvalue())
self.cached_images.create(filename=cache_filename)
@@ -840,10 +840,10 @@ class DocumentPage(models.Model):
cache_filename = self.cache_filename
logger.debug('Page cache filename: %s', cache_filename)
if not setting_disable_base_image_cache.value and cache_storage_backend.exists(cache_filename):
if not setting_disable_base_image_cache.value and documentimagecache_storage.exists(cache_filename):
logger.debug('Page cache file "%s" found', cache_filename)
converter = converter_class(
file_object=cache_storage_backend.open(cache_filename)
file_object=documentimagecache_storage.open(cache_filename)
)
converter.seek(0)
@@ -858,7 +858,7 @@ class DocumentPage(models.Model):
page_image = converter.get_page()
with cache_storage_backend.open(cache_filename, 'wb+') as file_object:
with documentimagecache_storage.open(cache_filename, 'wb+') as file_object:
file_object.write(page_image.getvalue())
except Exception as exception:
# Cleanup in case of error
@@ -866,7 +866,7 @@ class DocumentPage(models.Model):
'Error creating page cache file "%s"; %s',
cache_filename, exception
)
cache_storage_backend.delete(cache_filename)
documentimagecache_storage.delete(cache_filename)
raise
for transformation in transformations:
@@ -875,7 +875,7 @@ class DocumentPage(models.Model):
return converter.get_page()
def invalidate_cache(self):
cache_storage_backend.delete(self.cache_filename)
documentimagecache_storage.delete(self.cache_filename)
for cached_image in self.cached_images.all():
cached_image.delete()
@@ -906,7 +906,7 @@ class DocumentPageCachedImage(models.Model):
verbose_name_plural = _('Document page cached images')
def delete(self, *args, **kwargs):
cache_storage_backend.delete(self.filename)
documentimagecache_storage.delete(self.filename)
return super(DocumentPageCachedImage, self).delete(*args, **kwargs)
mayan/apps/documents/runtime.py
-6
View File
@@ -1,6 +0,0 @@
from django.utils.module_loading import import_string
from .settings import setting_cache_storage_backend, setting_storage_backend
storage_backend = import_string(setting_storage_backend.value)()
cache_storage_backend = import_string(setting_cache_storage_backend.value)()
mayan/apps/documents/settings.py
+11 -3
View File
@@ -45,7 +45,11 @@ setting_recent_count = namespace.add_setting(
)
setting_storage_backend = namespace.add_setting(
global_name='DOCUMENTS_STORAGE_BACKEND',
default='storage.backends.filebasedstorage.FileBasedStorage'
default='django.core.files.storage.FileSystemStorage'
)
setting_storage_backend_arguments = namespace.add_setting(
global_name='DOCUMENTS_STORAGE_BACKEND_ARGUMENTS',
default='{location: mayan/media/document_storage}'
)
setting_zoom_percent_step = namespace.add_setting(
global_name='DOCUMENTS_ZOOM_PERCENT_STEP', default=25,
@@ -74,9 +78,13 @@ setting_rotation_step = namespace.add_setting(
'Amount in degrees to rotate a document page per user interaction.'
)
)
setting_cache_storage_backend = namespace.add_setting(
setting_documentimagecache_storage = namespace.add_setting(
global_name='DOCUMENTS_CACHE_STORAGE_BACKEND',
default='documents.storage.LocalCacheFileStorage'
default='django.core.files.storage.FileSystemStorage'
)
setting_documentimagecache_storage_arguments = namespace.add_setting(
global_name='DOCUMENTS_CACHE_STORAGE_BACKEND_ARGUMENTS',
default='{location: mayan/media/document_cache}'
)
setting_language = namespace.add_setting(
global_name='DOCUMENTS_LANGUAGE', default='eng',
mayan/apps/documents/storage.py
-18
View File
@@ -1,18 +0,0 @@
from __future__ import unicode_literals
import os
from django.conf import settings
from django.core.files.storage import FileSystemStorage
from .literals import CACHE_PATH
class LocalCacheFileStorage(FileSystemStorage):
"""Simple wrapper for the stock Django FileSystemStorage class"""
def __init__(self, *args, **kwargs):
super(LocalCacheFileStorage, self).__init__(*args, **kwargs)
self.location = os.path.join(settings.MEDIA_ROOT, CACHE_PATH)
if not os.path.exists(os.path.dirname(self.location)):
os.makedirs(os.path.dirname(self.location))
mayan/apps/documents/storages.py
+26
View File
@@ -0,0 +1,26 @@
from __future__ import unicode_literals
import yaml
from django.utils.module_loading import import_string
from .settings import (
setting_documentimagecache_storage, setting_documentimagecache_storage_arguments,
setting_storage_backend, setting_storage_backend_arguments
)
documentversion_storage = import_string(
dotted_path=setting_storage_backend.value
)(
**yaml.safe_load(
setting_storage_backend_arguments.value or '{}'
)
)
documentimagecache_storage = import_string(
dotted_path=setting_documentimagecache_storage.value
)(
**yaml.safe_load(
setting_documentimagecache_storage_arguments.value or '{}'
)
)