Move file and storage code to the storage app
The setting COMMON_TEMPORARY_DIRECTORY is now STORAGE_TEMPORARY_DIRECTORY. Signed-off-by: Roberto Rosario <Roberto.Rosario@mayan-edms.com>
This commit is contained in:
@@ -227,6 +227,8 @@
|
|||||||
- Add a test mixin to generate random model primary keys.
|
- Add a test mixin to generate random model primary keys.
|
||||||
- Add support for checkout and check in multiple documents at
|
- Add support for checkout and check in multiple documents at
|
||||||
the same time.
|
the same time.
|
||||||
|
- Move file and storage code to the storage app. The setting
|
||||||
|
COMMON_TEMPORARY_DIRECTORY is now STORAGE_TEMPORARY_DIRECTORY.
|
||||||
|
|
||||||
3.1.9 (2018-11-01)
|
3.1.9 (2018-11-01)
|
||||||
==================
|
==================
|
||||||
|
|||||||
@@ -72,16 +72,6 @@ class ErrorLogNamespace(object):
|
|||||||
return ErrorLogEntry.objects.filter(namespace=self.name)
|
return ErrorLogEntry.objects.filter(namespace=self.name)
|
||||||
|
|
||||||
|
|
||||||
class FakeStorageSubclass(object):
|
|
||||||
"""
|
|
||||||
Placeholder class to allow serializing the real storage subclass to
|
|
||||||
support migrations.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __eq__(self, other):
|
|
||||||
return True
|
|
||||||
|
|
||||||
|
|
||||||
class MissingItem(object):
|
class MissingItem(object):
|
||||||
_registry = []
|
_registry = []
|
||||||
|
|
||||||
|
|||||||
@@ -11,8 +11,8 @@ from django.core.management.base import CommandError
|
|||||||
from django.utils.encoding import force_text
|
from django.utils.encoding import force_text
|
||||||
from django.utils.translation import ugettext_lazy as _
|
from django.utils.translation import ugettext_lazy as _
|
||||||
|
|
||||||
from mayan.apps.common.utils import fs_cleanup
|
|
||||||
from mayan.apps.documents.models import DocumentType
|
from mayan.apps.documents.models import DocumentType
|
||||||
|
from mayan.apps.storage.utils import fs_cleanup
|
||||||
|
|
||||||
CONVERTDB_FOLDER = 'convertdb'
|
CONVERTDB_FOLDER = 'convertdb'
|
||||||
CONVERTDB_OUTPUT_FILENAME = 'migrate.json'
|
CONVERTDB_OUTPUT_FILENAME = 'migrate.json'
|
||||||
|
|||||||
@@ -1,14 +1,13 @@
|
|||||||
# -*- coding: utf-8 -*-
|
|
||||||
# Generated by Django 1.11.16 on 2018-12-29 07:38
|
|
||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
from django.db import migrations, models
|
from django.db import migrations, models
|
||||||
|
|
||||||
import mayan.apps.common.classes
|
import mayan.apps.common.classes
|
||||||
import mayan.apps.common.models
|
import mayan.apps.common.models
|
||||||
|
import mayan.apps.storage.classes
|
||||||
|
|
||||||
|
|
||||||
class Migration(migrations.Migration):
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
dependencies = [
|
dependencies = [
|
||||||
('common', '0010_auto_20180403_0702_squashed_0011_auto_20180429_0758'),
|
('common', '0010_auto_20180403_0702_squashed_0011_auto_20180429_0758'),
|
||||||
]
|
]
|
||||||
@@ -17,6 +16,10 @@ class Migration(migrations.Migration):
|
|||||||
migrations.AlterField(
|
migrations.AlterField(
|
||||||
model_name='shareduploadedfile',
|
model_name='shareduploadedfile',
|
||||||
name='file',
|
name='file',
|
||||||
field=models.FileField(storage=mayan.apps.common.classes.FakeStorageSubclass(), upload_to=mayan.apps.common.models.upload_to, verbose_name='File'),
|
field=models.FileField(
|
||||||
|
storage=mayan.apps.storage.classes.FakeStorageSubclass(),
|
||||||
|
upload_to=mayan.apps.common.models.upload_to,
|
||||||
|
verbose_name='File'
|
||||||
|
),
|
||||||
),
|
),
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -1,6 +1,5 @@
|
|||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
from django.conf import settings
|
|
||||||
from django.contrib import messages
|
from django.contrib import messages
|
||||||
from django.contrib.contenttypes.models import ContentType
|
from django.contrib.contenttypes.models import ContentType
|
||||||
from django.core.exceptions import ImproperlyConfigured, PermissionDenied
|
from django.core.exceptions import ImproperlyConfigured, PermissionDenied
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
import os
|
import os
|
||||||
import tempfile
|
|
||||||
|
|
||||||
from django.conf import settings
|
from django.conf import settings
|
||||||
from django.utils.translation import ugettext_lazy as _
|
from django.utils.translation import ugettext_lazy as _
|
||||||
@@ -76,13 +75,6 @@ setting_shared_storage_arguments = namespace.add_setting(
|
|||||||
global_name='COMMON_SHARED_STORAGE_ARGUMENTS',
|
global_name='COMMON_SHARED_STORAGE_ARGUMENTS',
|
||||||
default={'location': os.path.join(settings.MEDIA_ROOT, 'shared_files')}
|
default={'location': os.path.join(settings.MEDIA_ROOT, 'shared_files')}
|
||||||
)
|
)
|
||||||
setting_temporary_directory = namespace.add_setting(
|
|
||||||
global_name='COMMON_TEMPORARY_DIRECTORY', default=tempfile.gettempdir(),
|
|
||||||
help_text=_(
|
|
||||||
'Temporary directory used site wide to store thumbnails, previews '
|
|
||||||
'and temporary files.'
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
namespace = Namespace(label=_('Django'), name='django')
|
namespace = Namespace(label=_('Django'), name='django')
|
||||||
|
|
||||||
|
|||||||
@@ -1,7 +1,8 @@
|
|||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
|
from mayan.apps.storage.utils import get_storage_subclass
|
||||||
|
|
||||||
from .settings import setting_shared_storage, setting_shared_storage_arguments
|
from .settings import setting_shared_storage, setting_shared_storage_arguments
|
||||||
from .utils import get_storage_subclass
|
|
||||||
|
|
||||||
storage_sharedupload = get_storage_subclass(
|
storage_sharedupload = get_storage_subclass(
|
||||||
dotted_path=setting_shared_storage.value
|
dotted_path=setting_shared_storage.value
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ from django.template import Context, Template
|
|||||||
from django.test.utils import ContextList
|
from django.test.utils import ContextList
|
||||||
from django.urls import clear_url_caches, reverse
|
from django.urls import clear_url_caches, reverse
|
||||||
|
|
||||||
from ..settings import setting_temporary_directory
|
from mayan.apps.storage.settings import setting_temporary_directory
|
||||||
|
|
||||||
from .literals import TEST_VIEW_NAME, TEST_VIEW_URL
|
from .literals import TEST_VIEW_NAME, TEST_VIEW_URL
|
||||||
from .utils import mute_stdout
|
from .utils import mute_stdout
|
||||||
|
|||||||
@@ -14,37 +14,33 @@ from .literals import TEST_ERROR_LOG_ENTRY_RESULT
|
|||||||
|
|
||||||
class CommonViewTestCase(GenericViewTestCase):
|
class CommonViewTestCase(GenericViewTestCase):
|
||||||
def test_about_view(self):
|
def test_about_view(self):
|
||||||
self.login_user()
|
|
||||||
|
|
||||||
response = self.get('common:about_view')
|
response = self.get('common:about_view')
|
||||||
self.assertContains(response, text='About', status_code=200)
|
self.assertContains(response, text='About', status_code=200)
|
||||||
|
|
||||||
def _create_error_log_entry(self):
|
def _create_error_log_entry(self):
|
||||||
ModelPermission.register(
|
ModelPermission.register(
|
||||||
model=get_user_model(), permission=permission_error_log_view
|
model=get_user_model(), permissions=(permission_error_log_view,)
|
||||||
)
|
)
|
||||||
ErrorLogEntry.objects.register(model=get_user_model())
|
ErrorLogEntry.objects.register(model=get_user_model())
|
||||||
|
|
||||||
self.error_log_entry = self.user.error_logs.create(
|
self.error_log_entry = self._test_case_user.error_logs.create(
|
||||||
result=TEST_ERROR_LOG_ENTRY_RESULT
|
result=TEST_ERROR_LOG_ENTRY_RESULT
|
||||||
)
|
)
|
||||||
|
|
||||||
def _request_object_error_log_list(self):
|
def _request_object_error_log_list(self):
|
||||||
content_type = ContentType.objects.get_for_model(model=self.user)
|
content_type = ContentType.objects.get_for_model(model=self._test_case_user)
|
||||||
|
|
||||||
return self.get(
|
return self.get(
|
||||||
'common:object_error_list', kwargs={
|
'common:object_error_list', kwargs={
|
||||||
'app_label': content_type.app_label,
|
'app_label': content_type.app_label,
|
||||||
'model': content_type.model,
|
'model': content_type.model,
|
||||||
'object_id': self.user.pk
|
'object_id': self._test_case_user.pk
|
||||||
}, follow=True
|
}, follow=True
|
||||||
)
|
)
|
||||||
|
|
||||||
def test_object_error_list_view_no_permissions(self):
|
def test_object_error_list_view_no_permissions(self):
|
||||||
self._create_error_log_entry()
|
self._create_error_log_entry()
|
||||||
|
|
||||||
self.login_user()
|
|
||||||
|
|
||||||
response = self._request_object_error_log_list()
|
response = self._request_object_error_log_list()
|
||||||
|
|
||||||
self.assertNotContains(
|
self.assertNotContains(
|
||||||
@@ -55,9 +51,8 @@ class CommonViewTestCase(GenericViewTestCase):
|
|||||||
def test_object_error_list_view_with_access(self):
|
def test_object_error_list_view_with_access(self):
|
||||||
self._create_error_log_entry()
|
self._create_error_log_entry()
|
||||||
|
|
||||||
self.login_user()
|
|
||||||
self.grant_access(
|
self.grant_access(
|
||||||
obj=self.user, permission=permission_error_log_view
|
obj=self._test_case_user, permission=permission_error_log_view
|
||||||
)
|
)
|
||||||
|
|
||||||
response = self._request_object_error_log_list()
|
response = self._request_object_error_log_list()
|
||||||
|
|||||||
@@ -1,9 +1,6 @@
|
|||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
import os
|
|
||||||
import shutil
|
|
||||||
import tempfile
|
|
||||||
|
|
||||||
from django.conf import settings
|
from django.conf import settings
|
||||||
from django.core.exceptions import FieldDoesNotExist
|
from django.core.exceptions import FieldDoesNotExist
|
||||||
@@ -13,7 +10,6 @@ from django.urls.base import get_script_prefix
|
|||||||
from django.utils.datastructures import MultiValueDict
|
from django.utils.datastructures import MultiValueDict
|
||||||
from django.utils.http import urlencode as django_urlencode
|
from django.utils.http import urlencode as django_urlencode
|
||||||
from django.utils.http import urlquote as django_urlquote
|
from django.utils.http import urlquote as django_urlquote
|
||||||
from django.utils.module_loading import import_string
|
|
||||||
from django.utils.six.moves import reduce as reduce_function
|
from django.utils.six.moves import reduce as reduce_function
|
||||||
from django.utils.six.moves import xmlrpc_client
|
from django.utils.six.moves import xmlrpc_client
|
||||||
|
|
||||||
@@ -21,7 +17,6 @@ import mayan
|
|||||||
|
|
||||||
from .exceptions import NotLatestVersion, UnknownLatestVersion
|
from .exceptions import NotLatestVersion, UnknownLatestVersion
|
||||||
from .literals import DJANGO_SQLITE_BACKEND, MAYAN_PYPI_NAME, PYPI_URL
|
from .literals import DJANGO_SQLITE_BACKEND, MAYAN_PYPI_NAME, PYPI_URL
|
||||||
from .settings import setting_temporary_directory
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -40,27 +35,6 @@ def check_version():
|
|||||||
raise NotLatestVersion(upstream_version=versions[0])
|
raise NotLatestVersion(upstream_version=versions[0])
|
||||||
|
|
||||||
|
|
||||||
# http://stackoverflow.com/questions/123198/how-do-i-copy-a-file-in-python
|
|
||||||
def copyfile(source, destination, buffer_size=1024 * 1024):
|
|
||||||
"""
|
|
||||||
Copy a file from source to dest. source and dest
|
|
||||||
can either be strings or any object with a read or
|
|
||||||
write method, like StringIO for example.
|
|
||||||
"""
|
|
||||||
source_descriptor = get_descriptor(source)
|
|
||||||
destination_descriptor = get_descriptor(destination, read=False)
|
|
||||||
|
|
||||||
while True:
|
|
||||||
copy_buffer = source_descriptor.read(buffer_size)
|
|
||||||
if copy_buffer:
|
|
||||||
destination_descriptor.write(copy_buffer)
|
|
||||||
else:
|
|
||||||
break
|
|
||||||
|
|
||||||
source_descriptor.close()
|
|
||||||
destination_descriptor.close()
|
|
||||||
|
|
||||||
|
|
||||||
def encapsulate(function):
|
def encapsulate(function):
|
||||||
# Workaround Django ticket 15791
|
# Workaround Django ticket 15791
|
||||||
# Changeset 16045
|
# Changeset 16045
|
||||||
@@ -69,25 +43,6 @@ def encapsulate(function):
|
|||||||
return lambda: function
|
return lambda: function
|
||||||
|
|
||||||
|
|
||||||
def fs_cleanup(filename, file_descriptor=None, suppress_exceptions=True):
|
|
||||||
"""
|
|
||||||
Tries to remove the given filename. Ignores non-existent files
|
|
||||||
"""
|
|
||||||
if file_descriptor:
|
|
||||||
os.close(file_descriptor)
|
|
||||||
|
|
||||||
try:
|
|
||||||
os.remove(filename)
|
|
||||||
except OSError:
|
|
||||||
try:
|
|
||||||
shutil.rmtree(filename)
|
|
||||||
except OSError:
|
|
||||||
if suppress_exceptions:
|
|
||||||
pass
|
|
||||||
else:
|
|
||||||
raise
|
|
||||||
|
|
||||||
|
|
||||||
def get_related_field(model, related_field_name):
|
def get_related_field(model, related_field_name):
|
||||||
try:
|
try:
|
||||||
local_field_name, remaining_field_path = related_field_name.split(
|
local_field_name, remaining_field_path = related_field_name.split(
|
||||||
@@ -108,41 +63,6 @@ def get_related_field(model, related_field_name):
|
|||||||
return related_field
|
return related_field
|
||||||
|
|
||||||
|
|
||||||
def get_descriptor(file_input, read=True):
|
|
||||||
try:
|
|
||||||
# Is it a file like object?
|
|
||||||
file_input.seek(0)
|
|
||||||
except AttributeError:
|
|
||||||
# If not, try open it.
|
|
||||||
if read:
|
|
||||||
return open(file_input, mode='rb')
|
|
||||||
else:
|
|
||||||
return open(file_input, mode='wb')
|
|
||||||
else:
|
|
||||||
return file_input
|
|
||||||
|
|
||||||
|
|
||||||
def get_storage_subclass(dotted_path):
|
|
||||||
"""
|
|
||||||
Import a storage class and return a subclass that will always return eq
|
|
||||||
True to avoid creating a new migration when for runtime storage class
|
|
||||||
changes.
|
|
||||||
"""
|
|
||||||
imported_storage_class = import_string(dotted_path=dotted_path)
|
|
||||||
|
|
||||||
class StorageSubclass(imported_storage_class):
|
|
||||||
def __init__(self, *args, **kwargs):
|
|
||||||
return super(StorageSubclass, self).__init__(*args, **kwargs)
|
|
||||||
|
|
||||||
def __eq__(self, other):
|
|
||||||
return True
|
|
||||||
|
|
||||||
def deconstruct(self):
|
|
||||||
return ('mayan.apps.common.classes.FakeStorageSubclass', (), {})
|
|
||||||
|
|
||||||
return StorageSubclass
|
|
||||||
|
|
||||||
|
|
||||||
def introspect_attribute(attribute_name, obj):
|
def introspect_attribute(attribute_name, obj):
|
||||||
try:
|
try:
|
||||||
# Try as a related field
|
# Try as a related field
|
||||||
@@ -168,21 +88,6 @@ def introspect_attribute(attribute_name, obj):
|
|||||||
return attribute_name, obj
|
return attribute_name, obj
|
||||||
|
|
||||||
|
|
||||||
def TemporaryFile(*args, **kwargs):
|
|
||||||
kwargs.update({'dir': setting_temporary_directory.value})
|
|
||||||
return tempfile.TemporaryFile(*args, **kwargs)
|
|
||||||
|
|
||||||
|
|
||||||
def mkdtemp(*args, **kwargs):
|
|
||||||
kwargs.update({'dir': setting_temporary_directory.value})
|
|
||||||
return tempfile.mkdtemp(*args, **kwargs)
|
|
||||||
|
|
||||||
|
|
||||||
def mkstemp(*args, **kwargs):
|
|
||||||
kwargs.update({'dir': setting_temporary_directory.value})
|
|
||||||
return tempfile.mkstemp(*args, **kwargs)
|
|
||||||
|
|
||||||
|
|
||||||
def resolve(path, urlconf=None):
|
def resolve(path, urlconf=None):
|
||||||
path = '/{}'.format(path.replace(get_script_prefix(), '', 1))
|
path = '/{}'.format(path.replace(get_script_prefix(), '', 1))
|
||||||
return django_resolve(path=path, urlconf=urlconf)
|
return django_resolve(path=path, urlconf=urlconf)
|
||||||
@@ -275,24 +180,3 @@ def urlquote(link=None, get=None):
|
|||||||
return '%s%s' % (link, django_urlencode(get, doseq=True))
|
return '%s%s' % (link, django_urlencode(get, doseq=True))
|
||||||
else:
|
else:
|
||||||
return django_urlquote(link)
|
return django_urlquote(link)
|
||||||
|
|
||||||
|
|
||||||
def validate_path(path):
|
|
||||||
if not os.path.exists(path):
|
|
||||||
# If doesn't exist try to create it
|
|
||||||
try:
|
|
||||||
os.mkdir(path)
|
|
||||||
except Exception as exception:
|
|
||||||
logger.debug('unhandled exception: %s', exception)
|
|
||||||
return False
|
|
||||||
|
|
||||||
# Check if it is writable
|
|
||||||
try:
|
|
||||||
fd, test_filepath = tempfile.mkstemp(dir=path)
|
|
||||||
os.close(fd)
|
|
||||||
os.unlink(test_filepath)
|
|
||||||
except Exception as exception:
|
|
||||||
logger.debug('unhandled exception: %s', exception)
|
|
||||||
return False
|
|
||||||
|
|
||||||
return True
|
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ import sh
|
|||||||
from django.utils.encoding import force_text
|
from django.utils.encoding import force_text
|
||||||
from django.utils.translation import ugettext_lazy as _
|
from django.utils.translation import ugettext_lazy as _
|
||||||
|
|
||||||
from mayan.apps.common.utils import fs_cleanup, mkstemp
|
from mayan.apps.storage.utils import fs_cleanup, mkstemp
|
||||||
|
|
||||||
from ..classes import ConverterBase
|
from ..classes import ConverterBase
|
||||||
from ..exceptions import PageCountError
|
from ..exceptions import PageCountError
|
||||||
|
|||||||
@@ -9,9 +9,9 @@ import sh
|
|||||||
|
|
||||||
from django.utils.translation import ugettext_lazy as _
|
from django.utils.translation import ugettext_lazy as _
|
||||||
|
|
||||||
from mayan.apps.common.settings import setting_temporary_directory
|
|
||||||
from mayan.apps.common.utils import fs_cleanup, mkdtemp, mkstemp
|
|
||||||
from mayan.apps.mimetype.api import get_mimetype
|
from mayan.apps.mimetype.api import get_mimetype
|
||||||
|
from mayan.apps.storage.settings import setting_temporary_directory
|
||||||
|
from mayan.apps.storage.utils import fs_cleanup, mkdtemp, mkstemp
|
||||||
|
|
||||||
from .exceptions import InvalidOfficeFormat, OfficeConversionError
|
from .exceptions import InvalidOfficeFormat, OfficeConversionError
|
||||||
from .literals import (
|
from .literals import (
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ from django.apps import apps
|
|||||||
from django.utils.encoding import force_bytes, force_text
|
from django.utils.encoding import force_bytes, force_text
|
||||||
from django.utils.functional import cached_property
|
from django.utils.functional import cached_property
|
||||||
|
|
||||||
from mayan.apps.common.utils import mkdtemp
|
from mayan.apps.storage.utils import mkdtemp
|
||||||
|
|
||||||
from .exceptions import DependenciesException
|
from .exceptions import DependenciesException
|
||||||
|
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ import gnupg
|
|||||||
|
|
||||||
from django.utils.translation import ugettext_lazy as _
|
from django.utils.translation import ugettext_lazy as _
|
||||||
|
|
||||||
from mayan.apps.common.utils import mkdtemp
|
from mayan.apps.storage.utils import mkdtemp
|
||||||
|
|
||||||
|
|
||||||
class GPGBackend(object):
|
class GPGBackend(object):
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ import os
|
|||||||
|
|
||||||
from django.db import models
|
from django.db import models
|
||||||
|
|
||||||
from mayan.apps.common.utils import mkstemp
|
from mayan.apps.storage.utils import mkstemp
|
||||||
|
|
||||||
from .classes import KeyStub, SignatureVerification
|
from .classes import KeyStub, SignatureVerification
|
||||||
from .exceptions import (
|
from .exceptions import (
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ import mock
|
|||||||
from django.utils.encoding import force_bytes
|
from django.utils.encoding import force_bytes
|
||||||
|
|
||||||
from mayan.apps.common.tests import BaseTestCase
|
from mayan.apps.common.tests import BaseTestCase
|
||||||
from mayan.apps.common.utils import TemporaryFile
|
from mayan.apps.storage.utils import TemporaryFile
|
||||||
|
|
||||||
from ..exceptions import (
|
from ..exceptions import (
|
||||||
DecryptionError, KeyDoesNotExist, NeedPassphrase, PassphraseError,
|
DecryptionError, KeyDoesNotExist, NeedPassphrase, PassphraseError,
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ import subprocess
|
|||||||
from django.apps import apps
|
from django.apps import apps
|
||||||
from django.utils.translation import ugettext_lazy as _
|
from django.utils.translation import ugettext_lazy as _
|
||||||
|
|
||||||
from mayan.apps.common.utils import copyfile, fs_cleanup, mkstemp
|
from mayan.apps.storage.utils import copyfile, fs_cleanup, mkstemp
|
||||||
|
|
||||||
from .exceptions import ParserError
|
from .exceptions import ParserError
|
||||||
from .settings import setting_pdftotext_path
|
from .settings import setting_pdftotext_path
|
||||||
|
|||||||
@@ -5,10 +5,10 @@ import os
|
|||||||
|
|
||||||
from django.db import models
|
from django.db import models
|
||||||
|
|
||||||
from mayan.apps.common.utils import mkstemp
|
|
||||||
from mayan.apps.django_gpg.exceptions import DecryptionError
|
from mayan.apps.django_gpg.exceptions import DecryptionError
|
||||||
from mayan.apps.django_gpg.models import Key
|
from mayan.apps.django_gpg.models import Key
|
||||||
from mayan.apps.documents.models import DocumentVersion
|
from mayan.apps.documents.models import DocumentVersion
|
||||||
|
from mayan.apps.storage.utils import mkstemp
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|||||||
@@ -1,14 +1,13 @@
|
|||||||
# -*- coding: utf-8 -*-
|
|
||||||
# Generated by Django 1.11.16 on 2018-12-29 07:37
|
|
||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
from django.db import migrations, models
|
from django.db import migrations, models
|
||||||
|
|
||||||
import mayan.apps.common.classes
|
import mayan.apps.common.classes
|
||||||
import mayan.apps.document_signatures.models
|
import mayan.apps.document_signatures.models
|
||||||
|
import mayan.apps.storage.classes
|
||||||
|
|
||||||
|
|
||||||
class Migration(migrations.Migration):
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
dependencies = [
|
dependencies = [
|
||||||
('document_signatures', '0008_auto_20180429_0759'),
|
('document_signatures', '0008_auto_20180429_0759'),
|
||||||
]
|
]
|
||||||
@@ -17,6 +16,11 @@ class Migration(migrations.Migration):
|
|||||||
migrations.AlterField(
|
migrations.AlterField(
|
||||||
model_name='detachedsignature',
|
model_name='detachedsignature',
|
||||||
name='signature_file',
|
name='signature_file',
|
||||||
field=models.FileField(blank=True, null=True, storage=mayan.apps.common.classes.FakeStorageSubclass(), upload_to=mayan.apps.document_signatures.models.upload_to, verbose_name='Signature file'),
|
field=models.FileField(
|
||||||
|
blank=True, null=True,
|
||||||
|
storage=mayan.apps.storage.classes.FakeStorageSubclass(),
|
||||||
|
upload_to=mayan.apps.document_signatures.models.upload_to,
|
||||||
|
verbose_name='Signature file'
|
||||||
|
),
|
||||||
),
|
),
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
from mayan.apps.common.utils import get_storage_subclass
|
from mayan.apps.storage.utils import get_storage_subclass
|
||||||
|
|
||||||
from .settings import (
|
from .settings import (
|
||||||
setting_storage_backend, setting_storage_backend_arguments
|
setting_storage_backend, setting_storage_backend_arguments
|
||||||
|
|||||||
@@ -15,9 +15,9 @@ from mayan.apps.common.generics import (
|
|||||||
SingleObjectDetailView, SingleObjectDownloadView, SingleObjectListView
|
SingleObjectDetailView, SingleObjectDownloadView, SingleObjectListView
|
||||||
)
|
)
|
||||||
from mayan.apps.common.mixins import ExternalObjectMixin
|
from mayan.apps.common.mixins import ExternalObjectMixin
|
||||||
from mayan.apps.common.utils import TemporaryFile
|
|
||||||
from mayan.apps.django_gpg.exceptions import NeedPassphrase, PassphraseError
|
from mayan.apps.django_gpg.exceptions import NeedPassphrase, PassphraseError
|
||||||
from mayan.apps.documents.models import DocumentVersion
|
from mayan.apps.documents.models import DocumentVersion
|
||||||
|
from mayan.apps.storage.utils import TemporaryFile
|
||||||
|
|
||||||
from .forms import (
|
from .forms import (
|
||||||
DocumentVersionSignatureCreateForm, DocumentVersionSignatureDetailForm
|
DocumentVersionSignatureCreateForm, DocumentVersionSignatureDetailForm
|
||||||
|
|||||||
@@ -1,10 +1,10 @@
|
|||||||
# -*- coding: utf-8 -*-
|
|
||||||
# Generated by Django 1.11.16 on 2018-12-29 07:45
|
|
||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
from django.db import migrations, models
|
from django.db import migrations, models
|
||||||
|
|
||||||
import mayan.apps.common.classes
|
import mayan.apps.common.classes
|
||||||
import mayan.apps.documents.utils
|
import mayan.apps.documents.utils
|
||||||
|
import mayan.apps.storage.classes
|
||||||
|
|
||||||
|
|
||||||
class Migration(migrations.Migration):
|
class Migration(migrations.Migration):
|
||||||
@@ -17,6 +17,10 @@ class Migration(migrations.Migration):
|
|||||||
migrations.AlterField(
|
migrations.AlterField(
|
||||||
model_name='documentversion',
|
model_name='documentversion',
|
||||||
name='file',
|
name='file',
|
||||||
field=models.FileField(storage=mayan.apps.common.classes.FakeStorageSubclass(), upload_to=mayan.apps.documents.utils.document_uuid_function, verbose_name='File'),
|
field=models.FileField(
|
||||||
|
storage=mayan.apps.storage.classes.FakeStorageSubclass(),
|
||||||
|
upload_to=mayan.apps.documents.utils.document_uuid_function,
|
||||||
|
verbose_name='File'
|
||||||
|
),
|
||||||
),
|
),
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -2,7 +2,7 @@ from __future__ import unicode_literals
|
|||||||
|
|
||||||
from django.utils.module_loading import import_string
|
from django.utils.module_loading import import_string
|
||||||
|
|
||||||
from mayan.apps.common.utils import get_storage_subclass
|
from mayan.apps.storage.utils import get_storage_subclass
|
||||||
|
|
||||||
from .settings import (
|
from .settings import (
|
||||||
setting_documentimagecache_storage,
|
setting_documentimagecache_storage,
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ import sh
|
|||||||
|
|
||||||
from django.utils.translation import ugettext_lazy as _
|
from django.utils.translation import ugettext_lazy as _
|
||||||
|
|
||||||
from mayan.apps.common.utils import fs_cleanup, mkstemp
|
from mayan.apps.storage.utils import fs_cleanup, mkstemp
|
||||||
|
|
||||||
from ..classes import FileMetadataDriver
|
from ..classes import FileMetadataDriver
|
||||||
from ..settings import setting_drivers_arguments
|
from ..settings import setting_drivers_arguments
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ from django.conf import settings
|
|||||||
from django.core.files import locks
|
from django.core.files import locks
|
||||||
from django.utils.encoding import force_bytes, force_text
|
from django.utils.encoding import force_bytes, force_text
|
||||||
|
|
||||||
from mayan.apps.common.settings import setting_temporary_directory
|
from mayan.apps.storage.settings import setting_temporary_directory
|
||||||
|
|
||||||
from ..exceptions import LockError
|
from ..exceptions import LockError
|
||||||
from ..settings import setting_default_lock_timeout
|
from ..settings import setting_default_lock_timeout
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ from django.conf import settings
|
|||||||
|
|
||||||
from mayan.apps.common.settings import setting_paginate_by
|
from mayan.apps.common.settings import setting_paginate_by
|
||||||
from mayan.apps.common.tests import BaseTestCase
|
from mayan.apps.common.tests import BaseTestCase
|
||||||
from mayan.apps.common.utils import fs_cleanup, mkstemp
|
from mayan.apps.storage.utils import fs_cleanup, mkstemp
|
||||||
|
|
||||||
from .literals import TEST_SETTING_NAME, TEST_SETTING_VALUE
|
from .literals import TEST_SETTING_NAME, TEST_SETTING_VALUE
|
||||||
|
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ from django.db import models
|
|||||||
from django.utils.timezone import now
|
from django.utils.timezone import now
|
||||||
from django.utils.translation import ugettext_lazy as _
|
from django.utils.translation import ugettext_lazy as _
|
||||||
|
|
||||||
from mayan.apps.common.utils import TemporaryFile
|
from mayan.apps.storage.utils import TemporaryFile
|
||||||
|
|
||||||
from ..classes import PseudoFile, SourceUploadedFile
|
from ..classes import PseudoFile, SourceUploadedFile
|
||||||
from ..exceptions import SourceException
|
from ..exceptions import SourceException
|
||||||
|
|||||||
@@ -4,8 +4,8 @@ import os
|
|||||||
import shutil
|
import shutil
|
||||||
|
|
||||||
from mayan.apps.common.tests import BaseTestCase
|
from mayan.apps.common.tests import BaseTestCase
|
||||||
from mayan.apps.common.utils import mkdtemp
|
|
||||||
from mayan.apps.documents.tests import TEST_NON_ASCII_DOCUMENT_PATH
|
from mayan.apps.documents.tests import TEST_NON_ASCII_DOCUMENT_PATH
|
||||||
|
from mayan.apps.storage.utils import mkdtemp
|
||||||
|
|
||||||
from ..classes import StagingFile
|
from ..classes import StagingFile
|
||||||
|
|
||||||
|
|||||||
@@ -9,7 +9,6 @@ from pathlib2 import Path
|
|||||||
from django.utils.encoding import force_text
|
from django.utils.encoding import force_text
|
||||||
|
|
||||||
from mayan.apps.common.tests import BaseTestCase
|
from mayan.apps.common.tests import BaseTestCase
|
||||||
from mayan.apps.common.utils import mkdtemp
|
|
||||||
from mayan.apps.documents.models import Document, DocumentType
|
from mayan.apps.documents.models import Document, DocumentType
|
||||||
from mayan.apps.documents.tests import (
|
from mayan.apps.documents.tests import (
|
||||||
TEST_COMPRESSED_DOCUMENT_PATH, TEST_DOCUMENT_TYPE_LABEL,
|
TEST_COMPRESSED_DOCUMENT_PATH, TEST_DOCUMENT_TYPE_LABEL,
|
||||||
@@ -17,6 +16,7 @@ from mayan.apps.documents.tests import (
|
|||||||
TEST_NON_ASCII_DOCUMENT_PATH, DocumentTestMixin
|
TEST_NON_ASCII_DOCUMENT_PATH, DocumentTestMixin
|
||||||
)
|
)
|
||||||
from mayan.apps.metadata.models import MetadataType
|
from mayan.apps.metadata.models import MetadataType
|
||||||
|
from mayan.apps.storage.utils import mkdtemp
|
||||||
|
|
||||||
from ..literals import SOURCE_UNCOMPRESS_CHOICE_Y
|
from ..literals import SOURCE_UNCOMPRESS_CHOICE_Y
|
||||||
from ..models import POP3Email, WatchFolderSource, WebFormSource
|
from ..models import POP3Email, WatchFolderSource, WebFormSource
|
||||||
|
|||||||
@@ -6,13 +6,13 @@ import shutil
|
|||||||
|
|
||||||
from mayan.apps.checkouts.models import NewVersionBlock
|
from mayan.apps.checkouts.models import NewVersionBlock
|
||||||
from mayan.apps.common.tests import GenericViewTestCase
|
from mayan.apps.common.tests import GenericViewTestCase
|
||||||
from mayan.apps.common.utils import fs_cleanup, mkdtemp
|
|
||||||
from mayan.apps.documents.models import Document
|
from mayan.apps.documents.models import Document
|
||||||
from mayan.apps.documents.permissions import permission_document_create
|
from mayan.apps.documents.permissions import permission_document_create
|
||||||
from mayan.apps.documents.tests import (
|
from mayan.apps.documents.tests import (
|
||||||
TEST_DOCUMENT_DESCRIPTION, TEST_SMALL_DOCUMENT_CHECKSUM,
|
TEST_DOCUMENT_DESCRIPTION, TEST_SMALL_DOCUMENT_CHECKSUM,
|
||||||
TEST_SMALL_DOCUMENT_PATH, GenericDocumentViewTestCase
|
TEST_SMALL_DOCUMENT_PATH, GenericDocumentViewTestCase
|
||||||
)
|
)
|
||||||
|
from mayan.apps.storage.utils import fs_cleanup, mkdtemp
|
||||||
|
|
||||||
from ..links import link_upload_version
|
from ..links import link_upload_version
|
||||||
from ..literals import SOURCE_CHOICE_WEB_FORM
|
from ..literals import SOURCE_CHOICE_WEB_FORM
|
||||||
|
|||||||
10
mayan/apps/storage/classes.py
Normal file
10
mayan/apps/storage/classes.py
Normal file
@@ -0,0 +1,10 @@
|
|||||||
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
|
|
||||||
|
class FakeStorageSubclass(object):
|
||||||
|
"""
|
||||||
|
Placeholder class to allow serializing the real storage subclass to
|
||||||
|
support migrations.
|
||||||
|
"""
|
||||||
|
def __eq__(self, other):
|
||||||
|
return True
|
||||||
17
mayan/apps/storage/settings.py
Normal file
17
mayan/apps/storage/settings.py
Normal file
@@ -0,0 +1,17 @@
|
|||||||
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
|
import tempfile
|
||||||
|
|
||||||
|
from django.utils.translation import ugettext_lazy as _
|
||||||
|
|
||||||
|
from mayan.apps.smart_settings import Namespace
|
||||||
|
|
||||||
|
namespace = Namespace(label=_('Storage'), name='storage')
|
||||||
|
|
||||||
|
setting_temporary_directory = namespace.add_setting(
|
||||||
|
global_name='STORAGE_TEMPORARY_DIRECTORY', default=tempfile.gettempdir(),
|
||||||
|
help_text=_(
|
||||||
|
'Temporary directory used site wide to store thumbnails, previews '
|
||||||
|
'and temporary files.'
|
||||||
|
)
|
||||||
|
)
|
||||||
123
mayan/apps/storage/utils.py
Normal file
123
mayan/apps/storage/utils.py
Normal file
@@ -0,0 +1,123 @@
|
|||||||
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
import tempfile
|
||||||
|
|
||||||
|
from django.utils.module_loading import import_string
|
||||||
|
|
||||||
|
from .settings import setting_temporary_directory
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def TemporaryFile(*args, **kwargs):
|
||||||
|
kwargs.update({'dir': setting_temporary_directory.value})
|
||||||
|
return tempfile.TemporaryFile(*args, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
# http://stackoverflow.com/questions/123198/how-do-i-copy-a-file-in-python
|
||||||
|
def copyfile(source, destination, buffer_size=1024 * 1024):
|
||||||
|
"""
|
||||||
|
Copy a file from source to dest. source and dest
|
||||||
|
can either be strings or any object with a read or
|
||||||
|
write method, like StringIO for example.
|
||||||
|
"""
|
||||||
|
source_descriptor = get_descriptor(source)
|
||||||
|
destination_descriptor = get_descriptor(destination, read=False)
|
||||||
|
|
||||||
|
while True:
|
||||||
|
copy_buffer = source_descriptor.read(buffer_size)
|
||||||
|
if copy_buffer:
|
||||||
|
destination_descriptor.write(copy_buffer)
|
||||||
|
else:
|
||||||
|
break
|
||||||
|
|
||||||
|
source_descriptor.close()
|
||||||
|
destination_descriptor.close()
|
||||||
|
|
||||||
|
|
||||||
|
def fs_cleanup(filename, file_descriptor=None, suppress_exceptions=True):
|
||||||
|
"""
|
||||||
|
Tries to remove the given filename. Ignores non-existent files
|
||||||
|
"""
|
||||||
|
if file_descriptor:
|
||||||
|
os.close(file_descriptor)
|
||||||
|
|
||||||
|
try:
|
||||||
|
os.remove(filename)
|
||||||
|
except OSError:
|
||||||
|
try:
|
||||||
|
shutil.rmtree(filename)
|
||||||
|
except OSError:
|
||||||
|
if suppress_exceptions:
|
||||||
|
pass
|
||||||
|
else:
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def get_descriptor(file_input, read=True):
|
||||||
|
try:
|
||||||
|
# Is it a file like object?
|
||||||
|
file_input.seek(0)
|
||||||
|
except AttributeError:
|
||||||
|
# If not, try open it.
|
||||||
|
if read:
|
||||||
|
return open(file_input, mode='rb')
|
||||||
|
else:
|
||||||
|
return open(file_input, mode='wb')
|
||||||
|
else:
|
||||||
|
return file_input
|
||||||
|
|
||||||
|
|
||||||
|
def get_storage_subclass(dotted_path):
|
||||||
|
"""
|
||||||
|
Import a storage class and return a subclass that will always return eq
|
||||||
|
True to avoid creating a new migration when for runtime storage class
|
||||||
|
changes.
|
||||||
|
"""
|
||||||
|
imported_storage_class = import_string(dotted_path=dotted_path)
|
||||||
|
|
||||||
|
class StorageSubclass(imported_storage_class):
|
||||||
|
def __init__(self, *args, **kwargs):
|
||||||
|
return super(StorageSubclass, self).__init__(*args, **kwargs)
|
||||||
|
|
||||||
|
def __eq__(self, other):
|
||||||
|
return True
|
||||||
|
|
||||||
|
def deconstruct(self):
|
||||||
|
return ('mayan.apps.storage.classes.FakeStorageSubclass', (), {})
|
||||||
|
|
||||||
|
return StorageSubclass
|
||||||
|
|
||||||
|
|
||||||
|
def mkdtemp(*args, **kwargs):
|
||||||
|
kwargs.update({'dir': setting_temporary_directory.value})
|
||||||
|
return tempfile.mkdtemp(*args, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
def mkstemp(*args, **kwargs):
|
||||||
|
kwargs.update({'dir': setting_temporary_directory.value})
|
||||||
|
return tempfile.mkstemp(*args, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
def validate_path(path):
|
||||||
|
if not os.path.exists(path):
|
||||||
|
# If doesn't exist try to create it
|
||||||
|
try:
|
||||||
|
os.mkdir(path)
|
||||||
|
except Exception as exception:
|
||||||
|
logger.debug('unhandled exception: %s', exception)
|
||||||
|
return False
|
||||||
|
|
||||||
|
# Check if it is writable
|
||||||
|
try:
|
||||||
|
fd, test_filepath = tempfile.mkstemp(dir=path)
|
||||||
|
os.close(fd)
|
||||||
|
os.unlink(test_filepath)
|
||||||
|
except Exception as exception:
|
||||||
|
logger.debug('unhandled exception: %s', exception)
|
||||||
|
return False
|
||||||
|
|
||||||
|
return True
|
||||||
Reference in New Issue
Block a user