Move file and storage code to the storage app

The setting COMMON_TEMPORARY_DIRECTORY is now
STORAGE_TEMPORARY_DIRECTORY.

Signed-off-by: Roberto Rosario <Roberto.Rosario@mayan-edms.com>
This commit is contained in:
Roberto Rosario
2019-01-31 22:26:07 -04:00
parent 125a4317f4
commit 8e66eefe7c
33 changed files with 202 additions and 178 deletions

View File

@@ -227,6 +227,8 @@
- Add a test mixin to generate random model primary keys. - Add a test mixin to generate random model primary keys.
- Add support for checkout and check in multiple documents at - Add support for checkout and check in multiple documents at
the same time. the same time.
- Move file and storage code to the storage app. The setting
COMMON_TEMPORARY_DIRECTORY is now STORAGE_TEMPORARY_DIRECTORY.
3.1.9 (2018-11-01) 3.1.9 (2018-11-01)
================== ==================

View File

@@ -72,16 +72,6 @@ class ErrorLogNamespace(object):
return ErrorLogEntry.objects.filter(namespace=self.name) return ErrorLogEntry.objects.filter(namespace=self.name)
class FakeStorageSubclass(object):
"""
Placeholder class to allow serializing the real storage subclass to
support migrations.
"""
def __eq__(self, other):
return True
class MissingItem(object): class MissingItem(object):
_registry = [] _registry = []

View File

@@ -11,8 +11,8 @@ from django.core.management.base import CommandError
from django.utils.encoding import force_text from django.utils.encoding import force_text
from django.utils.translation import ugettext_lazy as _ from django.utils.translation import ugettext_lazy as _
from mayan.apps.common.utils import fs_cleanup
from mayan.apps.documents.models import DocumentType from mayan.apps.documents.models import DocumentType
from mayan.apps.storage.utils import fs_cleanup
CONVERTDB_FOLDER = 'convertdb' CONVERTDB_FOLDER = 'convertdb'
CONVERTDB_OUTPUT_FILENAME = 'migrate.json' CONVERTDB_OUTPUT_FILENAME = 'migrate.json'

View File

@@ -1,14 +1,13 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.11.16 on 2018-12-29 07:38
from __future__ import unicode_literals from __future__ import unicode_literals
from django.db import migrations, models from django.db import migrations, models
import mayan.apps.common.classes import mayan.apps.common.classes
import mayan.apps.common.models import mayan.apps.common.models
import mayan.apps.storage.classes
class Migration(migrations.Migration): class Migration(migrations.Migration):
dependencies = [ dependencies = [
('common', '0010_auto_20180403_0702_squashed_0011_auto_20180429_0758'), ('common', '0010_auto_20180403_0702_squashed_0011_auto_20180429_0758'),
] ]
@@ -17,6 +16,10 @@ class Migration(migrations.Migration):
migrations.AlterField( migrations.AlterField(
model_name='shareduploadedfile', model_name='shareduploadedfile',
name='file', name='file',
field=models.FileField(storage=mayan.apps.common.classes.FakeStorageSubclass(), upload_to=mayan.apps.common.models.upload_to, verbose_name='File'), field=models.FileField(
storage=mayan.apps.storage.classes.FakeStorageSubclass(),
upload_to=mayan.apps.common.models.upload_to,
verbose_name='File'
),
), ),
] ]

View File

@@ -1,6 +1,5 @@
from __future__ import unicode_literals from __future__ import unicode_literals
from django.conf import settings
from django.contrib import messages from django.contrib import messages
from django.contrib.contenttypes.models import ContentType from django.contrib.contenttypes.models import ContentType
from django.core.exceptions import ImproperlyConfigured, PermissionDenied from django.core.exceptions import ImproperlyConfigured, PermissionDenied

View File

@@ -1,7 +1,6 @@
from __future__ import unicode_literals from __future__ import unicode_literals
import os import os
import tempfile
from django.conf import settings from django.conf import settings
from django.utils.translation import ugettext_lazy as _ from django.utils.translation import ugettext_lazy as _
@@ -76,13 +75,6 @@ setting_shared_storage_arguments = namespace.add_setting(
global_name='COMMON_SHARED_STORAGE_ARGUMENTS', global_name='COMMON_SHARED_STORAGE_ARGUMENTS',
default={'location': os.path.join(settings.MEDIA_ROOT, 'shared_files')} default={'location': os.path.join(settings.MEDIA_ROOT, 'shared_files')}
) )
setting_temporary_directory = namespace.add_setting(
global_name='COMMON_TEMPORARY_DIRECTORY', default=tempfile.gettempdir(),
help_text=_(
'Temporary directory used site wide to store thumbnails, previews '
'and temporary files.'
)
)
namespace = Namespace(label=_('Django'), name='django') namespace = Namespace(label=_('Django'), name='django')

View File

@@ -1,7 +1,8 @@
from __future__ import unicode_literals from __future__ import unicode_literals
from mayan.apps.storage.utils import get_storage_subclass
from .settings import setting_shared_storage, setting_shared_storage_arguments from .settings import setting_shared_storage, setting_shared_storage_arguments
from .utils import get_storage_subclass
storage_sharedupload = get_storage_subclass( storage_sharedupload = get_storage_subclass(
dotted_path=setting_shared_storage.value dotted_path=setting_shared_storage.value

View File

@@ -13,7 +13,7 @@ from django.template import Context, Template
from django.test.utils import ContextList from django.test.utils import ContextList
from django.urls import clear_url_caches, reverse from django.urls import clear_url_caches, reverse
from ..settings import setting_temporary_directory from mayan.apps.storage.settings import setting_temporary_directory
from .literals import TEST_VIEW_NAME, TEST_VIEW_URL from .literals import TEST_VIEW_NAME, TEST_VIEW_URL
from .utils import mute_stdout from .utils import mute_stdout

View File

@@ -14,37 +14,33 @@ from .literals import TEST_ERROR_LOG_ENTRY_RESULT
class CommonViewTestCase(GenericViewTestCase): class CommonViewTestCase(GenericViewTestCase):
def test_about_view(self): def test_about_view(self):
self.login_user()
response = self.get('common:about_view') response = self.get('common:about_view')
self.assertContains(response, text='About', status_code=200) self.assertContains(response, text='About', status_code=200)
def _create_error_log_entry(self): def _create_error_log_entry(self):
ModelPermission.register( ModelPermission.register(
model=get_user_model(), permission=permission_error_log_view model=get_user_model(), permissions=(permission_error_log_view,)
) )
ErrorLogEntry.objects.register(model=get_user_model()) ErrorLogEntry.objects.register(model=get_user_model())
self.error_log_entry = self.user.error_logs.create( self.error_log_entry = self._test_case_user.error_logs.create(
result=TEST_ERROR_LOG_ENTRY_RESULT result=TEST_ERROR_LOG_ENTRY_RESULT
) )
def _request_object_error_log_list(self): def _request_object_error_log_list(self):
content_type = ContentType.objects.get_for_model(model=self.user) content_type = ContentType.objects.get_for_model(model=self._test_case_user)
return self.get( return self.get(
'common:object_error_list', kwargs={ 'common:object_error_list', kwargs={
'app_label': content_type.app_label, 'app_label': content_type.app_label,
'model': content_type.model, 'model': content_type.model,
'object_id': self.user.pk 'object_id': self._test_case_user.pk
}, follow=True }, follow=True
) )
def test_object_error_list_view_no_permissions(self): def test_object_error_list_view_no_permissions(self):
self._create_error_log_entry() self._create_error_log_entry()
self.login_user()
response = self._request_object_error_log_list() response = self._request_object_error_log_list()
self.assertNotContains( self.assertNotContains(
@@ -55,9 +51,8 @@ class CommonViewTestCase(GenericViewTestCase):
def test_object_error_list_view_with_access(self): def test_object_error_list_view_with_access(self):
self._create_error_log_entry() self._create_error_log_entry()
self.login_user()
self.grant_access( self.grant_access(
obj=self.user, permission=permission_error_log_view obj=self._test_case_user, permission=permission_error_log_view
) )
response = self._request_object_error_log_list() response = self._request_object_error_log_list()

View File

@@ -1,9 +1,6 @@
from __future__ import unicode_literals from __future__ import unicode_literals
import logging import logging
import os
import shutil
import tempfile
from django.conf import settings from django.conf import settings
from django.core.exceptions import FieldDoesNotExist from django.core.exceptions import FieldDoesNotExist
@@ -13,7 +10,6 @@ from django.urls.base import get_script_prefix
from django.utils.datastructures import MultiValueDict from django.utils.datastructures import MultiValueDict
from django.utils.http import urlencode as django_urlencode from django.utils.http import urlencode as django_urlencode
from django.utils.http import urlquote as django_urlquote from django.utils.http import urlquote as django_urlquote
from django.utils.module_loading import import_string
from django.utils.six.moves import reduce as reduce_function from django.utils.six.moves import reduce as reduce_function
from django.utils.six.moves import xmlrpc_client from django.utils.six.moves import xmlrpc_client
@@ -21,7 +17,6 @@ import mayan
from .exceptions import NotLatestVersion, UnknownLatestVersion from .exceptions import NotLatestVersion, UnknownLatestVersion
from .literals import DJANGO_SQLITE_BACKEND, MAYAN_PYPI_NAME, PYPI_URL from .literals import DJANGO_SQLITE_BACKEND, MAYAN_PYPI_NAME, PYPI_URL
from .settings import setting_temporary_directory
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@@ -40,27 +35,6 @@ def check_version():
raise NotLatestVersion(upstream_version=versions[0]) raise NotLatestVersion(upstream_version=versions[0])
# http://stackoverflow.com/questions/123198/how-do-i-copy-a-file-in-python
def copyfile(source, destination, buffer_size=1024 * 1024):
"""
Copy a file from source to dest. source and dest
can either be strings or any object with a read or
write method, like StringIO for example.
"""
source_descriptor = get_descriptor(source)
destination_descriptor = get_descriptor(destination, read=False)
while True:
copy_buffer = source_descriptor.read(buffer_size)
if copy_buffer:
destination_descriptor.write(copy_buffer)
else:
break
source_descriptor.close()
destination_descriptor.close()
def encapsulate(function): def encapsulate(function):
# Workaround Django ticket 15791 # Workaround Django ticket 15791
# Changeset 16045 # Changeset 16045
@@ -69,25 +43,6 @@ def encapsulate(function):
return lambda: function return lambda: function
def fs_cleanup(filename, file_descriptor=None, suppress_exceptions=True):
"""
Tries to remove the given filename. Ignores non-existent files
"""
if file_descriptor:
os.close(file_descriptor)
try:
os.remove(filename)
except OSError:
try:
shutil.rmtree(filename)
except OSError:
if suppress_exceptions:
pass
else:
raise
def get_related_field(model, related_field_name): def get_related_field(model, related_field_name):
try: try:
local_field_name, remaining_field_path = related_field_name.split( local_field_name, remaining_field_path = related_field_name.split(
@@ -108,41 +63,6 @@ def get_related_field(model, related_field_name):
return related_field return related_field
def get_descriptor(file_input, read=True):
try:
# Is it a file like object?
file_input.seek(0)
except AttributeError:
# If not, try open it.
if read:
return open(file_input, mode='rb')
else:
return open(file_input, mode='wb')
else:
return file_input
def get_storage_subclass(dotted_path):
"""
Import a storage class and return a subclass that will always return eq
True to avoid creating a new migration when for runtime storage class
changes.
"""
imported_storage_class = import_string(dotted_path=dotted_path)
class StorageSubclass(imported_storage_class):
def __init__(self, *args, **kwargs):
return super(StorageSubclass, self).__init__(*args, **kwargs)
def __eq__(self, other):
return True
def deconstruct(self):
return ('mayan.apps.common.classes.FakeStorageSubclass', (), {})
return StorageSubclass
def introspect_attribute(attribute_name, obj): def introspect_attribute(attribute_name, obj):
try: try:
# Try as a related field # Try as a related field
@@ -168,21 +88,6 @@ def introspect_attribute(attribute_name, obj):
return attribute_name, obj return attribute_name, obj
def TemporaryFile(*args, **kwargs):
kwargs.update({'dir': setting_temporary_directory.value})
return tempfile.TemporaryFile(*args, **kwargs)
def mkdtemp(*args, **kwargs):
kwargs.update({'dir': setting_temporary_directory.value})
return tempfile.mkdtemp(*args, **kwargs)
def mkstemp(*args, **kwargs):
kwargs.update({'dir': setting_temporary_directory.value})
return tempfile.mkstemp(*args, **kwargs)
def resolve(path, urlconf=None): def resolve(path, urlconf=None):
path = '/{}'.format(path.replace(get_script_prefix(), '', 1)) path = '/{}'.format(path.replace(get_script_prefix(), '', 1))
return django_resolve(path=path, urlconf=urlconf) return django_resolve(path=path, urlconf=urlconf)
@@ -275,24 +180,3 @@ def urlquote(link=None, get=None):
return '%s%s' % (link, django_urlencode(get, doseq=True)) return '%s%s' % (link, django_urlencode(get, doseq=True))
else: else:
return django_urlquote(link) return django_urlquote(link)
def validate_path(path):
if not os.path.exists(path):
# If doesn't exist try to create it
try:
os.mkdir(path)
except Exception as exception:
logger.debug('unhandled exception: %s', exception)
return False
# Check if it is writable
try:
fd, test_filepath = tempfile.mkstemp(dir=path)
os.close(fd)
os.unlink(test_filepath)
except Exception as exception:
logger.debug('unhandled exception: %s', exception)
return False
return True

View File

@@ -11,7 +11,7 @@ import sh
from django.utils.encoding import force_text from django.utils.encoding import force_text
from django.utils.translation import ugettext_lazy as _ from django.utils.translation import ugettext_lazy as _
from mayan.apps.common.utils import fs_cleanup, mkstemp from mayan.apps.storage.utils import fs_cleanup, mkstemp
from ..classes import ConverterBase from ..classes import ConverterBase
from ..exceptions import PageCountError from ..exceptions import PageCountError

View File

@@ -9,9 +9,9 @@ import sh
from django.utils.translation import ugettext_lazy as _ from django.utils.translation import ugettext_lazy as _
from mayan.apps.common.settings import setting_temporary_directory
from mayan.apps.common.utils import fs_cleanup, mkdtemp, mkstemp
from mayan.apps.mimetype.api import get_mimetype from mayan.apps.mimetype.api import get_mimetype
from mayan.apps.storage.settings import setting_temporary_directory
from mayan.apps.storage.utils import fs_cleanup, mkdtemp, mkstemp
from .exceptions import InvalidOfficeFormat, OfficeConversionError from .exceptions import InvalidOfficeFormat, OfficeConversionError
from .literals import ( from .literals import (

View File

@@ -15,7 +15,7 @@ from django.apps import apps
from django.utils.encoding import force_bytes, force_text from django.utils.encoding import force_bytes, force_text
from django.utils.functional import cached_property from django.utils.functional import cached_property
from mayan.apps.common.utils import mkdtemp from mayan.apps.storage.utils import mkdtemp
from .exceptions import DependenciesException from .exceptions import DependenciesException

View File

@@ -8,7 +8,7 @@ import gnupg
from django.utils.translation import ugettext_lazy as _ from django.utils.translation import ugettext_lazy as _
from mayan.apps.common.utils import mkdtemp from mayan.apps.storage.utils import mkdtemp
class GPGBackend(object): class GPGBackend(object):

View File

@@ -6,7 +6,7 @@ import os
from django.db import models from django.db import models
from mayan.apps.common.utils import mkstemp from mayan.apps.storage.utils import mkstemp
from .classes import KeyStub, SignatureVerification from .classes import KeyStub, SignatureVerification
from .exceptions import ( from .exceptions import (

View File

@@ -8,7 +8,7 @@ import mock
from django.utils.encoding import force_bytes from django.utils.encoding import force_bytes
from mayan.apps.common.tests import BaseTestCase from mayan.apps.common.tests import BaseTestCase
from mayan.apps.common.utils import TemporaryFile from mayan.apps.storage.utils import TemporaryFile
from ..exceptions import ( from ..exceptions import (
DecryptionError, KeyDoesNotExist, NeedPassphrase, PassphraseError, DecryptionError, KeyDoesNotExist, NeedPassphrase, PassphraseError,

View File

@@ -7,7 +7,7 @@ import subprocess
from django.apps import apps from django.apps import apps
from django.utils.translation import ugettext_lazy as _ from django.utils.translation import ugettext_lazy as _
from mayan.apps.common.utils import copyfile, fs_cleanup, mkstemp from mayan.apps.storage.utils import copyfile, fs_cleanup, mkstemp
from .exceptions import ParserError from .exceptions import ParserError
from .settings import setting_pdftotext_path from .settings import setting_pdftotext_path

View File

@@ -5,10 +5,10 @@ import os
from django.db import models from django.db import models
from mayan.apps.common.utils import mkstemp
from mayan.apps.django_gpg.exceptions import DecryptionError from mayan.apps.django_gpg.exceptions import DecryptionError
from mayan.apps.django_gpg.models import Key from mayan.apps.django_gpg.models import Key
from mayan.apps.documents.models import DocumentVersion from mayan.apps.documents.models import DocumentVersion
from mayan.apps.storage.utils import mkstemp
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)

View File

@@ -1,14 +1,13 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.11.16 on 2018-12-29 07:37
from __future__ import unicode_literals from __future__ import unicode_literals
from django.db import migrations, models from django.db import migrations, models
import mayan.apps.common.classes import mayan.apps.common.classes
import mayan.apps.document_signatures.models import mayan.apps.document_signatures.models
import mayan.apps.storage.classes
class Migration(migrations.Migration): class Migration(migrations.Migration):
dependencies = [ dependencies = [
('document_signatures', '0008_auto_20180429_0759'), ('document_signatures', '0008_auto_20180429_0759'),
] ]
@@ -17,6 +16,11 @@ class Migration(migrations.Migration):
migrations.AlterField( migrations.AlterField(
model_name='detachedsignature', model_name='detachedsignature',
name='signature_file', name='signature_file',
field=models.FileField(blank=True, null=True, storage=mayan.apps.common.classes.FakeStorageSubclass(), upload_to=mayan.apps.document_signatures.models.upload_to, verbose_name='Signature file'), field=models.FileField(
blank=True, null=True,
storage=mayan.apps.storage.classes.FakeStorageSubclass(),
upload_to=mayan.apps.document_signatures.models.upload_to,
verbose_name='Signature file'
),
), ),
] ]

View File

@@ -1,6 +1,6 @@
from __future__ import unicode_literals from __future__ import unicode_literals
from mayan.apps.common.utils import get_storage_subclass from mayan.apps.storage.utils import get_storage_subclass
from .settings import ( from .settings import (
setting_storage_backend, setting_storage_backend_arguments setting_storage_backend, setting_storage_backend_arguments

View File

@@ -15,9 +15,9 @@ from mayan.apps.common.generics import (
SingleObjectDetailView, SingleObjectDownloadView, SingleObjectListView SingleObjectDetailView, SingleObjectDownloadView, SingleObjectListView
) )
from mayan.apps.common.mixins import ExternalObjectMixin from mayan.apps.common.mixins import ExternalObjectMixin
from mayan.apps.common.utils import TemporaryFile
from mayan.apps.django_gpg.exceptions import NeedPassphrase, PassphraseError from mayan.apps.django_gpg.exceptions import NeedPassphrase, PassphraseError
from mayan.apps.documents.models import DocumentVersion from mayan.apps.documents.models import DocumentVersion
from mayan.apps.storage.utils import TemporaryFile
from .forms import ( from .forms import (
DocumentVersionSignatureCreateForm, DocumentVersionSignatureDetailForm DocumentVersionSignatureCreateForm, DocumentVersionSignatureDetailForm

View File

@@ -1,10 +1,10 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.11.16 on 2018-12-29 07:45
from __future__ import unicode_literals from __future__ import unicode_literals
from django.db import migrations, models from django.db import migrations, models
import mayan.apps.common.classes import mayan.apps.common.classes
import mayan.apps.documents.utils import mayan.apps.documents.utils
import mayan.apps.storage.classes
class Migration(migrations.Migration): class Migration(migrations.Migration):
@@ -17,6 +17,10 @@ class Migration(migrations.Migration):
migrations.AlterField( migrations.AlterField(
model_name='documentversion', model_name='documentversion',
name='file', name='file',
field=models.FileField(storage=mayan.apps.common.classes.FakeStorageSubclass(), upload_to=mayan.apps.documents.utils.document_uuid_function, verbose_name='File'), field=models.FileField(
storage=mayan.apps.storage.classes.FakeStorageSubclass(),
upload_to=mayan.apps.documents.utils.document_uuid_function,
verbose_name='File'
),
), ),
] ]

View File

@@ -2,7 +2,7 @@ from __future__ import unicode_literals
from django.utils.module_loading import import_string from django.utils.module_loading import import_string
from mayan.apps.common.utils import get_storage_subclass from mayan.apps.storage.utils import get_storage_subclass
from .settings import ( from .settings import (
setting_documentimagecache_storage, setting_documentimagecache_storage,

View File

@@ -7,7 +7,7 @@ import sh
from django.utils.translation import ugettext_lazy as _ from django.utils.translation import ugettext_lazy as _
from mayan.apps.common.utils import fs_cleanup, mkstemp from mayan.apps.storage.utils import fs_cleanup, mkstemp
from ..classes import FileMetadataDriver from ..classes import FileMetadataDriver
from ..settings import setting_drivers_arguments from ..settings import setting_drivers_arguments

View File

@@ -12,7 +12,7 @@ from django.conf import settings
from django.core.files import locks from django.core.files import locks
from django.utils.encoding import force_bytes, force_text from django.utils.encoding import force_bytes, force_text
from mayan.apps.common.settings import setting_temporary_directory from mayan.apps.storage.settings import setting_temporary_directory
from ..exceptions import LockError from ..exceptions import LockError
from ..settings import setting_default_lock_timeout from ..settings import setting_default_lock_timeout

View File

@@ -8,7 +8,7 @@ from django.conf import settings
from mayan.apps.common.settings import setting_paginate_by from mayan.apps.common.settings import setting_paginate_by
from mayan.apps.common.tests import BaseTestCase from mayan.apps.common.tests import BaseTestCase
from mayan.apps.common.utils import fs_cleanup, mkstemp from mayan.apps.storage.utils import fs_cleanup, mkstemp
from .literals import TEST_SETTING_NAME, TEST_SETTING_VALUE from .literals import TEST_SETTING_NAME, TEST_SETTING_VALUE

View File

@@ -7,7 +7,7 @@ from django.db import models
from django.utils.timezone import now from django.utils.timezone import now
from django.utils.translation import ugettext_lazy as _ from django.utils.translation import ugettext_lazy as _
from mayan.apps.common.utils import TemporaryFile from mayan.apps.storage.utils import TemporaryFile
from ..classes import PseudoFile, SourceUploadedFile from ..classes import PseudoFile, SourceUploadedFile
from ..exceptions import SourceException from ..exceptions import SourceException

View File

@@ -4,8 +4,8 @@ import os
import shutil import shutil
from mayan.apps.common.tests import BaseTestCase from mayan.apps.common.tests import BaseTestCase
from mayan.apps.common.utils import mkdtemp
from mayan.apps.documents.tests import TEST_NON_ASCII_DOCUMENT_PATH from mayan.apps.documents.tests import TEST_NON_ASCII_DOCUMENT_PATH
from mayan.apps.storage.utils import mkdtemp
from ..classes import StagingFile from ..classes import StagingFile

View File

@@ -9,7 +9,6 @@ from pathlib2 import Path
from django.utils.encoding import force_text from django.utils.encoding import force_text
from mayan.apps.common.tests import BaseTestCase from mayan.apps.common.tests import BaseTestCase
from mayan.apps.common.utils import mkdtemp
from mayan.apps.documents.models import Document, DocumentType from mayan.apps.documents.models import Document, DocumentType
from mayan.apps.documents.tests import ( from mayan.apps.documents.tests import (
TEST_COMPRESSED_DOCUMENT_PATH, TEST_DOCUMENT_TYPE_LABEL, TEST_COMPRESSED_DOCUMENT_PATH, TEST_DOCUMENT_TYPE_LABEL,
@@ -17,6 +16,7 @@ from mayan.apps.documents.tests import (
TEST_NON_ASCII_DOCUMENT_PATH, DocumentTestMixin TEST_NON_ASCII_DOCUMENT_PATH, DocumentTestMixin
) )
from mayan.apps.metadata.models import MetadataType from mayan.apps.metadata.models import MetadataType
from mayan.apps.storage.utils import mkdtemp
from ..literals import SOURCE_UNCOMPRESS_CHOICE_Y from ..literals import SOURCE_UNCOMPRESS_CHOICE_Y
from ..models import POP3Email, WatchFolderSource, WebFormSource from ..models import POP3Email, WatchFolderSource, WebFormSource

View File

@@ -6,13 +6,13 @@ import shutil
from mayan.apps.checkouts.models import NewVersionBlock from mayan.apps.checkouts.models import NewVersionBlock
from mayan.apps.common.tests import GenericViewTestCase from mayan.apps.common.tests import GenericViewTestCase
from mayan.apps.common.utils import fs_cleanup, mkdtemp
from mayan.apps.documents.models import Document from mayan.apps.documents.models import Document
from mayan.apps.documents.permissions import permission_document_create from mayan.apps.documents.permissions import permission_document_create
from mayan.apps.documents.tests import ( from mayan.apps.documents.tests import (
TEST_DOCUMENT_DESCRIPTION, TEST_SMALL_DOCUMENT_CHECKSUM, TEST_DOCUMENT_DESCRIPTION, TEST_SMALL_DOCUMENT_CHECKSUM,
TEST_SMALL_DOCUMENT_PATH, GenericDocumentViewTestCase TEST_SMALL_DOCUMENT_PATH, GenericDocumentViewTestCase
) )
from mayan.apps.storage.utils import fs_cleanup, mkdtemp
from ..links import link_upload_version from ..links import link_upload_version
from ..literals import SOURCE_CHOICE_WEB_FORM from ..literals import SOURCE_CHOICE_WEB_FORM

View File

@@ -0,0 +1,10 @@
from __future__ import unicode_literals
class FakeStorageSubclass(object):
"""
Placeholder class to allow serializing the real storage subclass to
support migrations.
"""
def __eq__(self, other):
return True

View File

@@ -0,0 +1,17 @@
from __future__ import unicode_literals
import tempfile
from django.utils.translation import ugettext_lazy as _
from mayan.apps.smart_settings import Namespace
namespace = Namespace(label=_('Storage'), name='storage')
setting_temporary_directory = namespace.add_setting(
global_name='STORAGE_TEMPORARY_DIRECTORY', default=tempfile.gettempdir(),
help_text=_(
'Temporary directory used site wide to store thumbnails, previews '
'and temporary files.'
)
)

123
mayan/apps/storage/utils.py Normal file
View File

@@ -0,0 +1,123 @@
from __future__ import unicode_literals
import logging
import os
import shutil
import tempfile
from django.utils.module_loading import import_string
from .settings import setting_temporary_directory
logger = logging.getLogger(__name__)
def TemporaryFile(*args, **kwargs):
kwargs.update({'dir': setting_temporary_directory.value})
return tempfile.TemporaryFile(*args, **kwargs)
# http://stackoverflow.com/questions/123198/how-do-i-copy-a-file-in-python
def copyfile(source, destination, buffer_size=1024 * 1024):
"""
Copy a file from source to dest. source and dest
can either be strings or any object with a read or
write method, like StringIO for example.
"""
source_descriptor = get_descriptor(source)
destination_descriptor = get_descriptor(destination, read=False)
while True:
copy_buffer = source_descriptor.read(buffer_size)
if copy_buffer:
destination_descriptor.write(copy_buffer)
else:
break
source_descriptor.close()
destination_descriptor.close()
def fs_cleanup(filename, file_descriptor=None, suppress_exceptions=True):
"""
Tries to remove the given filename. Ignores non-existent files
"""
if file_descriptor:
os.close(file_descriptor)
try:
os.remove(filename)
except OSError:
try:
shutil.rmtree(filename)
except OSError:
if suppress_exceptions:
pass
else:
raise
def get_descriptor(file_input, read=True):
try:
# Is it a file like object?
file_input.seek(0)
except AttributeError:
# If not, try open it.
if read:
return open(file_input, mode='rb')
else:
return open(file_input, mode='wb')
else:
return file_input
def get_storage_subclass(dotted_path):
"""
Import a storage class and return a subclass that will always return eq
True to avoid creating a new migration when for runtime storage class
changes.
"""
imported_storage_class = import_string(dotted_path=dotted_path)
class StorageSubclass(imported_storage_class):
def __init__(self, *args, **kwargs):
return super(StorageSubclass, self).__init__(*args, **kwargs)
def __eq__(self, other):
return True
def deconstruct(self):
return ('mayan.apps.storage.classes.FakeStorageSubclass', (), {})
return StorageSubclass
def mkdtemp(*args, **kwargs):
kwargs.update({'dir': setting_temporary_directory.value})
return tempfile.mkdtemp(*args, **kwargs)
def mkstemp(*args, **kwargs):
kwargs.update({'dir': setting_temporary_directory.value})
return tempfile.mkstemp(*args, **kwargs)
def validate_path(path):
if not os.path.exists(path):
# If doesn't exist try to create it
try:
os.mkdir(path)
except Exception as exception:
logger.debug('unhandled exception: %s', exception)
return False
# Check if it is writable
try:
fd, test_filepath = tempfile.mkstemp(dir=path)
os.close(fd)
os.unlink(test_filepath)
except Exception as exception:
logger.debug('unhandled exception: %s', exception)
return False
return True