From 8e66eefe7ce1c36bdd60750c50b3599d909bb3ae Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Thu, 31 Jan 2019 22:26:07 -0400 Subject: [PATCH] Move file and storage code to the storage app The setting COMMON_TEMPORARY_DIRECTORY is now STORAGE_TEMPORARY_DIRECTORY. Signed-off-by: Roberto Rosario --- HISTORY.rst | 2 + mayan/apps/common/classes.py | 10 -- .../common/management/commands/convertdb.py | 2 +- .../migrations/0011_auto_20181229_0738.py | 11 +- mayan/apps/common/mixins.py | 1 - mayan/apps/common/settings.py | 8 -- mayan/apps/common/storages.py | 3 +- mayan/apps/common/tests/mixins.py | 2 +- mayan/apps/common/tests/test_views.py | 15 +-- mayan/apps/common/utils.py | 116 ----------------- mayan/apps/converter/backends/python.py | 2 +- mayan/apps/converter/classes.py | 4 +- mayan/apps/dependencies/javascript.py | 2 +- mayan/apps/django_gpg/classes.py | 2 +- mayan/apps/django_gpg/managers.py | 2 +- mayan/apps/django_gpg/tests/test_models.py | 2 +- mayan/apps/document_parsing/parsers.py | 2 +- mayan/apps/document_signatures/managers.py | 2 +- .../migrations/0009_auto_20181229_0737.py | 12 +- mayan/apps/document_signatures/storages.py | 2 +- mayan/apps/document_signatures/views.py | 2 +- .../migrations/0051_auto_20181229_0745.py | 10 +- mayan/apps/documents/storages.py | 2 +- mayan/apps/file_metadata/drivers/exiftool.py | 2 +- mayan/apps/lock_manager/backends/file_lock.py | 2 +- .../apps/smart_settings/tests/test_classes.py | 2 +- mayan/apps/sources/models/scanner_sources.py | 2 +- mayan/apps/sources/tests/test_classes.py | 2 +- mayan/apps/sources/tests/test_models.py | 2 +- mayan/apps/sources/tests/test_views.py | 2 +- mayan/apps/storage/classes.py | 10 ++ mayan/apps/storage/settings.py | 17 +++ mayan/apps/storage/utils.py | 123 ++++++++++++++++++ 33 files changed, 202 insertions(+), 178 deletions(-) create mode 100644 mayan/apps/storage/classes.py create mode 100644 mayan/apps/storage/settings.py create mode 100644 mayan/apps/storage/utils.py diff --git a/HISTORY.rst b/HISTORY.rst index e26036dd7e..4106ec19a0 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -227,6 +227,8 @@ - Add a test mixin to generate random model primary keys. - Add support for checkout and check in multiple documents at the same time. +- Move file and storage code to the storage app. The setting + COMMON_TEMPORARY_DIRECTORY is now STORAGE_TEMPORARY_DIRECTORY. 3.1.9 (2018-11-01) ================== diff --git a/mayan/apps/common/classes.py b/mayan/apps/common/classes.py index f8958a304e..161def4408 100644 --- a/mayan/apps/common/classes.py +++ b/mayan/apps/common/classes.py @@ -72,16 +72,6 @@ class ErrorLogNamespace(object): return ErrorLogEntry.objects.filter(namespace=self.name) -class FakeStorageSubclass(object): - """ - Placeholder class to allow serializing the real storage subclass to - support migrations. - """ - - def __eq__(self, other): - return True - - class MissingItem(object): _registry = [] diff --git a/mayan/apps/common/management/commands/convertdb.py b/mayan/apps/common/management/commands/convertdb.py index bd822aba4e..3d4fe0c063 100644 --- a/mayan/apps/common/management/commands/convertdb.py +++ b/mayan/apps/common/management/commands/convertdb.py @@ -11,8 +11,8 @@ from django.core.management.base import CommandError from django.utils.encoding import force_text from django.utils.translation import ugettext_lazy as _ -from mayan.apps.common.utils import fs_cleanup from mayan.apps.documents.models import DocumentType +from mayan.apps.storage.utils import fs_cleanup CONVERTDB_FOLDER = 'convertdb' CONVERTDB_OUTPUT_FILENAME = 'migrate.json' diff --git a/mayan/apps/common/migrations/0011_auto_20181229_0738.py b/mayan/apps/common/migrations/0011_auto_20181229_0738.py index b1796669a1..b9668e2b92 100644 --- a/mayan/apps/common/migrations/0011_auto_20181229_0738.py +++ b/mayan/apps/common/migrations/0011_auto_20181229_0738.py @@ -1,14 +1,13 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.16 on 2018-12-29 07:38 from __future__ import unicode_literals from django.db import migrations, models + import mayan.apps.common.classes import mayan.apps.common.models +import mayan.apps.storage.classes class Migration(migrations.Migration): - dependencies = [ ('common', '0010_auto_20180403_0702_squashed_0011_auto_20180429_0758'), ] @@ -17,6 +16,10 @@ class Migration(migrations.Migration): migrations.AlterField( model_name='shareduploadedfile', name='file', - field=models.FileField(storage=mayan.apps.common.classes.FakeStorageSubclass(), upload_to=mayan.apps.common.models.upload_to, verbose_name='File'), + field=models.FileField( + storage=mayan.apps.storage.classes.FakeStorageSubclass(), + upload_to=mayan.apps.common.models.upload_to, + verbose_name='File' + ), ), ] diff --git a/mayan/apps/common/mixins.py b/mayan/apps/common/mixins.py index 5eaa001d2b..e2592217e6 100644 --- a/mayan/apps/common/mixins.py +++ b/mayan/apps/common/mixins.py @@ -1,6 +1,5 @@ from __future__ import unicode_literals -from django.conf import settings from django.contrib import messages from django.contrib.contenttypes.models import ContentType from django.core.exceptions import ImproperlyConfigured, PermissionDenied diff --git a/mayan/apps/common/settings.py b/mayan/apps/common/settings.py index 2187fd1db5..1cdae268b7 100644 --- a/mayan/apps/common/settings.py +++ b/mayan/apps/common/settings.py @@ -1,7 +1,6 @@ from __future__ import unicode_literals import os -import tempfile from django.conf import settings from django.utils.translation import ugettext_lazy as _ @@ -76,13 +75,6 @@ setting_shared_storage_arguments = namespace.add_setting( global_name='COMMON_SHARED_STORAGE_ARGUMENTS', default={'location': os.path.join(settings.MEDIA_ROOT, 'shared_files')} ) -setting_temporary_directory = namespace.add_setting( - global_name='COMMON_TEMPORARY_DIRECTORY', default=tempfile.gettempdir(), - help_text=_( - 'Temporary directory used site wide to store thumbnails, previews ' - 'and temporary files.' - ) -) namespace = Namespace(label=_('Django'), name='django') diff --git a/mayan/apps/common/storages.py b/mayan/apps/common/storages.py index 3140cfad49..9af5517b65 100644 --- a/mayan/apps/common/storages.py +++ b/mayan/apps/common/storages.py @@ -1,7 +1,8 @@ from __future__ import unicode_literals +from mayan.apps.storage.utils import get_storage_subclass + from .settings import setting_shared_storage, setting_shared_storage_arguments -from .utils import get_storage_subclass storage_sharedupload = get_storage_subclass( dotted_path=setting_shared_storage.value diff --git a/mayan/apps/common/tests/mixins.py b/mayan/apps/common/tests/mixins.py index 565a25f569..1e70354454 100644 --- a/mayan/apps/common/tests/mixins.py +++ b/mayan/apps/common/tests/mixins.py @@ -13,7 +13,7 @@ from django.template import Context, Template from django.test.utils import ContextList from django.urls import clear_url_caches, reverse -from ..settings import setting_temporary_directory +from mayan.apps.storage.settings import setting_temporary_directory from .literals import TEST_VIEW_NAME, TEST_VIEW_URL from .utils import mute_stdout diff --git a/mayan/apps/common/tests/test_views.py b/mayan/apps/common/tests/test_views.py index 865c7d782e..b6a7bf35ac 100644 --- a/mayan/apps/common/tests/test_views.py +++ b/mayan/apps/common/tests/test_views.py @@ -14,37 +14,33 @@ from .literals import TEST_ERROR_LOG_ENTRY_RESULT class CommonViewTestCase(GenericViewTestCase): def test_about_view(self): - self.login_user() - response = self.get('common:about_view') self.assertContains(response, text='About', status_code=200) def _create_error_log_entry(self): ModelPermission.register( - model=get_user_model(), permission=permission_error_log_view + model=get_user_model(), permissions=(permission_error_log_view,) ) ErrorLogEntry.objects.register(model=get_user_model()) - self.error_log_entry = self.user.error_logs.create( + self.error_log_entry = self._test_case_user.error_logs.create( result=TEST_ERROR_LOG_ENTRY_RESULT ) def _request_object_error_log_list(self): - content_type = ContentType.objects.get_for_model(model=self.user) + content_type = ContentType.objects.get_for_model(model=self._test_case_user) return self.get( 'common:object_error_list', kwargs={ 'app_label': content_type.app_label, 'model': content_type.model, - 'object_id': self.user.pk + 'object_id': self._test_case_user.pk }, follow=True ) def test_object_error_list_view_no_permissions(self): self._create_error_log_entry() - self.login_user() - response = self._request_object_error_log_list() self.assertNotContains( @@ -55,9 +51,8 @@ class CommonViewTestCase(GenericViewTestCase): def test_object_error_list_view_with_access(self): self._create_error_log_entry() - self.login_user() self.grant_access( - obj=self.user, permission=permission_error_log_view + obj=self._test_case_user, permission=permission_error_log_view ) response = self._request_object_error_log_list() diff --git a/mayan/apps/common/utils.py b/mayan/apps/common/utils.py index fb3df43336..ba51d4d455 100644 --- a/mayan/apps/common/utils.py +++ b/mayan/apps/common/utils.py @@ -1,9 +1,6 @@ from __future__ import unicode_literals import logging -import os -import shutil -import tempfile from django.conf import settings from django.core.exceptions import FieldDoesNotExist @@ -13,7 +10,6 @@ from django.urls.base import get_script_prefix from django.utils.datastructures import MultiValueDict from django.utils.http import urlencode as django_urlencode from django.utils.http import urlquote as django_urlquote -from django.utils.module_loading import import_string from django.utils.six.moves import reduce as reduce_function from django.utils.six.moves import xmlrpc_client @@ -21,7 +17,6 @@ import mayan from .exceptions import NotLatestVersion, UnknownLatestVersion from .literals import DJANGO_SQLITE_BACKEND, MAYAN_PYPI_NAME, PYPI_URL -from .settings import setting_temporary_directory logger = logging.getLogger(__name__) @@ -40,27 +35,6 @@ def check_version(): raise NotLatestVersion(upstream_version=versions[0]) -# http://stackoverflow.com/questions/123198/how-do-i-copy-a-file-in-python -def copyfile(source, destination, buffer_size=1024 * 1024): - """ - Copy a file from source to dest. source and dest - can either be strings or any object with a read or - write method, like StringIO for example. - """ - source_descriptor = get_descriptor(source) - destination_descriptor = get_descriptor(destination, read=False) - - while True: - copy_buffer = source_descriptor.read(buffer_size) - if copy_buffer: - destination_descriptor.write(copy_buffer) - else: - break - - source_descriptor.close() - destination_descriptor.close() - - def encapsulate(function): # Workaround Django ticket 15791 # Changeset 16045 @@ -69,25 +43,6 @@ def encapsulate(function): return lambda: function -def fs_cleanup(filename, file_descriptor=None, suppress_exceptions=True): - """ - Tries to remove the given filename. Ignores non-existent files - """ - if file_descriptor: - os.close(file_descriptor) - - try: - os.remove(filename) - except OSError: - try: - shutil.rmtree(filename) - except OSError: - if suppress_exceptions: - pass - else: - raise - - def get_related_field(model, related_field_name): try: local_field_name, remaining_field_path = related_field_name.split( @@ -108,41 +63,6 @@ def get_related_field(model, related_field_name): return related_field -def get_descriptor(file_input, read=True): - try: - # Is it a file like object? - file_input.seek(0) - except AttributeError: - # If not, try open it. - if read: - return open(file_input, mode='rb') - else: - return open(file_input, mode='wb') - else: - return file_input - - -def get_storage_subclass(dotted_path): - """ - Import a storage class and return a subclass that will always return eq - True to avoid creating a new migration when for runtime storage class - changes. - """ - imported_storage_class = import_string(dotted_path=dotted_path) - - class StorageSubclass(imported_storage_class): - def __init__(self, *args, **kwargs): - return super(StorageSubclass, self).__init__(*args, **kwargs) - - def __eq__(self, other): - return True - - def deconstruct(self): - return ('mayan.apps.common.classes.FakeStorageSubclass', (), {}) - - return StorageSubclass - - def introspect_attribute(attribute_name, obj): try: # Try as a related field @@ -168,21 +88,6 @@ def introspect_attribute(attribute_name, obj): return attribute_name, obj -def TemporaryFile(*args, **kwargs): - kwargs.update({'dir': setting_temporary_directory.value}) - return tempfile.TemporaryFile(*args, **kwargs) - - -def mkdtemp(*args, **kwargs): - kwargs.update({'dir': setting_temporary_directory.value}) - return tempfile.mkdtemp(*args, **kwargs) - - -def mkstemp(*args, **kwargs): - kwargs.update({'dir': setting_temporary_directory.value}) - return tempfile.mkstemp(*args, **kwargs) - - def resolve(path, urlconf=None): path = '/{}'.format(path.replace(get_script_prefix(), '', 1)) return django_resolve(path=path, urlconf=urlconf) @@ -275,24 +180,3 @@ def urlquote(link=None, get=None): return '%s%s' % (link, django_urlencode(get, doseq=True)) else: return django_urlquote(link) - - -def validate_path(path): - if not os.path.exists(path): - # If doesn't exist try to create it - try: - os.mkdir(path) - except Exception as exception: - logger.debug('unhandled exception: %s', exception) - return False - - # Check if it is writable - try: - fd, test_filepath = tempfile.mkstemp(dir=path) - os.close(fd) - os.unlink(test_filepath) - except Exception as exception: - logger.debug('unhandled exception: %s', exception) - return False - - return True diff --git a/mayan/apps/converter/backends/python.py b/mayan/apps/converter/backends/python.py index fd6243a48a..5027c54123 100644 --- a/mayan/apps/converter/backends/python.py +++ b/mayan/apps/converter/backends/python.py @@ -11,7 +11,7 @@ import sh from django.utils.encoding import force_text from django.utils.translation import ugettext_lazy as _ -from mayan.apps.common.utils import fs_cleanup, mkstemp +from mayan.apps.storage.utils import fs_cleanup, mkstemp from ..classes import ConverterBase from ..exceptions import PageCountError diff --git a/mayan/apps/converter/classes.py b/mayan/apps/converter/classes.py index d6b7b7a35f..2a19212296 100644 --- a/mayan/apps/converter/classes.py +++ b/mayan/apps/converter/classes.py @@ -9,9 +9,9 @@ import sh from django.utils.translation import ugettext_lazy as _ -from mayan.apps.common.settings import setting_temporary_directory -from mayan.apps.common.utils import fs_cleanup, mkdtemp, mkstemp from mayan.apps.mimetype.api import get_mimetype +from mayan.apps.storage.settings import setting_temporary_directory +from mayan.apps.storage.utils import fs_cleanup, mkdtemp, mkstemp from .exceptions import InvalidOfficeFormat, OfficeConversionError from .literals import ( diff --git a/mayan/apps/dependencies/javascript.py b/mayan/apps/dependencies/javascript.py index 3a879c9ae4..d104e6f361 100644 --- a/mayan/apps/dependencies/javascript.py +++ b/mayan/apps/dependencies/javascript.py @@ -15,7 +15,7 @@ from django.apps import apps from django.utils.encoding import force_bytes, force_text from django.utils.functional import cached_property -from mayan.apps.common.utils import mkdtemp +from mayan.apps.storage.utils import mkdtemp from .exceptions import DependenciesException diff --git a/mayan/apps/django_gpg/classes.py b/mayan/apps/django_gpg/classes.py index 7243b63744..840952eb95 100644 --- a/mayan/apps/django_gpg/classes.py +++ b/mayan/apps/django_gpg/classes.py @@ -8,7 +8,7 @@ import gnupg from django.utils.translation import ugettext_lazy as _ -from mayan.apps.common.utils import mkdtemp +from mayan.apps.storage.utils import mkdtemp class GPGBackend(object): diff --git a/mayan/apps/django_gpg/managers.py b/mayan/apps/django_gpg/managers.py index e6b498f32c..d565e8ae2a 100644 --- a/mayan/apps/django_gpg/managers.py +++ b/mayan/apps/django_gpg/managers.py @@ -6,7 +6,7 @@ import os from django.db import models -from mayan.apps.common.utils import mkstemp +from mayan.apps.storage.utils import mkstemp from .classes import KeyStub, SignatureVerification from .exceptions import ( diff --git a/mayan/apps/django_gpg/tests/test_models.py b/mayan/apps/django_gpg/tests/test_models.py index 9273cfd065..51c94e7a85 100644 --- a/mayan/apps/django_gpg/tests/test_models.py +++ b/mayan/apps/django_gpg/tests/test_models.py @@ -8,7 +8,7 @@ import mock from django.utils.encoding import force_bytes from mayan.apps.common.tests import BaseTestCase -from mayan.apps.common.utils import TemporaryFile +from mayan.apps.storage.utils import TemporaryFile from ..exceptions import ( DecryptionError, KeyDoesNotExist, NeedPassphrase, PassphraseError, diff --git a/mayan/apps/document_parsing/parsers.py b/mayan/apps/document_parsing/parsers.py index fd70926255..ea6ed8dc60 100644 --- a/mayan/apps/document_parsing/parsers.py +++ b/mayan/apps/document_parsing/parsers.py @@ -7,7 +7,7 @@ import subprocess from django.apps import apps from django.utils.translation import ugettext_lazy as _ -from mayan.apps.common.utils import copyfile, fs_cleanup, mkstemp +from mayan.apps.storage.utils import copyfile, fs_cleanup, mkstemp from .exceptions import ParserError from .settings import setting_pdftotext_path diff --git a/mayan/apps/document_signatures/managers.py b/mayan/apps/document_signatures/managers.py index 8f0eb8f161..3c58e69abe 100644 --- a/mayan/apps/document_signatures/managers.py +++ b/mayan/apps/document_signatures/managers.py @@ -5,10 +5,10 @@ import os from django.db import models -from mayan.apps.common.utils import mkstemp from mayan.apps.django_gpg.exceptions import DecryptionError from mayan.apps.django_gpg.models import Key from mayan.apps.documents.models import DocumentVersion +from mayan.apps.storage.utils import mkstemp logger = logging.getLogger(__name__) diff --git a/mayan/apps/document_signatures/migrations/0009_auto_20181229_0737.py b/mayan/apps/document_signatures/migrations/0009_auto_20181229_0737.py index 9f3d388e6b..e354d69d8a 100644 --- a/mayan/apps/document_signatures/migrations/0009_auto_20181229_0737.py +++ b/mayan/apps/document_signatures/migrations/0009_auto_20181229_0737.py @@ -1,14 +1,13 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.16 on 2018-12-29 07:37 from __future__ import unicode_literals from django.db import migrations, models + import mayan.apps.common.classes import mayan.apps.document_signatures.models +import mayan.apps.storage.classes class Migration(migrations.Migration): - dependencies = [ ('document_signatures', '0008_auto_20180429_0759'), ] @@ -17,6 +16,11 @@ class Migration(migrations.Migration): migrations.AlterField( model_name='detachedsignature', name='signature_file', - field=models.FileField(blank=True, null=True, storage=mayan.apps.common.classes.FakeStorageSubclass(), upload_to=mayan.apps.document_signatures.models.upload_to, verbose_name='Signature file'), + field=models.FileField( + blank=True, null=True, + storage=mayan.apps.storage.classes.FakeStorageSubclass(), + upload_to=mayan.apps.document_signatures.models.upload_to, + verbose_name='Signature file' + ), ), ] diff --git a/mayan/apps/document_signatures/storages.py b/mayan/apps/document_signatures/storages.py index 1b88c818ca..423f11c1e5 100644 --- a/mayan/apps/document_signatures/storages.py +++ b/mayan/apps/document_signatures/storages.py @@ -1,6 +1,6 @@ from __future__ import unicode_literals -from mayan.apps.common.utils import get_storage_subclass +from mayan.apps.storage.utils import get_storage_subclass from .settings import ( setting_storage_backend, setting_storage_backend_arguments diff --git a/mayan/apps/document_signatures/views.py b/mayan/apps/document_signatures/views.py index 571343db07..b723cbc304 100644 --- a/mayan/apps/document_signatures/views.py +++ b/mayan/apps/document_signatures/views.py @@ -15,9 +15,9 @@ from mayan.apps.common.generics import ( SingleObjectDetailView, SingleObjectDownloadView, SingleObjectListView ) from mayan.apps.common.mixins import ExternalObjectMixin -from mayan.apps.common.utils import TemporaryFile from mayan.apps.django_gpg.exceptions import NeedPassphrase, PassphraseError from mayan.apps.documents.models import DocumentVersion +from mayan.apps.storage.utils import TemporaryFile from .forms import ( DocumentVersionSignatureCreateForm, DocumentVersionSignatureDetailForm diff --git a/mayan/apps/documents/migrations/0051_auto_20181229_0745.py b/mayan/apps/documents/migrations/0051_auto_20181229_0745.py index d95527ab42..e423157903 100644 --- a/mayan/apps/documents/migrations/0051_auto_20181229_0745.py +++ b/mayan/apps/documents/migrations/0051_auto_20181229_0745.py @@ -1,10 +1,10 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.16 on 2018-12-29 07:45 from __future__ import unicode_literals from django.db import migrations, models + import mayan.apps.common.classes import mayan.apps.documents.utils +import mayan.apps.storage.classes class Migration(migrations.Migration): @@ -17,6 +17,10 @@ class Migration(migrations.Migration): migrations.AlterField( model_name='documentversion', name='file', - field=models.FileField(storage=mayan.apps.common.classes.FakeStorageSubclass(), upload_to=mayan.apps.documents.utils.document_uuid_function, verbose_name='File'), + field=models.FileField( + storage=mayan.apps.storage.classes.FakeStorageSubclass(), + upload_to=mayan.apps.documents.utils.document_uuid_function, + verbose_name='File' + ), ), ] diff --git a/mayan/apps/documents/storages.py b/mayan/apps/documents/storages.py index a71ba1c097..4bfb1ec4f1 100644 --- a/mayan/apps/documents/storages.py +++ b/mayan/apps/documents/storages.py @@ -2,7 +2,7 @@ from __future__ import unicode_literals from django.utils.module_loading import import_string -from mayan.apps.common.utils import get_storage_subclass +from mayan.apps.storage.utils import get_storage_subclass from .settings import ( setting_documentimagecache_storage, diff --git a/mayan/apps/file_metadata/drivers/exiftool.py b/mayan/apps/file_metadata/drivers/exiftool.py index 8a75b7c0fe..9b60e0c1cc 100644 --- a/mayan/apps/file_metadata/drivers/exiftool.py +++ b/mayan/apps/file_metadata/drivers/exiftool.py @@ -7,7 +7,7 @@ import sh from django.utils.translation import ugettext_lazy as _ -from mayan.apps.common.utils import fs_cleanup, mkstemp +from mayan.apps.storage.utils import fs_cleanup, mkstemp from ..classes import FileMetadataDriver from ..settings import setting_drivers_arguments diff --git a/mayan/apps/lock_manager/backends/file_lock.py b/mayan/apps/lock_manager/backends/file_lock.py index 459d91e044..43b6a743d7 100644 --- a/mayan/apps/lock_manager/backends/file_lock.py +++ b/mayan/apps/lock_manager/backends/file_lock.py @@ -12,7 +12,7 @@ from django.conf import settings from django.core.files import locks from django.utils.encoding import force_bytes, force_text -from mayan.apps.common.settings import setting_temporary_directory +from mayan.apps.storage.settings import setting_temporary_directory from ..exceptions import LockError from ..settings import setting_default_lock_timeout diff --git a/mayan/apps/smart_settings/tests/test_classes.py b/mayan/apps/smart_settings/tests/test_classes.py index 55c595102a..6be2ae1df0 100644 --- a/mayan/apps/smart_settings/tests/test_classes.py +++ b/mayan/apps/smart_settings/tests/test_classes.py @@ -8,7 +8,7 @@ from django.conf import settings from mayan.apps.common.settings import setting_paginate_by from mayan.apps.common.tests import BaseTestCase -from mayan.apps.common.utils import fs_cleanup, mkstemp +from mayan.apps.storage.utils import fs_cleanup, mkstemp from .literals import TEST_SETTING_NAME, TEST_SETTING_VALUE diff --git a/mayan/apps/sources/models/scanner_sources.py b/mayan/apps/sources/models/scanner_sources.py index 099d6ca913..b4ca0bbc63 100644 --- a/mayan/apps/sources/models/scanner_sources.py +++ b/mayan/apps/sources/models/scanner_sources.py @@ -7,7 +7,7 @@ from django.db import models from django.utils.timezone import now from django.utils.translation import ugettext_lazy as _ -from mayan.apps.common.utils import TemporaryFile +from mayan.apps.storage.utils import TemporaryFile from ..classes import PseudoFile, SourceUploadedFile from ..exceptions import SourceException diff --git a/mayan/apps/sources/tests/test_classes.py b/mayan/apps/sources/tests/test_classes.py index 7466b33fad..319c3eeb37 100644 --- a/mayan/apps/sources/tests/test_classes.py +++ b/mayan/apps/sources/tests/test_classes.py @@ -4,8 +4,8 @@ import os import shutil from mayan.apps.common.tests import BaseTestCase -from mayan.apps.common.utils import mkdtemp from mayan.apps.documents.tests import TEST_NON_ASCII_DOCUMENT_PATH +from mayan.apps.storage.utils import mkdtemp from ..classes import StagingFile diff --git a/mayan/apps/sources/tests/test_models.py b/mayan/apps/sources/tests/test_models.py index 97794ecb19..9947edc461 100644 --- a/mayan/apps/sources/tests/test_models.py +++ b/mayan/apps/sources/tests/test_models.py @@ -9,7 +9,6 @@ from pathlib2 import Path from django.utils.encoding import force_text from mayan.apps.common.tests import BaseTestCase -from mayan.apps.common.utils import mkdtemp from mayan.apps.documents.models import Document, DocumentType from mayan.apps.documents.tests import ( TEST_COMPRESSED_DOCUMENT_PATH, TEST_DOCUMENT_TYPE_LABEL, @@ -17,6 +16,7 @@ from mayan.apps.documents.tests import ( TEST_NON_ASCII_DOCUMENT_PATH, DocumentTestMixin ) from mayan.apps.metadata.models import MetadataType +from mayan.apps.storage.utils import mkdtemp from ..literals import SOURCE_UNCOMPRESS_CHOICE_Y from ..models import POP3Email, WatchFolderSource, WebFormSource diff --git a/mayan/apps/sources/tests/test_views.py b/mayan/apps/sources/tests/test_views.py index 7d02866212..5778283ca6 100644 --- a/mayan/apps/sources/tests/test_views.py +++ b/mayan/apps/sources/tests/test_views.py @@ -6,13 +6,13 @@ import shutil from mayan.apps.checkouts.models import NewVersionBlock from mayan.apps.common.tests import GenericViewTestCase -from mayan.apps.common.utils import fs_cleanup, mkdtemp from mayan.apps.documents.models import Document from mayan.apps.documents.permissions import permission_document_create from mayan.apps.documents.tests import ( TEST_DOCUMENT_DESCRIPTION, TEST_SMALL_DOCUMENT_CHECKSUM, TEST_SMALL_DOCUMENT_PATH, GenericDocumentViewTestCase ) +from mayan.apps.storage.utils import fs_cleanup, mkdtemp from ..links import link_upload_version from ..literals import SOURCE_CHOICE_WEB_FORM diff --git a/mayan/apps/storage/classes.py b/mayan/apps/storage/classes.py new file mode 100644 index 0000000000..0c4bdd2cd5 --- /dev/null +++ b/mayan/apps/storage/classes.py @@ -0,0 +1,10 @@ +from __future__ import unicode_literals + + +class FakeStorageSubclass(object): + """ + Placeholder class to allow serializing the real storage subclass to + support migrations. + """ + def __eq__(self, other): + return True diff --git a/mayan/apps/storage/settings.py b/mayan/apps/storage/settings.py new file mode 100644 index 0000000000..e17acbd76c --- /dev/null +++ b/mayan/apps/storage/settings.py @@ -0,0 +1,17 @@ +from __future__ import unicode_literals + +import tempfile + +from django.utils.translation import ugettext_lazy as _ + +from mayan.apps.smart_settings import Namespace + +namespace = Namespace(label=_('Storage'), name='storage') + +setting_temporary_directory = namespace.add_setting( + global_name='STORAGE_TEMPORARY_DIRECTORY', default=tempfile.gettempdir(), + help_text=_( + 'Temporary directory used site wide to store thumbnails, previews ' + 'and temporary files.' + ) +) diff --git a/mayan/apps/storage/utils.py b/mayan/apps/storage/utils.py new file mode 100644 index 0000000000..09ba4f6c22 --- /dev/null +++ b/mayan/apps/storage/utils.py @@ -0,0 +1,123 @@ +from __future__ import unicode_literals + +import logging +import os +import shutil +import tempfile + +from django.utils.module_loading import import_string + +from .settings import setting_temporary_directory + +logger = logging.getLogger(__name__) + + +def TemporaryFile(*args, **kwargs): + kwargs.update({'dir': setting_temporary_directory.value}) + return tempfile.TemporaryFile(*args, **kwargs) + + +# http://stackoverflow.com/questions/123198/how-do-i-copy-a-file-in-python +def copyfile(source, destination, buffer_size=1024 * 1024): + """ + Copy a file from source to dest. source and dest + can either be strings or any object with a read or + write method, like StringIO for example. + """ + source_descriptor = get_descriptor(source) + destination_descriptor = get_descriptor(destination, read=False) + + while True: + copy_buffer = source_descriptor.read(buffer_size) + if copy_buffer: + destination_descriptor.write(copy_buffer) + else: + break + + source_descriptor.close() + destination_descriptor.close() + + +def fs_cleanup(filename, file_descriptor=None, suppress_exceptions=True): + """ + Tries to remove the given filename. Ignores non-existent files + """ + if file_descriptor: + os.close(file_descriptor) + + try: + os.remove(filename) + except OSError: + try: + shutil.rmtree(filename) + except OSError: + if suppress_exceptions: + pass + else: + raise + + +def get_descriptor(file_input, read=True): + try: + # Is it a file like object? + file_input.seek(0) + except AttributeError: + # If not, try open it. + if read: + return open(file_input, mode='rb') + else: + return open(file_input, mode='wb') + else: + return file_input + + +def get_storage_subclass(dotted_path): + """ + Import a storage class and return a subclass that will always return eq + True to avoid creating a new migration when for runtime storage class + changes. + """ + imported_storage_class = import_string(dotted_path=dotted_path) + + class StorageSubclass(imported_storage_class): + def __init__(self, *args, **kwargs): + return super(StorageSubclass, self).__init__(*args, **kwargs) + + def __eq__(self, other): + return True + + def deconstruct(self): + return ('mayan.apps.storage.classes.FakeStorageSubclass', (), {}) + + return StorageSubclass + + +def mkdtemp(*args, **kwargs): + kwargs.update({'dir': setting_temporary_directory.value}) + return tempfile.mkdtemp(*args, **kwargs) + + +def mkstemp(*args, **kwargs): + kwargs.update({'dir': setting_temporary_directory.value}) + return tempfile.mkstemp(*args, **kwargs) + + +def validate_path(path): + if not os.path.exists(path): + # If doesn't exist try to create it + try: + os.mkdir(path) + except Exception as exception: + logger.debug('unhandled exception: %s', exception) + return False + + # Check if it is writable + try: + fd, test_filepath = tempfile.mkstemp(dir=path) + os.close(fd) + os.unlink(test_filepath) + except Exception as exception: + logger.debug('unhandled exception: %s', exception) + return False + + return True