matthias/mayan-edms
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Move file and storage code to the storage app

Browse Source 
The setting COMMON_TEMPORARY_DIRECTORY is now
STORAGE_TEMPORARY_DIRECTORY.

Signed-off-by: Roberto Rosario <Roberto.Rosario@mayan-edms.com>
This commit is contained in:
Roberto Rosario
2019-01-31 22:26:07 -04:00
parent 125a4317f4
commit 8e66eefe7c
33 changed files with 202 additions and 178 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
HISTORY.rst
View File

@@ -227,6 +227,8 @@
- Add a test mixin to generate random model primary keys.
- Add support for checkout and check in multiple documents at
the same time.
- Move file and storage code to the storage app. The setting
COMMON_TEMPORARY_DIRECTORY is now STORAGE_TEMPORARY_DIRECTORY.
3.1.9 (2018-11-01)
==================

10
mayan/apps/common/classes.py
View File

@@ -72,16 +72,6 @@ class ErrorLogNamespace(object):
return ErrorLogEntry.objects.filter(namespace=self.name)
class FakeStorageSubclass(object):
"""
Placeholder class to allow serializing the real storage subclass to
support migrations.
"""
def __eq__(self, other):
return True
class MissingItem(object):
_registry = []

2
mayan/apps/common/management/commands/convertdb.py
View File

@@ -11,8 +11,8 @@ from django.core.management.base import CommandError
from django.utils.encoding import force_text
from django.utils.translation import ugettext_lazy as _
from mayan.apps.common.utils import fs_cleanup
from mayan.apps.documents.models import DocumentType
from mayan.apps.storage.utils import fs_cleanup
CONVERTDB_FOLDER = 'convertdb'
CONVERTDB_OUTPUT_FILENAME = 'migrate.json'

11
mayan/apps/common/migrations/0011_auto_20181229_0738.py
View File

@@ -1,14 +1,13 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.11.16 on 2018-12-29 07:38
from __future__ import unicode_literals
from django.db import migrations, models
import mayan.apps.common.classes
import mayan.apps.common.models
import mayan.apps.storage.classes
class Migration(migrations.Migration):
dependencies = [
('common', '0010_auto_20180403_0702_squashed_0011_auto_20180429_0758'),
]
@@ -17,6 +16,10 @@ class Migration(migrations.Migration):
migrations.AlterField(
model_name='shareduploadedfile',
name='file',
field=models.FileField(storage=mayan.apps.common.classes.FakeStorageSubclass(), upload_to=mayan.apps.common.models.upload_to, verbose_name='File'),
field=models.FileField(
storage=mayan.apps.storage.classes.FakeStorageSubclass(),
upload_to=mayan.apps.common.models.upload_to,
verbose_name='File'
),
),
]

1
mayan/apps/common/mixins.py
View File

@@ -1,6 +1,5 @@
from __future__ import unicode_literals
from django.conf import settings
from django.contrib import messages
from django.contrib.contenttypes.models import ContentType
from django.core.exceptions import ImproperlyConfigured, PermissionDenied

8
mayan/apps/common/settings.py
View File

@@ -1,7 +1,6 @@
from __future__ import unicode_literals
import os
import tempfile
from django.conf import settings
from django.utils.translation import ugettext_lazy as _
@@ -76,13 +75,6 @@ setting_shared_storage_arguments = namespace.add_setting(
global_name='COMMON_SHARED_STORAGE_ARGUMENTS',
default={'location': os.path.join(settings.MEDIA_ROOT, 'shared_files')}
)
setting_temporary_directory = namespace.add_setting(
global_name='COMMON_TEMPORARY_DIRECTORY', default=tempfile.gettempdir(),
help_text=_(
'Temporary directory used site wide to store thumbnails, previews '
'and temporary files.'
)
)
namespace = Namespace(label=_('Django'), name='django')

3
mayan/apps/common/storages.py
View File

@@ -1,7 +1,8 @@
from __future__ import unicode_literals
from mayan.apps.storage.utils import get_storage_subclass
from .settings import setting_shared_storage, setting_shared_storage_arguments
from .utils import get_storage_subclass
storage_sharedupload = get_storage_subclass(
dotted_path=setting_shared_storage.value

2
mayan/apps/common/tests/mixins.py
View File

@@ -13,7 +13,7 @@ from django.template import Context, Template
from django.test.utils import ContextList
from django.urls import clear_url_caches, reverse
from ..settings import setting_temporary_directory
from mayan.apps.storage.settings import setting_temporary_directory
from .literals import TEST_VIEW_NAME, TEST_VIEW_URL
from .utils import mute_stdout

15
mayan/apps/common/tests/test_views.py
View File

@@ -14,37 +14,33 @@ from .literals import TEST_ERROR_LOG_ENTRY_RESULT
class CommonViewTestCase(GenericViewTestCase):
def test_about_view(self):
self.login_user()
response = self.get('common:about_view')
self.assertContains(response, text='About', status_code=200)
def _create_error_log_entry(self):
ModelPermission.register(
model=get_user_model(), permission=permission_error_log_view
model=get_user_model(), permissions=(permission_error_log_view,)
)
ErrorLogEntry.objects.register(model=get_user_model())
self.error_log_entry = self.user.error_logs.create(
self.error_log_entry = self._test_case_user.error_logs.create(
result=TEST_ERROR_LOG_ENTRY_RESULT
)
def _request_object_error_log_list(self):
content_type = ContentType.objects.get_for_model(model=self.user)
content_type = ContentType.objects.get_for_model(model=self._test_case_user)
return self.get(
'common:object_error_list', kwargs={
'app_label': content_type.app_label,
'model': content_type.model,
'object_id': self.user.pk
'object_id': self._test_case_user.pk
}, follow=True
)
def test_object_error_list_view_no_permissions(self):
self._create_error_log_entry()
self.login_user()
response = self._request_object_error_log_list()
self.assertNotContains(
@@ -55,9 +51,8 @@ class CommonViewTestCase(GenericViewTestCase):
def test_object_error_list_view_with_access(self):
self._create_error_log_entry()
self.login_user()
self.grant_access(
obj=self.user, permission=permission_error_log_view
obj=self._test_case_user, permission=permission_error_log_view
)
response = self._request_object_error_log_list()

116
mayan/apps/common/utils.py
View File

@@ -1,9 +1,6 @@
from __future__ import unicode_literals
import logging
import os
import shutil
import tempfile
from django.conf import settings
from django.core.exceptions import FieldDoesNotExist
@@ -13,7 +10,6 @@ from django.urls.base import get_script_prefix
from django.utils.datastructures import MultiValueDict
from django.utils.http import urlencode as django_urlencode
from django.utils.http import urlquote as django_urlquote
from django.utils.module_loading import import_string
from django.utils.six.moves import reduce as reduce_function
from django.utils.six.moves import xmlrpc_client
@@ -21,7 +17,6 @@ import mayan
from .exceptions import NotLatestVersion, UnknownLatestVersion
from .literals import DJANGO_SQLITE_BACKEND, MAYAN_PYPI_NAME, PYPI_URL
from .settings import setting_temporary_directory
logger = logging.getLogger(__name__)
@@ -40,27 +35,6 @@ def check_version():
raise NotLatestVersion(upstream_version=versions[0])
# http://stackoverflow.com/questions/123198/how-do-i-copy-a-file-in-python
def copyfile(source, destination, buffer_size=1024 * 1024):
"""
Copy a file from source to dest. source and dest
can either be strings or any object with a read or
write method, like StringIO for example.
"""
source_descriptor = get_descriptor(source)
destination_descriptor = get_descriptor(destination, read=False)
while True:
copy_buffer = source_descriptor.read(buffer_size)
if copy_buffer:
destination_descriptor.write(copy_buffer)
else:
break
source_descriptor.close()
destination_descriptor.close()
def encapsulate(function):
# Workaround Django ticket 15791
# Changeset 16045
@@ -69,25 +43,6 @@ def encapsulate(function):
return lambda: function
def fs_cleanup(filename, file_descriptor=None, suppress_exceptions=True):
"""
Tries to remove the given filename. Ignores non-existent files
"""
if file_descriptor:
os.close(file_descriptor)
try:
os.remove(filename)
except OSError:
try:
shutil.rmtree(filename)
except OSError:
if suppress_exceptions:
pass
else:
raise
def get_related_field(model, related_field_name):
try:
local_field_name, remaining_field_path = related_field_name.split(
@@ -108,41 +63,6 @@ def get_related_field(model, related_field_name):
return related_field
def get_descriptor(file_input, read=True):
try:
# Is it a file like object?
file_input.seek(0)
except AttributeError:
# If not, try open it.
if read:
return open(file_input, mode='rb')
else:
return open(file_input, mode='wb')
else:
return file_input
def get_storage_subclass(dotted_path):
"""
Import a storage class and return a subclass that will always return eq
True to avoid creating a new migration when for runtime storage class
changes.
"""
imported_storage_class = import_string(dotted_path=dotted_path)
class StorageSubclass(imported_storage_class):
def __init__(self, *args, **kwargs):
return super(StorageSubclass, self).__init__(*args, **kwargs)
def __eq__(self, other):
return True
def deconstruct(self):
return ('mayan.apps.common.classes.FakeStorageSubclass', (), {})
return StorageSubclass
def introspect_attribute(attribute_name, obj):
try:
# Try as a related field
@@ -168,21 +88,6 @@ def introspect_attribute(attribute_name, obj):
return attribute_name, obj
def TemporaryFile(*args, **kwargs):
kwargs.update({'dir': setting_temporary_directory.value})
return tempfile.TemporaryFile(*args, **kwargs)
def mkdtemp(*args, **kwargs):
kwargs.update({'dir': setting_temporary_directory.value})
return tempfile.mkdtemp(*args, **kwargs)
def mkstemp(*args, **kwargs):
kwargs.update({'dir': setting_temporary_directory.value})
return tempfile.mkstemp(*args, **kwargs)
def resolve(path, urlconf=None):
path = '/{}'.format(path.replace(get_script_prefix(), '', 1))
return django_resolve(path=path, urlconf=urlconf)
@@ -275,24 +180,3 @@ def urlquote(link=None, get=None):
return '%s%s' % (link, django_urlencode(get, doseq=True))
else:
return django_urlquote(link)
def validate_path(path):
if not os.path.exists(path):
# If doesn't exist try to create it
try:
os.mkdir(path)
except Exception as exception:
logger.debug('unhandled exception: %s', exception)
return False
# Check if it is writable
try:
fd, test_filepath = tempfile.mkstemp(dir=path)
os.close(fd)
os.unlink(test_filepath)
except Exception as exception:
logger.debug('unhandled exception: %s', exception)
return False
return True

2
mayan/apps/converter/backends/python.py
View File

@@ -11,7 +11,7 @@ import sh
from django.utils.encoding import force_text
from django.utils.translation import ugettext_lazy as _
from mayan.apps.common.utils import fs_cleanup, mkstemp
from mayan.apps.storage.utils import fs_cleanup, mkstemp
from ..classes import ConverterBase
from ..exceptions import PageCountError

4
mayan/apps/converter/classes.py
View File

@@ -9,9 +9,9 @@ import sh
from django.utils.translation import ugettext_lazy as _
from mayan.apps.common.settings import setting_temporary_directory
from mayan.apps.common.utils import fs_cleanup, mkdtemp, mkstemp
from mayan.apps.mimetype.api import get_mimetype
from mayan.apps.storage.settings import setting_temporary_directory
from mayan.apps.storage.utils import fs_cleanup, mkdtemp, mkstemp
from .exceptions import InvalidOfficeFormat, OfficeConversionError
from .literals import (

2
mayan/apps/dependencies/javascript.py
View File

@@ -15,7 +15,7 @@ from django.apps import apps
from django.utils.encoding import force_bytes, force_text
from django.utils.functional import cached_property
from mayan.apps.common.utils import mkdtemp
from mayan.apps.storage.utils import mkdtemp
from .exceptions import DependenciesException

2
mayan/apps/django_gpg/classes.py
View File

@@ -8,7 +8,7 @@ import gnupg
from django.utils.translation import ugettext_lazy as _
from mayan.apps.common.utils import mkdtemp
from mayan.apps.storage.utils import mkdtemp
class GPGBackend(object):

2
mayan/apps/django_gpg/managers.py
View File

@@ -6,7 +6,7 @@ import os
from django.db import models
from mayan.apps.common.utils import mkstemp
from mayan.apps.storage.utils import mkstemp
from .classes import KeyStub, SignatureVerification
from .exceptions import (

2
mayan/apps/django_gpg/tests/test_models.py
View File

@@ -8,7 +8,7 @@ import mock
from django.utils.encoding import force_bytes
from mayan.apps.common.tests import BaseTestCase
from mayan.apps.common.utils import TemporaryFile
from mayan.apps.storage.utils import TemporaryFile
from ..exceptions import (
DecryptionError, KeyDoesNotExist, NeedPassphrase, PassphraseError,

2
mayan/apps/document_parsing/parsers.py
View File

@@ -7,7 +7,7 @@ import subprocess
from django.apps import apps
from django.utils.translation import ugettext_lazy as _
from mayan.apps.common.utils import copyfile, fs_cleanup, mkstemp
from mayan.apps.storage.utils import copyfile, fs_cleanup, mkstemp
from .exceptions import ParserError
from .settings import setting_pdftotext_path

2
mayan/apps/document_signatures/managers.py
View File

@@ -5,10 +5,10 @@ import os
from django.db import models
from mayan.apps.common.utils import mkstemp
from mayan.apps.django_gpg.exceptions import DecryptionError
from mayan.apps.django_gpg.models import Key
from mayan.apps.documents.models import DocumentVersion
from mayan.apps.storage.utils import mkstemp
logger = logging.getLogger(__name__)

12
mayan/apps/document_signatures/migrations/0009_auto_20181229_0737.py
View File

@@ -1,14 +1,13 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.11.16 on 2018-12-29 07:37
from __future__ import unicode_literals
from django.db import migrations, models
import mayan.apps.common.classes
import mayan.apps.document_signatures.models
import mayan.apps.storage.classes
class Migration(migrations.Migration):
dependencies = [
('document_signatures', '0008_auto_20180429_0759'),
]
@@ -17,6 +16,11 @@ class Migration(migrations.Migration):
migrations.AlterField(
model_name='detachedsignature',
name='signature_file',
field=models.FileField(blank=True, null=True, storage=mayan.apps.common.classes.FakeStorageSubclass(), upload_to=mayan.apps.document_signatures.models.upload_to, verbose_name='Signature file'),
field=models.FileField(
blank=True, null=True,
storage=mayan.apps.storage.classes.FakeStorageSubclass(),
upload_to=mayan.apps.document_signatures.models.upload_to,
verbose_name='Signature file'
),
),
]

2
mayan/apps/document_signatures/storages.py
View File

@@ -1,6 +1,6 @@
from __future__ import unicode_literals
from mayan.apps.common.utils import get_storage_subclass
from mayan.apps.storage.utils import get_storage_subclass
from .settings import (
setting_storage_backend, setting_storage_backend_arguments

2
mayan/apps/document_signatures/views.py
View File

@@ -15,9 +15,9 @@ from mayan.apps.common.generics import (
SingleObjectDetailView, SingleObjectDownloadView, SingleObjectListView
)
from mayan.apps.common.mixins import ExternalObjectMixin
from mayan.apps.common.utils import TemporaryFile
from mayan.apps.django_gpg.exceptions import NeedPassphrase, PassphraseError
from mayan.apps.documents.models import DocumentVersion
from mayan.apps.storage.utils import TemporaryFile
from .forms import (
DocumentVersionSignatureCreateForm, DocumentVersionSignatureDetailForm

10
mayan/apps/documents/migrations/0051_auto_20181229_0745.py
View File

@@ -1,10 +1,10 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.11.16 on 2018-12-29 07:45
from __future__ import unicode_literals
from django.db import migrations, models
import mayan.apps.common.classes
import mayan.apps.documents.utils
import mayan.apps.storage.classes
class Migration(migrations.Migration):
@@ -17,6 +17,10 @@ class Migration(migrations.Migration):
migrations.AlterField(
model_name='documentversion',
name='file',
field=models.FileField(storage=mayan.apps.common.classes.FakeStorageSubclass(), upload_to=mayan.apps.documents.utils.document_uuid_function, verbose_name='File'),
field=models.FileField(
storage=mayan.apps.storage.classes.FakeStorageSubclass(),
upload_to=mayan.apps.documents.utils.document_uuid_function,
verbose_name='File'
),
),
]

2
mayan/apps/documents/storages.py
View File

@@ -2,7 +2,7 @@ from __future__ import unicode_literals
from django.utils.module_loading import import_string
from mayan.apps.common.utils import get_storage_subclass
from mayan.apps.storage.utils import get_storage_subclass
from .settings import (
setting_documentimagecache_storage,

2
mayan/apps/file_metadata/drivers/exiftool.py
View File

@@ -7,7 +7,7 @@ import sh
from django.utils.translation import ugettext_lazy as _
from mayan.apps.common.utils import fs_cleanup, mkstemp
from mayan.apps.storage.utils import fs_cleanup, mkstemp
from ..classes import FileMetadataDriver
from ..settings import setting_drivers_arguments

2
mayan/apps/lock_manager/backends/file_lock.py
View File

@@ -12,7 +12,7 @@ from django.conf import settings
from django.core.files import locks
from django.utils.encoding import force_bytes, force_text
from mayan.apps.common.settings import setting_temporary_directory
from mayan.apps.storage.settings import setting_temporary_directory
from ..exceptions import LockError
from ..settings import setting_default_lock_timeout

2
mayan/apps/smart_settings/tests/test_classes.py
View File

@@ -8,7 +8,7 @@ from django.conf import settings
from mayan.apps.common.settings import setting_paginate_by
from mayan.apps.common.tests import BaseTestCase
from mayan.apps.common.utils import fs_cleanup, mkstemp
from mayan.apps.storage.utils import fs_cleanup, mkstemp
from .literals import TEST_SETTING_NAME, TEST_SETTING_VALUE

2
mayan/apps/sources/models/scanner_sources.py
View File

@@ -7,7 +7,7 @@ from django.db import models
from django.utils.timezone import now
from django.utils.translation import ugettext_lazy as _
from mayan.apps.common.utils import TemporaryFile
from mayan.apps.storage.utils import TemporaryFile
from ..classes import PseudoFile, SourceUploadedFile
from ..exceptions import SourceException

2
mayan/apps/sources/tests/test_classes.py
View File

@@ -4,8 +4,8 @@ import os
import shutil
from mayan.apps.common.tests import BaseTestCase
from mayan.apps.common.utils import mkdtemp
from mayan.apps.documents.tests import TEST_NON_ASCII_DOCUMENT_PATH
from mayan.apps.storage.utils import mkdtemp
from ..classes import StagingFile

2
mayan/apps/sources/tests/test_models.py
View File

@@ -9,7 +9,6 @@ from pathlib2 import Path
from django.utils.encoding import force_text
from mayan.apps.common.tests import BaseTestCase
from mayan.apps.common.utils import mkdtemp
from mayan.apps.documents.models import Document, DocumentType
from mayan.apps.documents.tests import (
TEST_COMPRESSED_DOCUMENT_PATH, TEST_DOCUMENT_TYPE_LABEL,
@@ -17,6 +16,7 @@ from mayan.apps.documents.tests import (
TEST_NON_ASCII_DOCUMENT_PATH, DocumentTestMixin
)
from mayan.apps.metadata.models import MetadataType
from mayan.apps.storage.utils import mkdtemp
from ..literals import SOURCE_UNCOMPRESS_CHOICE_Y
from ..models import POP3Email, WatchFolderSource, WebFormSource

2
mayan/apps/sources/tests/test_views.py
View File

@@ -6,13 +6,13 @@ import shutil
from mayan.apps.checkouts.models import NewVersionBlock
from mayan.apps.common.tests import GenericViewTestCase
from mayan.apps.common.utils import fs_cleanup, mkdtemp
from mayan.apps.documents.models import Document
from mayan.apps.documents.permissions import permission_document_create
from mayan.apps.documents.tests import (
TEST_DOCUMENT_DESCRIPTION, TEST_SMALL_DOCUMENT_CHECKSUM,
TEST_SMALL_DOCUMENT_PATH, GenericDocumentViewTestCase
)
from mayan.apps.storage.utils import fs_cleanup, mkdtemp
from ..links import link_upload_version
from ..literals import SOURCE_CHOICE_WEB_FORM

10
mayan/apps/storage/classes.py Normal file
View File

@@ -0,0 +1,10 @@
from __future__ import unicode_literals
class FakeStorageSubclass(object):
"""
Placeholder class to allow serializing the real storage subclass to
support migrations.
"""
def __eq__(self, other):
return True

17
mayan/apps/storage/settings.py Normal file
View File

@@ -0,0 +1,17 @@
from __future__ import unicode_literals
import tempfile
from django.utils.translation import ugettext_lazy as _
from mayan.apps.smart_settings import Namespace
namespace = Namespace(label=_('Storage'), name='storage')
setting_temporary_directory = namespace.add_setting(
global_name='STORAGE_TEMPORARY_DIRECTORY', default=tempfile.gettempdir(),
help_text=_(
'Temporary directory used site wide to store thumbnails, previews '
'and temporary files.'
)
)

123
mayan/apps/storage/utils.py Normal file
View File

@@ -0,0 +1,123 @@
from __future__ import unicode_literals
import logging
import os
import shutil
import tempfile
from django.utils.module_loading import import_string
from .settings import setting_temporary_directory
logger = logging.getLogger(__name__)
def TemporaryFile(*args, **kwargs):
kwargs.update({'dir': setting_temporary_directory.value})
return tempfile.TemporaryFile(*args, **kwargs)
# http://stackoverflow.com/questions/123198/how-do-i-copy-a-file-in-python
def copyfile(source, destination, buffer_size=1024 * 1024):
"""
Copy a file from source to dest. source and dest
can either be strings or any object with a read or
write method, like StringIO for example.
"""
source_descriptor = get_descriptor(source)
destination_descriptor = get_descriptor(destination, read=False)
while True:
copy_buffer = source_descriptor.read(buffer_size)
if copy_buffer:
destination_descriptor.write(copy_buffer)
else:
break
source_descriptor.close()
destination_descriptor.close()
def fs_cleanup(filename, file_descriptor=None, suppress_exceptions=True):
"""
Tries to remove the given filename. Ignores non-existent files
"""
if file_descriptor:
os.close(file_descriptor)
try:
os.remove(filename)
except OSError:
try:
shutil.rmtree(filename)
except OSError:
if suppress_exceptions:
pass
else:
raise
def get_descriptor(file_input, read=True):
try:
# Is it a file like object?
file_input.seek(0)
except AttributeError:
# If not, try open it.
if read:
return open(file_input, mode='rb')
else:
return open(file_input, mode='wb')
else:
return file_input
def get_storage_subclass(dotted_path):
"""
Import a storage class and return a subclass that will always return eq
True to avoid creating a new migration when for runtime storage class
changes.
"""
imported_storage_class = import_string(dotted_path=dotted_path)
class StorageSubclass(imported_storage_class):
def __init__(self, *args, **kwargs):
return super(StorageSubclass, self).__init__(*args, **kwargs)
def __eq__(self, other):
return True
def deconstruct(self):
return ('mayan.apps.storage.classes.FakeStorageSubclass', (), {})
return StorageSubclass
def mkdtemp(*args, **kwargs):
kwargs.update({'dir': setting_temporary_directory.value})
return tempfile.mkdtemp(*args, **kwargs)
def mkstemp(*args, **kwargs):
kwargs.update({'dir': setting_temporary_directory.value})
return tempfile.mkstemp(*args, **kwargs)
def validate_path(path):
if not os.path.exists(path):
# If doesn't exist try to create it
try:
os.mkdir(path)
except Exception as exception:
logger.debug('unhandled exception: %s', exception)
return False
# Check if it is writable
try:
fd, test_filepath = tempfile.mkstemp(dir=path)
os.close(fd)
os.unlink(test_filepath)
except Exception as exception:
logger.debug('unhandled exception: %s', exception)
return False
return True