diff --git a/HISTORY.rst b/HISTORY.rst index 0d04e872b8..6522e0d7da 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -36,6 +36,9 @@ https://github.com/tesseract-ocr/tesseract/issues/1670 https://github.com/tesseract-ocr/tesseract/commit/3292484f67af8bdda23aa5e510918d0115785291 https://gitlab.gnome.org/World/OpenPaperwork/pyocr/issues/104 +* Move setting COMMON_TEMPORARY_DIRECTORY to the storage app. + The setting is now STORAGE_TEMPORARY_DIRECTORY. +* Move file related utilities to the storage app. 3.1.11 (2019-04-XX) =================== diff --git a/docs/releases/3.2.rst b/docs/releases/3.2.rst index 78ffed157b..b6b6ca93c4 100644 --- a/docs/releases/3.2.rst +++ b/docs/releases/3.2.rst @@ -60,6 +60,9 @@ Other changes https://github.com/tesseract-ocr/tesseract/issues/1670 https://github.com/tesseract-ocr/tesseract/commit/3292484f67af8bdda23aa5e510918d0115785291 https://gitlab.gnome.org/World/OpenPaperwork/pyocr/issues/104 +* Move setting COMMON_TEMPORARY_DIRECTORY to the storage app. + The setting is now STORAGE_TEMPORARY_DIRECTORY. +* Move file related utilities to the storage app. Removals diff --git a/mayan/apps/common/management/commands/convertdb.py b/mayan/apps/common/management/commands/convertdb.py index bd822aba4e..3d4fe0c063 100644 --- a/mayan/apps/common/management/commands/convertdb.py +++ b/mayan/apps/common/management/commands/convertdb.py @@ -11,8 +11,8 @@ from django.core.management.base import CommandError from django.utils.encoding import force_text from django.utils.translation import ugettext_lazy as _ -from mayan.apps.common.utils import fs_cleanup from mayan.apps.documents.models import DocumentType +from mayan.apps.storage.utils import fs_cleanup CONVERTDB_FOLDER = 'convertdb' CONVERTDB_OUTPUT_FILENAME = 'migrate.json' diff --git a/mayan/apps/common/settings.py b/mayan/apps/common/settings.py index 77a5b25be8..50d7dfad11 100644 --- a/mayan/apps/common/settings.py +++ b/mayan/apps/common/settings.py @@ -80,14 +80,6 @@ setting_shared_storage_arguments = namespace.add_setting( os.path.join(settings.MEDIA_ROOT, 'shared_files') ), quoted=True ) -setting_temporary_directory = namespace.add_setting( - global_name='COMMON_TEMPORARY_DIRECTORY', default=tempfile.gettempdir(), - help_text=_( - 'Temporary directory used site wide to store thumbnails, previews ' - 'and temporary files.' - ), - is_path=True -) namespace = Namespace(name='django', label=_('Django')) diff --git a/mayan/apps/common/tests/mixins.py b/mayan/apps/common/tests/mixins.py index f28dd1df85..cd7179f735 100644 --- a/mayan/apps/common/tests/mixins.py +++ b/mayan/apps/common/tests/mixins.py @@ -18,9 +18,10 @@ from django.template import Context, Template from django.test.utils import ContextList from django.urls import clear_url_caches, reverse +from mayan.apps.storage.settings import setting_temporary_directory + from .literals import TEST_VIEW_NAME, TEST_VIEW_URL -from ..settings import setting_temporary_directory if getattr(settings, 'COMMON_TEST_FILE_HANDLES', False): import psutil diff --git a/mayan/apps/common/utils.py b/mayan/apps/common/utils.py index a9791343e0..24cdbcd433 100644 --- a/mayan/apps/common/utils.py +++ b/mayan/apps/common/utils.py @@ -21,7 +21,6 @@ from mayan.apps.common.compat import dict_type, dictionary_type from .exceptions import NotLatestVersion, UnknownLatestVersion from .literals import DJANGO_SQLITE_BACKEND, MAYAN_PYPI_NAME, PYPI_URL -from .settings import setting_temporary_directory logger = logging.getLogger(__name__) @@ -40,27 +39,6 @@ def check_version(): raise NotLatestVersion(upstream_version=versions[0]) -# http://stackoverflow.com/questions/123198/how-do-i-copy-a-file-in-python -def copyfile(source, destination, buffer_size=1024 * 1024): - """ - Copy a file from source to dest. source and dest - can either be strings or any object with a read or - write method, like StringIO for example. - """ - source_descriptor = get_descriptor(source) - destination_descriptor = get_descriptor(destination, read=False) - - while True: - copy_buffer = source_descriptor.read(buffer_size) - if copy_buffer: - destination_descriptor.write(copy_buffer) - else: - break - - source_descriptor.close() - destination_descriptor.close() - - def encapsulate(function): # Workaround Django ticket 15791 # Changeset 16045 @@ -76,54 +54,6 @@ def get_user_label_text(context): return context['request'].user.get_full_name() or context['request'].user -def fs_cleanup(filename, file_descriptor=None, suppress_exceptions=True): - """ - Tries to remove the given filename. Ignores non-existent files - """ - if file_descriptor: - os.close(file_descriptor) - - try: - os.remove(filename) - except OSError: - try: - shutil.rmtree(filename) - except OSError: - if suppress_exceptions: - pass - else: - raise - - -def get_descriptor(file_input, read=True): - try: - # Is it a file like object? - file_input.seek(0) - except AttributeError: - # If not, try open it. - if read: - return open(file_input, mode='rb') - else: - return open(file_input, mode='wb') - else: - return file_input - - -def TemporaryFile(*args, **kwargs): - kwargs.update({'dir': setting_temporary_directory.value}) - return tempfile.TemporaryFile(*args, **kwargs) - - -def mkdtemp(*args, **kwargs): - kwargs.update({'dir': setting_temporary_directory.value}) - return tempfile.mkdtemp(*args, **kwargs) - - -def mkstemp(*args, **kwargs): - kwargs.update({'dir': setting_temporary_directory.value}) - return tempfile.mkstemp(*args, **kwargs) - - def resolve(path, urlconf=None): path = '/{}'.format(path.replace(get_script_prefix(), '', 1)) return django_resolve(path=path, urlconf=urlconf) @@ -201,24 +131,3 @@ def urlquote(link=None, get=None): return '%s%s' % (link, django_urlencode(get, doseq=True)) else: return django_urlquote(link) - - -def validate_path(path): - if not os.path.exists(path): - # If doesn't exist try to create it - try: - os.mkdir(path) - except Exception as exception: - logger.debug('unhandled exception: %s', exception) - return False - - # Check if it is writable - try: - fd, test_filepath = tempfile.mkstemp(dir=path) - os.close(fd) - os.unlink(test_filepath) - except Exception as exception: - logger.debug('unhandled exception: %s', exception) - return False - - return True diff --git a/mayan/apps/converter/backends/python.py b/mayan/apps/converter/backends/python.py index 51ba5914b1..cb1a72d05e 100644 --- a/mayan/apps/converter/backends/python.py +++ b/mayan/apps/converter/backends/python.py @@ -16,7 +16,7 @@ except ImportError: from django.utils.encoding import force_text from django.utils.translation import ugettext_lazy as _ -from mayan.apps.common.utils import fs_cleanup, mkstemp +from mayan.apps.storage.utils import fs_cleanup, mkstemp from ..classes import ConverterBase from ..exceptions import PageCountError diff --git a/mayan/apps/converter/classes.py b/mayan/apps/converter/classes.py index 5f64fe5ee2..5b0b98175e 100644 --- a/mayan/apps/converter/classes.py +++ b/mayan/apps/converter/classes.py @@ -16,9 +16,9 @@ except ImportError: from django.utils.translation import ugettext_lazy as _ -from mayan.apps.common.settings import setting_temporary_directory -from mayan.apps.common.utils import fs_cleanup, mkdtemp, mkstemp from mayan.apps.mimetype.api import get_mimetype +from mayan.apps.storage.settings import setting_temporary_directory +from mayan.apps.storage.utils import fs_cleanup, mkdtemp, mkstemp from .exceptions import InvalidOfficeFormat, OfficeConversionError from .literals import ( diff --git a/mayan/apps/django_gpg/classes.py b/mayan/apps/django_gpg/classes.py index 494fe86438..ba0f6cb631 100644 --- a/mayan/apps/django_gpg/classes.py +++ b/mayan/apps/django_gpg/classes.py @@ -6,7 +6,7 @@ import shutil import gnupg -from mayan.apps.common.utils import mkdtemp +from mayan.apps.storage.utils import mkdtemp class GPGBackend(object): diff --git a/mayan/apps/django_gpg/managers.py b/mayan/apps/django_gpg/managers.py index e6b498f32c..d565e8ae2a 100644 --- a/mayan/apps/django_gpg/managers.py +++ b/mayan/apps/django_gpg/managers.py @@ -6,7 +6,7 @@ import os from django.db import models -from mayan.apps.common.utils import mkstemp +from mayan.apps.storage.utils import mkstemp from .classes import KeyStub, SignatureVerification from .exceptions import ( diff --git a/mayan/apps/django_gpg/tests/test_models.py b/mayan/apps/django_gpg/tests/test_models.py index 763166f905..5b805e5468 100644 --- a/mayan/apps/django_gpg/tests/test_models.py +++ b/mayan/apps/django_gpg/tests/test_models.py @@ -8,7 +8,7 @@ import mock from django.utils.encoding import force_bytes from mayan.apps.common.tests import BaseTestCase -from mayan.apps.common.utils import TemporaryFile +from mayan.apps.storage.utils import TemporaryFile from ..exceptions import ( DecryptionError, KeyDoesNotExist, NeedPassphrase, PassphraseError, diff --git a/mayan/apps/document_parsing/parsers.py b/mayan/apps/document_parsing/parsers.py index fd70926255..ea6ed8dc60 100644 --- a/mayan/apps/document_parsing/parsers.py +++ b/mayan/apps/document_parsing/parsers.py @@ -7,7 +7,7 @@ import subprocess from django.apps import apps from django.utils.translation import ugettext_lazy as _ -from mayan.apps.common.utils import copyfile, fs_cleanup, mkstemp +from mayan.apps.storage.utils import copyfile, fs_cleanup, mkstemp from .exceptions import ParserError from .settings import setting_pdftotext_path diff --git a/mayan/apps/document_signatures/managers.py b/mayan/apps/document_signatures/managers.py index 0425fa7d18..c8ace7c6c3 100644 --- a/mayan/apps/document_signatures/managers.py +++ b/mayan/apps/document_signatures/managers.py @@ -5,10 +5,10 @@ import os from django.db import models -from mayan.apps.common.utils import mkstemp from mayan.apps.django_gpg.exceptions import DecryptionError from mayan.apps.django_gpg.models import Key from mayan.apps.documents.models import DocumentVersion +from mayan.apps.storage.utils import mkstemp logger = logging.getLogger(__name__) diff --git a/mayan/apps/document_signatures/views.py b/mayan/apps/document_signatures/views.py index 4c0aa5d6fb..51055ca89f 100644 --- a/mayan/apps/document_signatures/views.py +++ b/mayan/apps/document_signatures/views.py @@ -16,10 +16,10 @@ from mayan.apps.common.generics import ( ConfirmView, FormView, SingleObjectCreateView, SingleObjectDeleteView, SingleObjectDetailView, SingleObjectDownloadView, SingleObjectListView ) -from mayan.apps.common.utils import TemporaryFile from mayan.apps.django_gpg.exceptions import NeedPassphrase, PassphraseError from mayan.apps.django_gpg.permissions import permission_key_sign from mayan.apps.documents.models import DocumentVersion +from mayan.apps.storage.utils import TemporaryFile from .forms import ( DocumentVersionSignatureCreateForm, diff --git a/mayan/apps/lock_manager/backends/file_lock.py b/mayan/apps/lock_manager/backends/file_lock.py index 29e4b16f79..16a8fd96c3 100644 --- a/mayan/apps/lock_manager/backends/file_lock.py +++ b/mayan/apps/lock_manager/backends/file_lock.py @@ -12,7 +12,7 @@ from django.conf import settings from django.core.files import locks from django.utils.encoding import force_bytes, force_text -from mayan.apps.common.settings import setting_temporary_directory +from mayan.apps.storage.settings import setting_temporary_directory from ..exceptions import LockError from ..settings import setting_default_lock_timeout diff --git a/mayan/apps/sources/models/scanner_sources.py b/mayan/apps/sources/models/scanner_sources.py index 099d6ca913..b4ca0bbc63 100644 --- a/mayan/apps/sources/models/scanner_sources.py +++ b/mayan/apps/sources/models/scanner_sources.py @@ -7,7 +7,7 @@ from django.db import models from django.utils.timezone import now from django.utils.translation import ugettext_lazy as _ -from mayan.apps.common.utils import TemporaryFile +from mayan.apps.storage.utils import TemporaryFile from ..classes import PseudoFile, SourceUploadedFile from ..exceptions import SourceException diff --git a/mayan/apps/sources/tests/test_classes.py b/mayan/apps/sources/tests/test_classes.py index 7466b33fad..319c3eeb37 100644 --- a/mayan/apps/sources/tests/test_classes.py +++ b/mayan/apps/sources/tests/test_classes.py @@ -4,8 +4,8 @@ import os import shutil from mayan.apps.common.tests import BaseTestCase -from mayan.apps.common.utils import mkdtemp from mayan.apps.documents.tests import TEST_NON_ASCII_DOCUMENT_PATH +from mayan.apps.storage.utils import mkdtemp from ..classes import StagingFile diff --git a/mayan/apps/sources/tests/test_models.py b/mayan/apps/sources/tests/test_models.py index 6c45999d7d..b0ebd81c45 100644 --- a/mayan/apps/sources/tests/test_models.py +++ b/mayan/apps/sources/tests/test_models.py @@ -8,7 +8,6 @@ from pathlib2 import Path from django.test import override_settings from django.utils.encoding import force_text -from mayan.apps.common.utils import mkdtemp from mayan.apps.common.tests import BaseTestCase from mayan.apps.documents.models import Document, DocumentType from mayan.apps.documents.tests import ( @@ -17,6 +16,7 @@ from mayan.apps.documents.tests import ( TEST_NON_ASCII_COMPRESSED_DOCUMENT_PATH ) from mayan.apps.metadata.models import MetadataType +from mayan.apps.storage.utils import mkdtemp from ..literals import SOURCE_UNCOMPRESS_CHOICE_Y from ..models import POP3Email, WatchFolderSource, WebFormSource diff --git a/mayan/apps/sources/tests/test_views.py b/mayan/apps/sources/tests/test_views.py index dd5a9838c1..7936fcdd93 100644 --- a/mayan/apps/sources/tests/test_views.py +++ b/mayan/apps/sources/tests/test_views.py @@ -7,7 +7,6 @@ from django.test import override_settings from mayan.apps.checkouts.models import NewVersionBlock from mayan.apps.common.tests import GenericViewTestCase -from mayan.apps.common.utils import fs_cleanup, mkdtemp from mayan.apps.documents.models import Document, DocumentType from mayan.apps.documents.permissions import permission_document_create from mayan.apps.documents.tests import ( @@ -15,6 +14,7 @@ from mayan.apps.documents.tests import ( TEST_DOCUMENT_DESCRIPTION, TEST_DOCUMENT_TYPE_LABEL, TEST_SMALL_DOCUMENT_CHECKSUM, TEST_SMALL_DOCUMENT_PATH, ) +from mayan.apps.storage.utils import fs_cleanup, mkdtemp from ..links import link_upload_version from ..literals import SOURCE_CHOICE_WEB_FORM, SOURCE_UNCOMPRESS_CHOICE_Y diff --git a/mayan/apps/storage/settings.py b/mayan/apps/storage/settings.py new file mode 100644 index 0000000000..e17acbd76c --- /dev/null +++ b/mayan/apps/storage/settings.py @@ -0,0 +1,17 @@ +from __future__ import unicode_literals + +import tempfile + +from django.utils.translation import ugettext_lazy as _ + +from mayan.apps.smart_settings import Namespace + +namespace = Namespace(label=_('Storage'), name='storage') + +setting_temporary_directory = namespace.add_setting( + global_name='STORAGE_TEMPORARY_DIRECTORY', default=tempfile.gettempdir(), + help_text=_( + 'Temporary directory used site wide to store thumbnails, previews ' + 'and temporary files.' + ) +) diff --git a/mayan/apps/storage/utils.py b/mayan/apps/storage/utils.py new file mode 100644 index 0000000000..da99718149 --- /dev/null +++ b/mayan/apps/storage/utils.py @@ -0,0 +1,114 @@ +from __future__ import unicode_literals + +import logging +import os +import shutil +import tempfile +import types + +from django.conf import settings +from django.urls import resolve as django_resolve +from django.urls.base import get_script_prefix +from django.utils.datastructures import MultiValueDict +from django.utils.http import ( + urlencode as django_urlencode, urlquote as django_urlquote +) +from django.utils.six.moves import reduce as reduce_function, xmlrpc_client +from django.utils.translation import ugettext_lazy as _ + +import mayan +from mayan.apps.common.compat import dict_type, dictionary_type + +from .settings import setting_temporary_directory + +logger = logging.getLogger(__name__) + + +def TemporaryFile(*args, **kwargs): + kwargs.update({'dir': setting_temporary_directory.value}) + return tempfile.TemporaryFile(*args, **kwargs) + + +# http://stackoverflow.com/questions/123198/how-do-i-copy-a-file-in-python +def copyfile(source, destination, buffer_size=1024 * 1024): + """ + Copy a file from source to dest. source and dest + can either be strings or any object with a read or + write method, like StringIO for example. + """ + source_descriptor = get_descriptor(source) + destination_descriptor = get_descriptor(destination, read=False) + + while True: + copy_buffer = source_descriptor.read(buffer_size) + if copy_buffer: + destination_descriptor.write(copy_buffer) + else: + break + + source_descriptor.close() + destination_descriptor.close() + + +def fs_cleanup(filename, file_descriptor=None, suppress_exceptions=True): + """ + Tries to remove the given filename. Ignores non-existent files + """ + if file_descriptor: + os.close(file_descriptor) + + try: + os.remove(filename) + except OSError: + try: + shutil.rmtree(filename) + except OSError: + if suppress_exceptions: + pass + else: + raise + + +def get_descriptor(file_input, read=True): + try: + # Is it a file like object? + file_input.seek(0) + except AttributeError: + # If not, try open it. + if read: + return open(file_input, mode='rb') + else: + return open(file_input, mode='wb') + else: + return file_input + + +def mkdtemp(*args, **kwargs): + kwargs.update({'dir': setting_temporary_directory.value}) + return tempfile.mkdtemp(*args, **kwargs) + + +def mkstemp(*args, **kwargs): + kwargs.update({'dir': setting_temporary_directory.value}) + return tempfile.mkstemp(*args, **kwargs) + + +def validate_path(path): + if not os.path.exists(path): + # If doesn't exist try to create it + try: + os.mkdir(path) + except Exception as exception: + logger.debug('unhandled exception: %s', exception) + return False + + # Check if it is writable + try: + fd, test_filepath = tempfile.mkstemp(dir=path) + os.close(fd) + os.unlink(test_filepath) + except Exception as exception: + logger.debug('unhandled exception: %s', exception) + return False + + return True