Move setting to the storage app

The setting COMMON_TEMPORARY_DIRECTORY is now
STORAGE_TEMPORARY_DIRECTORY.

Move file related utilities to the storage app.

Signed-off-by: Roberto Rosario <roberto.rosario.gonzalez@gmail.com>

HISTORY.rst

@@ -36,6 +36,9 @@
   https://github.com/tesseract-ocr/tesseract/issues/1670
   https://github.com/tesseract-ocr/tesseract/commit/3292484f67af8bdda23aa5e510918d0115785291
   https://gitlab.gnome.org/World/OpenPaperwork/pyocr/issues/104
+* Move setting COMMON_TEMPORARY_DIRECTORY to the storage app.
+  The setting is now STORAGE_TEMPORARY_DIRECTORY.
+* Move file related utilities to the storage app.
 
 3.1.11 (2019-04-XX)
 ===================

docs/releases/3.2.rst

@@ -60,6 +60,9 @@ Other changes
   https://github.com/tesseract-ocr/tesseract/issues/1670
   https://github.com/tesseract-ocr/tesseract/commit/3292484f67af8bdda23aa5e510918d0115785291
   https://gitlab.gnome.org/World/OpenPaperwork/pyocr/issues/104
+* Move setting COMMON_TEMPORARY_DIRECTORY to the storage app.
+  The setting is now STORAGE_TEMPORARY_DIRECTORY.
+* Move file related utilities to the storage app.
 
 
 Removals

mayan/apps/common/management/commands/convertdb.py

@@ -11,8 +11,8 @@ from django.core.management.base import CommandError
 from django.utils.encoding import force_text
 from django.utils.translation import ugettext_lazy as _
 
-from mayan.apps.common.utils import fs_cleanup
 from mayan.apps.documents.models import DocumentType
+from mayan.apps.storage.utils import fs_cleanup
 
 CONVERTDB_FOLDER = 'convertdb'
 CONVERTDB_OUTPUT_FILENAME = 'migrate.json'

mayan/apps/common/settings.py

@@ -80,14 +80,6 @@ setting_shared_storage_arguments = namespace.add_setting(
         os.path.join(settings.MEDIA_ROOT, 'shared_files')
     ), quoted=True
 )
-setting_temporary_directory = namespace.add_setting(
-    global_name='COMMON_TEMPORARY_DIRECTORY', default=tempfile.gettempdir(),
-    help_text=_(
-        'Temporary directory used site wide to store thumbnails, previews '
-        'and temporary files.'
-    ),
-    is_path=True
-)
 
 namespace = Namespace(name='django', label=_('Django'))
 

mayan/apps/common/tests/mixins.py

@@ -18,9 +18,10 @@ from django.template import Context, Template
 from django.test.utils import ContextList
 from django.urls import clear_url_caches, reverse
 
+from mayan.apps.storage.settings import setting_temporary_directory
+
 from .literals import TEST_VIEW_NAME, TEST_VIEW_URL
 
-from ..settings import setting_temporary_directory
 
 if getattr(settings, 'COMMON_TEST_FILE_HANDLES', False):
     import psutil

mayan/apps/common/utils.py

@@ -21,7 +21,6 @@ from mayan.apps.common.compat import dict_type, dictionary_type
 
 from .exceptions import NotLatestVersion, UnknownLatestVersion
 from .literals import DJANGO_SQLITE_BACKEND, MAYAN_PYPI_NAME, PYPI_URL
-from .settings import setting_temporary_directory
 
 logger = logging.getLogger(__name__)
 
@@ -40,27 +39,6 @@ def check_version():
             raise NotLatestVersion(upstream_version=versions[0])
 
 
-# http://stackoverflow.com/questions/123198/how-do-i-copy-a-file-in-python
-def copyfile(source, destination, buffer_size=1024 * 1024):
-    """
-    Copy a file from source to dest. source and dest
-    can either be strings or any object with a read or
-    write method, like StringIO for example.
-    """
-    source_descriptor = get_descriptor(source)
-    destination_descriptor = get_descriptor(destination, read=False)
-
-    while True:
-        copy_buffer = source_descriptor.read(buffer_size)
-        if copy_buffer:
-            destination_descriptor.write(copy_buffer)
-        else:
-            break
-
-    source_descriptor.close()
-    destination_descriptor.close()
-
-
 def encapsulate(function):
     # Workaround Django ticket 15791
     # Changeset 16045
@@ -76,54 +54,6 @@ def get_user_label_text(context):
         return context['request'].user.get_full_name() or context['request'].user
 
 
-def fs_cleanup(filename, file_descriptor=None, suppress_exceptions=True):
-    """
-    Tries to remove the given filename. Ignores non-existent files
-    """
-    if file_descriptor:
-        os.close(file_descriptor)
-
-    try:
-        os.remove(filename)
-    except OSError:
-        try:
-            shutil.rmtree(filename)
-        except OSError:
-            if suppress_exceptions:
-                pass
-            else:
-                raise
-
-
-def get_descriptor(file_input, read=True):
-    try:
-        # Is it a file like object?
-        file_input.seek(0)
-    except AttributeError:
-        # If not, try open it.
-        if read:
-            return open(file_input, mode='rb')
-        else:
-            return open(file_input, mode='wb')
-    else:
-        return file_input
-
-
-def TemporaryFile(*args, **kwargs):
-    kwargs.update({'dir': setting_temporary_directory.value})
-    return tempfile.TemporaryFile(*args, **kwargs)
-
-
-def mkdtemp(*args, **kwargs):
-    kwargs.update({'dir': setting_temporary_directory.value})
-    return tempfile.mkdtemp(*args, **kwargs)
-
-
-def mkstemp(*args, **kwargs):
-    kwargs.update({'dir': setting_temporary_directory.value})
-    return tempfile.mkstemp(*args, **kwargs)
-
-
 def resolve(path, urlconf=None):
     path = '/{}'.format(path.replace(get_script_prefix(), '', 1))
     return django_resolve(path=path, urlconf=urlconf)
@@ -201,24 +131,3 @@ def urlquote(link=None, get=None):
         return '%s%s' % (link, django_urlencode(get, doseq=True))
     else:
         return django_urlquote(link)
-
-
-def validate_path(path):
-    if not os.path.exists(path):
-        # If doesn't exist try to create it
-        try:
-            os.mkdir(path)
-        except Exception as exception:
-            logger.debug('unhandled exception: %s', exception)
-            return False
-
-    # Check if it is writable
-    try:
-        fd, test_filepath = tempfile.mkstemp(dir=path)
-        os.close(fd)
-        os.unlink(test_filepath)
-    except Exception as exception:
-        logger.debug('unhandled exception: %s', exception)
-        return False
-
-    return True

mayan/apps/converter/backends/python.py

@@ -16,7 +16,7 @@ except ImportError:
 from django.utils.encoding import force_text
 from django.utils.translation import ugettext_lazy as _
 
-from mayan.apps.common.utils import fs_cleanup, mkstemp
+from mayan.apps.storage.utils import fs_cleanup, mkstemp
 
 from ..classes import ConverterBase
 from ..exceptions import PageCountError

mayan/apps/converter/classes.py

@@ -16,9 +16,9 @@ except ImportError:
 
 from django.utils.translation import ugettext_lazy as _
 
-from mayan.apps.common.settings import setting_temporary_directory
-from mayan.apps.common.utils import fs_cleanup, mkdtemp, mkstemp
 from mayan.apps.mimetype.api import get_mimetype
+from mayan.apps.storage.settings import setting_temporary_directory
+from mayan.apps.storage.utils import fs_cleanup, mkdtemp, mkstemp
 
 from .exceptions import InvalidOfficeFormat, OfficeConversionError
 from .literals import (

mayan/apps/django_gpg/classes.py

@@ -6,7 +6,7 @@ import shutil
 
 import gnupg
 
-from mayan.apps.common.utils import mkdtemp
+from mayan.apps.storage.utils import mkdtemp
 
 
 class GPGBackend(object):

mayan/apps/django_gpg/managers.py

@@ -6,7 +6,7 @@ import os
 
 from django.db import models
 
-from mayan.apps.common.utils import mkstemp
+from mayan.apps.storage.utils import mkstemp
 
 from .classes import KeyStub, SignatureVerification
 from .exceptions import (

mayan/apps/django_gpg/tests/test_models.py

@@ -8,7 +8,7 @@ import mock
 from django.utils.encoding import force_bytes
 
 from mayan.apps.common.tests import BaseTestCase
-from mayan.apps.common.utils import TemporaryFile
+from mayan.apps.storage.utils import TemporaryFile
 
 from ..exceptions import (
     DecryptionError, KeyDoesNotExist, NeedPassphrase, PassphraseError,

mayan/apps/document_parsing/parsers.py

@@ -7,7 +7,7 @@ import subprocess
 from django.apps import apps
 from django.utils.translation import ugettext_lazy as _
 
-from mayan.apps.common.utils import copyfile, fs_cleanup, mkstemp
+from mayan.apps.storage.utils import copyfile, fs_cleanup, mkstemp
 
 from .exceptions import ParserError
 from .settings import setting_pdftotext_path

mayan/apps/document_signatures/managers.py

@@ -5,10 +5,10 @@ import os
 
 from django.db import models
 
-from mayan.apps.common.utils import mkstemp
 from mayan.apps.django_gpg.exceptions import DecryptionError
 from mayan.apps.django_gpg.models import Key
 from mayan.apps.documents.models import DocumentVersion
+from mayan.apps.storage.utils import mkstemp
 
 logger = logging.getLogger(__name__)
 

mayan/apps/document_signatures/views.py

@@ -16,10 +16,10 @@ from mayan.apps.common.generics import (
     ConfirmView, FormView, SingleObjectCreateView, SingleObjectDeleteView,
     SingleObjectDetailView, SingleObjectDownloadView, SingleObjectListView
 )
-from mayan.apps.common.utils import TemporaryFile
 from mayan.apps.django_gpg.exceptions import NeedPassphrase, PassphraseError
 from mayan.apps.django_gpg.permissions import permission_key_sign
 from mayan.apps.documents.models import DocumentVersion
+from mayan.apps.storage.utils import TemporaryFile
 
 from .forms import (
     DocumentVersionSignatureCreateForm,

mayan/apps/lock_manager/backends/file_lock.py

@@ -12,7 +12,7 @@ from django.conf import settings
 from django.core.files import locks
 from django.utils.encoding import force_bytes, force_text
 
-from mayan.apps.common.settings import setting_temporary_directory
+from mayan.apps.storage.settings import setting_temporary_directory
 
 from ..exceptions import LockError
 from ..settings import setting_default_lock_timeout

mayan/apps/sources/models/scanner_sources.py

@@ -7,7 +7,7 @@ from django.db import models
 from django.utils.timezone import now
 from django.utils.translation import ugettext_lazy as _
 
-from mayan.apps.common.utils import TemporaryFile
+from mayan.apps.storage.utils import TemporaryFile
 
 from ..classes import PseudoFile, SourceUploadedFile
 from ..exceptions import SourceException

mayan/apps/sources/tests/test_classes.py

@@ -4,8 +4,8 @@ import os
 import shutil
 
 from mayan.apps.common.tests import BaseTestCase
-from mayan.apps.common.utils import mkdtemp
 from mayan.apps.documents.tests import TEST_NON_ASCII_DOCUMENT_PATH
+from mayan.apps.storage.utils import mkdtemp
 
 from ..classes import StagingFile
 

mayan/apps/sources/tests/test_models.py

@@ -8,7 +8,6 @@ from pathlib2 import Path
 from django.test import override_settings
 from django.utils.encoding import force_text
 
-from mayan.apps.common.utils import mkdtemp
 from mayan.apps.common.tests import BaseTestCase
 from mayan.apps.documents.models import Document, DocumentType
 from mayan.apps.documents.tests import (
@@ -17,6 +16,7 @@ from mayan.apps.documents.tests import (
     TEST_NON_ASCII_COMPRESSED_DOCUMENT_PATH
 )
 from mayan.apps.metadata.models import MetadataType
+from mayan.apps.storage.utils import mkdtemp
 
 from ..literals import SOURCE_UNCOMPRESS_CHOICE_Y
 from ..models import POP3Email, WatchFolderSource, WebFormSource

mayan/apps/sources/tests/test_views.py

@@ -7,7 +7,6 @@ from django.test import override_settings
 
 from mayan.apps.checkouts.models import NewVersionBlock
 from mayan.apps.common.tests import GenericViewTestCase
-from mayan.apps.common.utils import fs_cleanup, mkdtemp
 from mayan.apps.documents.models import Document, DocumentType
 from mayan.apps.documents.permissions import permission_document_create
 from mayan.apps.documents.tests import (
@@ -15,6 +14,7 @@ from mayan.apps.documents.tests import (
     TEST_DOCUMENT_DESCRIPTION, TEST_DOCUMENT_TYPE_LABEL,
     TEST_SMALL_DOCUMENT_CHECKSUM, TEST_SMALL_DOCUMENT_PATH,
 )
+from mayan.apps.storage.utils import fs_cleanup, mkdtemp
 
 from ..links import link_upload_version
 from ..literals import SOURCE_CHOICE_WEB_FORM, SOURCE_UNCOMPRESS_CHOICE_Y

mayan/apps/storage/settings.py

@@ -0,0 +1,17 @@
+from __future__ import unicode_literals
+
+import tempfile
+
+from django.utils.translation import ugettext_lazy as _
+
+from mayan.apps.smart_settings import Namespace
+
+namespace = Namespace(label=_('Storage'), name='storage')
+
+setting_temporary_directory = namespace.add_setting(
+    global_name='STORAGE_TEMPORARY_DIRECTORY', default=tempfile.gettempdir(),
+    help_text=_(
+        'Temporary directory used site wide to store thumbnails, previews '
+        'and temporary files.'
+    )
+)

mayan/apps/storage/utils.py

@@ -0,0 +1,114 @@
+from __future__ import unicode_literals
+
+import logging
+import os
+import shutil
+import tempfile
+import types
+
+from django.conf import settings
+from django.urls import resolve as django_resolve
+from django.urls.base import get_script_prefix
+from django.utils.datastructures import MultiValueDict
+from django.utils.http import (
+    urlencode as django_urlencode, urlquote as django_urlquote
+)
+from django.utils.six.moves import reduce as reduce_function, xmlrpc_client
+from django.utils.translation import ugettext_lazy as _
+
+import mayan
+from mayan.apps.common.compat import dict_type, dictionary_type
+
+from .settings import setting_temporary_directory
+
+logger = logging.getLogger(__name__)
+
+
+def TemporaryFile(*args, **kwargs):
+    kwargs.update({'dir': setting_temporary_directory.value})
+    return tempfile.TemporaryFile(*args, **kwargs)
+
+
+# http://stackoverflow.com/questions/123198/how-do-i-copy-a-file-in-python
+def copyfile(source, destination, buffer_size=1024 * 1024):
+    """
+    Copy a file from source to dest. source and dest
+    can either be strings or any object with a read or
+    write method, like StringIO for example.
+    """
+    source_descriptor = get_descriptor(source)
+    destination_descriptor = get_descriptor(destination, read=False)
+
+    while True:
+        copy_buffer = source_descriptor.read(buffer_size)
+        if copy_buffer:
+            destination_descriptor.write(copy_buffer)
+        else:
+            break
+
+    source_descriptor.close()
+    destination_descriptor.close()
+
+
+def fs_cleanup(filename, file_descriptor=None, suppress_exceptions=True):
+    """
+    Tries to remove the given filename. Ignores non-existent files
+    """
+    if file_descriptor:
+        os.close(file_descriptor)
+
+    try:
+        os.remove(filename)
+    except OSError:
+        try:
+            shutil.rmtree(filename)
+        except OSError:
+            if suppress_exceptions:
+                pass
+            else:
+                raise
+
+
+def get_descriptor(file_input, read=True):
+    try:
+        # Is it a file like object?
+        file_input.seek(0)
+    except AttributeError:
+        # If not, try open it.
+        if read:
+            return open(file_input, mode='rb')
+        else:
+            return open(file_input, mode='wb')
+    else:
+        return file_input
+
+
+def mkdtemp(*args, **kwargs):
+    kwargs.update({'dir': setting_temporary_directory.value})
+    return tempfile.mkdtemp(*args, **kwargs)
+
+
+def mkstemp(*args, **kwargs):
+    kwargs.update({'dir': setting_temporary_directory.value})
+    return tempfile.mkstemp(*args, **kwargs)
+
+
+def validate_path(path):
+    if not os.path.exists(path):
+        # If doesn't exist try to create it
+        try:
+            os.mkdir(path)
+        except Exception as exception:
+            logger.debug('unhandled exception: %s', exception)
+            return False
+
+    # Check if it is writable
+    try:
+        fd, test_filepath = tempfile.mkstemp(dir=path)
+        os.close(fd)
+        os.unlink(test_filepath)
+    except Exception as exception:
+        logger.debug('unhandled exception: %s', exception)
+        return False
+
+    return True