Merge branch 'master' into feature/merge2

HISTORY.rst

@@ -1,3 +1,11 @@
+2.1.3 (2016-XX-XX)
+- Add help message when initialsetup migration phase fails. Relates to GitLab issue #296.
+- Start using self.setdout instead of print as per documentation.
+- Fix GitLab issue #295, "When editing a user the top bar jumps to the name of the user".
+- Normalize handling of temporary file and directory creation.
+- Fix GitLab issue #309, "Temp files quickly filling-up my /tmp (1GB tmpfs)".
+- Explicitly check for residual temporary files in tests.
+
 2.1.2 (2016-05-20)
 ==================
 - Sort document languages and user profile locale language lists. GitLab issue #292.

mayan/apps/common/settings.py

@@ -11,8 +11,7 @@ setting_temporary_directory = namespace.add_setting(
     global_name='COMMON_TEMPORARY_DIRECTORY', default=tempfile.gettempdir(),
     help_text=_(
         'Temporary directory used site wide to store thumbnails, previews '
-        'and temporary files.  If none is specified, one will be created '
-        'using tempfile.mkdtemp().'
+        'and temporary files.'
     ),
     is_path=True
 )

mayan/apps/common/tests/test_views.py

@@ -1,5 +1,7 @@
 from __future__ import absolute_import, unicode_literals
 
+import os
+
 from django.conf.urls import url
 from django.contrib.auth import get_user_model
 from django.core.urlresolvers import clear_url_caches, reverse
@@ -17,12 +19,16 @@ from user_management.tests import (
     TEST_USER_EMAIL, TEST_USER_USERNAME, TEST_USER_PASSWORD
 )
 
+from ..settings import setting_temporary_directory
+
 from .literals import TEST_VIEW_NAME, TEST_VIEW_URL
 
 
 class GenericViewTestCase(OrganizationTestCase):
     def setUp(self):
         super(GenericViewTestCase, self).setUp()
+        self.temp_items = len(os.listdir(setting_temporary_directory.value))
+
         self.has_test_view = False
         self.admin_user = get_user_model().on_organization.create_superuser(
             username=TEST_ADMIN_USERNAME, email=TEST_ADMIN_EMAIL,
@@ -47,6 +53,10 @@ class GenericViewTestCase(OrganizationTestCase):
         if self.has_test_view:
             urlpatterns.pop(0)
 
+        self.assertEqual(
+            self.temp_items, len(os.listdir(setting_temporary_directory.value))
+        )
+
         super(GenericViewTestCase, self).tearDown()
 
     def add_test_view(self, test_object):

mayan/apps/common/utils.py

@@ -2,6 +2,7 @@ from __future__ import unicode_literals
 
 import logging
 import os
+import shutil
 import tempfile
 import types
 
@@ -10,6 +11,8 @@ from django.utils.datastructures import MultiValueDict
 from django.utils.http import urlquote as django_urlquote
 from django.utils.http import urlencode as django_urlencode
 
+from .settings import setting_temporary_directory
+
 logger = logging.getLogger(__name__)
 
 
@@ -49,10 +52,13 @@ def fs_cleanup(filename, suppress_exceptions=True):
     try:
         os.remove(filename)
     except OSError:
-        if suppress_exceptions:
-            pass
-        else:
-            raise
+        try:
+            shutil.rmtree(filename)
+        except OSError:
+            if suppress_exceptions:
+                pass
+            else:
+                raise
 
 
 def get_descriptor(file_input, read=True):
@@ -69,6 +75,21 @@ def get_descriptor(file_input, read=True):
         return file_input
 
 
+def TemporaryFile(*args, **kwargs):
+    kwargs.update({'dir': setting_temporary_directory.value})
+    return tempfile.TemporaryFile(*args, **kwargs)
+
+
+def mkdtemp(*args, **kwargs):
+    kwargs.update({'dir': setting_temporary_directory.value})
+    return tempfile.mkdtemp(*args, **kwargs)
+
+
+def mkstemp(*args, **kwargs):
+    kwargs.update({'dir': setting_temporary_directory.value})
+    return tempfile.mkstemp(*args, **kwargs)
+
+
 def return_attrib(obj, attrib, arguments=None):
     try:
         if isinstance(attrib, types.FunctionType):

mayan/apps/converter/backends/python.py

@@ -3,7 +3,6 @@ from __future__ import unicode_literals
 import io
 import logging
 import os
-import tempfile
 
 try:
     from cStringIO import StringIO
@@ -16,7 +15,7 @@ import sh
 
 from django.utils.translation import ugettext_lazy as _
 
-from common.utils import fs_cleanup
+from common.utils import fs_cleanup, mkstemp
 
 from ..classes import ConverterBase
 from ..exceptions import PageCountError
@@ -50,7 +49,7 @@ class Python(ConverterBase):
 
         if self.mime_type == 'application/pdf' and pdftoppm:
 
-            new_file_object, input_filepath = tempfile.mkstemp()
+            new_file_object, input_filepath = mkstemp()
             self.file_object.seek(0)
             os.write(new_file_object, self.file_object.read())
             self.file_object.seek(0)

mayan/apps/converter/classes.py

@@ -3,7 +3,6 @@ from __future__ import unicode_literals
 import base64
 import logging
 import os
-import tempfile
 
 try:
     from cStringIO import StringIO
@@ -16,7 +15,7 @@ import sh
 from django.utils.translation import string_concat, ugettext_lazy as _
 
 from common.settings import setting_temporary_directory
-from common.utils import fs_cleanup
+from common.utils import fs_cleanup, mkstemp
 from mimetype.api import get_mimetype
 
 from .exceptions import InvalidOfficeFormat, OfficeConversionError
@@ -122,7 +121,7 @@ class ConverterBase(object):
                 ) % setting_libreoffice_path.value
             )
 
-        new_file_object, input_filepath = tempfile.mkstemp()
+        new_file_object, input_filepath = mkstemp()
         self.file_object.seek(0)
         os.write(new_file_object, self.file_object.read())
         self.file_object.seek(0)

mayan/apps/django_gpg/managers.py

@@ -4,12 +4,13 @@ import io
 import logging
 import os
 import shutil
-import tempfile
 
 import gnupg
 
 from django.db import models
 
+from common.utils import mkdtemp, mkstemp
+
 from .classes import KeyStub, SignatureVerification
 from .exceptions import (
     DecryptionError, KeyDoesNotExist, KeyFetchingError, VerificationError
@@ -22,7 +23,7 @@ logger = logging.getLogger(__name__)
 
 class KeyManager(models.Manager):
     def decrypt_file(self, file_object, all_keys=False, key_fingerprint=None, key_id=None):
-        temporary_directory = tempfile.mkdtemp()
+        temporary_directory = mkdtemp()
 
         os.chmod(temporary_directory, 0x1C0)
 
@@ -71,7 +72,7 @@ class KeyManager(models.Manager):
         return io.BytesIO(decrypt_result.data)
 
     def receive_key(self, key_id):
-        temporary_directory = tempfile.mkdtemp()
+        temporary_directory = mkdtemp()
 
         os.chmod(temporary_directory, 0x1C0)
 
@@ -92,7 +93,7 @@ class KeyManager(models.Manager):
             return self.create(key_data=key_data)
 
     def search(self, query):
-        temporary_directory = tempfile.mkdtemp()
+        temporary_directory = mkdtemp()
 
         os.chmod(temporary_directory, 0x1C0)
 
@@ -118,7 +119,7 @@ class KeyManager(models.Manager):
         return self.filter(key_type=KEY_TYPE_SECRET)
 
     def verify_file(self, file_object, signature_file=None, all_keys=False, key_fingerprint=None, key_id=None):
-        temporary_directory = tempfile.mkdtemp()
+        temporary_directory = mkdtemp()
 
         os.chmod(temporary_directory, 0x1C0)
 
@@ -156,7 +157,7 @@ class KeyManager(models.Manager):
         if signature_file:
             # Save the original data and invert the argument order
             # Signature first, file second
-            temporary_file_object, temporary_filename = tempfile.mkstemp()
+            temporary_file_object, temporary_filename = mkstemp()
             os.write(temporary_file_object, file_object.read())
             os.close(temporary_file_object)
 

mayan/apps/django_gpg/models.py

@@ -4,7 +4,6 @@ from datetime import date
 import logging
 import os
 import shutil
-import tempfile
 
 import gnupg
 
@@ -14,6 +13,8 @@ from django.db import models
 from django.utils.encoding import python_2_unicode_compatible
 from django.utils.translation import ugettext_lazy as _
 
+from common.utils import mkdtemp
+
 from .exceptions import NeedPassphrase, PassphraseError
 from .literals import (
     ERROR_MSG_NEED_PASSPHRASE, ERROR_MSG_BAD_PASSPHRASE,
@@ -27,7 +28,7 @@ logger = logging.getLogger(__name__)
 
 
 def gpg_command(function):
-    temporary_directory = tempfile.mkdtemp()
+    temporary_directory = mkdtemp()
     os.chmod(temporary_directory, 0x1C0)
 
     gpg = gnupg.GPG(
@@ -92,7 +93,7 @@ class Key(models.Model):
         return reverse('django_gpg:key_detail', args=(self.pk,))
 
     def save(self, *args, **kwargs):
-        temporary_directory = tempfile.mkdtemp()
+        temporary_directory = mkdtemp()
 
         os.chmod(temporary_directory, 0x1C0)
 
@@ -133,7 +134,7 @@ class Key(models.Model):
         # file, and appear to be due to random data being inserted in the
         # output data stream."
 
-        temporary_directory = tempfile.mkdtemp()
+        temporary_directory = mkdtemp()
 
         os.chmod(temporary_directory, 0x1C0)
 

mayan/apps/django_gpg/tests/test_models.py

@@ -1,13 +1,14 @@
 from __future__ import unicode_literals
 
 import StringIO
-import tempfile
 
 import gnupg
 import mock
 
 from django.test import TestCase
 
+from common.utils import TemporaryFile
+
 from ..exceptions import (
     DecryptionError, KeyDoesNotExist, NeedPassphrase, PassphraseError,
     VerificationError
@@ -74,7 +75,7 @@ class KeyTestCase(TestCase):
         )
 
     def test_cleartext_file_verification(self):
-        cleartext_file = tempfile.TemporaryFile()
+        cleartext_file = TemporaryFile()
         cleartext_file.write('test')
         cleartext_file.seek(0)
 
@@ -124,7 +125,7 @@ class KeyTestCase(TestCase):
         self.assertEqual(result.read(), TEST_SIGNED_FILE_CONTENT)
 
     def test_cleartext_file_decryption(self):
-        cleartext_file = tempfile.TemporaryFile()
+        cleartext_file = TemporaryFile()
         cleartext_file.write('test')
         cleartext_file.seek(0)
 

mayan/apps/document_signatures/managers.py

@@ -2,10 +2,10 @@ from __future__ import unicode_literals
 
 import logging
 import os
-import tempfile
 
 from django.db import models
 
+from common.utils import mkstemp
 from django_gpg.exceptions import DecryptionError
 from django_gpg.models import Key
 from documents.models import DocumentVersion
@@ -34,7 +34,7 @@ class EmbeddedSignatureManager(models.Manager):
         )
 
     def sign_document_version(self, document_version, key, passphrase=None, user=None):
-        temporary_file_object, temporary_filename = tempfile.mkstemp()
+        temporary_file_object, temporary_filename = mkstemp()
 
         try:
             with document_version.open() as file_object:

mayan/apps/document_signatures/views.py

@@ -1,6 +1,5 @@
 from __future__ import absolute_import, unicode_literals
 
-import tempfile
 import logging
 
 from django.contrib import messages
@@ -16,6 +15,7 @@ from common.generics import (
     ConfirmView, FormView, SingleObjectCreateView, SingleObjectDeleteView,
     SingleObjectDetailView, SingleObjectDownloadView, SingleObjectListView
 )
+from common.utils import TemporaryFile
 from django_gpg.exceptions import NeedPassphrase, PassphraseError
 from django_gpg.permissions import permission_key_sign
 from documents.models import DocumentVersion
@@ -83,7 +83,7 @@ class DocumentVersionDetachedSignatureCreateView(FormView):
                 )
             )
         else:
-            temporary_file_object = tempfile.TemporaryFile()
+            temporary_file_object = TemporaryFile()
             temporary_file_object.write(detached_signature.data)
             temporary_file_object.seek(0)
 
@@ -188,7 +188,7 @@ class DocumentVersionEmbeddedSignatureCreateView(FormView):
                 )
             )
         else:
-            temporary_file_object = tempfile.TemporaryFile()
+            temporary_file_object = TemporaryFile()
             temporary_file_object.write(signature_result.data)
             temporary_file_object.seek(0)
 

mayan/apps/ocr/parsers.py

@@ -8,12 +8,11 @@ from pdfminer.pdfpage import PDFPage
 from pdfminer.converter import TextConverter
 from pdfminer.layout import LAParams
 import subprocess
-import tempfile
 
 from django.utils.translation import ugettext_lazy as _
 
 from common.settings import setting_temporary_directory
-from common.utils import copyfile
+from common.utils import copyfile, fs_cleanup, mkstemp
 
 from .exceptions import ParserError, NoMIMETypeMatch
 from .models import DocumentPageContent
@@ -137,9 +136,7 @@ class PopplerParser(Parser):
     def execute(self, file_object, page_number):
         logger.debug('Parsing PDF page: %d', page_number)
 
-        destination_descriptor, temp_filepath = tempfile.mkstemp(
-            dir=setting_temporary_directory.value
-        )
+        destination_descriptor, temp_filepath = mkstemp()
         copyfile(file_object, temp_filepath)
 
         command = []
@@ -158,9 +155,12 @@ class PopplerParser(Parser):
         return_code = proc.wait()
         if return_code != 0:
             logger.error(proc.stderr.readline())
+            fs_cleanup(temp_filepath)
+
             raise ParserError
 
         output = proc.stdout.read()
+        fs_cleanup(temp_filepath)
 
         if output == b'\x0c':
             logger.debug('Parser didn\'t return any output')

mayan/apps/ocr/tests/test_parsers.py

@@ -2,6 +2,7 @@ from __future__ import unicode_literals
 
 from django.test import override_settings
 
+from common.settings import setting_temporary_directory
 from documents.models import DocumentType
 from documents.tests import (
     TEST_DOCUMENT_PATH, TEST_DOCUMENT_TYPE, TEST_HYBRID_DOCUMENT_PATH
@@ -16,7 +17,6 @@ from ..parsers import PDFMinerParser, PopplerParser
 class ParserTestCase(OrganizationTestCase):
     def setUp(self):
         super(ParserTestCase, self).setUp()
-
         self.document_type = DocumentType.on_organization.create(
             label=TEST_DOCUMENT_TYPE
         )

mayan/apps/sources/tests/test_classes.py

@@ -2,18 +2,18 @@ from __future__ import unicode_literals
 
 import os
 import shutil
-import tempfile
 
 from django.test import TestCase
 
 from documents.tests import TEST_NON_ASCII_DOCUMENT_PATH
+from common.utils import mkdtemp
 
 from ..classes import StagingFile
 
 
 class StagingFileTestCase(TestCase):
     def test_unicode_staging_file(self):
-        temporary_directory = tempfile.mkdtemp()
+        temporary_directory = mkdtemp()
         shutil.copy(TEST_NON_ASCII_DOCUMENT_PATH, temporary_directory)
 
         filename = os.path.basename(TEST_NON_ASCII_DOCUMENT_PATH)

mayan/apps/sources/tests/test_models.py

@@ -1,13 +1,13 @@
 from __future__ import unicode_literals
 
 import shutil
-import tempfile
 
 from django.contrib.auth import get_user_model
 from django.core.files.base import File
 from django.test import TestCase, override_settings
 from django.test.client import Client
 
+from common.utils import mkdtemp
 from documents.models import Document, DocumentType
 from documents.tests import (
     TEST_COMPRESSED_DOCUMENT_PATH, TEST_DOCUMENT_TYPE,
@@ -55,7 +55,7 @@ class UploadDocumentTestCase(TestCase):
         gh-issue #163 https://github.com/mayan-edms/mayan-edms/issues/163
         """
 
-        temporary_directory = tempfile.mkdtemp()
+        temporary_directory = mkdtemp()
         shutil.copy(TEST_NON_ASCII_DOCUMENT_PATH, temporary_directory)
 
         watch_folder = WatchFolderSource.on_organization.create(

mayan/apps/sources/tests/test_views.py

@@ -2,7 +2,6 @@ from __future__ import unicode_literals
 
 import os
 import shutil
-import tempfile
 
 from django.contrib.auth import get_user_model
 from django.core.urlresolvers import reverse
@@ -11,6 +10,7 @@ from django.test import TestCase, override_settings
 
 from acls.models import AccessControlList
 from common.tests.test_views import GenericViewTestCase
+from common.utils import fs_cleanup, mkdtemp
 from documents.models import Document, DocumentType, NewVersionBlock
 from documents.permissions import permission_document_create
 from documents.tests import (
@@ -230,15 +230,14 @@ class NewDocumentVersionViewTestCase(GenericDocumentViewTestCase):
 class StagingFolderTestCase(GenericViewTestCase):
     def setUp(self):
         super(StagingFolderTestCase, self).setUp()
-        self.temporary_directory = tempfile.mkdtemp()
-        # TODO: remove temp directory after test
+        self.temporary_directory = mkdtemp()
         shutil.copy(TEST_SMALL_DOCUMENT_PATH, self.temporary_directory)
 
         self.filename = os.path.basename(TEST_SMALL_DOCUMENT_PATH)
 
     def tearDown(self):
+        fs_cleanup(self.temporary_directory)
         super(StagingFolderTestCase, self).tearDown()
-        shutil.rmtree(self.temporary_directory)
 
     def test_staging_folder_delete_no_permission(self):
         self.login(