diff --git a/mayan/apps/common/utils.py b/mayan/apps/common/utils.py index fa82b89ce5..8fe68a21b7 100644 --- a/mayan/apps/common/utils.py +++ b/mayan/apps/common/utils.py @@ -10,6 +10,8 @@ from django.utils.datastructures import MultiValueDict from django.utils.http import urlquote as django_urlquote from django.utils.http import urlencode as django_urlencode +from .settings import setting_temporary_directory + logger = logging.getLogger(__name__) @@ -69,6 +71,21 @@ def get_descriptor(file_input, read=True): return file_input +def TemporaryFile(*args, **kwargs): + kwargs.update({'dir': setting_temporary_directory.value}) + return tempfile.TemporaryFile(*args, **kwargs) + + +def mkdtemp(*args, **kwargs): + kwargs.update({'dir': setting_temporary_directory.value}) + return tempfile.mkdtemp(*args, **kwargs) + + +def mkstemp(*args, **kwargs): + kwargs.update({'dir': setting_temporary_directory.value}) + return tempfile.mkstemp(*args, **kwargs) + + def return_attrib(obj, attrib, arguments=None): try: if isinstance(attrib, types.FunctionType): diff --git a/mayan/apps/converter/backends/python.py b/mayan/apps/converter/backends/python.py index 8d37a4265a..5ef3b216f9 100644 --- a/mayan/apps/converter/backends/python.py +++ b/mayan/apps/converter/backends/python.py @@ -3,7 +3,6 @@ from __future__ import unicode_literals import io import logging import os -import tempfile try: from cStringIO import StringIO @@ -16,7 +15,7 @@ import sh from django.utils.translation import ugettext_lazy as _ -from common.utils import fs_cleanup +from common.utils import fs_cleanup, mkstemp from ..classes import ConverterBase from ..exceptions import PageCountError @@ -50,7 +49,7 @@ class Python(ConverterBase): if self.mime_type == 'application/pdf' and pdftoppm: - new_file_object, input_filepath = tempfile.mkstemp() + new_file_object, input_filepath = mkstemp() self.file_object.seek(0) os.write(new_file_object, self.file_object.read()) self.file_object.seek(0) diff --git a/mayan/apps/converter/classes.py b/mayan/apps/converter/classes.py index a7024ebdc3..69bdf5b6bf 100644 --- a/mayan/apps/converter/classes.py +++ b/mayan/apps/converter/classes.py @@ -3,7 +3,6 @@ from __future__ import unicode_literals import base64 import logging import os -import tempfile try: from cStringIO import StringIO @@ -16,7 +15,7 @@ import sh from django.utils.translation import string_concat, ugettext_lazy as _ from common.settings import setting_temporary_directory -from common.utils import fs_cleanup +from common.utils import fs_cleanup, mkstemp from mimetype.api import get_mimetype from .exceptions import InvalidOfficeFormat, OfficeConversionError @@ -122,7 +121,7 @@ class ConverterBase(object): ) % setting_libreoffice_path.value ) - new_file_object, input_filepath = tempfile.mkstemp() + new_file_object, input_filepath = mkstemp() self.file_object.seek(0) os.write(new_file_object, self.file_object.read()) self.file_object.seek(0) diff --git a/mayan/apps/django_gpg/managers.py b/mayan/apps/django_gpg/managers.py index 1542dfa940..4247529e49 100644 --- a/mayan/apps/django_gpg/managers.py +++ b/mayan/apps/django_gpg/managers.py @@ -4,12 +4,13 @@ import io import logging import os import shutil -import tempfile import gnupg from django.db import models +from common.utils import mkdtemp, mkstemp + from .classes import KeyStub, SignatureVerification from .exceptions import ( DecryptionError, KeyDoesNotExist, KeyFetchingError, VerificationError @@ -22,7 +23,7 @@ logger = logging.getLogger(__name__) class KeyManager(models.Manager): def decrypt_file(self, file_object, all_keys=False, key_fingerprint=None, key_id=None): - temporary_directory = tempfile.mkdtemp() + temporary_directory = mkdtemp() os.chmod(temporary_directory, 0x1C0) @@ -71,7 +72,7 @@ class KeyManager(models.Manager): return io.BytesIO(decrypt_result.data) def receive_key(self, key_id): - temporary_directory = tempfile.mkdtemp() + temporary_directory = mkdtemp() os.chmod(temporary_directory, 0x1C0) @@ -92,7 +93,7 @@ class KeyManager(models.Manager): return self.create(key_data=key_data) def search(self, query): - temporary_directory = tempfile.mkdtemp() + temporary_directory = mkdtemp() os.chmod(temporary_directory, 0x1C0) @@ -118,7 +119,7 @@ class KeyManager(models.Manager): return self.filter(key_type=KEY_TYPE_SECRET) def verify_file(self, file_object, signature_file=None, all_keys=False, key_fingerprint=None, key_id=None): - temporary_directory = tempfile.mkdtemp() + temporary_directory = mkdtemp() os.chmod(temporary_directory, 0x1C0) @@ -156,7 +157,7 @@ class KeyManager(models.Manager): if signature_file: # Save the original data and invert the argument order # Signature first, file second - temporary_file_object, temporary_filename = tempfile.mkstemp() + temporary_file_object, temporary_filename = mkstemp() os.write(temporary_file_object, file_object.read()) os.close(temporary_file_object) diff --git a/mayan/apps/django_gpg/models.py b/mayan/apps/django_gpg/models.py index a4f1b814cb..075d11e249 100644 --- a/mayan/apps/django_gpg/models.py +++ b/mayan/apps/django_gpg/models.py @@ -4,7 +4,6 @@ from datetime import date import logging import os import shutil -import tempfile import gnupg @@ -14,6 +13,8 @@ from django.db import models from django.utils.encoding import python_2_unicode_compatible from django.utils.translation import ugettext_lazy as _ +from common.utils import mkdtemp + from .exceptions import NeedPassphrase, PassphraseError from .literals import ( ERROR_MSG_NEED_PASSPHRASE, ERROR_MSG_BAD_PASSPHRASE, @@ -27,7 +28,7 @@ logger = logging.getLogger(__name__) def gpg_command(function): - temporary_directory = tempfile.mkdtemp() + temporary_directory = mkdtemp() os.chmod(temporary_directory, 0x1C0) gpg = gnupg.GPG( @@ -92,7 +93,7 @@ class Key(models.Model): return reverse('django_gpg:key_detail', args=(self.pk,)) def save(self, *args, **kwargs): - temporary_directory = tempfile.mkdtemp() + temporary_directory = mkdtemp() os.chmod(temporary_directory, 0x1C0) @@ -133,7 +134,7 @@ class Key(models.Model): # file, and appear to be due to random data being inserted in the # output data stream." - temporary_directory = tempfile.mkdtemp() + temporary_directory = mkdtemp() os.chmod(temporary_directory, 0x1C0) diff --git a/mayan/apps/django_gpg/tests/test_models.py b/mayan/apps/django_gpg/tests/test_models.py index bcec133f40..019fe4ba7c 100644 --- a/mayan/apps/django_gpg/tests/test_models.py +++ b/mayan/apps/django_gpg/tests/test_models.py @@ -1,13 +1,14 @@ from __future__ import unicode_literals import StringIO -import tempfile import gnupg import mock from django.test import TestCase +from common.utils import TemporaryFile + from ..exceptions import ( DecryptionError, KeyDoesNotExist, NeedPassphrase, PassphraseError, VerificationError @@ -74,7 +75,7 @@ class KeyTestCase(TestCase): ) def test_cleartext_file_verification(self): - cleartext_file = tempfile.TemporaryFile() + cleartext_file = TemporaryFile() cleartext_file.write('test') cleartext_file.seek(0) @@ -124,7 +125,7 @@ class KeyTestCase(TestCase): self.assertEqual(result.read(), TEST_SIGNED_FILE_CONTENT) def test_cleartext_file_decryption(self): - cleartext_file = tempfile.TemporaryFile() + cleartext_file = TemporaryFile() cleartext_file.write('test') cleartext_file.seek(0) diff --git a/mayan/apps/document_signatures/managers.py b/mayan/apps/document_signatures/managers.py index 8601473589..6224cc40be 100644 --- a/mayan/apps/document_signatures/managers.py +++ b/mayan/apps/document_signatures/managers.py @@ -2,10 +2,10 @@ from __future__ import unicode_literals import logging import os -import tempfile from django.db import models +from common.utils import mkstemp from django_gpg.exceptions import DecryptionError from django_gpg.models import Key from documents.models import DocumentVersion @@ -34,7 +34,7 @@ class EmbeddedSignatureManager(models.Manager): ) def sign_document_version(self, document_version, key, passphrase=None, user=None): - temporary_file_object, temporary_filename = tempfile.mkstemp() + temporary_file_object, temporary_filename = mkstemp() try: with document_version.open() as file_object: diff --git a/mayan/apps/document_signatures/views.py b/mayan/apps/document_signatures/views.py index 8e483062f6..3057086c44 100644 --- a/mayan/apps/document_signatures/views.py +++ b/mayan/apps/document_signatures/views.py @@ -1,6 +1,5 @@ from __future__ import absolute_import, unicode_literals -import tempfile import logging from django.contrib import messages @@ -16,6 +15,7 @@ from common.generics import ( ConfirmView, FormView, SingleObjectCreateView, SingleObjectDeleteView, SingleObjectDetailView, SingleObjectDownloadView, SingleObjectListView ) +from common.utils import TemporaryFile from django_gpg.exceptions import NeedPassphrase, PassphraseError from django_gpg.permissions import permission_key_sign from documents.models import DocumentVersion @@ -83,7 +83,7 @@ class DocumentVersionDetachedSignatureCreateView(FormView): ) ) else: - temporary_file_object = tempfile.TemporaryFile() + temporary_file_object = TemporaryFile() temporary_file_object.write(detached_signature.data) temporary_file_object.seek(0) @@ -188,7 +188,7 @@ class DocumentVersionEmbeddedSignatureCreateView(FormView): ) ) else: - temporary_file_object = tempfile.TemporaryFile() + temporary_file_object = TemporaryFile() temporary_file_object.write(signature_result.data) temporary_file_object.seek(0) diff --git a/mayan/apps/ocr/parsers.py b/mayan/apps/ocr/parsers.py index bc21031037..4a47422d21 100644 --- a/mayan/apps/ocr/parsers.py +++ b/mayan/apps/ocr/parsers.py @@ -8,12 +8,11 @@ from pdfminer.pdfpage import PDFPage from pdfminer.converter import TextConverter from pdfminer.layout import LAParams import subprocess -import tempfile from django.utils.translation import ugettext_lazy as _ from common.settings import setting_temporary_directory -from common.utils import copyfile +from common.utils import copyfile, mkstemp from .exceptions import ParserError, NoMIMETypeMatch from .models import DocumentPageContent @@ -137,9 +136,7 @@ class PopplerParser(Parser): def execute(self, file_object, page_number): logger.debug('Parsing PDF page: %d', page_number) - destination_descriptor, temp_filepath = tempfile.mkstemp( - dir=setting_temporary_directory.value - ) + destination_descriptor, temp_filepath = mkstemp() copyfile(file_object, temp_filepath) command = [] @@ -158,6 +155,7 @@ class PopplerParser(Parser): return_code = proc.wait() if return_code != 0: logger.error(proc.stderr.readline()) + raise ParserError output = proc.stdout.read() diff --git a/mayan/apps/ocr/tests/test_parsers.py b/mayan/apps/ocr/tests/test_parsers.py index ff2ddb629f..f11c6ec12c 100644 --- a/mayan/apps/ocr/tests/test_parsers.py +++ b/mayan/apps/ocr/tests/test_parsers.py @@ -1,8 +1,11 @@ from __future__ import unicode_literals +import os + from django.core.files.base import File from django.test import TestCase, override_settings +from common.settings import setting_temporary_directory from documents.models import DocumentType from documents.tests import ( TEST_DOCUMENT_PATH, TEST_DOCUMENT_TYPE, TEST_HYBRID_DOCUMENT_PATH @@ -15,7 +18,6 @@ from ..parsers import PDFMinerParser, PopplerParser @override_settings(OCR_AUTO_OCR=False) class ParserTestCase(TestCase): def setUp(self): - self.document_type = DocumentType.objects.create( label=TEST_DOCUMENT_TYPE ) @@ -46,6 +48,17 @@ class ParserTestCase(TestCase): 'Mayan EDMS Documentation' in self.document.pages.first().ocr_content.content ) + def test_poppler_parser_cleanup(self): + temp_items = len(os.listdir(setting_temporary_directory.value)) + + parser = PopplerParser() + + parser.process_document_version(self.document.latest_version) + + self.assertEqual( + temp_items, len(os.listdir(setting_temporary_directory.value)) + ) + @override_settings(OCR_AUTO_OCR=False) class TextExtractorTestCase(TestCase): diff --git a/mayan/apps/sources/tests/test_classes.py b/mayan/apps/sources/tests/test_classes.py index e3ae3395a2..3406eed8c8 100644 --- a/mayan/apps/sources/tests/test_classes.py +++ b/mayan/apps/sources/tests/test_classes.py @@ -2,18 +2,18 @@ from __future__ import unicode_literals import os import shutil -import tempfile from django.test import TestCase from documents.tests import TEST_NON_ASCII_DOCUMENT_PATH +from common.utils import mkdtemp from ..classes import StagingFile class StagingFileTestCase(TestCase): def test_unicode_staging_file(self): - temporary_directory = tempfile.mkdtemp() + temporary_directory = mkdtemp() shutil.copy(TEST_NON_ASCII_DOCUMENT_PATH, temporary_directory) filename = os.path.basename(TEST_NON_ASCII_DOCUMENT_PATH) diff --git a/mayan/apps/sources/tests/test_models.py b/mayan/apps/sources/tests/test_models.py index d5713455b6..6c8fc94b67 100644 --- a/mayan/apps/sources/tests/test_models.py +++ b/mayan/apps/sources/tests/test_models.py @@ -1,13 +1,13 @@ from __future__ import unicode_literals import shutil -import tempfile from django.contrib.auth import get_user_model from django.core.files.base import File from django.test import TestCase, override_settings from django.test.client import Client +from common.utils import mkdtemp from documents.models import Document, DocumentType from documents.tests import ( TEST_COMPRESSED_DOCUMENT_PATH, TEST_DOCUMENT_TYPE, @@ -49,7 +49,7 @@ class UploadDocumentTestCase(TestCase): gh-issue #163 https://github.com/mayan-edms/mayan-edms/issues/163 """ - temporary_directory = tempfile.mkdtemp() + temporary_directory = mkdtemp() shutil.copy(TEST_NON_ASCII_DOCUMENT_PATH, temporary_directory) watch_folder = WatchFolderSource.objects.create( diff --git a/mayan/apps/sources/tests/test_views.py b/mayan/apps/sources/tests/test_views.py index 2ad5038f77..97a49d05ac 100644 --- a/mayan/apps/sources/tests/test_views.py +++ b/mayan/apps/sources/tests/test_views.py @@ -2,7 +2,6 @@ from __future__ import unicode_literals import os import shutil -import tempfile from django.contrib.auth import get_user_model from django.core.urlresolvers import reverse @@ -11,6 +10,7 @@ from django.test import TestCase, override_settings from acls.models import AccessControlList from common.tests.test_views import GenericViewTestCase +from common.utils import mkdtemp from documents.models import Document, DocumentType, NewVersionBlock from documents.permissions import permission_document_create from documents.tests import ( @@ -225,7 +225,7 @@ class NewDocumentVersionViewTestCase(GenericDocumentViewTestCase): class StagingFolderTestCase(GenericViewTestCase): def setUp(self): super(StagingFolderTestCase, self).setUp() - self.temporary_directory = tempfile.mkdtemp() + self.temporary_directory = mkdtemp() # TODO: remove temp directory after test shutil.copy(TEST_SMALL_DOCUMENT_PATH, self.temporary_directory)