Merge remote-tracking branch 'origin/master' into feature/merge_test

HISTORY.rst

@@ -1,10 +1,12 @@
-2.1.3 (2016-XX-XX)
+2.1.3 (2016-06-29)
 - Add help message when initialsetup migration phase fails. Relates to GitLab issue #296.
 - Start using self.setdout instead of print as per documentation.
 - Fix GitLab issue #295, "When editing a user the top bar jumps to the name of the user".
 - Normalize handling of temporary file and directory creation.
 - Fix GitLab issue #309, "Temp files quickly filling-up my /tmp (1GB tmpfs)".
 - Explicitly check for residual temporary files in tests.
+- Add missing temporary file cleanup for office documents.
+- Fix file descriptor leak in the document signature download test.
 
 2.1.2 (2016-05-20)
 ==================

README.rst

@@ -1,4 +1,4 @@
-|PyPI badge| |Build Status| |Coverage badge| |Installs badge| |License badge|
+|PyPI badge| |Build Status| |Coverage badge| |License badge|
 
 |Logo|
 
@@ -63,8 +63,6 @@ Contribute
    :target: https://gitlab.com/mayan-edms/mayan-edms/commits/master
 .. |Logo| image:: https://gitlab.com/mayan-edms/mayan-edms/raw/master/docs/_static/mayan_logo.png
 .. |Animation| image:: https://gitlab.com/mayan-edms/mayan-edms/raw/master/docs/_static/overview.gif
-.. |Installs badge| image:: http://img.shields.io/pypi/dm/mayan-edms.svg?style=flat
-   :target: https://crate.io/packages/mayan-edms/
 .. |PyPI badge| image:: http://img.shields.io/pypi/v/mayan-edms.svg?style=flat
    :target: http://badge.fury.io/py/mayan-edms
 .. |License badge| image:: http://img.shields.io/badge/license-Apache%202.0-green.svg?style=flat

docs/releases/2.1.3.rst

@@ -0,0 +1,87 @@
+===============================
+Mayan EDMS v2.1.3 release notes
+===============================
+
+Released: June 29, 2016
+
+What's new
+==========
+
+This is a bug-fix release and all users are encouraged to upgrade.
+
+Temporary files cleanup
+-----------------------
+When uploading PDF files that had been OCRed by previous software, the text
+parser backend that uses Poppler, would leave behind some temporary files in
+the /tmp folder. The issue has been resolved and from the fix a test mixin
+system check has been devised that will identify places in the codebase with
+similar behaviors, reducing the recurrence of similar issues in the future.
+
+Other changes
+-------------
+- Add help message when initialsetup migration phase fails. Relates to GitLab issue #296
+- Start using self.setdout instead of print as per documentation.
+- Fix GitLab issue #295, "When editing a user the top bar jumps to the name of the user".
+- Normalize handling of temporary file and directory creation.
+- Explicitly check for residual temporary files in tests.
+- Add missing temporary file cleanup for office documents.
+- Fix file descriptor leak in the document signature download test.
+
+Removals
+--------
+* None
+
+Upgrading from a previous version
+---------------------------------
+
+Using PIP
+~~~~~~~~~
+
+Type in the console::
+
+    $ pip install -U mayan-edms
+
+the requirements will also be updated automatically.
+
+Using Git
+~~~~~~~~~
+
+If you installed Mayan EDMS by cloning the Git repository issue the commands::
+
+    $ git reset --hard HEAD
+    $ git pull
+
+otherwise download the compressed archived and uncompress it overriding the
+existing installation.
+
+Next upgrade/add the new requirements::
+
+    $ pip install --upgrade -r requirements.txt
+
+Common steps
+~~~~~~~~~~~~
+
+Migrate existing database schema with::
+
+    $ mayan-edms.py performupgrade
+
+Add new static media::
+
+    $ mayan-edms.py collectstatic --noinput
+
+The upgrade procedure is now complete.
+
+
+Backward incompatible changes
+=============================
+
+* None
+
+Bugs fixed or issues closed
+===========================
+
+* `GitLab issue #295 <https://gitlab.com/mayan-edms/mayan-edms/issues/295>`_   When editing a user the top bar jumps to the name of the user
+* `GitLab issue #309 <https://gitlab.com/mayan-edms/mayan-edms/issues/309>`_   Temp files quickly filling-up my /tmp (1GB tmpfs)
+
+
+.. _PyPI: https://pypi.python.org/pypi/mayan-edms/

mayan/__init__.py

@@ -1,8 +1,8 @@
 from __future__ import unicode_literals
 
 __title__ = 'Mayan EDMS'
-__version__ = '2.1.2'
-__build__ = 0x020102
+__version__ = '2.1.3'
+__build__ = 0x020103
 __author__ = 'Roberto Rosario'
 __author_email__ = 'roberto.rosario@mayan-edms.com'
 __description__ = 'Free Open Source Electronic Document Management System'

mayan/apps/common/tests/__init__.py

@@ -0,0 +1,2 @@
+from .base import BaseTestCase  # NOQA
+from .decorators import skip_file_descriptor_check  # NOQA

mayan/apps/common/tests/base.py

@@ -0,0 +1,11 @@
+from __future__ import unicode_literals
+
+from django.test import TestCase
+
+from .mixins import OpenFileCheckMixin, TempfileCheckMixin
+
+
+class BaseTestCase(OpenFileCheckMixin, TempfileCheckMixin, TestCase):
+    """
+    This is the most basic test case class any test in the project should use.
+    """

mayan/apps/common/tests/decorators.py

@@ -0,0 +1,5 @@
+def skip_file_descriptor_check(func):
+   def func_wrapper(item):
+       item._skip_file_descriptor_test = True
+       return func(item)
+   return func_wrapper

mayan/apps/common/tests/mixins.py

@@ -0,0 +1,51 @@
+from __future__ import unicode_literals
+
+import os
+
+import psutil
+
+from ..settings import setting_temporary_directory
+
+
+class TempfileCheckMixin(object):
+    def _get_temporary_entries_count(self):
+        return len(os.listdir(setting_temporary_directory.value))
+
+    def setUp(self):
+        super(TempfileCheckMixin, self).setUp()
+        self._temporary_items = self._get_temporary_entries_count()
+
+    def tearDown(self):
+        self.assertEqual(
+            self._temporary_items, self._get_temporary_entries_count(),
+            msg='Orphan temporary file. The number of temporary file and '
+            'directories at the start and at the end of the test are not the '
+            'same.'
+        )
+        super(TempfileCheckMixin, self).tearDown()
+
+
+class OpenFileCheckMixin(object):
+    def _get_descriptor_count(self):
+        process = psutil.Process()
+        return process.num_fds()
+
+    def _get_open_files(self):
+        process = psutil.Process()
+        return process.open_files()
+
+    def setUp(self):
+        super(OpenFileCheckMixin, self).setUp()
+        self._open_files = self._get_open_files()
+
+    def tearDown(self):
+        if not getattr(self, '_skip_file_descriptor_test', False):
+            for new_open_file in self._get_open_files():
+                self.assertFalse(new_open_file not in self._open_files,
+                    msg='File descriptor leak. The number of file descriptors '
+                    'at the start and at the end of the test are not the same.'
+                )
+
+            self._skip_file_descriptor_test = False
+
+        super(OpenFileCheckMixin, self).tearDown()

mayan/apps/common/tests/test_views.py

@@ -1,7 +1,5 @@
 from __future__ import absolute_import, unicode_literals
 
-import os
-
 from django.conf.urls import url
 from django.contrib.auth import get_user_model
 from django.core.urlresolvers import clear_url_caches, reverse
@@ -19,15 +17,12 @@ from user_management.tests import (
     TEST_USER_EMAIL, TEST_USER_USERNAME, TEST_USER_PASSWORD
 )
 
-from ..settings import setting_temporary_directory
-
 from .literals import TEST_VIEW_NAME, TEST_VIEW_URL
 
 
 class GenericViewTestCase(OrganizationTestCase):
     def setUp(self):
         super(GenericViewTestCase, self).setUp()
-        self.temp_items = len(os.listdir(setting_temporary_directory.value))
 
         self.has_test_view = False
         self.admin_user = get_user_model().on_organization.create_superuser(
@@ -52,11 +47,6 @@ class GenericViewTestCase(OrganizationTestCase):
         self.client.logout()
         if self.has_test_view:
             urlpatterns.pop(0)
-
-        self.assertEqual(
-            self.temp_items, len(os.listdir(setting_temporary_directory.value))
-        )
-
         super(GenericViewTestCase, self).tearDown()
 
     def add_test_view(self, test_object):

mayan/apps/common/utils.py

@@ -45,10 +45,13 @@ def encapsulate(function):
     return lambda: function
 
 
-def fs_cleanup(filename, suppress_exceptions=True):
+def fs_cleanup(filename, file_descriptor=None, suppress_exceptions=True):
     """
     Tries to remove the given filename. Ignores non-existent files
     """
+    if file_descriptor:
+        os.close(file_descriptor)
+
     try:
         os.remove(filename)
     except OSError:

mayan/apps/converter/classes.py

@@ -167,6 +167,7 @@ class ConverterBase(object):
                 yield data
 
         fs_cleanup(input_filepath)
+        fs_cleanup(converted_output)
 
     def get_page(self, output_format=DEFAULT_FILE_FORMAT, as_base64=False):
         if not self.image:

mayan/apps/document_signatures/tests/test_views.py

@@ -218,9 +218,10 @@ class SignaturesViewTestCase(GenericDocumentViewTestCase):
             args=(signature.pk,),
         )
 
-        assert_download_response(
-            self, response=response, content=signature.signature_file.read(),
-        )
+        with signature.signature_file as file_object:
+            assert_download_response(
+                self, response=response, content=file_object.read(),
+            )
 
     def test_signature_delete_view_no_permission(self):
         with open(TEST_KEY_FILE) as file_object:

mayan/apps/documents/tests/test_events.py

@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 
 from actstream.models import Action
 
+from common.tests import skip_file_descriptor_check
 from user_management.tests.literals import (
     TEST_USER_PASSWORD, TEST_USER_USERNAME
 )
@@ -38,7 +39,10 @@ class DocumentEventsTestCase(GenericDocumentViewTestCase):
         self.assertEqual(response.status_code, 302)
         self.assertEqual(list(Action.objects.any(obj=self.document)), [])
 
+    @skip_file_descriptor_check
     def test_document_download_event_with_permissions(self):
+        # TODO: Skip this test's file descriptor check until it gets migrate
+        # SingleObjectDownloadView CBV
         self.login(
             username=TEST_USER_USERNAME, password=TEST_USER_PASSWORD
         )

mayan/apps/documents/tests/test_models.py

@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 from datetime import timedelta
 import time
 
+from common.tests import BaseTestCase
 from django.test import TestCase, override_settings
 
 from organizations.tests.base import OrganizationTestCase

mayan/apps/documents/tests/test_views.py

@@ -6,6 +6,7 @@ from django.contrib.contenttypes.models import ContentType
 from django.test import override_settings
 from django.utils.six import BytesIO
 
+from common.tests import skip_file_descriptor_check
 from common.tests.test_views import GenericViewTestCase
 from converter.models import Transformation
 from converter.permissions import permission_transformation_delete
@@ -221,7 +222,11 @@ class DocumentsViewsTestCase(GenericDocumentViewTestCase):
             Document.on_organization.first().document_type, document_type
         )
 
+    @skip_file_descriptor_check
     def test_document_download_user_view(self):
+        # TODO: Skip this test's file descriptor check until it gets migrate
+        # SingleObjectDownloadView CBV
+
         self.login(
             username=TEST_USER_USERNAME, password=TEST_USER_PASSWORD
         )
@@ -253,7 +258,11 @@ class DocumentsViewsTestCase(GenericDocumentViewTestCase):
 
         del(buf)
 
+    @skip_file_descriptor_check
     def test_document_multiple_download_user_view(self):
+        # TODO: Skip this test's file descriptor check until it gets migrate
+        # SingleObjectDownloadView CBV
+
         self.login(
             username=TEST_USER_USERNAME, password=TEST_USER_PASSWORD
         )
@@ -287,7 +296,11 @@ class DocumentsViewsTestCase(GenericDocumentViewTestCase):
 
         del(buf)
 
+    @skip_file_descriptor_check
     def test_document_version_download_user_view(self):
+        # TODO: Skip this test's file descriptor check until it gets migrate
+        # SingleObjectDownloadView CBV
+
         self.login(
             username=TEST_USER_USERNAME, password=TEST_USER_PASSWORD
         )
@@ -354,6 +367,7 @@ class DocumentsViewsTestCase(GenericDocumentViewTestCase):
         self.assertContains(response, text='queued', status_code=200)
         self.assertEqual(self.document.pages.count(), page_count)
 
+
     def test_document_multiple_update_page_count_view_no_permission(self):
         self.login(username=TEST_USER_USERNAME, password=TEST_USER_PASSWORD)
 

mayan/apps/ocr/parsers.py

@@ -155,12 +155,12 @@ class PopplerParser(Parser):
         return_code = proc.wait()
         if return_code != 0:
             logger.error(proc.stderr.readline())
-            fs_cleanup(temp_filepath)
+            fs_cleanup(temp_filepath, file_descriptor=destination_descriptor)
 
             raise ParserError
 
         output = proc.stdout.read()
-        fs_cleanup(temp_filepath)
+        fs_cleanup(temp_filepath, file_descriptor=destination_descriptor)
 
         if output == b'\x0c':
             logger.debug('Parser didn\'t return any output')

mayan/apps/ocr/tests/test_models.py

@@ -2,6 +2,8 @@
 
 from __future__ import unicode_literals
 
+from django.core.files.base import File
+
 from documents.models import DocumentType
 from documents.settings import setting_language_choices
 from documents.tests import (

mayan/apps/ocr/tests/test_parsers.py

@@ -1,8 +1,10 @@
 from __future__ import unicode_literals
 
+from django.core.files.base import File
 from django.test import override_settings
 
 from common.settings import setting_temporary_directory
+from common.tests import BaseTestCase
 from documents.models import DocumentType
 from documents.tests import (
     TEST_DOCUMENT_PATH, TEST_DOCUMENT_TYPE, TEST_HYBRID_DOCUMENT_PATH

mayan/apps/organizations/tests/base.py

@@ -1,12 +1,12 @@
 from __future__ import unicode_literals
 
-from django.test import TestCase
+from common.tests import BaseTestCase
 
 from ..models import Organization
 from ..utils import create_default_organization
 
 
-class OrganizationTestCase(TestCase):
+class OrganizationTestCase(BaseTestCase):
     def setUp(self):
         create_default_organization()
 

requirements/testing-base.txt

@@ -4,4 +4,4 @@ coveralls==0.5
 django-test-without-migrations==0.2
 mock==2.0.0
 tox==2.1.1
-
+psutil==4.3.0