diff --git a/HISTORY.rst b/HISTORY.rst index 2f3ac73d21..93d192eea4 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,10 +1,12 @@ -2.1.3 (2016-XX-XX) +2.1.3 (2016-06-29) - Add help message when initialsetup migration phase fails. Relates to GitLab issue #296. - Start using self.setdout instead of print as per documentation. - Fix GitLab issue #295, "When editing a user the top bar jumps to the name of the user". - Normalize handling of temporary file and directory creation. - Fix GitLab issue #309, "Temp files quickly filling-up my /tmp (1GB tmpfs)". - Explicitly check for residual temporary files in tests. +- Add missing temporary file cleanup for office documents. +- Fix file descriptor leak in the document signature download test. 2.1.2 (2016-05-20) ================== diff --git a/README.rst b/README.rst index 518b9e1ef4..2ba44c413a 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,4 @@ -|PyPI badge| |Build Status| |Coverage badge| |Installs badge| |License badge| +|PyPI badge| |Build Status| |Coverage badge| |License badge| |Logo| @@ -63,8 +63,6 @@ Contribute :target: https://gitlab.com/mayan-edms/mayan-edms/commits/master .. |Logo| image:: https://gitlab.com/mayan-edms/mayan-edms/raw/master/docs/_static/mayan_logo.png .. |Animation| image:: https://gitlab.com/mayan-edms/mayan-edms/raw/master/docs/_static/overview.gif -.. |Installs badge| image:: http://img.shields.io/pypi/dm/mayan-edms.svg?style=flat - :target: https://crate.io/packages/mayan-edms/ .. |PyPI badge| image:: http://img.shields.io/pypi/v/mayan-edms.svg?style=flat :target: http://badge.fury.io/py/mayan-edms .. |License badge| image:: http://img.shields.io/badge/license-Apache%202.0-green.svg?style=flat diff --git a/docs/releases/2.1.3.rst b/docs/releases/2.1.3.rst new file mode 100644 index 0000000000..f125db4843 --- /dev/null +++ b/docs/releases/2.1.3.rst @@ -0,0 +1,87 @@ +=============================== +Mayan EDMS v2.1.3 release notes +=============================== + +Released: June 29, 2016 + +What's new +========== + +This is a bug-fix release and all users are encouraged to upgrade. + +Temporary files cleanup +----------------------- +When uploading PDF files that had been OCRed by previous software, the text +parser backend that uses Poppler, would leave behind some temporary files in +the /tmp folder. The issue has been resolved and from the fix a test mixin +system check has been devised that will identify places in the codebase with +similar behaviors, reducing the recurrence of similar issues in the future. + +Other changes +------------- +- Add help message when initialsetup migration phase fails. Relates to GitLab issue #296 +- Start using self.setdout instead of print as per documentation. +- Fix GitLab issue #295, "When editing a user the top bar jumps to the name of the user". +- Normalize handling of temporary file and directory creation. +- Explicitly check for residual temporary files in tests. +- Add missing temporary file cleanup for office documents. +- Fix file descriptor leak in the document signature download test. + +Removals +-------- +* None + +Upgrading from a previous version +--------------------------------- + +Using PIP +~~~~~~~~~ + +Type in the console:: + + $ pip install -U mayan-edms + +the requirements will also be updated automatically. + +Using Git +~~~~~~~~~ + +If you installed Mayan EDMS by cloning the Git repository issue the commands:: + + $ git reset --hard HEAD + $ git pull + +otherwise download the compressed archived and uncompress it overriding the +existing installation. + +Next upgrade/add the new requirements:: + + $ pip install --upgrade -r requirements.txt + +Common steps +~~~~~~~~~~~~ + +Migrate existing database schema with:: + + $ mayan-edms.py performupgrade + +Add new static media:: + + $ mayan-edms.py collectstatic --noinput + +The upgrade procedure is now complete. + + +Backward incompatible changes +============================= + +* None + +Bugs fixed or issues closed +=========================== + +* `GitLab issue #295 `_ When editing a user the top bar jumps to the name of the user +* `GitLab issue #309 `_ Temp files quickly filling-up my /tmp (1GB tmpfs) + + +.. _PyPI: https://pypi.python.org/pypi/mayan-edms/ diff --git a/mayan/__init__.py b/mayan/__init__.py index 011981549a..601f4c5e76 100644 --- a/mayan/__init__.py +++ b/mayan/__init__.py @@ -1,8 +1,8 @@ from __future__ import unicode_literals __title__ = 'Mayan EDMS' -__version__ = '2.1.2' -__build__ = 0x020102 +__version__ = '2.1.3' +__build__ = 0x020103 __author__ = 'Roberto Rosario' __author_email__ = 'roberto.rosario@mayan-edms.com' __description__ = 'Free Open Source Electronic Document Management System' diff --git a/mayan/apps/common/tests/__init__.py b/mayan/apps/common/tests/__init__.py index e69de29bb2..2da5088d15 100644 --- a/mayan/apps/common/tests/__init__.py +++ b/mayan/apps/common/tests/__init__.py @@ -0,0 +1,2 @@ +from .base import BaseTestCase # NOQA +from .decorators import skip_file_descriptor_check # NOQA diff --git a/mayan/apps/common/tests/base.py b/mayan/apps/common/tests/base.py new file mode 100644 index 0000000000..086179b17b --- /dev/null +++ b/mayan/apps/common/tests/base.py @@ -0,0 +1,11 @@ +from __future__ import unicode_literals + +from django.test import TestCase + +from .mixins import OpenFileCheckMixin, TempfileCheckMixin + + +class BaseTestCase(OpenFileCheckMixin, TempfileCheckMixin, TestCase): + """ + This is the most basic test case class any test in the project should use. + """ diff --git a/mayan/apps/common/tests/decorators.py b/mayan/apps/common/tests/decorators.py new file mode 100644 index 0000000000..9c1b5b8186 --- /dev/null +++ b/mayan/apps/common/tests/decorators.py @@ -0,0 +1,5 @@ +def skip_file_descriptor_check(func): + def func_wrapper(item): + item._skip_file_descriptor_test = True + return func(item) + return func_wrapper diff --git a/mayan/apps/common/tests/mixins.py b/mayan/apps/common/tests/mixins.py new file mode 100644 index 0000000000..844585ecef --- /dev/null +++ b/mayan/apps/common/tests/mixins.py @@ -0,0 +1,51 @@ +from __future__ import unicode_literals + +import os + +import psutil + +from ..settings import setting_temporary_directory + + +class TempfileCheckMixin(object): + def _get_temporary_entries_count(self): + return len(os.listdir(setting_temporary_directory.value)) + + def setUp(self): + super(TempfileCheckMixin, self).setUp() + self._temporary_items = self._get_temporary_entries_count() + + def tearDown(self): + self.assertEqual( + self._temporary_items, self._get_temporary_entries_count(), + msg='Orphan temporary file. The number of temporary file and ' + 'directories at the start and at the end of the test are not the ' + 'same.' + ) + super(TempfileCheckMixin, self).tearDown() + + +class OpenFileCheckMixin(object): + def _get_descriptor_count(self): + process = psutil.Process() + return process.num_fds() + + def _get_open_files(self): + process = psutil.Process() + return process.open_files() + + def setUp(self): + super(OpenFileCheckMixin, self).setUp() + self._open_files = self._get_open_files() + + def tearDown(self): + if not getattr(self, '_skip_file_descriptor_test', False): + for new_open_file in self._get_open_files(): + self.assertFalse(new_open_file not in self._open_files, + msg='File descriptor leak. The number of file descriptors ' + 'at the start and at the end of the test are not the same.' + ) + + self._skip_file_descriptor_test = False + + super(OpenFileCheckMixin, self).tearDown() diff --git a/mayan/apps/common/tests/test_views.py b/mayan/apps/common/tests/test_views.py index dd2f8e7682..cccfa83ade 100644 --- a/mayan/apps/common/tests/test_views.py +++ b/mayan/apps/common/tests/test_views.py @@ -1,7 +1,5 @@ from __future__ import absolute_import, unicode_literals -import os - from django.conf.urls import url from django.contrib.auth import get_user_model from django.core.urlresolvers import clear_url_caches, reverse @@ -19,15 +17,12 @@ from user_management.tests import ( TEST_USER_EMAIL, TEST_USER_USERNAME, TEST_USER_PASSWORD ) -from ..settings import setting_temporary_directory - from .literals import TEST_VIEW_NAME, TEST_VIEW_URL class GenericViewTestCase(OrganizationTestCase): def setUp(self): super(GenericViewTestCase, self).setUp() - self.temp_items = len(os.listdir(setting_temporary_directory.value)) self.has_test_view = False self.admin_user = get_user_model().on_organization.create_superuser( @@ -52,11 +47,6 @@ class GenericViewTestCase(OrganizationTestCase): self.client.logout() if self.has_test_view: urlpatterns.pop(0) - - self.assertEqual( - self.temp_items, len(os.listdir(setting_temporary_directory.value)) - ) - super(GenericViewTestCase, self).tearDown() def add_test_view(self, test_object): diff --git a/mayan/apps/common/utils.py b/mayan/apps/common/utils.py index 56889d2774..1f03bc2e23 100644 --- a/mayan/apps/common/utils.py +++ b/mayan/apps/common/utils.py @@ -45,10 +45,13 @@ def encapsulate(function): return lambda: function -def fs_cleanup(filename, suppress_exceptions=True): +def fs_cleanup(filename, file_descriptor=None, suppress_exceptions=True): """ Tries to remove the given filename. Ignores non-existent files """ + if file_descriptor: + os.close(file_descriptor) + try: os.remove(filename) except OSError: diff --git a/mayan/apps/converter/classes.py b/mayan/apps/converter/classes.py index 69bdf5b6bf..72d3eaf61a 100644 --- a/mayan/apps/converter/classes.py +++ b/mayan/apps/converter/classes.py @@ -167,6 +167,7 @@ class ConverterBase(object): yield data fs_cleanup(input_filepath) + fs_cleanup(converted_output) def get_page(self, output_format=DEFAULT_FILE_FORMAT, as_base64=False): if not self.image: diff --git a/mayan/apps/document_signatures/tests/test_views.py b/mayan/apps/document_signatures/tests/test_views.py index adaeebe697..2b0ff6b0f0 100644 --- a/mayan/apps/document_signatures/tests/test_views.py +++ b/mayan/apps/document_signatures/tests/test_views.py @@ -218,9 +218,10 @@ class SignaturesViewTestCase(GenericDocumentViewTestCase): args=(signature.pk,), ) - assert_download_response( - self, response=response, content=signature.signature_file.read(), - ) + with signature.signature_file as file_object: + assert_download_response( + self, response=response, content=file_object.read(), + ) def test_signature_delete_view_no_permission(self): with open(TEST_KEY_FILE) as file_object: diff --git a/mayan/apps/documents/tests/test_events.py b/mayan/apps/documents/tests/test_events.py index a7e20e6c28..3824de40ee 100644 --- a/mayan/apps/documents/tests/test_events.py +++ b/mayan/apps/documents/tests/test_events.py @@ -4,6 +4,7 @@ from __future__ import unicode_literals from actstream.models import Action +from common.tests import skip_file_descriptor_check from user_management.tests.literals import ( TEST_USER_PASSWORD, TEST_USER_USERNAME ) @@ -38,7 +39,10 @@ class DocumentEventsTestCase(GenericDocumentViewTestCase): self.assertEqual(response.status_code, 302) self.assertEqual(list(Action.objects.any(obj=self.document)), []) + @skip_file_descriptor_check def test_document_download_event_with_permissions(self): + # TODO: Skip this test's file descriptor check until it gets migrate + # SingleObjectDownloadView CBV self.login( username=TEST_USER_USERNAME, password=TEST_USER_PASSWORD ) diff --git a/mayan/apps/documents/tests/test_models.py b/mayan/apps/documents/tests/test_models.py index 370f5215ba..dba9eea4f3 100644 --- a/mayan/apps/documents/tests/test_models.py +++ b/mayan/apps/documents/tests/test_models.py @@ -3,6 +3,7 @@ from __future__ import unicode_literals from datetime import timedelta import time +from common.tests import BaseTestCase from django.test import TestCase, override_settings from organizations.tests.base import OrganizationTestCase diff --git a/mayan/apps/documents/tests/test_views.py b/mayan/apps/documents/tests/test_views.py index 2ba41f804f..222fbb0c12 100644 --- a/mayan/apps/documents/tests/test_views.py +++ b/mayan/apps/documents/tests/test_views.py @@ -6,6 +6,7 @@ from django.contrib.contenttypes.models import ContentType from django.test import override_settings from django.utils.six import BytesIO +from common.tests import skip_file_descriptor_check from common.tests.test_views import GenericViewTestCase from converter.models import Transformation from converter.permissions import permission_transformation_delete @@ -221,7 +222,11 @@ class DocumentsViewsTestCase(GenericDocumentViewTestCase): Document.on_organization.first().document_type, document_type ) + @skip_file_descriptor_check def test_document_download_user_view(self): + # TODO: Skip this test's file descriptor check until it gets migrate + # SingleObjectDownloadView CBV + self.login( username=TEST_USER_USERNAME, password=TEST_USER_PASSWORD ) @@ -253,7 +258,11 @@ class DocumentsViewsTestCase(GenericDocumentViewTestCase): del(buf) + @skip_file_descriptor_check def test_document_multiple_download_user_view(self): + # TODO: Skip this test's file descriptor check until it gets migrate + # SingleObjectDownloadView CBV + self.login( username=TEST_USER_USERNAME, password=TEST_USER_PASSWORD ) @@ -287,7 +296,11 @@ class DocumentsViewsTestCase(GenericDocumentViewTestCase): del(buf) + @skip_file_descriptor_check def test_document_version_download_user_view(self): + # TODO: Skip this test's file descriptor check until it gets migrate + # SingleObjectDownloadView CBV + self.login( username=TEST_USER_USERNAME, password=TEST_USER_PASSWORD ) @@ -354,6 +367,7 @@ class DocumentsViewsTestCase(GenericDocumentViewTestCase): self.assertContains(response, text='queued', status_code=200) self.assertEqual(self.document.pages.count(), page_count) + def test_document_multiple_update_page_count_view_no_permission(self): self.login(username=TEST_USER_USERNAME, password=TEST_USER_PASSWORD) diff --git a/mayan/apps/ocr/parsers.py b/mayan/apps/ocr/parsers.py index ee690ed6a0..088072f566 100644 --- a/mayan/apps/ocr/parsers.py +++ b/mayan/apps/ocr/parsers.py @@ -155,12 +155,12 @@ class PopplerParser(Parser): return_code = proc.wait() if return_code != 0: logger.error(proc.stderr.readline()) - fs_cleanup(temp_filepath) + fs_cleanup(temp_filepath, file_descriptor=destination_descriptor) raise ParserError output = proc.stdout.read() - fs_cleanup(temp_filepath) + fs_cleanup(temp_filepath, file_descriptor=destination_descriptor) if output == b'\x0c': logger.debug('Parser didn\'t return any output') diff --git a/mayan/apps/ocr/tests/test_models.py b/mayan/apps/ocr/tests/test_models.py index e68eb5aabc..c6c7a2955c 100644 --- a/mayan/apps/ocr/tests/test_models.py +++ b/mayan/apps/ocr/tests/test_models.py @@ -2,6 +2,8 @@ from __future__ import unicode_literals +from django.core.files.base import File + from documents.models import DocumentType from documents.settings import setting_language_choices from documents.tests import ( diff --git a/mayan/apps/ocr/tests/test_parsers.py b/mayan/apps/ocr/tests/test_parsers.py index edb3f0dab8..b17670422a 100644 --- a/mayan/apps/ocr/tests/test_parsers.py +++ b/mayan/apps/ocr/tests/test_parsers.py @@ -1,8 +1,10 @@ from __future__ import unicode_literals +from django.core.files.base import File from django.test import override_settings from common.settings import setting_temporary_directory +from common.tests import BaseTestCase from documents.models import DocumentType from documents.tests import ( TEST_DOCUMENT_PATH, TEST_DOCUMENT_TYPE, TEST_HYBRID_DOCUMENT_PATH diff --git a/mayan/apps/organizations/tests/base.py b/mayan/apps/organizations/tests/base.py index 36133f0abd..dc5f31b249 100644 --- a/mayan/apps/organizations/tests/base.py +++ b/mayan/apps/organizations/tests/base.py @@ -1,12 +1,12 @@ from __future__ import unicode_literals -from django.test import TestCase +from common.tests import BaseTestCase from ..models import Organization from ..utils import create_default_organization -class OrganizationTestCase(TestCase): +class OrganizationTestCase(BaseTestCase): def setUp(self): create_default_organization() diff --git a/requirements/testing-base.txt b/requirements/testing-base.txt index 5b7560fd79..48f9064c42 100644 --- a/requirements/testing-base.txt +++ b/requirements/testing-base.txt @@ -4,4 +4,4 @@ coveralls==0.5 django-test-without-migrations==0.2 mock==2.0.0 tox==2.1.1 - +psutil==4.3.0