diff --git a/HISTORY.rst b/HISTORY.rst index 06355887cc..6813c3d257 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -7,7 +7,15 @@ even when the source is disabled and to not deleted processed files during a check. * Switch to full app paths. + +3.1.11 (2019-04-XX) +=================== * Fix multiple tag selection wizard step. +* Change the required permission for the checkout info link from + document check in to document checkout details view. +* Lower the log severity when links don't resolve. +* Add DOCUMENTS_HASH_BLOCK_SIZE to control the size of the file + block when calculating a document's checksum. 3.1.10 (2019-04-04) =================== diff --git a/docs/releases/3.1.11.rst b/docs/releases/3.1.11.rst new file mode 100644 index 0000000000..2efe86bb2d --- /dev/null +++ b/docs/releases/3.1.11.rst @@ -0,0 +1,148 @@ +Version 3.1.11 +============== + +Released: April XX, 2019 + + +Changes +------- + +Memory usage +^^^^^^^^^^^^ + +The ``DOCUMENTS_HASH_BLOCK_SIZE`` setting was added to limit the number of +bytes that will be read into memory when calculating the checksum of a new +document. For compatibility with the current bevahor this setting defaults to +0 which means that it is disabled. Disabling the setting will cause the +entire document's file to be loaded into memory. If documents are not +processing due to out of memory errors (large documents or devices with +limited memory), set ``DOCUMENTS_HASH_BLOCK_SIZE`` to a value other than 0. +Limited tests suggest 65535 to be a good alternative. + + +Tag wizard step +^^^^^^^^^^^^^^^ + +The tag wizard step was fixed and will now allow attaching multple tags to a +new document. + + +Permissions +^^^^^^^^^^^ + +Previously the document checkout information link required one of the following +permissions: document check in, document check in override, or document +checkout. Meanwhile the document checkout information view would require the +document checkout detail view permission. This difference in permissions +has been eliminated and the link will now required the document checkout +detail view permission, same as the view. Update your user role permissions +accordingly. + + +Other changes +^^^^^^^^^^^^^ + +* Lower the log severity when links don't resolve. + + +Removals +-------- + +* None + + +Upgrading from a previous version +--------------------------------- + +If installed via Python's PIP +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Remove deprecated requirements:: + + $ curl https://gitlab.com/mayan-edms/mayan-edms/raw/master/removals.txt | pip uninstall -r /dev/stdin + +Type in the console:: + + $ pip install mayan-edms==3.1.11 + +the requirements will also be updated automatically. + +Migrate existing database schema with:: + + $ mayan-edms.py performupgrade + +Add new static media:: + + $ mayan-edms.py collectstatic --noinput + +The upgrade procedure is now complete. + + +If installed using a direct deployment +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Remove deprecated requirements:: + + $ curl https://gitlab.com/mayan-edms/mayan-edms/raw/master/removals.txt | sudo -u mayan /opt/mayan-edms/bin/pip uninstall -r /dev/stdin + +Download and install the new version:: + + $ sudo -u mayan /opt/mayan-edms/bin/pip install --no-cache-dir --no-use-pep517 mayan-edms==3.1.11 + +the requirements will also be updated automatically. + +Run the upgrade command:: + + $ sudo -u mayan MAYAN_DATABASE_ENGINE=django.db.backends.postgresql \ + MAYAN_DATABASE_NAME=mayan MAYAN_DATABASE_PASSWORD=mayanuserpass \ + MAYAN_DATABASE_USER=mayan MAYAN_DATABASE_HOST=127.0.0.1 \ + MAYAN_MEDIA_ROOT=/opt/mayan-edms/media /opt/mayan-edms/bin/mayan-edms.py performupgrade + +Add any new static files:: + + $ sudo -u mayan MAYAN_MEDIA_ROOT=/opt/mayan-edms/media /opt/mayan-edms/bin/mayan-edms.py collectstatic --noinput + +The upgrade procedure is now complete. + +Using Git +^^^^^^^^^ + +If you installed Mayan EDMS by cloning the Git repository issue the commands:: + + $ git reset --hard HEAD + $ git pull + +otherwise download the compressed archived and uncompress it overriding the +existing installation. + +Remove deprecated requirements:: + + $ pip uninstall -y -r removals.txt + +Next upgrade/add the new requirements:: + + $ pip install --upgrade -r requirements.txt + +Migrate existing database schema with:: + + $ mayan-edms.py performupgrade + +Add new static media:: + + $ mayan-edms.py collectstatic --noinput + +The upgrade procedure is now complete. + + +Backward incompatible changes +----------------------------- + +* None + + +Bugs fixed or issues closed +--------------------------- + +* None + +.. _PyPI: https://pypi.python.org/pypi/mayan-edms/ diff --git a/docs/releases/index.rst b/docs/releases/index.rst index d3b2470bfa..922a5647fa 100644 --- a/docs/releases/index.rst +++ b/docs/releases/index.rst @@ -20,6 +20,7 @@ versions of the documentation contain the release notes for any later releases. .. toctree:: :maxdepth: 1 + 3.1.11 3.1.10 3.1.9 3.1.8 diff --git a/mayan/apps/checkouts/links.py b/mayan/apps/checkouts/links.py index cb77c54477..d45fca201a 100644 --- a/mayan/apps/checkouts/links.py +++ b/mayan/apps/checkouts/links.py @@ -7,7 +7,8 @@ from mayan.apps.navigation import Link from .icons import icon_checkout_info from .permissions import ( permission_document_checkout, permission_document_checkin, - permission_document_checkin_override + permission_document_checkin_override, + permission_document_checkout_detail_view ) @@ -40,11 +41,9 @@ link_checkin_document = Link( args='object.pk', condition=is_checked_out, permissions=( permission_document_checkin, permission_document_checkin_override ), text=_('Check in document'), view='checkouts:checkin_document', - ) link_checkout_info = Link( args='resolved_object.pk', icon_class=icon_checkout_info, permissions=( - permission_document_checkin, permission_document_checkin_override, - permission_document_checkout + permission_document_checkout_detail_view, ), text=_('Check in/out'), view='checkouts:checkout_info', ) diff --git a/mayan/apps/checkouts/tests/mixins.py b/mayan/apps/checkouts/tests/mixins.py new file mode 100644 index 0000000000..ad931f3a45 --- /dev/null +++ b/mayan/apps/checkouts/tests/mixins.py @@ -0,0 +1,18 @@ +from __future__ import unicode_literals + +import datetime + +from django.utils.timezone import now + +from ..models import DocumentCheckout + + +class DocumentCheckoutTestMixin(object): + def _checkout_document(self): + expiration_datetime = now() + datetime.timedelta(days=1) + + DocumentCheckout.objects.checkout_document( + document=self.document, expiration_datetime=expiration_datetime, + user=self.user, block_new_version=True + ) + self.assertTrue(self.document.is_checked_out()) diff --git a/mayan/apps/checkouts/tests/test_links.py b/mayan/apps/checkouts/tests/test_links.py new file mode 100644 index 0000000000..a060cfc740 --- /dev/null +++ b/mayan/apps/checkouts/tests/test_links.py @@ -0,0 +1,50 @@ +from __future__ import unicode_literals + +from documents.tests import GenericDocumentViewTestCase + +from ..links import link_checkout_document, link_checkout_info +from ..permissions import ( + permission_document_checkout, permission_document_checkout_detail_view +) + +from .mixins import DocumentCheckoutTestMixin + + +class CheckoutLinksTestCase(DocumentCheckoutTestMixin, GenericDocumentViewTestCase): + def setUp(self): + super(CheckoutLinksTestCase, self).setUp() + self.login_user() + + def _resolve_checkout_link(self): + self.add_test_view(test_object=self.document) + context = self.get_test_view() + context['user'] = self.user + return link_checkout_document.resolve(context=context) + + def test_checkout_link_no_access(self): + resolved_link = self._resolve_checkout_link() + self.assertEqual(resolved_link, None) + + def test_checkout_link_with_access(self): + self.grant_access( + obj=self.document, permission=permission_document_checkout + ) + resolved_link = self._resolve_checkout_link() + self.assertNotEqual(resolved_link, None) + + def _resolve_checkout_info_link(self): + self.add_test_view(test_object=self.document) + context = self.get_test_view() + context['user'] = self.user + return link_checkout_info.resolve(context=context) + + def test_checkout_info_link_no_access(self): + resolved_link = self._resolve_checkout_info_link() + self.assertEqual(resolved_link, None) + + def test_checkout_info_link_with_access(self): + self.grant_access( + obj=self.document, permission=permission_document_checkout_detail_view + ) + resolved_link = self._resolve_checkout_info_link() + self.assertNotEqual(resolved_link, None) diff --git a/mayan/apps/checkouts/tests/test_views.py b/mayan/apps/checkouts/tests/test_views.py index 3daa2ab210..b1a16755ec 100644 --- a/mayan/apps/checkouts/tests/test_views.py +++ b/mayan/apps/checkouts/tests/test_views.py @@ -13,22 +13,27 @@ from mayan.apps.user_management.tests.literals import ( TEST_ADMIN_USERNAME, ) +from ..literals import STATE_CHECKED_OUT, STATE_LABELS from ..models import DocumentCheckout from ..permissions import ( permission_document_checkin, permission_document_checkin_override, permission_document_checkout, permission_document_checkout_detail_view ) +from .mixins import DocumentCheckoutTestMixin + + +class DocumentCheckoutViewTestCase(DocumentCheckoutTestMixin, GenericDocumentViewTestCase): + def setUp(self): + super(DocumentCheckoutViewTestCase, self).setUp() + self.login_user() -class DocumentCheckoutViewTestCase(GenericDocumentViewTestCase): def _request_document_check_in_view(self): return self.post( viewname='checkouts:checkin_document', args=(self.document.pk,), ) def test_checkin_document_view_no_permission(self): - self.login_user() - expiration_datetime = now() + datetime.timedelta(days=1) DocumentCheckout.objects.checkout_document( @@ -43,8 +48,6 @@ class DocumentCheckoutViewTestCase(GenericDocumentViewTestCase): self.assertTrue(self.document.is_checked_out()) def test_checkin_document_view_with_access(self): - self.login_user() - expiration_datetime = now() + datetime.timedelta(days=1) DocumentCheckout.objects.checkout_document( @@ -82,14 +85,11 @@ class DocumentCheckoutViewTestCase(GenericDocumentViewTestCase): ) def test_checkout_document_view_no_permission(self): - self.login_user() - response = self._request_document_checkout_view() self.assertEquals(response.status_code, 403) self.assertFalse(self.document.is_checked_out()) def test_checkout_document_view_with_access(self): - self.login_user() self.grant_access( obj=self.document, permission=permission_document_checkout ) @@ -102,6 +102,36 @@ class DocumentCheckoutViewTestCase(GenericDocumentViewTestCase): self.assertEquals(response.status_code, 302) self.assertTrue(self.document.is_checked_out()) + def _request_checkout_detail_view(self): + return self.get( + viewname='checkouts:checkout_info', args=(self.document.pk,), + ) + + def test_checkout_detail_view_no_permission(self): + self._checkout_document() + self.grant_access( + obj=self.document, + permission=permission_document_checkout + ) + + response = self._request_checkout_detail_view() + + self.assertNotContains( + response, text=STATE_LABELS[STATE_CHECKED_OUT], status_code=403 + ) + + def test_checkout_detail_view_with_access(self): + self._checkout_document() + + self.grant_access( + obj=self.document, + permission=permission_document_checkout_detail_view + ) + + response = self._request_checkout_detail_view() + + self.assertContains(response, text=STATE_LABELS[STATE_CHECKED_OUT], status_code=200) + def test_document_new_version_after_checkout(self): """ Gitlab issue #231 diff --git a/mayan/apps/documents/models.py b/mayan/apps/documents/models.py index 08a034cbba..a76e6fadf8 100644 --- a/mayan/apps/documents/models.py +++ b/mayan/apps/documents/models.py @@ -45,7 +45,8 @@ from .permissions import permission_document_view from .settings import ( setting_disable_base_image_cache, setting_disable_transformed_image_cache, setting_display_width, setting_display_height, setting_fix_orientation, - setting_language, setting_zoom_max_level, setting_zoom_min_level + setting_hash_block_size, setting_language, setting_zoom_max_level, + setting_zoom_min_level ) from .signals import ( post_document_created, post_document_type_change, post_version_upload @@ -56,8 +57,8 @@ logger = logging.getLogger(__name__) # document image cache name hash function -def HASH_FUNCTION(data): - return hashlib.sha256(data).hexdigest() +def hash_function(): + return hashlib.sha256() def UUID_FUNCTION(*args, **kwargs): @@ -697,10 +698,25 @@ class DocumentVersion(models.Model): Open a document version's file and update the checksum field using the user provided checksum function """ + block_size = setting_hash_block_size.value + if block_size == 0: + # If the setting value is 0 that means disable read limit. To disable + # the read limit passing None won't work, we pass -1 instead as per + # the Python documentation. + # https://docs.python.org/2/tutorial/inputoutput.html#methods-of-file-objects + block_size = -1 + if self.exists(): - source = self.open() - self.checksum = force_text(HASH_FUNCTION(source.read())) - source.close() + hash_object = hash_function() + with self.open() as file_object: + while (True): + data = file_object.read(block_size) + if not data: + break + + hash_object.update(data) + + self.checksum = force_text(hash_object.hexdigest()) if save: self.save() diff --git a/mayan/apps/documents/settings.py b/mayan/apps/documents/settings.py index 331f12f7ff..8d51f29e70 100644 --- a/mayan/apps/documents/settings.py +++ b/mayan/apps/documents/settings.py @@ -62,6 +62,14 @@ setting_fix_orientation = namespace.add_setting( 'feature and it is disabled by default.' ) ) +setting_hash_block_size = namespace.add_setting( + global_name='DOCUMENTS_HASH_BLOCK_SIZE', default=0, + help_text=_( + 'Size of blocks to use when calculating the document file\'s ' + 'checksum. A value of 0 disables the block calculation and the entire ' + 'file will be loaded into memory.' + ) +) setting_language = namespace.add_setting( global_name='DOCUMENTS_LANGUAGE', default=DEFAULT_LANGUAGE, help_text=_('Default documents language (in ISO639-3 format).') diff --git a/mayan/apps/navigation/classes.py b/mayan/apps/navigation/classes.py index b9a77ab85e..338e74b50b 100644 --- a/mayan/apps/navigation/classes.py +++ b/mayan/apps/navigation/classes.py @@ -419,7 +419,7 @@ class Link(object): try: resolved_link.url = node.render(context) except Exception as exception: - logger.error( + logger.debug( 'Error resolving link "%s" URL; %s', self.text, exception ) elif self.url: diff --git a/mayan/apps/tags/wizard_steps.py b/mayan/apps/tags/wizard_steps.py index b1aab0a426..0f87893e10 100644 --- a/mayan/apps/tags/wizard_steps.py +++ b/mayan/apps/tags/wizard_steps.py @@ -45,7 +45,12 @@ class WizardStepTags(WizardStep): furl_instance = furl(querystring) Tag = apps.get_model(app_label='tags', model_name='Tag') - for tag in Tag.objects.filter(pk__in=furl_instance.args['tags'].split(',')): + tag_id_list = furl_instance.args.get('tags', '') + + if tag_id_list: + tag_id_list = tag_id_list.split(',') + + for tag in Tag.objects.filter(pk__in=tag_id_list): tag.documents.add(document)