Merge branch 'hotfix' into hotfix_merge

Signed-off-by: Roberto Rosario <roberto.rosario.gonzalez@gmail.com>

HISTORY.rst

@@ -7,7 +7,15 @@
   even when the source is disabled and to not deleted processed files
   during a check.
 * Switch to full app paths.
+
+3.1.11 (2019-04-XX)
+===================
 * Fix multiple tag selection wizard step.
+* Change the required permission for the checkout info link from
+  document check in to document checkout details view.
+* Lower the log severity when links don't resolve.
+* Add DOCUMENTS_HASH_BLOCK_SIZE to control the size of the file
+  block when calculating a document's checksum.
 
 3.1.10 (2019-04-04)
 ===================

docs/releases/3.1.11.rst

@@ -0,0 +1,148 @@
+Version 3.1.11
+==============
+
+Released: April XX, 2019
+
+
+Changes
+-------
+
+Memory usage
+^^^^^^^^^^^^
+
+The ``DOCUMENTS_HASH_BLOCK_SIZE`` setting was added to limit the number of
+bytes that will be read into memory when calculating the checksum of a new
+document. For compatibility with the current bevahor this setting defaults to
+0 which means that it is disabled. Disabling the setting will cause the
+entire document's file to be loaded into memory. If documents are not
+processing due to out of memory errors (large documents or devices with
+limited memory), set ``DOCUMENTS_HASH_BLOCK_SIZE`` to a value other than 0.
+Limited tests suggest 65535 to be a good alternative.
+
+
+Tag wizard step
+^^^^^^^^^^^^^^^
+
+The tag wizard step was fixed and will now allow attaching multple tags to a
+new document.
+
+
+Permissions
+^^^^^^^^^^^
+
+Previously the document checkout information link required one of the following
+permissions: document check in, document check in override, or document
+checkout. Meanwhile the document checkout information view would require the
+document checkout detail view permission. This difference in permissions
+has been eliminated and the link will now required the document checkout
+detail view permission, same as the view. Update your user role permissions
+accordingly.
+
+
+Other changes
+^^^^^^^^^^^^^
+
+* Lower the log severity when links don't resolve.
+
+
+Removals
+--------
+
+* None
+
+
+Upgrading from a previous version
+---------------------------------
+
+If installed via Python's PIP
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Remove deprecated requirements::
+
+    $ curl https://gitlab.com/mayan-edms/mayan-edms/raw/master/removals.txt | pip uninstall -r /dev/stdin
+
+Type in the console::
+
+    $ pip install mayan-edms==3.1.11
+
+the requirements will also be updated automatically.
+
+Migrate existing database schema with::
+
+    $ mayan-edms.py performupgrade
+
+Add new static media::
+
+    $ mayan-edms.py collectstatic --noinput
+
+The upgrade procedure is now complete.
+
+
+If installed using a direct deployment
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Remove deprecated requirements::
+
+    $ curl https://gitlab.com/mayan-edms/mayan-edms/raw/master/removals.txt | sudo -u mayan /opt/mayan-edms/bin/pip uninstall -r /dev/stdin
+
+Download and install the new version::
+
+    $ sudo -u mayan /opt/mayan-edms/bin/pip install --no-cache-dir --no-use-pep517 mayan-edms==3.1.11
+
+the requirements will also be updated automatically.
+
+Run the upgrade command::
+
+    $ sudo -u mayan MAYAN_DATABASE_ENGINE=django.db.backends.postgresql \
+    MAYAN_DATABASE_NAME=mayan MAYAN_DATABASE_PASSWORD=mayanuserpass \
+    MAYAN_DATABASE_USER=mayan MAYAN_DATABASE_HOST=127.0.0.1 \
+    MAYAN_MEDIA_ROOT=/opt/mayan-edms/media /opt/mayan-edms/bin/mayan-edms.py performupgrade
+
+Add any new static files::
+
+    $ sudo -u mayan MAYAN_MEDIA_ROOT=/opt/mayan-edms/media /opt/mayan-edms/bin/mayan-edms.py collectstatic --noinput
+
+The upgrade procedure is now complete.
+
+Using Git
+^^^^^^^^^
+
+If you installed Mayan EDMS by cloning the Git repository issue the commands::
+
+    $ git reset --hard HEAD
+    $ git pull
+
+otherwise download the compressed archived and uncompress it overriding the
+existing installation.
+
+Remove deprecated requirements::
+
+    $ pip uninstall -y -r removals.txt
+
+Next upgrade/add the new requirements::
+
+    $ pip install --upgrade -r requirements.txt
+
+Migrate existing database schema with::
+
+    $ mayan-edms.py performupgrade
+
+Add new static media::
+
+    $ mayan-edms.py collectstatic --noinput
+
+The upgrade procedure is now complete.
+
+
+Backward incompatible changes
+-----------------------------
+
+* None
+
+
+Bugs fixed or issues closed
+---------------------------
+
+* None
+
+.. _PyPI: https://pypi.python.org/pypi/mayan-edms/

docs/releases/index.rst

@@ -20,6 +20,7 @@ versions of the documentation contain the release notes for any later releases.
 .. toctree::
    :maxdepth: 1
 
+   3.1.11
    3.1.10
    3.1.9
    3.1.8

mayan/apps/checkouts/links.py

@@ -7,7 +7,8 @@ from mayan.apps.navigation import Link
 from .icons import icon_checkout_info
 from .permissions import (
     permission_document_checkout, permission_document_checkin,
-    permission_document_checkin_override
+    permission_document_checkin_override,
+    permission_document_checkout_detail_view
 )
 
 
@@ -40,11 +41,9 @@ link_checkin_document = Link(
     args='object.pk', condition=is_checked_out, permissions=(
         permission_document_checkin, permission_document_checkin_override
     ), text=_('Check in document'), view='checkouts:checkin_document',
-
 )
 link_checkout_info = Link(
     args='resolved_object.pk', icon_class=icon_checkout_info, permissions=(
-        permission_document_checkin, permission_document_checkin_override,
+        permission_document_checkout_detail_view,
-        permission_document_checkout
     ), text=_('Check in/out'), view='checkouts:checkout_info',
 )

mayan/apps/checkouts/tests/mixins.py

@@ -0,0 +1,18 @@
+from __future__ import unicode_literals
+
+import datetime
+
+from django.utils.timezone import now
+
+from ..models import DocumentCheckout
+
+
+class DocumentCheckoutTestMixin(object):
+    def _checkout_document(self):
+        expiration_datetime = now() + datetime.timedelta(days=1)
+
+        DocumentCheckout.objects.checkout_document(
+            document=self.document, expiration_datetime=expiration_datetime,
+            user=self.user, block_new_version=True
+        )
+        self.assertTrue(self.document.is_checked_out())

mayan/apps/checkouts/tests/test_links.py

@@ -0,0 +1,50 @@
+from __future__ import unicode_literals
+
+from documents.tests import GenericDocumentViewTestCase
+
+from ..links import link_checkout_document, link_checkout_info
+from ..permissions import (
+    permission_document_checkout, permission_document_checkout_detail_view
+)
+
+from .mixins import DocumentCheckoutTestMixin
+
+
+class CheckoutLinksTestCase(DocumentCheckoutTestMixin, GenericDocumentViewTestCase):
+    def setUp(self):
+        super(CheckoutLinksTestCase, self).setUp()
+        self.login_user()
+
+    def _resolve_checkout_link(self):
+        self.add_test_view(test_object=self.document)
+        context = self.get_test_view()
+        context['user'] = self.user
+        return link_checkout_document.resolve(context=context)
+
+    def test_checkout_link_no_access(self):
+        resolved_link = self._resolve_checkout_link()
+        self.assertEqual(resolved_link, None)
+
+    def test_checkout_link_with_access(self):
+        self.grant_access(
+            obj=self.document, permission=permission_document_checkout
+        )
+        resolved_link = self._resolve_checkout_link()
+        self.assertNotEqual(resolved_link, None)
+
+    def _resolve_checkout_info_link(self):
+        self.add_test_view(test_object=self.document)
+        context = self.get_test_view()
+        context['user'] = self.user
+        return link_checkout_info.resolve(context=context)
+
+    def test_checkout_info_link_no_access(self):
+        resolved_link = self._resolve_checkout_info_link()
+        self.assertEqual(resolved_link, None)
+
+    def test_checkout_info_link_with_access(self):
+        self.grant_access(
+            obj=self.document, permission=permission_document_checkout_detail_view
+        )
+        resolved_link = self._resolve_checkout_info_link()
+        self.assertNotEqual(resolved_link, None)

mayan/apps/checkouts/tests/test_views.py

@@ -13,22 +13,27 @@ from mayan.apps.user_management.tests.literals import (
     TEST_ADMIN_USERNAME,
 )
 
+from ..literals import STATE_CHECKED_OUT, STATE_LABELS
 from ..models import DocumentCheckout
 from ..permissions import (
     permission_document_checkin, permission_document_checkin_override,
     permission_document_checkout, permission_document_checkout_detail_view
 )
 
+from .mixins import DocumentCheckoutTestMixin
+
+
+class DocumentCheckoutViewTestCase(DocumentCheckoutTestMixin, GenericDocumentViewTestCase):
+    def setUp(self):
+        super(DocumentCheckoutViewTestCase, self).setUp()
+        self.login_user()
 
-class DocumentCheckoutViewTestCase(GenericDocumentViewTestCase):
     def _request_document_check_in_view(self):
         return self.post(
             viewname='checkouts:checkin_document', args=(self.document.pk,),
         )
 
     def test_checkin_document_view_no_permission(self):
-        self.login_user()
-
         expiration_datetime = now() + datetime.timedelta(days=1)
 
         DocumentCheckout.objects.checkout_document(
@@ -43,8 +48,6 @@ class DocumentCheckoutViewTestCase(GenericDocumentViewTestCase):
         self.assertTrue(self.document.is_checked_out())
 
     def test_checkin_document_view_with_access(self):
-        self.login_user()
-
         expiration_datetime = now() + datetime.timedelta(days=1)
 
         DocumentCheckout.objects.checkout_document(
@@ -82,14 +85,11 @@ class DocumentCheckoutViewTestCase(GenericDocumentViewTestCase):
         )
 
     def test_checkout_document_view_no_permission(self):
-        self.login_user()
-
         response = self._request_document_checkout_view()
         self.assertEquals(response.status_code, 403)
         self.assertFalse(self.document.is_checked_out())
 
     def test_checkout_document_view_with_access(self):
-        self.login_user()
         self.grant_access(
             obj=self.document, permission=permission_document_checkout
         )
@@ -102,6 +102,36 @@ class DocumentCheckoutViewTestCase(GenericDocumentViewTestCase):
         self.assertEquals(response.status_code, 302)
         self.assertTrue(self.document.is_checked_out())
 
+    def _request_checkout_detail_view(self):
+        return self.get(
+            viewname='checkouts:checkout_info', args=(self.document.pk,),
+        )
+
+    def test_checkout_detail_view_no_permission(self):
+        self._checkout_document()
+        self.grant_access(
+            obj=self.document,
+            permission=permission_document_checkout
+        )
+
+        response = self._request_checkout_detail_view()
+
+        self.assertNotContains(
+            response, text=STATE_LABELS[STATE_CHECKED_OUT], status_code=403
+        )
+
+    def test_checkout_detail_view_with_access(self):
+        self._checkout_document()
+
+        self.grant_access(
+            obj=self.document,
+            permission=permission_document_checkout_detail_view
+        )
+
+        response = self._request_checkout_detail_view()
+
+        self.assertContains(response, text=STATE_LABELS[STATE_CHECKED_OUT], status_code=200)
+
     def test_document_new_version_after_checkout(self):
         """
         Gitlab issue #231

mayan/apps/documents/models.py

@@ -45,7 +45,8 @@ from .permissions import permission_document_view
 from .settings import (
     setting_disable_base_image_cache, setting_disable_transformed_image_cache,
     setting_display_width, setting_display_height, setting_fix_orientation,
-    setting_language, setting_zoom_max_level, setting_zoom_min_level
+    setting_hash_block_size, setting_language, setting_zoom_max_level,
+    setting_zoom_min_level
 )
 from .signals import (
     post_document_created, post_document_type_change, post_version_upload
@@ -56,8 +57,8 @@ logger = logging.getLogger(__name__)
 
 
 # document image cache name hash function
-def HASH_FUNCTION(data):
+def hash_function():
-    return hashlib.sha256(data).hexdigest()
+    return hashlib.sha256()
 
 
 def UUID_FUNCTION(*args, **kwargs):
@@ -697,10 +698,25 @@ class DocumentVersion(models.Model):
         Open a document version's file and update the checksum field using
         the user provided checksum function
         """
+        block_size = setting_hash_block_size.value
+        if block_size == 0:
+            # If the setting value is 0 that means disable read limit. To disable
+            # the read limit passing None won't work, we pass -1 instead as per
+            # the Python documentation.
+            # https://docs.python.org/2/tutorial/inputoutput.html#methods-of-file-objects
+            block_size = -1
+
         if self.exists():
-            source = self.open()
+            hash_object = hash_function()
-            self.checksum = force_text(HASH_FUNCTION(source.read()))
+            with self.open() as file_object:
-            source.close()
+                while (True):
+                    data = file_object.read(block_size)
+                    if not data:
+                        break
+
+                    hash_object.update(data)
+
+            self.checksum = force_text(hash_object.hexdigest())
             if save:
                 self.save()
 

mayan/apps/documents/settings.py

@@ -62,6 +62,14 @@ setting_fix_orientation = namespace.add_setting(
         'feature and it is disabled by default.'
     )
 )
+setting_hash_block_size = namespace.add_setting(
+    global_name='DOCUMENTS_HASH_BLOCK_SIZE', default=0,
+    help_text=_(
+        'Size of blocks to use when calculating the document file\'s '
+        'checksum. A value of 0 disables the block calculation and the entire '
+        'file will be loaded into memory.'
+    )
+)
 setting_language = namespace.add_setting(
     global_name='DOCUMENTS_LANGUAGE', default=DEFAULT_LANGUAGE,
     help_text=_('Default documents language (in ISO639-3 format).')

mayan/apps/navigation/classes.py

@@ -419,7 +419,7 @@ class Link(object):
             try:
                 resolved_link.url = node.render(context)
             except Exception as exception:
-                logger.error(
+                logger.debug(
                     'Error resolving link "%s" URL; %s', self.text, exception
                 )
         elif self.url:

mayan/apps/tags/wizard_steps.py

@@ -45,7 +45,12 @@ class WizardStepTags(WizardStep):
         furl_instance = furl(querystring)
         Tag = apps.get_model(app_label='tags', model_name='Tag')
 
-        for tag in Tag.objects.filter(pk__in=furl_instance.args['tags'].split(',')):
+        tag_id_list = furl_instance.args.get('tags', '')
+
+        if tag_id_list:
+            tag_id_list = tag_id_list.split(',')
+
+        for tag in Tag.objects.filter(pk__in=tag_id_list):
             tag.documents.add(document)