Merge remote-tracking branch 'origin/features/move_yaml_code' into merge_features

Signed-off-by: Roberto Rosario <roberto.rosario.gonzalez@gmail.com>

HISTORY.rst

@@ -29,6 +29,16 @@
 - Added support for YAML encoded environment variables to the platform
   templates apps.
 
+3.2.6 (2019-07-10)
+==================
+* Remove the smart settings app * import.
+* Encode settings YAML before hashing.
+* Fix document icon used in the workflow runtime links.
+* Add trashed date time label.
+* Fix thumbnail generation issue. GitLab issue #637.
+  Thanks to Giacomo Cariello (@giacomocariello) for the report
+  and the merge request fixing the issue.
+
 3.2.5 (2019-07-05)
 ==================
 - Don't error out if the EXTRA_APPS or the DISABLED_APPS settings

docker/rootfs/version

@@ -1 +1 @@
-3.2.5
+3.2.6

docs/releases/3.2.6.rst

@@ -0,0 +1,111 @@
+Version 3.2.6
+=============
+
+Released: July 10, 2019
+
+
+Changes
+-------
+
+- Remove the smart settings app * import. Following MERC 0005.
+- Encode settings YAML before hashing. Avoids unicode issues with Python 3.
+- Fix document icon used in the workflow runtime links.
+- Add trashed date time label.
+- Fix thumbnail generation issue. GitLab issue #637.
+  Thanks to Giacomo Cariello (@giacomocariello) for the report
+  and the merge request fixing the issue.
+
+Removals
+--------
+
+- None
+
+
+Upgrading from a previous version
+---------------------------------
+
+If installed via Python's PIP
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Remove deprecated requirements::
+
+    sudo -u mayan curl https://gitlab.com/mayan-edms/mayan-edms/raw/master/removals.txt -o /tmp/removals.txt && sudo -u mayan /opt/mayan-edms/bin/pip uninstall -y -r /tmp/removals.txt
+
+Type in the console::
+
+    sudo -u mayan /opt/mayan-edms/bin/pip install mayan-edms==3.2.6
+
+the requirements will also be updated automatically.
+
+
+Using Git
+^^^^^^^^^
+
+If you installed Mayan EDMS by cloning the Git repository issue the commands::
+
+    $ git reset --hard HEAD
+    $ git pull
+
+otherwise download the compressed archived and uncompress it overriding the
+existing installation.
+
+Remove deprecated requirements::
+
+    $ pip uninstall -y -r removals.txt
+
+Next upgrade/add the new requirements::
+
+    $ pip install --upgrade -r requirements.txt
+
+
+Common steps
+^^^^^^^^^^^^
+
+Perform these steps after updating the code from either step above.
+
+Make a backup of your supervisord file::
+
+    sudo cp /etc/supervisor/conf.d/mayan.conf /etc/supervisor/conf.d/mayan.conf.bck
+
+Update the supervisord configuration file. Replace the environment
+variables values show here with your respective settings. This step will refresh
+the supervisord configuration file with the new queues and the latest
+recommended layout::
+
+    sudo MAYAN_DATABASE_ENGINE=django.db.backends.postgresql MAYAN_DATABASE_NAME=mayan \
+    MAYAN_DATABASE_PASSWORD=mayanuserpass MAYAN_DATABASE_USER=mayan \
+    MAYAN_DATABASE_HOST=127.0.0.1 MAYAN_MEDIA_ROOT=/opt/mayan-edms/media \
+    /opt/mayan-edms/bin/mayan-edms.py platformtemplate supervisord > /etc/supervisor/conf.d/mayan.conf
+
+Edit the supervisord configuration file and update any setting the template
+generator missed::
+
+    sudo vi /etc/supervisor/conf.d/mayan.conf
+
+Migrate existing database schema with::
+
+    sudo -u mayan MAYAN_DATABASE_ENGINE=django.db.backends.postgresql MAYAN_DATABASE_NAME=mayan \
+    MAYAN_DATABASE_PASSWORD=mayanuserpass MAYAN_DATABASE_USER=mayan \
+    MAYAN_DATABASE_HOST=127.0.0.1 MAYAN_MEDIA_ROOT=/opt/mayan-edms/media \
+    /opt/mayan-edms/bin/mayan-edms.py performupgrade
+
+Add new static media::
+
+    sudo -u mayan MAYAN_MEDIA_ROOT=/opt/mayan-edms/media \
+    /opt/mayan-edms/bin/mayan-edms.py preparestatic --noinput
+
+The upgrade procedure is now complete.
+
+
+Backward incompatible changes
+-----------------------------
+
+- None
+
+
+Bugs fixed or issues closed
+---------------------------
+
+- :gitlab-issue:`637` Thumbnail generation bug
+
+.. _PyPI: https://pypi.python.org/pypi/mayan-edms/

docs/releases/index.rst

@@ -21,6 +21,7 @@ versions of the documentation contain the release notes for any later releases.
    :maxdepth: 1
 
    3.3
+   3.2.6
    3.2.5
    3.2.4
    3.2.3

mayan/__init__.py

@@ -1,9 +1,9 @@
 from __future__ import unicode_literals
 
 __title__ = 'Mayan EDMS'
-__version__ = '3.2.5'
+__version__ = '3.2.6'
-__build__ = 0x030205
+__build__ = 0x030206
-__build_string__ = 'v3.2.5_Fri Jul 5 16:39:17 2019 -0400'
+__build_string__ = 'v3.2.6_Wed Jul 10 03:18:15 2019 -0400'
 __django_version__ = '1.11'
 __author__ = 'Roberto Rosario'
 __author_email__ = 'roberto.rosario@mayan-edms.com'

mayan/apps/common/serialization.py

@@ -0,0 +1,22 @@
+from __future__ import unicode_literals
+
+import yaml
+
+try:
+    from yaml import CSafeLoader as SafeLoader, CSafeDumper as SafeDumper
+except ImportError:
+    from yaml import SafeLoader, SafeDumper
+
+
+def yaml_dump(*args, **kwargs):
+    defaults = {'Dumper': SafeDumper}
+    defaults.update(kwargs)
+
+    return yaml.dump(*args, **defaults)
+
+
+def yaml_load(*args, **kwargs):
+    defaults = {'Loader': SafeLoader}
+    defaults.update(kwargs)
+
+    return yaml.load(*args, **defaults)

mayan/apps/converter/classes.py

@@ -10,6 +10,7 @@ import sh
 
 from django.utils.translation import ugettext_lazy as _
 
+from mayan.apps.common.serialization import yaml_load
 from mayan.apps.mimetype.api import get_mimetype
 from mayan.apps.storage.settings import setting_temporary_directory
 from mayan.apps.storage.utils import (

mayan/apps/converter/forms.py

@@ -2,15 +2,12 @@ from __future__ import unicode_literals
 
 import yaml
 
-try:
-    from yaml import CSafeLoader as SafeLoader
-except ImportError:
-    from yaml import SafeLoader
-
 from django import forms
 from django.core.exceptions import ValidationError
 from django.utils.translation import ugettext_lazy as _
 
+from mayan.apps.common.serialization import yaml_load
+
 from .models import Transformation
 
 
@@ -21,7 +18,7 @@ class TransformationForm(forms.ModelForm):
 
     def clean(self):
         try:
-            yaml.load(stream=self.cleaned_data['arguments'], Loader=SafeLoader)
+            yaml_load(stream=self.cleaned_data['arguments'])
         except yaml.YAMLError:
             raise ValidationError(
                 _(

mayan/apps/converter/managers.py

@@ -2,16 +2,11 @@ from __future__ import unicode_literals
 
 import logging
 
-import yaml
-
-try:
-    from yaml import CSafeLoader as SafeLoader, CSafeDumper as SafeDumper
-except ImportError:
-    from yaml import SafeLoader, SafeDumper
-
 from django.contrib.contenttypes.models import ContentType
 from django.db import models, transaction
 
+from mayan.apps.common.serialization import yaml_dump, yaml_load
+
 from .transformations import BaseTransformation
 
 logger = logging.getLogger(__name__)
@@ -23,8 +18,8 @@ class TransformationManager(models.Manager):
 
         self.create(
             content_type=content_type, object_id=obj.pk,
-            name=transformation.name, arguments=yaml.dump(
+            name=transformation.name, arguments=yaml_dump(
-                data=arguments, Dumper=SafeDumper
+                data=arguments
             )
         )
 
@@ -96,9 +91,8 @@ class TransformationManager(models.Manager):
                         # Some transformations don't require arguments
                         # return an empty dictionary as ** doesn't allow None
                         if transformation.arguments:
-                            kwargs = yaml.load(
+                            kwargs = yaml_load(
                                 stream=transformation.arguments,
-                                Loader=SafeLoader
                             )
                         else:
                             kwargs = {}

mayan/apps/converter/validators.py

@@ -2,15 +2,12 @@ from __future__ import unicode_literals
 
 import yaml
 
-try:
-    from yaml import CSafeLoader as SafeLoader
-except ImportError:
-    from yaml import SafeLoader
-
 from django.core.exceptions import ValidationError
 from django.utils.deconstruct import deconstructible
 from django.utils.translation import ugettext_lazy as _
 
+from mayan.apps.common.serialization import yaml_load
+
 
 @deconstructible
 class YAMLValidator(object):
@@ -20,7 +17,7 @@ class YAMLValidator(object):
     def __call__(self, value):
         value = value.strip()
         try:
-            yaml.load(stream=value, Loader=SafeLoader)
+            yaml_load(stream=value)
         except yaml.error.YAMLError:
             raise ValidationError(
                 _('Enter a valid YAML value.'),

mayan/apps/document_indexing/tests/test_views.py

@@ -17,7 +17,9 @@ from .literals import (
 from .mixins import IndexTestMixin, IndexViewTestMixin
 
 
-class IndexViewTestCase(IndexTestMixin, IndexViewTestMixin, GenericDocumentViewTestCase):
+class IndexViewTestCase(
+    IndexTestMixin, IndexViewTestMixin, GenericDocumentViewTestCase
+):
     auto_upload_document = False
 
     def test_index_create_view_no_permission(self):
@@ -113,7 +115,7 @@ class IndexViewTestCase(IndexTestMixin, IndexViewTestMixin, GenericDocumentViewT
         self.assertNotEqual(IndexInstanceNode.objects.count(), 0)
 
 
-class IndexInstaceViewTestCase(
+class IndexInstanceViewTestCase(
     IndexTestMixin, IndexViewTestMixin, GenericDocumentViewTestCase
 ):
     def _request_index_instance_node_view(self, index_instance_node):
@@ -148,12 +150,12 @@ class IndexInstaceViewTestCase(
 class IndexToolsViewTestCase(
     IndexTestMixin, IndexViewTestMixin, GenericDocumentViewTestCase
 ):
-    def _request_index_rebuild_get_view(self):
+    def _request_indexes_rebuild_get_view(self):
         return self.get(
             viewname='indexing:rebuild_index_instances'
         )
 
-    def _request_index_rebuild_post_view(self):
+    def _request_indexes_rebuild_post_view(self):
         return self.post(
             viewname='indexing:rebuild_index_instances', data={
                 'index_templates': self.test_index.pk
@@ -163,12 +165,12 @@ class IndexToolsViewTestCase(
     def test_indexes_rebuild_no_permission(self):
         self._create_test_index(rebuild=False)
 
-        response = self._request_index_rebuild_get_view()
+        response = self._request_indexes_rebuild_get_view()
         self.assertNotContains(
             response=response, text=self.test_index.label, status_code=200
         )
 
-        response = self._request_index_rebuild_post_view()
+        response = self._request_indexes_rebuild_post_view()
         # No error since we just don't see the index
         self.assertEqual(response.status_code, 200)
 
@@ -184,12 +186,12 @@ class IndexToolsViewTestCase(
             permission=permission_document_indexing_rebuild
         )
 
-        response = self._request_index_rebuild_get_view()
+        response = self._request_indexes_rebuild_get_view()
         self.assertContains(
             response=response, text=self.test_index.label, status_code=200
         )
 
-        response = self._request_index_rebuild_post_view()
+        response = self._request_indexes_rebuild_post_view()
         self.assertEqual(response.status_code, 302)
 
         # An instance root exists

mayan/apps/documents/models/document_page_models.py

@@ -229,7 +229,7 @@ class DocumentPage(models.Model):
                     for transformation in transformations:
                         converter.transform(transformation=transformation)
 
-                    return page_image
+                    return converter.get_page()
             except Exception as exception:
                 # Cleanup in case of error
                 logger.error(

mayan/apps/platform/classes.py

@@ -2,17 +2,12 @@ from __future__ import absolute_import, unicode_literals
 
 import os
 
-import yaml
-try:
-    from yaml import CSafeLoader as SafeLoader, CSafeDumper as SafeDumper
-except ImportError:
-    from yaml import SafeLoader, SafeDumper
-
 from django.template import loader
 from django.utils.html import mark_safe
 from django.utils.encoding import force_text, python_2_unicode_compatible
 from django.utils.translation import ugettext_lazy as _
 
+from mayan.apps.common.serialization import yaml_dump, yaml_load
 from mayan.apps.common.settings import (
     setting_celery_broker_url, setting_celery_result_backend
 )
@@ -36,12 +31,12 @@ class YAMLVariable(Variable):
     def _get_value(self):
         value = os.environ.get(self.environment_name)
         if value:
-            value = yaml.load(stream=value, Loader=SafeLoader)
+            value = yaml_load(stream=value)
         else:
             value = self.default
 
-        return yaml.dump(
+        return yaml_dump(
-            data=value, allow_unicode=True, default_flow_style=True, width=999, Dumper=SafeDumper
+            data=value, allow_unicode=True, default_flow_style=True, width=999
         ).replace('...\n', '').replace('\n', '')
 
 
@@ -112,9 +107,7 @@ class PlatformTemplate(object):
 
         if context_string:
             context.update(
-                yaml.load(
+                yaml_load(stream=context_string)
-                    stream=context_string, Loader=SafeLoader
-                )
             )
         return loader.render_to_string(
             template_name=self.get_template_name(),
@@ -158,7 +151,13 @@ class PlatformTemplateSupervisord(PlatformTemplate):
         ),
         YAMLVariable(
             name='DATABASES',
-            default={'default':{'ENGINE':'django.db.backends.postgresql','NAME':'mayan','PASSWORD':'mayanuserpass','USER':'mayan','HOST':'127.0.0.1'}},
+            default={
+                'default': {
+                    'ENGINE': 'django.db.backends.postgresql',
+                    'NAME': 'mayan', 'PASSWORD':'mayanuserpass',
+                    'USER': 'mayan', 'HOST':'127.0.0.1'
+                }
+            },
             environment_name='MAYAN_DATABASES'
         ),
         YAMLVariable

mayan/apps/smart_settings/classes.py

@@ -9,11 +9,6 @@ import sys
 
 import yaml
 
-try:
-    from yaml import CSafeLoader as SafeLoader, CSafeDumper as SafeDumper
-except ImportError:
-    from yaml import SafeLoader, SafeDumper
-
 from django.apps import apps
 from django.conf import settings
 from django.utils.functional import Promise
@@ -21,6 +16,8 @@ from django.utils.encoding import (
     force_bytes, force_text, python_2_unicode_compatible
 )
 
+from mayan.apps.common.serialization import yaml_dump, yaml_load
+
 from .utils import read_configuration_file
 
 logger = logging.getLogger(__name__)
@@ -88,7 +85,7 @@ class Setting(object):
 
     @staticmethod
     def deserialize_value(value):
-        return yaml.load(stream=value, Loader=SafeLoader)
+        return yaml_load(stream=value)
 
     @staticmethod
     def express_promises(value):
@@ -104,10 +101,9 @@ class Setting(object):
 
     @staticmethod
     def serialize_value(value):
-        result = yaml.dump(
+        result = yaml_dump(
             data=Setting.express_promises(value), allow_unicode=True,
             default_flow_style=False,
-            Dumper=SafeDumper
         )
         # safe_dump returns bytestrings
         # Disregard the last 3 dots that mark the end of the YAML document
@@ -132,8 +128,8 @@ class Setting(object):
                 if (filter_term and filter_term.lower() in setting.global_name.lower()) or not filter_term:
                     dictionary[setting.global_name] = Setting.express_promises(setting.value)
 
-        return yaml.dump(
+        return yaml_dump(
-            data=dictionary, default_flow_style=False, Dumper=SafeDumper
+            data=dictionary, default_flow_style=False
         )
 
     @classmethod
@@ -199,7 +195,7 @@ class Setting(object):
         if environment_value:
             self.environment_variable = True
             try:
-                self.raw_value = yaml.load(stream=environment_value, Loader=SafeLoader)
+                self.raw_value = yaml_load(stream=environment_value)
             except yaml.YAMLError as exception:
                 raise type(exception)(
                     'Error interpreting environment variable: {} with '

mayan/apps/smart_settings/forms.py

@@ -2,15 +2,12 @@ from __future__ import unicode_literals
 
 import yaml
 
-try:
-    from yaml import CSafeLoader as SafeLoader
-except ImportError:
-    from yaml import SafeLoader
-
 from django import forms
 from django.core.exceptions import ValidationError
 from django.utils.translation import ugettext_lazy as _
 
+from mayan.apps.common.serialization import yaml_load
+
 
 class SettingForm(forms.Form):
     value = forms.CharField(
@@ -26,7 +23,7 @@ class SettingForm(forms.Form):
 
     def clean(self):
         try:
-            yaml.load(stream=self.cleaned_data['value'], Loader=SafeLoader)
+            yaml_load(stream=self.cleaned_data['value'])
         except yaml.YAMLError:
             raise ValidationError(
                 _(

mayan/apps/smart_settings/tests/test_classes.py

@@ -48,6 +48,7 @@ class ClassesTestCase(SmartSettingTestMixin, BaseTestCase):
             default='test value'
         )
         # Initialize hash cache
+        Setting._cache_hash = None
         Setting.check_changed()
         self.assertFalse(Setting.check_changed())
         test_setting.value = 'test value edited'

mayan/apps/sources/models/email_sources.py

@@ -4,18 +4,13 @@ import imaplib
 import logging
 import poplib
 
-import yaml
-try:
-    from yaml import CSafeLoader as SafeLoader
-except ImportError:
-    from yaml import SafeLoader
-
 from django.core.exceptions import ValidationError
 from django.core.files.base import ContentFile
 from django.db import models
 from django.utils.encoding import force_bytes
 from django.utils.translation import ugettext_lazy as _
 
+from mayan.apps.common.serialization import yaml_load
 from mayan.apps.documents.models import Document
 from mayan.apps.metadata.api import set_bulk_metadata
 from mayan.apps.metadata.models import MetadataType
@@ -142,8 +137,8 @@ class EmailBaseModel(IntervalBaseModel):
 
                 with ContentFile(content=message.body, name=label) as file_object:
                     if label == source.metadata_attachment_name:
-                        metadata_dictionary = yaml.load(
+                        metadata_dictionary = yaml_load(
-                            stream=file_object.read(), Loader=SafeLoader
+                            stream=file_object.read()
                         )
                         logger.debug(
                             'Got metadata dictionary: %s',

mayan/apps/sources/tests/test_models.py

@@ -6,15 +6,11 @@ import shutil
 
 import mock
 from pathlib2 import Path
-import yaml
-try:
-    from yaml import CSafeDumper as SafeDumper
-except ImportError:
-    from yaml import SafeDumper
 
 from django.core import mail
 from django.utils.encoding import force_text
 
+from mayan.apps.common.serialization import yaml_dump
 from mayan.apps.documents.models import Document
 from mayan.apps.documents.tests import (
     GenericDocumentTestCase, TEST_COMPRESSED_DOCUMENT_PATH,
@@ -213,8 +209,8 @@ class EmailBaseTestCase(GenericDocumentTestCase):
             metadata_type=test_metadata_type_2
         )
 
-        test_metadata_yaml = yaml.dump(
+        test_metadata_yaml = yaml_dump(
-            Dumper=SafeDumper, data={
+            data={
                 test_metadata_type_1.name: TEST_METADATA_VALUE_1,
                 test_metadata_type_2.name: TEST_METADATA_VALUE_2,
             }