From 929cc7d37470e35485548f0c35ab2703d15793c3 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Sat, 6 Jul 2019 17:14:44 -0400 Subject: [PATCH 01/13] Encode settings YAML before hashing Signed-off-by: Roberto Rosario --- mayan/apps/smart_settings/classes.py | 6 ++++-- .../apps/smart_settings/tests/test_classes.py | 18 ++++++++++++++++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/mayan/apps/smart_settings/classes.py b/mayan/apps/smart_settings/classes.py index ccedbd5083..9f35323608 100644 --- a/mayan/apps/smart_settings/classes.py +++ b/mayan/apps/smart_settings/classes.py @@ -17,7 +17,9 @@ except ImportError: from django.apps import apps from django.conf import settings from django.utils.functional import Promise -from django.utils.encoding import force_text, python_2_unicode_compatible +from django.utils.encoding import ( + force_bytes, force_text, python_2_unicode_compatible +) logger = logging.getLogger(__name__) @@ -141,7 +143,7 @@ class Setting(object): @classmethod def get_hash(cls): return force_text( - hashlib.sha256(cls.dump_data()).hexdigest() + hashlib.sha256(force_bytes(cls.dump_data())).hexdigest() ) @classmethod diff --git a/mayan/apps/smart_settings/tests/test_classes.py b/mayan/apps/smart_settings/tests/test_classes.py index e4c919acce..6e88607f7f 100644 --- a/mayan/apps/smart_settings/tests/test_classes.py +++ b/mayan/apps/smart_settings/tests/test_classes.py @@ -11,12 +11,13 @@ from mayan.apps.common.settings import setting_paginate_by from mayan.apps.common.tests import BaseTestCase from mayan.apps.storage.utils import fs_cleanup -from ..classes import Setting +from ..classes import Namespace, Setting from .literals import ENVIRONMENT_TEST_NAME, ENVIRONMENT_TEST_VALUE +from .mixins import SmartSettingTestMixin -class ClassesTestCase(BaseTestCase): +class ClassesTestCase(SmartSettingTestMixin, BaseTestCase): def test_environment_variable(self): os.environ[ 'MAYAN_{}'.format(ENVIRONMENT_TEST_NAME) @@ -39,3 +40,16 @@ class ClassesTestCase(BaseTestCase): with path_config_backup.open(mode='r') as file_object: self.assertFalse('!!python/' in file_object.read()) + + def test_setting_check_changed(self): + self._create_test_settings_namespace() + test_setting = self.test_settings_namespace.add_setting( + global_name='SMART_SETTINGS_TEST_SETTING', + default='test value' + ) + # Initialize hash cache + Setting.check_changed() + self.assertFalse(Setting.check_changed()) + test_setting.value = 'test value edited' + self.assertTrue(Setting.check_changed()) + From f6febdd2210e12d5affd6aa2f6299e5c7cd78398 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 10 Jul 2019 01:40:13 -0400 Subject: [PATCH 02/13] Remove the smart settings app * import Signed-off-by: Roberto Rosario --- HISTORY.rst | 4 ++++ mayan/apps/appearance/settings.py | 2 +- mayan/apps/authentication/settings.py | 2 +- mayan/apps/autoadmin/settings.py | 2 +- mayan/apps/common/settings.py | 2 +- mayan/apps/converter/settings.py | 2 +- mayan/apps/django_gpg/settings.py | 2 +- mayan/apps/document_parsing/settings.py | 2 +- mayan/apps/document_signatures/settings.py | 2 +- mayan/apps/documents/settings.py | 2 +- mayan/apps/file_metadata/settings.py | 2 +- mayan/apps/lock_manager/settings.py | 2 +- mayan/apps/mailer/settings.py | 2 +- mayan/apps/metadata/settings.py | 2 +- mayan/apps/mirroring/settings.py | 2 +- mayan/apps/ocr/settings.py | 2 +- mayan/apps/smart_settings/__init__.py | 2 -- mayan/apps/sources/settings.py | 2 +- mayan/apps/storage/settings.py | 2 +- mayan/conf.py | 2 +- 20 files changed, 22 insertions(+), 20 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 33c632cde1..fe107128ca 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,3 +1,7 @@ +3.2.6 (2019-07-XX) +================== +* Remove the smart settings app * import. + 3.2.5 (2019-07-05) ================== * Don't error out if the EXTRA_APPS or the DISABLED_APPS settings diff --git a/mayan/apps/appearance/settings.py b/mayan/apps/appearance/settings.py index 5b93d43525..ad0927530a 100644 --- a/mayan/apps/appearance/settings.py +++ b/mayan/apps/appearance/settings.py @@ -2,7 +2,7 @@ from __future__ import unicode_literals from django.utils.translation import ugettext_lazy as _ -from mayan.apps.smart_settings import Namespace +from mayan.apps.smart_settings.classes import Namespace from .literals import DEFAULT_MAXIMUM_TITLE_LENGTH diff --git a/mayan/apps/authentication/settings.py b/mayan/apps/authentication/settings.py index de693442d5..ec264c7933 100644 --- a/mayan/apps/authentication/settings.py +++ b/mayan/apps/authentication/settings.py @@ -2,7 +2,7 @@ from __future__ import unicode_literals from django.utils.translation import ugettext_lazy as _ -from mayan.apps.smart_settings import Namespace +from mayan.apps.smart_settings.classes import Namespace from .literals import DEFAULT_LOGIN_METHOD, DEFAULT_MAXIMUM_SESSION_LENGTH diff --git a/mayan/apps/autoadmin/settings.py b/mayan/apps/autoadmin/settings.py index d74b9b0d74..3c21d27339 100644 --- a/mayan/apps/autoadmin/settings.py +++ b/mayan/apps/autoadmin/settings.py @@ -2,7 +2,7 @@ from __future__ import unicode_literals from django.utils.translation import ugettext_lazy as _ -from mayan.apps.smart_settings import Namespace +from mayan.apps.smart_settings.classes import Namespace from .literals import DEFAULT_EMAIL, DEFAULT_PASSWORD, DEFAULT_USERNAME diff --git a/mayan/apps/common/settings.py b/mayan/apps/common/settings.py index 841549b38f..84009fe170 100644 --- a/mayan/apps/common/settings.py +++ b/mayan/apps/common/settings.py @@ -6,7 +6,7 @@ from django.conf import settings from django.utils.translation import ugettext_lazy as _ import mayan -from mayan.apps.smart_settings import Namespace +from mayan.apps.smart_settings.classes import Namespace from .literals import DEFAULT_COMMON_HOME_VIEW diff --git a/mayan/apps/converter/settings.py b/mayan/apps/converter/settings.py index 14ce07a295..b680fa6d47 100644 --- a/mayan/apps/converter/settings.py +++ b/mayan/apps/converter/settings.py @@ -2,7 +2,7 @@ from __future__ import unicode_literals from django.utils.translation import ugettext_lazy as _ -from mayan.apps.smart_settings import Namespace +from mayan.apps.smart_settings.classes import Namespace from .literals import ( DEFAULT_LIBREOFFICE_PATH, DEFAULT_PDFTOPPM_DPI, DEFAULT_PDFTOPPM_FORMAT, diff --git a/mayan/apps/django_gpg/settings.py b/mayan/apps/django_gpg/settings.py index b7c763f45f..651f54bcbc 100644 --- a/mayan/apps/django_gpg/settings.py +++ b/mayan/apps/django_gpg/settings.py @@ -5,7 +5,7 @@ import os from django.conf import settings from django.utils.translation import ugettext_lazy as _ -from mayan.apps.smart_settings import Namespace +from mayan.apps.smart_settings.classes import Namespace namespace = Namespace(label=_('Signatures'), name='django_gpg') diff --git a/mayan/apps/document_parsing/settings.py b/mayan/apps/document_parsing/settings.py index c6626fbbd1..7bf2b78d3a 100644 --- a/mayan/apps/document_parsing/settings.py +++ b/mayan/apps/document_parsing/settings.py @@ -2,7 +2,7 @@ from __future__ import unicode_literals from django.utils.translation import ugettext_lazy as _ -from mayan.apps.smart_settings import Namespace +from mayan.apps.smart_settings.classes import Namespace namespace = Namespace(label=_('Document parsing'), name='document_parsing') diff --git a/mayan/apps/document_signatures/settings.py b/mayan/apps/document_signatures/settings.py index 6e0b75ec23..e146ec667a 100644 --- a/mayan/apps/document_signatures/settings.py +++ b/mayan/apps/document_signatures/settings.py @@ -5,7 +5,7 @@ import os from django.conf import settings from django.utils.translation import ugettext_lazy as _ -from mayan.apps.smart_settings import Namespace +from mayan.apps.smart_settings.classes import Namespace namespace = Namespace(label=_('Document signatures'), name='signatures') diff --git a/mayan/apps/documents/settings.py b/mayan/apps/documents/settings.py index 19c0162ca8..ca27496599 100644 --- a/mayan/apps/documents/settings.py +++ b/mayan/apps/documents/settings.py @@ -5,7 +5,7 @@ import os from django.conf import settings from django.utils.translation import ugettext_lazy as _ -from mayan.apps.smart_settings import Namespace +from mayan.apps.smart_settings.classes import Namespace from .literals import ( DEFAULT_DOCUMENTS_HASH_BLOCK_SIZE, DEFAULT_LANGUAGE, DEFAULT_LANGUAGE_CODES diff --git a/mayan/apps/file_metadata/settings.py b/mayan/apps/file_metadata/settings.py index 247f0b72ec..4fa78da06f 100644 --- a/mayan/apps/file_metadata/settings.py +++ b/mayan/apps/file_metadata/settings.py @@ -2,7 +2,7 @@ from __future__ import unicode_literals from django.utils.translation import ugettext_lazy as _ -from mayan.apps.smart_settings import Namespace +from mayan.apps.smart_settings.classes import Namespace from .literals import DEFAULT_EXIF_PATH diff --git a/mayan/apps/lock_manager/settings.py b/mayan/apps/lock_manager/settings.py index d401b4c462..30448fa1fb 100644 --- a/mayan/apps/lock_manager/settings.py +++ b/mayan/apps/lock_manager/settings.py @@ -2,7 +2,7 @@ from __future__ import unicode_literals from django.utils.translation import ugettext_lazy as _ -from mayan.apps.smart_settings import Namespace +from mayan.apps.smart_settings.classes import Namespace from .literals import DEFAULT_BACKEND, DEFAULT_LOCK_TIMEOUT_VALUE diff --git a/mayan/apps/mailer/settings.py b/mayan/apps/mailer/settings.py index ed689b60a3..6ad1772a95 100644 --- a/mayan/apps/mailer/settings.py +++ b/mayan/apps/mailer/settings.py @@ -2,7 +2,7 @@ from __future__ import unicode_literals from django.utils.translation import ugettext_lazy as _ -from mayan.apps.smart_settings import Namespace +from mayan.apps.smart_settings.classes import Namespace from .literals import ( DEFAULT_DOCUMENT_BODY_TEMPLATE, DEFAULT_LINK_BODY_TEMPLATE diff --git a/mayan/apps/metadata/settings.py b/mayan/apps/metadata/settings.py index ce7aa6de35..0387f45309 100644 --- a/mayan/apps/metadata/settings.py +++ b/mayan/apps/metadata/settings.py @@ -2,7 +2,7 @@ from __future__ import unicode_literals from django.utils.translation import ugettext_lazy as _ -from mayan.apps.smart_settings import Namespace +from mayan.apps.smart_settings.classes import Namespace from .parsers import MetadataParser from .validators import MetadataValidator diff --git a/mayan/apps/mirroring/settings.py b/mayan/apps/mirroring/settings.py index aa16aa1335..41f9b372a5 100644 --- a/mayan/apps/mirroring/settings.py +++ b/mayan/apps/mirroring/settings.py @@ -2,7 +2,7 @@ from __future__ import unicode_literals from django.utils.translation import ugettext_lazy as _ -from mayan.apps.smart_settings import Namespace +from mayan.apps.smart_settings.classes import Namespace namespace = Namespace(label=_('Mirroring'), name='mirroring') diff --git a/mayan/apps/ocr/settings.py b/mayan/apps/ocr/settings.py index f2aa1052a3..4293c3ac24 100644 --- a/mayan/apps/ocr/settings.py +++ b/mayan/apps/ocr/settings.py @@ -2,7 +2,7 @@ from __future__ import unicode_literals from django.utils.translation import ugettext_lazy as _ -from mayan.apps.smart_settings import Namespace +from mayan.apps.smart_settings.classes import Namespace namespace = Namespace(label=_('OCR'), name='ocr') diff --git a/mayan/apps/smart_settings/__init__.py b/mayan/apps/smart_settings/__init__.py index 143550fa7b..54f25908f0 100644 --- a/mayan/apps/smart_settings/__init__.py +++ b/mayan/apps/smart_settings/__init__.py @@ -1,5 +1,3 @@ from __future__ import unicode_literals -from .classes import Namespace, Setting # NOQA - default_app_config = 'mayan.apps.smart_settings.apps.SmartSettingsApp' diff --git a/mayan/apps/sources/settings.py b/mayan/apps/sources/settings.py index 8f0156d427..8f09ae99b6 100644 --- a/mayan/apps/sources/settings.py +++ b/mayan/apps/sources/settings.py @@ -5,7 +5,7 @@ import os from django.conf import settings from django.utils.translation import ugettext_lazy as _ -from mayan.apps.smart_settings import Namespace +from mayan.apps.smart_settings.classes import Namespace namespace = Namespace(label=_('Sources'), name='sources') diff --git a/mayan/apps/storage/settings.py b/mayan/apps/storage/settings.py index e17acbd76c..362aa9e702 100644 --- a/mayan/apps/storage/settings.py +++ b/mayan/apps/storage/settings.py @@ -4,7 +4,7 @@ import tempfile from django.utils.translation import ugettext_lazy as _ -from mayan.apps.smart_settings import Namespace +from mayan.apps.smart_settings.classes import Namespace namespace = Namespace(label=_('Storage'), name='storage') diff --git a/mayan/conf.py b/mayan/conf.py index 212322bd5d..898ccdd7da 100644 --- a/mayan/conf.py +++ b/mayan/conf.py @@ -6,7 +6,7 @@ from __future__ import unicode_literals from django.utils.translation import ugettext_lazy as _ -from mayan.apps.smart_settings import Namespace +from mayan.apps.smart_settings.classes import Namespace namespace = Namespace(name='mayan', label=_('Mayan')) From 644cd57a0ff96fa4a097efaca29108d31068fd5f Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 10 Jul 2019 01:44:19 -0400 Subject: [PATCH 03/13] Fix document icon used in workflow runtime links Links are using the document type instead of the document icon. Signed-off-by: Roberto Rosario --- HISTORY.rst | 2 ++ mayan/apps/document_states/icons.py | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index fe107128ca..ccd73e65f7 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,6 +1,8 @@ 3.2.6 (2019-07-XX) ================== * Remove the smart settings app * import. +* Encode settings YAML before hashing. +* Fix document icon used in the workflow runtime links. 3.2.5 (2019-07-05) ================== diff --git a/mayan/apps/document_states/icons.py b/mayan/apps/document_states/icons.py index 8ae3b8e990..ec6aaae232 100644 --- a/mayan/apps/document_states/icons.py +++ b/mayan/apps/document_states/icons.py @@ -1,7 +1,7 @@ from __future__ import absolute_import, unicode_literals from mayan.apps.appearance.classes import Icon -from mayan.apps.documents.icons import icon_document_type +from mayan.apps.documents.icons import icon_document, icon_document_type icon_workflow = Icon(driver_name='fontawesome', symbol='sitemap') @@ -31,11 +31,11 @@ icon_workflow_instance_transition = Icon( driver_name='fontawesome', symbol='arrows-alt-h' ) -icon_workflow_runtime_proxy_document_list = icon_document_type +icon_workflow_runtime_proxy_document_list = icon_document icon_workflow_runtime_proxy_list = Icon( driver_name='fontawesome', symbol='sitemap' ) -icon_workflow_runtime_proxy_state_document_list = icon_document_type +icon_workflow_runtime_proxy_state_document_list = icon_document icon_workflow_runtime_proxy_state_list = Icon( driver_name='fontawesome', symbol='circle' ) From 4fcd60194eeb95e412ef5ff9b51d42812301fc7a Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Sun, 7 Jul 2019 00:38:47 -0400 Subject: [PATCH 04/13] Add trashed date time label and position Signed-off-by: Roberto Rosario --- mayan/apps/documents/apps.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mayan/apps/documents/apps.py b/mayan/apps/documents/apps.py index 5e8639c466..d0443f1578 100644 --- a/mayan/apps/documents/apps.py +++ b/mayan/apps/documents/apps.py @@ -319,7 +319,8 @@ class DocumentsApp(MayanAppConfig): attribute='document_type', is_sortable=True, source=DeletedDocument ) SourceColumn( - attribute='deleted_date_time', source=DeletedDocument + attribute='deleted_date_time', include_label=True, order=99, + source=DeletedDocument ) # DocumentVersion From 627e9cb1f02be731e947a5130c4b866d28157de3 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 10 Jul 2019 01:47:35 -0400 Subject: [PATCH 05/13] Update changelog Signed-off-by: Roberto Rosario --- HISTORY.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/HISTORY.rst b/HISTORY.rst index ccd73e65f7..40902f1315 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,6 +3,7 @@ * Remove the smart settings app * import. * Encode settings YAML before hashing. * Fix document icon used in the workflow runtime links. +* Add trashed date time label. 3.2.5 (2019-07-05) ================== From 6d0b7b1fc0ea66015203321d58fb4fdf268051b3 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 10 Jul 2019 01:59:23 -0400 Subject: [PATCH 06/13] Speed up document indexing view tests Signed-off-by: Roberto Rosario --- .../document_indexing/tests/test_views.py | 26 ++++++++++++------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/mayan/apps/document_indexing/tests/test_views.py b/mayan/apps/document_indexing/tests/test_views.py index 633939b576..8ce8230d83 100644 --- a/mayan/apps/document_indexing/tests/test_views.py +++ b/mayan/apps/document_indexing/tests/test_views.py @@ -14,7 +14,11 @@ from .literals import TEST_INDEX_LABEL, TEST_INDEX_LABEL_EDITED from .mixins import IndexTestMixin, IndexViewTestMixin -class IndexViewTestCase(IndexTestMixin, IndexViewTestMixin, GenericDocumentViewTestCase): +class IndexViewTestCase( + IndexTestMixin, IndexViewTestMixin, GenericDocumentViewTestCase +): + auto_upload_document = False + def test_index_create_view_no_permission(self): response = self._request_test_index_create_view() self.assertEqual(response.status_code, 403) @@ -72,6 +76,10 @@ class IndexViewTestCase(IndexTestMixin, IndexViewTestMixin, GenericDocumentViewT self.test_index.refresh_from_db() self.assertEqual(self.test_index.label, TEST_INDEX_LABEL_EDITED) + +class IndexInstanceViewTestCase( + IndexTestMixin, IndexViewTestMixin, GenericDocumentViewTestCase +): def _request_index_instance_node_view(self, index_instance_node): return self.get( viewname='indexing:index_instance_node_view', kwargs={ @@ -100,27 +108,27 @@ class IndexViewTestCase(IndexTestMixin, IndexViewTestMixin, GenericDocumentViewT ) self.assertContains(response, text=TEST_INDEX_LABEL, status_code=200) - def _request_index_rebuild_get_view(self): + def _request_indexes_rebuild_get_view(self): return self.get( viewname='indexing:rebuild_index_instances', ) - def _request_index_rebuild_post_view(self): + def _request_indexes_rebuild_post_view(self): return self.post( viewname='indexing:rebuild_index_instances', data={ 'index_templates': self.test_index.pk } ) - def test_index_rebuild_no_permission(self): + def test_indexes_rebuild_no_permission(self): self._create_test_index(rebuild=False) - response = self._request_index_rebuild_get_view() + response = self._request_indexes_rebuild_get_view() self.assertNotContains( response=response, text=self.test_index.label, status_code=200 ) - response = self._request_index_rebuild_post_view() + response = self._request_indexes_rebuild_post_view() # No error since we just don't see the index self.assertEqual(response.status_code, 200) @@ -128,7 +136,7 @@ class IndexViewTestCase(IndexTestMixin, IndexViewTestMixin, GenericDocumentViewT self.test_index.instance_root.get_children_count(), 0 ) - def test_index_rebuild_with_access(self): + def test_indexes_rebuild_with_access(self): self._create_test_index(rebuild=False) self.grant_access( @@ -136,12 +144,12 @@ class IndexViewTestCase(IndexTestMixin, IndexViewTestMixin, GenericDocumentViewT permission=permission_document_indexing_rebuild ) - response = self._request_index_rebuild_get_view() + response = self._request_indexes_rebuild_get_view() self.assertContains( response=response, text=self.test_index.label, status_code=200 ) - response = self._request_index_rebuild_post_view() + response = self._request_indexes_rebuild_post_view() self.assertEqual(response.status_code, 302) # An instance root exists From 5c91ab2d730aa1e14538c4cccaa811c578755ea3 Mon Sep 17 00:00:00 2001 From: Giacomo Cariello Date: Mon, 8 Jul 2019 17:33:30 +0200 Subject: [PATCH 07/13] Fix thumbnail generation issue (#637). --- mayan/apps/documents/models/document_page_models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mayan/apps/documents/models/document_page_models.py b/mayan/apps/documents/models/document_page_models.py index b5a1661bd8..afbf34cef3 100644 --- a/mayan/apps/documents/models/document_page_models.py +++ b/mayan/apps/documents/models/document_page_models.py @@ -229,7 +229,7 @@ class DocumentPage(models.Model): for transformation in transformations: converter.transform(transformation=transformation) - return page_image + return converter.get_page() except Exception as exception: # Cleanup in case of error logger.error( From efb8a35ea4fb259140aebbe676ed44509136808f Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 10 Jul 2019 02:57:53 -0400 Subject: [PATCH 08/13] Update changelog, add release notes for v3.2.6 Signed-off-by: Roberto Rosario --- HISTORY.rst | 5 +- docs/releases/3.2.6.rst | 111 ++++++++++++++++++++++++++++++++++++++++ docs/releases/index.rst | 1 + 3 files changed, 116 insertions(+), 1 deletion(-) create mode 100644 docs/releases/3.2.6.rst diff --git a/HISTORY.rst b/HISTORY.rst index 40902f1315..8f70bcdaac 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,9 +1,12 @@ -3.2.6 (2019-07-XX) +3.2.6 (2019-07-10) ================== * Remove the smart settings app * import. * Encode settings YAML before hashing. * Fix document icon used in the workflow runtime links. * Add trashed date time label. +* Fix thumbnail generation issue. GitLab issue #637. + Thanks to Giacomo Cariello (@giacomocariello) for the report + and the merge request fixing the issue. 3.2.5 (2019-07-05) ================== diff --git a/docs/releases/3.2.6.rst b/docs/releases/3.2.6.rst new file mode 100644 index 0000000000..0e897a24c9 --- /dev/null +++ b/docs/releases/3.2.6.rst @@ -0,0 +1,111 @@ +Version 3.2.6 +============= + +Released: July 10, 2019 + + +Changes +------- + +- Remove the smart settings app * import. Following MERC 0005. +- Encode settings YAML before hashing. Avoids unicode issues with Python 3. +- Fix document icon used in the workflow runtime links. +- Add trashed date time label. +- Fix thumbnail generation issue. GitLab issue #637. + Thanks to Giacomo Cariello (@giacomocariello) for the report + and the merge request fixing the issue. + +Removals +-------- + +- None + + +Upgrading from a previous version +--------------------------------- + +If installed via Python's PIP +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Remove deprecated requirements:: + + sudo -u mayan curl https://gitlab.com/mayan-edms/mayan-edms/raw/master/removals.txt -o /tmp/removals.txt && sudo -u mayan /opt/mayan-edms/bin/pip uninstall -y -r /tmp/removals.txt + +Type in the console:: + + sudo -u mayan /opt/mayan-edms/bin/pip install mayan-edms==3.2.6 + +the requirements will also be updated automatically. + + +Using Git +^^^^^^^^^ + +If you installed Mayan EDMS by cloning the Git repository issue the commands:: + + $ git reset --hard HEAD + $ git pull + +otherwise download the compressed archived and uncompress it overriding the +existing installation. + +Remove deprecated requirements:: + + $ pip uninstall -y -r removals.txt + +Next upgrade/add the new requirements:: + + $ pip install --upgrade -r requirements.txt + + +Common steps +^^^^^^^^^^^^ + +Perform these steps after updating the code from either step above. + +Make a backup of your supervisord file:: + + sudo cp /etc/supervisor/conf.d/mayan.conf /etc/supervisor/conf.d/mayan.conf.bck + +Update the supervisord configuration file. Replace the environment +variables values show here with your respective settings. This step will refresh +the supervisord configuration file with the new queues and the latest +recommended layout:: + + sudo MAYAN_DATABASE_ENGINE=django.db.backends.postgresql MAYAN_DATABASE_NAME=mayan \ + MAYAN_DATABASE_PASSWORD=mayanuserpass MAYAN_DATABASE_USER=mayan \ + MAYAN_DATABASE_HOST=127.0.0.1 MAYAN_MEDIA_ROOT=/opt/mayan-edms/media \ + /opt/mayan-edms/bin/mayan-edms.py platformtemplate supervisord > /etc/supervisor/conf.d/mayan.conf + +Edit the supervisord configuration file and update any setting the template +generator missed:: + + sudo vi /etc/supervisor/conf.d/mayan.conf + +Migrate existing database schema with:: + + sudo -u mayan MAYAN_DATABASE_ENGINE=django.db.backends.postgresql MAYAN_DATABASE_NAME=mayan \ + MAYAN_DATABASE_PASSWORD=mayanuserpass MAYAN_DATABASE_USER=mayan \ + MAYAN_DATABASE_HOST=127.0.0.1 MAYAN_MEDIA_ROOT=/opt/mayan-edms/media \ + /opt/mayan-edms/bin/mayan-edms.py performupgrade + +Add new static media:: + + sudo -u mayan MAYAN_MEDIA_ROOT=/opt/mayan-edms/media \ + /opt/mayan-edms/bin/mayan-edms.py preparestatic --noinput + +The upgrade procedure is now complete. + + +Backward incompatible changes +----------------------------- + +- None + + +Bugs fixed or issues closed +--------------------------- + +- :gitlab-issue:`637` Thumbnail generation bug + +.. _PyPI: https://pypi.python.org/pypi/mayan-edms/ diff --git a/docs/releases/index.rst b/docs/releases/index.rst index 80bdb19c1e..452c02cdb6 100644 --- a/docs/releases/index.rst +++ b/docs/releases/index.rst @@ -20,6 +20,7 @@ versions of the documentation contain the release notes for any later releases. .. toctree:: :maxdepth: 1 + 3.2.6 3.2.5 3.2.4 3.2.3 From 6dde3e1ac3b81b3e21b7a898fc4553a9cd9b3a88 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 10 Jul 2019 03:14:12 -0400 Subject: [PATCH 09/13] PEP8 cleanup Signed-off-by: Roberto Rosario --- mayan/apps/smart_settings/tests/test_classes.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mayan/apps/smart_settings/tests/test_classes.py b/mayan/apps/smart_settings/tests/test_classes.py index 6e88607f7f..dc472dd245 100644 --- a/mayan/apps/smart_settings/tests/test_classes.py +++ b/mayan/apps/smart_settings/tests/test_classes.py @@ -11,7 +11,7 @@ from mayan.apps.common.settings import setting_paginate_by from mayan.apps.common.tests import BaseTestCase from mayan.apps.storage.utils import fs_cleanup -from ..classes import Namespace, Setting +from ..classes import Setting from .literals import ENVIRONMENT_TEST_NAME, ENVIRONMENT_TEST_VALUE from .mixins import SmartSettingTestMixin @@ -52,4 +52,3 @@ class ClassesTestCase(SmartSettingTestMixin, BaseTestCase): self.assertFalse(Setting.check_changed()) test_setting.value = 'test value edited' self.assertTrue(Setting.check_changed()) - From d04f1eb9a5054dc134744562d224435416f6e7a1 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 10 Jul 2019 03:17:04 -0400 Subject: [PATCH 10/13] Improve Setting check changed test Signed-off-by: Roberto Rosario --- mayan/apps/smart_settings/tests/test_classes.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mayan/apps/smart_settings/tests/test_classes.py b/mayan/apps/smart_settings/tests/test_classes.py index dc472dd245..4eb6839ea9 100644 --- a/mayan/apps/smart_settings/tests/test_classes.py +++ b/mayan/apps/smart_settings/tests/test_classes.py @@ -48,6 +48,7 @@ class ClassesTestCase(SmartSettingTestMixin, BaseTestCase): default='test value' ) # Initialize hash cache + Setting._cache_hash = None Setting.check_changed() self.assertFalse(Setting.check_changed()) test_setting.value = 'test value edited' From 1e1e930e88a1508a195292001033dc666a0c7131 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 10 Jul 2019 03:18:07 -0400 Subject: [PATCH 11/13] Bump version to 3.2.6 Signed-off-by: Roberto Rosario --- docker/rootfs/version | 2 +- mayan/__init__.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/rootfs/version b/docker/rootfs/version index 5ae69bd5f0..34cde5690e 100755 --- a/docker/rootfs/version +++ b/docker/rootfs/version @@ -1 +1 @@ -3.2.5 +3.2.6 diff --git a/mayan/__init__.py b/mayan/__init__.py index 3c4a65b10d..971d0e8e2e 100644 --- a/mayan/__init__.py +++ b/mayan/__init__.py @@ -1,9 +1,9 @@ from __future__ import unicode_literals __title__ = 'Mayan EDMS' -__version__ = '3.2.5' -__build__ = 0x030205 -__build_string__ = 'v3.2.5_Fri Jul 5 16:39:17 2019 -0400' +__version__ = '3.2.6' +__build__ = 0x030206 +__build_string__ = 'v3.2.5-11-gd04f1eb9a5_Wed Jul 10 03:17:04 2019 -0400' __django_version__ = '1.11' __author__ = 'Roberto Rosario' __author_email__ = 'roberto.rosario@mayan-edms.com' From 3621af7e7dc7773142e5b42a12c3513632c80126 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 10 Jul 2019 03:19:09 -0400 Subject: [PATCH 12/13] Update build string Signed-off-by: Roberto Rosario --- mayan/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mayan/__init__.py b/mayan/__init__.py index 971d0e8e2e..5c740e43d9 100644 --- a/mayan/__init__.py +++ b/mayan/__init__.py @@ -3,7 +3,7 @@ from __future__ import unicode_literals __title__ = 'Mayan EDMS' __version__ = '3.2.6' __build__ = 0x030206 -__build_string__ = 'v3.2.5-11-gd04f1eb9a5_Wed Jul 10 03:17:04 2019 -0400' +__build_string__ = 'v3.2.6_Wed Jul 10 03:18:15 2019 -0400' __django_version__ = '1.11' __author__ = 'Roberto Rosario' __author_email__ = 'roberto.rosario@mayan-edms.com' From 8bc4b6a95e43d33a4ede40dd41a1523117dd34e7 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 10 Jul 2019 19:35:42 -0400 Subject: [PATCH 13/13] Move YAML code to its own module Code now resides in common.serialization in the form of two new functions: yaml_load and yaml_dump. Signed-off-by: Roberto Rosario --- mayan/apps/common/serialization.py | 22 +++++++++++++++++++ mayan/apps/common/storages.py | 12 +++------- mayan/apps/converter/backends/python.py | 23 +++++++++----------- mayan/apps/converter/classes.py | 15 +++++-------- mayan/apps/converter/forms.py | 9 +++----- mayan/apps/converter/managers.py | 16 +++++--------- mayan/apps/converter/validators.py | 9 +++----- mayan/apps/document_signatures/storages.py | 14 ++++-------- mayan/apps/documents/storages.py | 15 ++++--------- mayan/apps/file_metadata/drivers/exiftool.py | 11 +++------- mayan/apps/ocr/backends/tesseract.py | 9 ++------ mayan/apps/ocr/runtime.py | 13 ++++------- mayan/apps/platform/classes.py | 11 ++-------- mayan/apps/smart_settings/classes.py | 16 +++++--------- mayan/apps/smart_settings/forms.py | 9 +++----- mayan/apps/sources/models/email_sources.py | 11 +++------- mayan/apps/sources/storages.py | 11 +++------- mayan/apps/sources/tests/test_models.py | 10 +++------ 18 files changed, 88 insertions(+), 148 deletions(-) create mode 100644 mayan/apps/common/serialization.py diff --git a/mayan/apps/common/serialization.py b/mayan/apps/common/serialization.py new file mode 100644 index 0000000000..359500a622 --- /dev/null +++ b/mayan/apps/common/serialization.py @@ -0,0 +1,22 @@ +from __future__ import unicode_literals + +import yaml + +try: + from yaml import CSafeLoader as SafeLoader, CSafeDumper as SafeDumper +except ImportError: + from yaml import SafeLoader, SafeDumper + + +def yaml_dump(*args, **kwargs): + defaults = {'Dumper': SafeDumper} + defaults.update(kwargs) + + return yaml.dump(*args, **defaults) + + +def yaml_load(*args, **kwargs): + defaults = {'Loader': SafeLoader} + defaults.update(kwargs) + + return yaml.load(*args, **defaults) diff --git a/mayan/apps/common/storages.py b/mayan/apps/common/storages.py index ded073d4e8..44a97a08f6 100644 --- a/mayan/apps/common/storages.py +++ b/mayan/apps/common/storages.py @@ -1,14 +1,9 @@ from __future__ import unicode_literals -import yaml - -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader - from django.utils.module_loading import import_string +from mayan.apps.common.serialization import yaml_load + from .settings import ( setting_shared_storage, setting_shared_storage_arguments ) @@ -16,8 +11,7 @@ from .settings import ( storage_sharedupload = import_string( dotted_path=setting_shared_storage.value )( - **yaml.load( + **yaml_load( stream=setting_shared_storage_arguments.value or '{}', - Loader=SafeLoader ) ) diff --git a/mayan/apps/converter/backends/python.py b/mayan/apps/converter/backends/python.py index 672997f879..35cb0b31dc 100644 --- a/mayan/apps/converter/backends/python.py +++ b/mayan/apps/converter/backends/python.py @@ -7,15 +7,12 @@ import shutil from PIL import Image import PyPDF2 import sh -import yaml -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader + from django.utils.encoding import force_text from django.utils.translation import ugettext_lazy as _ +from mayan.apps.common.serialization import yaml_load from mayan.apps.storage.utils import NamedTemporaryFile from ..classes import ConverterBase @@ -27,8 +24,8 @@ from ..literals import ( DEFAULT_PDFINFO_PATH ) -pdftoppm_path = yaml.load( - stream=setting_graphics_backend_config.value, Loader=SafeLoader +pdftoppm_path = yaml_load( + stream=setting_graphics_backend_config.value ).get( 'pdftoppm_path', DEFAULT_PDFTOPPM_PATH ) @@ -39,16 +36,16 @@ except sh.CommandNotFound: pdftoppm = None else: pdftoppm_format = '-{}'.format( - yaml.load( - stream=setting_graphics_backend_config.value, Loader=SafeLoader + yaml_load( + stream=setting_graphics_backend_config.value ).get( 'pdftoppm_format', DEFAULT_PDFTOPPM_FORMAT ) ) pdftoppm_dpi = format( - yaml.load( - stream=setting_graphics_backend_config.value, Loader=SafeLoader + yaml_load( + stream=setting_graphics_backend_config.value ).get( 'pdftoppm_dpi', DEFAULT_PDFTOPPM_DPI ) @@ -56,8 +53,8 @@ else: pdftoppm = pdftoppm.bake(pdftoppm_format, '-r', pdftoppm_dpi) -pdfinfo_path = yaml.load( - stream=setting_graphics_backend_config.value, Loader=SafeLoader +pdfinfo_path = yaml_load( + stream=setting_graphics_backend_config.value ).get( 'pdfinfo_path', DEFAULT_PDFINFO_PATH ) diff --git a/mayan/apps/converter/classes.py b/mayan/apps/converter/classes.py index 517b04ac51..f248240561 100644 --- a/mayan/apps/converter/classes.py +++ b/mayan/apps/converter/classes.py @@ -7,15 +7,10 @@ import shutil from PIL import Image import sh -import yaml - -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader from django.utils.translation import ugettext_lazy as _ +from mayan.apps.common.serialization import yaml_load from mayan.apps.mimetype.api import get_mimetype from mayan.apps.storage.settings import setting_temporary_directory from mayan.apps.storage.utils import ( @@ -30,8 +25,8 @@ from .literals import ( from .settings import setting_graphics_backend_config logger = logging.getLogger(__name__) -BACKEND_CONFIG = yaml.load( - stream=setting_graphics_backend_config.value, Loader=SafeLoader +BACKEND_CONFIG = yaml_load( + stream=setting_graphics_backend_config.value ) libreoffice_path = BACKEND_CONFIG.get( 'libreoffice_path', DEFAULT_LIBREOFFICE_PATH @@ -62,8 +57,8 @@ class ConverterBase(object): pass def get_page(self, output_format=None): - output_format = output_format or yaml.load( - stream=setting_graphics_backend_config.value, Loader=SafeLoader + output_format = output_format or yaml_load( + stream=setting_graphics_backend_config.value ).get( 'pillow_format', DEFAULT_PILLOW_FORMAT ) diff --git a/mayan/apps/converter/forms.py b/mayan/apps/converter/forms.py index 73749d2897..3137f0dc80 100644 --- a/mayan/apps/converter/forms.py +++ b/mayan/apps/converter/forms.py @@ -2,15 +2,12 @@ from __future__ import unicode_literals import yaml -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader - from django import forms from django.core.exceptions import ValidationError from django.utils.translation import ugettext_lazy as _ +from mayan.apps.common.serialization import yaml_load + from .models import Transformation @@ -21,7 +18,7 @@ class TransformationForm(forms.ModelForm): def clean(self): try: - yaml.load(stream=self.cleaned_data['arguments'], Loader=SafeLoader) + yaml_load(stream=self.cleaned_data['arguments']) except yaml.YAMLError: raise ValidationError( _( diff --git a/mayan/apps/converter/managers.py b/mayan/apps/converter/managers.py index 5856231203..45c4d02a10 100644 --- a/mayan/apps/converter/managers.py +++ b/mayan/apps/converter/managers.py @@ -2,16 +2,11 @@ from __future__ import unicode_literals import logging -import yaml - -try: - from yaml import CSafeLoader as SafeLoader, CSafeDumper as SafeDumper -except ImportError: - from yaml import SafeLoader, SafeDumper - from django.contrib.contenttypes.models import ContentType from django.db import models, transaction +from mayan.apps.common.serialization import yaml_dump, yaml_load + from .transformations import BaseTransformation logger = logging.getLogger(__name__) @@ -23,8 +18,8 @@ class TransformationManager(models.Manager): self.create( content_type=content_type, object_id=obj.pk, - name=transformation.name, arguments=yaml.dump( - data=arguments, Dumper=SafeDumper + name=transformation.name, arguments=yaml_dump( + data=arguments ) ) @@ -96,9 +91,8 @@ class TransformationManager(models.Manager): # Some transformations don't require arguments # return an empty dictionary as ** doesn't allow None if transformation.arguments: - kwargs = yaml.load( + kwargs = yaml_load( stream=transformation.arguments, - Loader=SafeLoader ) else: kwargs = {} diff --git a/mayan/apps/converter/validators.py b/mayan/apps/converter/validators.py index 49c45d0cae..dd8562a945 100644 --- a/mayan/apps/converter/validators.py +++ b/mayan/apps/converter/validators.py @@ -2,15 +2,12 @@ from __future__ import unicode_literals import yaml -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader - from django.core.exceptions import ValidationError from django.utils.deconstruct import deconstructible from django.utils.translation import ugettext_lazy as _ +from mayan.apps.common.serialization import yaml_load + @deconstructible class YAMLValidator(object): @@ -20,7 +17,7 @@ class YAMLValidator(object): def __call__(self, value): value = value.strip() try: - yaml.load(stream=value, Loader=SafeLoader) + yaml_load(stream=value) except yaml.error.YAMLError: raise ValidationError( _('Enter a valid YAML value.'), diff --git a/mayan/apps/document_signatures/storages.py b/mayan/apps/document_signatures/storages.py index 45d5d63d5e..6ec00d087d 100644 --- a/mayan/apps/document_signatures/storages.py +++ b/mayan/apps/document_signatures/storages.py @@ -1,14 +1,9 @@ from __future__ import unicode_literals -import yaml - -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader - from django.utils.module_loading import import_string +from mayan.apps.common.serialization import yaml_load + from .settings import ( setting_storage_backend, setting_storage_backend_arguments ) @@ -16,8 +11,7 @@ from .settings import ( storage_detachedsignature = import_string( dotted_path=setting_storage_backend.value )( - **yaml.load( - stream=setting_storage_backend_arguments.value or '{}', - Loader=SafeLoader + **yaml_load( + stream=setting_storage_backend_arguments.value or '{}' ) ) diff --git a/mayan/apps/documents/storages.py b/mayan/apps/documents/storages.py index 95405e9242..5d2ca25472 100644 --- a/mayan/apps/documents/storages.py +++ b/mayan/apps/documents/storages.py @@ -1,14 +1,9 @@ from __future__ import unicode_literals -import yaml - -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader - from django.utils.module_loading import import_string +from mayan.apps.common.serialization import yaml_load + from .settings import ( setting_documentimagecache_storage, setting_documentimagecache_storage_arguments, @@ -18,17 +13,15 @@ from .settings import ( storage_documentversion = import_string( dotted_path=setting_storage_backend.value )( - **yaml.load( + **yaml_load( stream=setting_storage_backend_arguments.value or '{}', - Loader=SafeLoader ) ) storage_documentimagecache = import_string( dotted_path=setting_documentimagecache_storage.value )( - **yaml.load( + **yaml_load( stream=setting_documentimagecache_storage_arguments.value or '{}', - Loader=SafeLoader ) ) diff --git a/mayan/apps/file_metadata/drivers/exiftool.py b/mayan/apps/file_metadata/drivers/exiftool.py index 79a0993916..1c4df94ed8 100644 --- a/mayan/apps/file_metadata/drivers/exiftool.py +++ b/mayan/apps/file_metadata/drivers/exiftool.py @@ -4,15 +4,10 @@ import json import logging import sh -import yaml - -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader from django.utils.translation import ugettext_lazy as _ +from mayan.apps.common.serialization import yaml_load from mayan.apps.storage.utils import NamedTemporaryFile from ..literals import DEFAULT_EXIF_PATH @@ -57,8 +52,8 @@ class EXIFToolDriver(FileMetadataDriver): ) def read_settings(self): - driver_arguments = yaml.load( - stream=setting_drivers_arguments.value, Loader=SafeLoader + driver_arguments = yaml_load( + stream=setting_drivers_arguments.value ) self.exiftool_path = driver_arguments.get( diff --git a/mayan/apps/ocr/backends/tesseract.py b/mayan/apps/ocr/backends/tesseract.py index 3444198e4d..6a84510d43 100644 --- a/mayan/apps/ocr/backends/tesseract.py +++ b/mayan/apps/ocr/backends/tesseract.py @@ -4,15 +4,11 @@ import logging import shutil import sh -import yaml -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader from django.utils.encoding import force_text from django.utils.translation import ugettext_lazy as _ +from mayan.apps.common.serialization import yaml_load from mayan.apps.storage.utils import TemporaryFile from ..classes import OCRBackendBase @@ -115,8 +111,7 @@ class Tesseract(OCRBackendBase): logger.debug('Available languages: %s', ', '.join(self.languages)) def read_settings(self): - backend_arguments = yaml.load( - Loader=SafeLoader, + backend_arguments = yaml_load( stream=setting_ocr_backend_arguments.value or '{}', ) diff --git a/mayan/apps/ocr/runtime.py b/mayan/apps/ocr/runtime.py index 1d8643819b..6e2c3ed95c 100644 --- a/mayan/apps/ocr/runtime.py +++ b/mayan/apps/ocr/runtime.py @@ -1,20 +1,15 @@ from __future__ import unicode_literals -import yaml - -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader - from django.utils.module_loading import import_string +from mayan.apps.common.serialization import yaml_load + from .settings import setting_ocr_backend, setting_ocr_backend_arguments ocr_backend = import_string( dotted_path=setting_ocr_backend.value )( - **yaml.load( - stream=setting_ocr_backend_arguments.value or '{}', Loader=SafeLoader + **yaml_load( + stream=setting_ocr_backend_arguments.value or '{}' ) ) diff --git a/mayan/apps/platform/classes.py b/mayan/apps/platform/classes.py index 1d0152442f..e50cb0a477 100644 --- a/mayan/apps/platform/classes.py +++ b/mayan/apps/platform/classes.py @@ -2,16 +2,11 @@ from __future__ import absolute_import, unicode_literals import os -import yaml -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader - from django.template import loader from django.utils.encoding import force_text, python_2_unicode_compatible from django.utils.translation import ugettext_lazy as _ +from mayan.apps.common.serialization import yaml_load from mayan.apps.common.settings import ( setting_celery_broker_url, setting_celery_result_backend ) @@ -95,9 +90,7 @@ class PlatformTemplate(object): if context_string: context.update( - yaml.load( - stream=context_string, Loader=SafeLoader - ) + yaml_load(stream=context_string) ) return loader.render_to_string( template_name=self.get_template_name(), diff --git a/mayan/apps/smart_settings/classes.py b/mayan/apps/smart_settings/classes.py index 9f35323608..85b040feaa 100644 --- a/mayan/apps/smart_settings/classes.py +++ b/mayan/apps/smart_settings/classes.py @@ -9,11 +9,6 @@ import sys import yaml -try: - from yaml import CSafeLoader as SafeLoader, CSafeDumper as SafeDumper -except ImportError: - from yaml import SafeLoader, SafeDumper - from django.apps import apps from django.conf import settings from django.utils.functional import Promise @@ -21,6 +16,8 @@ from django.utils.encoding import ( force_bytes, force_text, python_2_unicode_compatible ) +from mayan.apps.common.serialization import yaml_dump, yaml_load + logger = logging.getLogger(__name__) @@ -85,7 +82,7 @@ class Setting(object): @staticmethod def deserialize_value(value): - return yaml.load(stream=value, Loader=SafeLoader) + return yaml_load(stream=value) @staticmethod def express_promises(value): @@ -101,9 +98,8 @@ class Setting(object): @staticmethod def serialize_value(value): - result = yaml.dump( + result = yaml_dump( data=Setting.express_promises(value), allow_unicode=True, - Dumper=SafeDumper ) # safe_dump returns bytestrings # Disregard the last 3 dots that mark the end of the YAML document @@ -128,8 +124,8 @@ class Setting(object): if (filter_term and filter_term.lower() in setting.global_name.lower()) or not filter_term: dictionary[setting.global_name] = Setting.express_promises(setting.value) - return yaml.dump( - data=dictionary, default_flow_style=False, Dumper=SafeDumper + return yaml_dump( + data=dictionary, default_flow_style=False ) @classmethod diff --git a/mayan/apps/smart_settings/forms.py b/mayan/apps/smart_settings/forms.py index d72548316d..d00aa2a6f2 100644 --- a/mayan/apps/smart_settings/forms.py +++ b/mayan/apps/smart_settings/forms.py @@ -2,15 +2,12 @@ from __future__ import unicode_literals import yaml -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader - from django import forms from django.core.exceptions import ValidationError from django.utils.translation import ugettext_lazy as _ +from mayan.apps.common.serialization import yaml_load + class SettingForm(forms.Form): value = forms.CharField( @@ -38,7 +35,7 @@ class SettingForm(forms.Form): ) try: - yaml.load(stream=self.cleaned_data['value'], Loader=SafeLoader) + yaml_load(stream=self.cleaned_data['value']) except yaml.YAMLError: raise ValidationError( _( diff --git a/mayan/apps/sources/models/email_sources.py b/mayan/apps/sources/models/email_sources.py index 4c07219088..fa8ba913f8 100644 --- a/mayan/apps/sources/models/email_sources.py +++ b/mayan/apps/sources/models/email_sources.py @@ -4,18 +4,13 @@ import imaplib import logging import poplib -import yaml -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader - from django.core.exceptions import ValidationError from django.core.files.base import ContentFile from django.db import models from django.utils.encoding import force_bytes from django.utils.translation import ugettext_lazy as _ +from mayan.apps.common.serialization import yaml_load from mayan.apps.documents.models import Document from mayan.apps.metadata.api import set_bulk_metadata from mayan.apps.metadata.models import MetadataType @@ -142,8 +137,8 @@ class EmailBaseModel(IntervalBaseModel): with ContentFile(content=message.body, name=label) as file_object: if label == source.metadata_attachment_name: - metadata_dictionary = yaml.load( - stream=file_object.read(), Loader=SafeLoader + metadata_dictionary = yaml_load( + stream=file_object.read() ) logger.debug( 'Got metadata dictionary: %s', diff --git a/mayan/apps/sources/storages.py b/mayan/apps/sources/storages.py index 30e23ab7c5..791d4b12bc 100644 --- a/mayan/apps/sources/storages.py +++ b/mayan/apps/sources/storages.py @@ -1,13 +1,9 @@ from __future__ import unicode_literals -import yaml -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader - from django.utils.module_loading import import_string +from mayan.apps.common.serialization import yaml_load + from .settings import ( setting_staging_file_image_cache_storage, setting_staging_file_image_cache_storage_arguments, @@ -16,8 +12,7 @@ from .settings import ( storage_staging_file_image_cache = import_string( dotted_path=setting_staging_file_image_cache_storage.value )( - **yaml.load( + **yaml_load( stream=setting_staging_file_image_cache_storage_arguments.value or '{}', - Loader=SafeLoader ) ) diff --git a/mayan/apps/sources/tests/test_models.py b/mayan/apps/sources/tests/test_models.py index 9288ff57be..a49b00ce42 100644 --- a/mayan/apps/sources/tests/test_models.py +++ b/mayan/apps/sources/tests/test_models.py @@ -6,15 +6,11 @@ import shutil import mock from pathlib2 import Path -import yaml -try: - from yaml import CSafeDumper as SafeDumper -except ImportError: - from yaml import SafeDumper from django.core import mail from django.utils.encoding import force_text +from mayan.apps.common.serialization import yaml_dump from mayan.apps.documents.models import Document from mayan.apps.documents.tests import ( GenericDocumentTestCase, TEST_COMPRESSED_DOCUMENT_PATH, @@ -213,8 +209,8 @@ class EmailBaseTestCase(GenericDocumentTestCase): metadata_type=test_metadata_type_2 ) - test_metadata_yaml = yaml.dump( - Dumper=SafeDumper, data={ + test_metadata_yaml = yaml_dump( + data={ test_metadata_type_1.name: TEST_METADATA_VALUE_1, test_metadata_type_2.name: TEST_METADATA_VALUE_2, }