From 40c94360712b156f722bc7e50adb25b947d85a25 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Fri, 12 Apr 2019 03:09:07 -0400 Subject: [PATCH] Update PyYAML to version 5.1 Update use of safe_load and safe_dump to load and dump using the CSafeLoader with SafeLoader as a fallback. Signed-off-by: Roberto Rosario --- HISTORY.rst | 3 +++ docs/releases/3.2.rst | 2 ++ mayan/apps/common/storages.py | 10 ++++++++-- mayan/apps/converter/backends/python.py | 20 ++++++++++++++++---- mayan/apps/converter/classes.py | 11 +++++++++-- mayan/apps/converter/forms.py | 7 ++++++- mayan/apps/converter/managers.py | 14 ++++++++++++-- mayan/apps/converter/validators.py | 7 ++++++- mayan/apps/document_signatures/storages.py | 10 ++++++++-- mayan/apps/documents/storages.py | 15 +++++++++++---- mayan/apps/ocr/runtime.py | 9 +++++++-- mayan/apps/smart_settings/classes.py | 13 ++++++++++--- mayan/apps/smart_settings/forms.py | 7 ++++++- mayan/apps/sources/models/email_sources.py | 8 ++++++-- mayan/apps/sources/storages.py | 9 +++++++-- requirements/.base.txt.swp | Bin 0 -> 12288 bytes requirements/base.txt | 2 +- 17 files changed, 118 insertions(+), 29 deletions(-) create mode 100644 requirements/.base.txt.swp diff --git a/HISTORY.rst b/HISTORY.rst index f048e7a9bb..92b4e48ee5 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -23,6 +23,9 @@ * Remove development URLs from main URL file. * Move API documentation generation from the root URLs module to the API app's URLs module. +* Update Pillow to version 6.0.0 +* Update PyYAML to version 5.1. Update use of safe_load and + safe_dump to load and dump using the SafeLoader. 3.1.11 (2019-04-XX) =================== diff --git a/docs/releases/3.2.rst b/docs/releases/3.2.rst index 9af04d6946..e24e607ebe 100644 --- a/docs/releases/3.2.rst +++ b/docs/releases/3.2.rst @@ -48,6 +48,8 @@ Other changes * Remove Django suit from requirements. * Move API documentation generation from the root URLs module to the API app's URLs module. +* Update PyYAML to version 5.1. Update use of safe_load and + safe_dump to load and dump using the CSafeLoader and SafeLoader as fallback. Removals -------- diff --git a/mayan/apps/common/storages.py b/mayan/apps/common/storages.py index 371ce84d76..ded073d4e8 100644 --- a/mayan/apps/common/storages.py +++ b/mayan/apps/common/storages.py @@ -2,6 +2,11 @@ from __future__ import unicode_literals import yaml +try: + from yaml import CSafeLoader as SafeLoader +except ImportError: + from yaml import SafeLoader + from django.utils.module_loading import import_string from .settings import ( @@ -11,7 +16,8 @@ from .settings import ( storage_sharedupload = import_string( dotted_path=setting_shared_storage.value )( - **yaml.safe_load( - setting_shared_storage_arguments.value or '{}' + **yaml.load( + stream=setting_shared_storage_arguments.value or '{}', + Loader=SafeLoader ) ) diff --git a/mayan/apps/converter/backends/python.py b/mayan/apps/converter/backends/python.py index d9f25261e9..51ba5914b1 100644 --- a/mayan/apps/converter/backends/python.py +++ b/mayan/apps/converter/backends/python.py @@ -8,6 +8,10 @@ from PIL import Image import PyPDF2 import sh import yaml +try: + from yaml import CSafeLoader as SafeLoader +except ImportError: + from yaml import SafeLoader from django.utils.encoding import force_text from django.utils.translation import ugettext_lazy as _ @@ -25,7 +29,9 @@ from ..literals import ( try: pdftoppm = sh.Command( - yaml.load(setting_graphics_backend_config.value).get( + yaml.load( + stream=setting_graphics_backend_config.value, Loader=SafeLoader + ).get( 'pdftoppm_path', DEFAULT_PDFTOPPM_PATH ) ) @@ -33,13 +39,17 @@ except sh.CommandNotFound: pdftoppm = None else: pdftoppm_format = '-{}'.format( - yaml.load(setting_graphics_backend_config.value).get( + yaml.load( + stream=setting_graphics_backend_config.value, Loader=SafeLoader + ).get( 'pdftoppm_format', DEFAULT_PDFTOPPM_FORMAT ) ) pdftoppm_dpi = format( - yaml.load(setting_graphics_backend_config.value).get( + yaml.load( + stream=setting_graphics_backend_config.value, Loader=SafeLoader + ).get( 'pdftoppm_dpi', DEFAULT_PDFTOPPM_DPI ) ) @@ -48,7 +58,9 @@ else: try: pdfinfo = sh.Command( - yaml.load(setting_graphics_backend_config.value).get( + yaml.load( + stream=setting_graphics_backend_config.value, Loader=SafeLoader + ).get( 'pdfinfo_path', DEFAULT_PDFINFO_PATH ) ) diff --git a/mayan/apps/converter/classes.py b/mayan/apps/converter/classes.py index 1aebf12d6e..5f64fe5ee2 100644 --- a/mayan/apps/converter/classes.py +++ b/mayan/apps/converter/classes.py @@ -9,6 +9,11 @@ from PIL import Image import sh import yaml +try: + from yaml import CSafeLoader as SafeLoader +except ImportError: + from yaml import SafeLoader + from django.utils.translation import ugettext_lazy as _ from mayan.apps.common.settings import setting_temporary_directory @@ -26,7 +31,9 @@ logger = logging.getLogger(__name__) try: LIBREOFFICE = sh.Command( - yaml.load(setting_graphics_backend_config.value).get( + yaml.load( + stream=setting_graphics_backend_config.value, Loader=SafeLoader + ).get( 'libreoffice_path', DEFAULT_LIBREOFFICE_PATH ) ).bake('--headless', '--convert-to', 'pdf:writer_pdf_Export') @@ -181,7 +188,7 @@ class ConverterBase(object): def get_page(self, output_format=None, as_base64=False): output_format = output_format or yaml.load( - setting_graphics_backend_config.value + stream=setting_graphics_backend_config.value, Loader=SafeLoader ).get( 'pillow_format', DEFAULT_PILLOW_FORMAT ) diff --git a/mayan/apps/converter/forms.py b/mayan/apps/converter/forms.py index 41dd34b4f4..73749d2897 100644 --- a/mayan/apps/converter/forms.py +++ b/mayan/apps/converter/forms.py @@ -2,6 +2,11 @@ from __future__ import unicode_literals import yaml +try: + from yaml import CSafeLoader as SafeLoader +except ImportError: + from yaml import SafeLoader + from django import forms from django.core.exceptions import ValidationError from django.utils.translation import ugettext_lazy as _ @@ -16,7 +21,7 @@ class TransformationForm(forms.ModelForm): def clean(self): try: - yaml.safe_load(self.cleaned_data['arguments']) + yaml.load(stream=self.cleaned_data['arguments'], Loader=SafeLoader) except yaml.YAMLError: raise ValidationError( _( diff --git a/mayan/apps/converter/managers.py b/mayan/apps/converter/managers.py index 484fe45b24..b71e580e8b 100644 --- a/mayan/apps/converter/managers.py +++ b/mayan/apps/converter/managers.py @@ -4,6 +4,11 @@ import logging import yaml +try: + from yaml import CSafeLoader as SafeLoader, CDumper as Dumper +except ImportError: + from yaml import SafeLoader, Dumper + from django.contrib.contenttypes.models import ContentType from django.db import models, transaction @@ -18,7 +23,9 @@ class TransformationManager(models.Manager): self.create( content_type=content_type, object_id=obj.pk, - name=transformation.name, arguments=yaml.safe_dump(arguments) + name=transformation.name, arguments=yaml.dump( + data=arguments, Dumper=Dumper + ) ) def copy(self, source, targets): @@ -89,7 +96,10 @@ class TransformationManager(models.Manager): # Some transformations don't require arguments # return an empty dictionary as ** doesn't allow None if transformation.arguments: - kwargs = yaml.safe_load(transformation.arguments) + kwargs = yaml.load( + stream=transformation.arguments, + Loader=SafeLoader + ) else: kwargs = {} diff --git a/mayan/apps/converter/validators.py b/mayan/apps/converter/validators.py index a851bf4821..49c45d0cae 100644 --- a/mayan/apps/converter/validators.py +++ b/mayan/apps/converter/validators.py @@ -2,6 +2,11 @@ from __future__ import unicode_literals import yaml +try: + from yaml import CSafeLoader as SafeLoader +except ImportError: + from yaml import SafeLoader + from django.core.exceptions import ValidationError from django.utils.deconstruct import deconstructible from django.utils.translation import ugettext_lazy as _ @@ -15,7 +20,7 @@ class YAMLValidator(object): def __call__(self, value): value = value.strip() try: - yaml.safe_load(value) + yaml.load(stream=value, Loader=SafeLoader) except yaml.error.YAMLError: raise ValidationError( _('Enter a valid YAML value.'), diff --git a/mayan/apps/document_signatures/storages.py b/mayan/apps/document_signatures/storages.py index 0393501def..45d5d63d5e 100644 --- a/mayan/apps/document_signatures/storages.py +++ b/mayan/apps/document_signatures/storages.py @@ -2,6 +2,11 @@ from __future__ import unicode_literals import yaml +try: + from yaml import CSafeLoader as SafeLoader +except ImportError: + from yaml import SafeLoader + from django.utils.module_loading import import_string from .settings import ( @@ -11,7 +16,8 @@ from .settings import ( storage_detachedsignature = import_string( dotted_path=setting_storage_backend.value )( - **yaml.safe_load( - setting_storage_backend_arguments.value or '{}' + **yaml.load( + stream=setting_storage_backend_arguments.value or '{}', + Loader=SafeLoader ) ) diff --git a/mayan/apps/documents/storages.py b/mayan/apps/documents/storages.py index ae61eed877..f29ebd17bb 100644 --- a/mayan/apps/documents/storages.py +++ b/mayan/apps/documents/storages.py @@ -2,6 +2,11 @@ from __future__ import unicode_literals import yaml +try: + from yaml import CSafeLoader as SafeLoader +except ImportError: + from yaml import SafeLoader + from django.utils.module_loading import import_string from .settings import ( @@ -12,15 +17,17 @@ from .settings import ( storage_documentversion = import_string( dotted_path=setting_storage_backend.value )( - **yaml.safe_load( - setting_storage_backend_arguments.value or '{}' + **yaml.load( + stream=setting_storage_backend_arguments.value or '{}', + Loader=SafeLoader ) ) storage_documentimagecache = import_string( dotted_path=setting_documentimagecache_storage.value )( - **yaml.safe_load( - setting_documentimagecache_storage_arguments.value or '{}' + **yaml.load( + stream=setting_documentimagecache_storage_arguments.value or '{}', + Loader=SafeLoader ) ) diff --git a/mayan/apps/ocr/runtime.py b/mayan/apps/ocr/runtime.py index 69819029e7..06d20559a8 100644 --- a/mayan/apps/ocr/runtime.py +++ b/mayan/apps/ocr/runtime.py @@ -2,6 +2,11 @@ from __future__ import unicode_literals import yaml +try: + from yaml import CSafeLoader as SafeLoader +except ImportError: + from yaml import SafeLoader + from django.utils.module_loading import import_string from .settings import setting_ocr_backend, setting_ocr_backend_arguments @@ -9,7 +14,7 @@ from .settings import setting_ocr_backend, setting_ocr_backend_arguments ocr_backend = import_string( setting_ocr_backend.value )( - **yaml.safe_load( - setting_ocr_backend_arguments.value or '{}' + **yaml.load( + stream=setting_ocr_backend_arguments.value or '{}', Loader=SafeLoader ) ) diff --git a/mayan/apps/smart_settings/classes.py b/mayan/apps/smart_settings/classes.py index 32d1973b4c..96c559d6b5 100644 --- a/mayan/apps/smart_settings/classes.py +++ b/mayan/apps/smart_settings/classes.py @@ -8,6 +8,11 @@ import sys import yaml +try: + from yaml import CSafeLoader as SafeLoader, CDumper as Dumper +except ImportError: + from yaml import SafeLoader, Dumper + from django.apps import apps from django.conf import settings from django.utils.functional import Promise @@ -76,14 +81,14 @@ class Setting(object): @staticmethod def deserialize_value(value): - return yaml.safe_load(value) + return yaml.load(stream=value, Loader=SafeLoader) @staticmethod def serialize_value(value): if isinstance(value, Promise): value = force_text(value) - result = yaml.safe_dump(value, allow_unicode=True) + result = yaml.dump(data=value, allow_unicode=True, Dumper=Dumper) # safe_dump returns bytestrings # Disregard the last 3 dots that mark the end of the YAML document if force_text(result).endswith('...\n'): @@ -103,7 +108,9 @@ class Setting(object): else: dictionary[setting.global_name] = setting.value - return yaml.safe_dump(dictionary, default_flow_style=False) + return yaml.dump( + data=dictionary, default_flow_style=False, Dumper=Dumper + ) @classmethod def get(cls, global_name): diff --git a/mayan/apps/smart_settings/forms.py b/mayan/apps/smart_settings/forms.py index 3f008ff1ca..d72548316d 100644 --- a/mayan/apps/smart_settings/forms.py +++ b/mayan/apps/smart_settings/forms.py @@ -2,6 +2,11 @@ from __future__ import unicode_literals import yaml +try: + from yaml import CSafeLoader as SafeLoader +except ImportError: + from yaml import SafeLoader + from django import forms from django.core.exceptions import ValidationError from django.utils.translation import ugettext_lazy as _ @@ -33,7 +38,7 @@ class SettingForm(forms.Form): ) try: - yaml.safe_load(self.cleaned_data['value']) + yaml.load(stream=self.cleaned_data['value'], Loader=SafeLoader) except yaml.YAMLError: raise ValidationError( _( diff --git a/mayan/apps/sources/models/email_sources.py b/mayan/apps/sources/models/email_sources.py index 825ad4c55e..4227f9d4bb 100644 --- a/mayan/apps/sources/models/email_sources.py +++ b/mayan/apps/sources/models/email_sources.py @@ -5,6 +5,10 @@ import logging import poplib import yaml +try: + from yaml import CSafeLoader as SafeLoader +except ImportError: + from yaml import SafeLoader from django.core.exceptions import ValidationError from django.core.files.base import ContentFile @@ -129,8 +133,8 @@ class EmailBaseModel(IntervalBaseModel): label = message.detected_file_name or 'attachment-{}'.format(counter) with ContentFile(content=message.body, name=label) as file_object: if label == source.metadata_attachment_name: - metadata_dictionary = yaml.safe_load( - file_object.read() + metadata_dictionary = yaml.load( + stream=file_object.read(), Loader=SafeLoader ) logger.debug( 'Got metadata dictionary: %s', metadata_dictionary diff --git a/mayan/apps/sources/storages.py b/mayan/apps/sources/storages.py index e3bedf4311..30e23ab7c5 100644 --- a/mayan/apps/sources/storages.py +++ b/mayan/apps/sources/storages.py @@ -1,6 +1,10 @@ from __future__ import unicode_literals import yaml +try: + from yaml import CSafeLoader as SafeLoader +except ImportError: + from yaml import SafeLoader from django.utils.module_loading import import_string @@ -12,7 +16,8 @@ from .settings import ( storage_staging_file_image_cache = import_string( dotted_path=setting_staging_file_image_cache_storage.value )( - **yaml.safe_load( - setting_staging_file_image_cache_storage_arguments.value or '{}' + **yaml.load( + stream=setting_staging_file_image_cache_storage_arguments.value or '{}', + Loader=SafeLoader ) ) diff --git a/requirements/.base.txt.swp b/requirements/.base.txt.swp new file mode 100644 index 0000000000000000000000000000000000000000..94982803e2f6750084e9be06e02c3930feeeadc1 GIT binary patch literal 12288 zcmeI2J%|)Z6vs;=L03gZFgRH?6S}rP_A_Wu6j4yvL_v^)+U}a3a=W|Qs_GfXKtnY) zRK(DaKqCX;PB1jSgA)wzxV!K!xIjVt*Ua=Th^(>4YxvRBN55C~d$qx^wH97FcVTv! zhYZhN#;zQHaplmXgY4ccV@6wHls0~7tEb7LC|TZW#$@bf7jQ0@16k^(lC_(w#7NYE z%qtsMC)P?iXXLe38RfDDiUGC&5% z02v?yWPl8OhXzE>*z28)J==xF;^O7(+p$(fIq=I@EdpwUV$gzF?ay(gFE0h7=j#J1ZP16 zegHebN38z?_#6BM-h-dPPr&c(DWHc8kO4A42FL&zAOmE843L5E#=x+zoUFC7GEEbn z@Q5)x6h$FTU>lhQ15v76IBn82=JPyZwx6aEk76F-SB<|^EUmMCFk0s5X-4h_S|M7A zZ}fE?RH9H>nuhp3TIf29x@`*Inedga4~b;!RO!Pljlb6!5{G<&$82@9diqT4lgEp= zS>22>-PX>G(6j6(EV4$peyP?lCq~qmv0CRcu(BG+ag@Yg$gXs@?k9x4QM9$nw5bte zg|DEeFrw)X6t)m@zn=nks%z(yCVYXhUTbB8jQ;a^v<#Mt^{`$kuczrEk9^u*DeCn} zMjw*prWcHa#bzU{Gi3QyQ5V|4ThE9}4z*bijLceN)c{opxsUjjNxnc!d}%rq3{_r8 z7r3Dm>nK13yWMBF^KEop^mUm}40lO`YZlkRo$X2zjWtYi1y$p)(-0n_E=L7{W#CJK z3^*K{87jE$snSV<5ntJj6isVn(4geCa7sh1FY(4`Pn*hV?c?Fdba`YwP+0LK(n8mp z>tS7Lkq?v{`VFph11Omd`cmZM>@i0)176ql>VTgI-5B~>kzwAzWCVL u##%(?)IhnBmncQ$*R<$szZnD?aMD*tmrq_eKh~14RaKT6i^Ms^T0R4z@l&e+ literal 0 HcmV?d00001 diff --git a/requirements/base.txt b/requirements/base.txt index 80e9a31a31..84c9cea39f 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,5 +1,5 @@ Pillow==6.0.0 -PyYAML==3.13 +PyYAML==5.1 celery==3.1.24