diff --git a/HISTORY.rst b/HISTORY.rst index f048e7a9bb..92b4e48ee5 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -23,6 +23,9 @@ * Remove development URLs from main URL file. * Move API documentation generation from the root URLs module to the API app's URLs module. +* Update Pillow to version 6.0.0 +* Update PyYAML to version 5.1. Update use of safe_load and + safe_dump to load and dump using the SafeLoader. 3.1.11 (2019-04-XX) =================== diff --git a/docs/releases/3.2.rst b/docs/releases/3.2.rst index 9af04d6946..e24e607ebe 100644 --- a/docs/releases/3.2.rst +++ b/docs/releases/3.2.rst @@ -48,6 +48,8 @@ Other changes * Remove Django suit from requirements. * Move API documentation generation from the root URLs module to the API app's URLs module. +* Update PyYAML to version 5.1. Update use of safe_load and + safe_dump to load and dump using the CSafeLoader and SafeLoader as fallback. Removals -------- diff --git a/mayan/apps/common/storages.py b/mayan/apps/common/storages.py index 371ce84d76..ded073d4e8 100644 --- a/mayan/apps/common/storages.py +++ b/mayan/apps/common/storages.py @@ -2,6 +2,11 @@ from __future__ import unicode_literals import yaml +try: + from yaml import CSafeLoader as SafeLoader +except ImportError: + from yaml import SafeLoader + from django.utils.module_loading import import_string from .settings import ( @@ -11,7 +16,8 @@ from .settings import ( storage_sharedupload = import_string( dotted_path=setting_shared_storage.value )( - **yaml.safe_load( - setting_shared_storage_arguments.value or '{}' + **yaml.load( + stream=setting_shared_storage_arguments.value or '{}', + Loader=SafeLoader ) ) diff --git a/mayan/apps/converter/backends/python.py b/mayan/apps/converter/backends/python.py index d9f25261e9..51ba5914b1 100644 --- a/mayan/apps/converter/backends/python.py +++ b/mayan/apps/converter/backends/python.py @@ -8,6 +8,10 @@ from PIL import Image import PyPDF2 import sh import yaml +try: + from yaml import CSafeLoader as SafeLoader +except ImportError: + from yaml import SafeLoader from django.utils.encoding import force_text from django.utils.translation import ugettext_lazy as _ @@ -25,7 +29,9 @@ from ..literals import ( try: pdftoppm = sh.Command( - yaml.load(setting_graphics_backend_config.value).get( + yaml.load( + stream=setting_graphics_backend_config.value, Loader=SafeLoader + ).get( 'pdftoppm_path', DEFAULT_PDFTOPPM_PATH ) ) @@ -33,13 +39,17 @@ except sh.CommandNotFound: pdftoppm = None else: pdftoppm_format = '-{}'.format( - yaml.load(setting_graphics_backend_config.value).get( + yaml.load( + stream=setting_graphics_backend_config.value, Loader=SafeLoader + ).get( 'pdftoppm_format', DEFAULT_PDFTOPPM_FORMAT ) ) pdftoppm_dpi = format( - yaml.load(setting_graphics_backend_config.value).get( + yaml.load( + stream=setting_graphics_backend_config.value, Loader=SafeLoader + ).get( 'pdftoppm_dpi', DEFAULT_PDFTOPPM_DPI ) ) @@ -48,7 +58,9 @@ else: try: pdfinfo = sh.Command( - yaml.load(setting_graphics_backend_config.value).get( + yaml.load( + stream=setting_graphics_backend_config.value, Loader=SafeLoader + ).get( 'pdfinfo_path', DEFAULT_PDFINFO_PATH ) ) diff --git a/mayan/apps/converter/classes.py b/mayan/apps/converter/classes.py index 1aebf12d6e..5f64fe5ee2 100644 --- a/mayan/apps/converter/classes.py +++ b/mayan/apps/converter/classes.py @@ -9,6 +9,11 @@ from PIL import Image import sh import yaml +try: + from yaml import CSafeLoader as SafeLoader +except ImportError: + from yaml import SafeLoader + from django.utils.translation import ugettext_lazy as _ from mayan.apps.common.settings import setting_temporary_directory @@ -26,7 +31,9 @@ logger = logging.getLogger(__name__) try: LIBREOFFICE = sh.Command( - yaml.load(setting_graphics_backend_config.value).get( + yaml.load( + stream=setting_graphics_backend_config.value, Loader=SafeLoader + ).get( 'libreoffice_path', DEFAULT_LIBREOFFICE_PATH ) ).bake('--headless', '--convert-to', 'pdf:writer_pdf_Export') @@ -181,7 +188,7 @@ class ConverterBase(object): def get_page(self, output_format=None, as_base64=False): output_format = output_format or yaml.load( - setting_graphics_backend_config.value + stream=setting_graphics_backend_config.value, Loader=SafeLoader ).get( 'pillow_format', DEFAULT_PILLOW_FORMAT ) diff --git a/mayan/apps/converter/forms.py b/mayan/apps/converter/forms.py index 41dd34b4f4..73749d2897 100644 --- a/mayan/apps/converter/forms.py +++ b/mayan/apps/converter/forms.py @@ -2,6 +2,11 @@ from __future__ import unicode_literals import yaml +try: + from yaml import CSafeLoader as SafeLoader +except ImportError: + from yaml import SafeLoader + from django import forms from django.core.exceptions import ValidationError from django.utils.translation import ugettext_lazy as _ @@ -16,7 +21,7 @@ class TransformationForm(forms.ModelForm): def clean(self): try: - yaml.safe_load(self.cleaned_data['arguments']) + yaml.load(stream=self.cleaned_data['arguments'], Loader=SafeLoader) except yaml.YAMLError: raise ValidationError( _( diff --git a/mayan/apps/converter/managers.py b/mayan/apps/converter/managers.py index 484fe45b24..b71e580e8b 100644 --- a/mayan/apps/converter/managers.py +++ b/mayan/apps/converter/managers.py @@ -4,6 +4,11 @@ import logging import yaml +try: + from yaml import CSafeLoader as SafeLoader, CDumper as Dumper +except ImportError: + from yaml import SafeLoader, Dumper + from django.contrib.contenttypes.models import ContentType from django.db import models, transaction @@ -18,7 +23,9 @@ class TransformationManager(models.Manager): self.create( content_type=content_type, object_id=obj.pk, - name=transformation.name, arguments=yaml.safe_dump(arguments) + name=transformation.name, arguments=yaml.dump( + data=arguments, Dumper=Dumper + ) ) def copy(self, source, targets): @@ -89,7 +96,10 @@ class TransformationManager(models.Manager): # Some transformations don't require arguments # return an empty dictionary as ** doesn't allow None if transformation.arguments: - kwargs = yaml.safe_load(transformation.arguments) + kwargs = yaml.load( + stream=transformation.arguments, + Loader=SafeLoader + ) else: kwargs = {} diff --git a/mayan/apps/converter/validators.py b/mayan/apps/converter/validators.py index a851bf4821..49c45d0cae 100644 --- a/mayan/apps/converter/validators.py +++ b/mayan/apps/converter/validators.py @@ -2,6 +2,11 @@ from __future__ import unicode_literals import yaml +try: + from yaml import CSafeLoader as SafeLoader +except ImportError: + from yaml import SafeLoader + from django.core.exceptions import ValidationError from django.utils.deconstruct import deconstructible from django.utils.translation import ugettext_lazy as _ @@ -15,7 +20,7 @@ class YAMLValidator(object): def __call__(self, value): value = value.strip() try: - yaml.safe_load(value) + yaml.load(stream=value, Loader=SafeLoader) except yaml.error.YAMLError: raise ValidationError( _('Enter a valid YAML value.'), diff --git a/mayan/apps/document_signatures/storages.py b/mayan/apps/document_signatures/storages.py index 0393501def..45d5d63d5e 100644 --- a/mayan/apps/document_signatures/storages.py +++ b/mayan/apps/document_signatures/storages.py @@ -2,6 +2,11 @@ from __future__ import unicode_literals import yaml +try: + from yaml import CSafeLoader as SafeLoader +except ImportError: + from yaml import SafeLoader + from django.utils.module_loading import import_string from .settings import ( @@ -11,7 +16,8 @@ from .settings import ( storage_detachedsignature = import_string( dotted_path=setting_storage_backend.value )( - **yaml.safe_load( - setting_storage_backend_arguments.value or '{}' + **yaml.load( + stream=setting_storage_backend_arguments.value or '{}', + Loader=SafeLoader ) ) diff --git a/mayan/apps/documents/storages.py b/mayan/apps/documents/storages.py index ae61eed877..f29ebd17bb 100644 --- a/mayan/apps/documents/storages.py +++ b/mayan/apps/documents/storages.py @@ -2,6 +2,11 @@ from __future__ import unicode_literals import yaml +try: + from yaml import CSafeLoader as SafeLoader +except ImportError: + from yaml import SafeLoader + from django.utils.module_loading import import_string from .settings import ( @@ -12,15 +17,17 @@ from .settings import ( storage_documentversion = import_string( dotted_path=setting_storage_backend.value )( - **yaml.safe_load( - setting_storage_backend_arguments.value or '{}' + **yaml.load( + stream=setting_storage_backend_arguments.value or '{}', + Loader=SafeLoader ) ) storage_documentimagecache = import_string( dotted_path=setting_documentimagecache_storage.value )( - **yaml.safe_load( - setting_documentimagecache_storage_arguments.value or '{}' + **yaml.load( + stream=setting_documentimagecache_storage_arguments.value or '{}', + Loader=SafeLoader ) ) diff --git a/mayan/apps/ocr/runtime.py b/mayan/apps/ocr/runtime.py index 69819029e7..06d20559a8 100644 --- a/mayan/apps/ocr/runtime.py +++ b/mayan/apps/ocr/runtime.py @@ -2,6 +2,11 @@ from __future__ import unicode_literals import yaml +try: + from yaml import CSafeLoader as SafeLoader +except ImportError: + from yaml import SafeLoader + from django.utils.module_loading import import_string from .settings import setting_ocr_backend, setting_ocr_backend_arguments @@ -9,7 +14,7 @@ from .settings import setting_ocr_backend, setting_ocr_backend_arguments ocr_backend = import_string( setting_ocr_backend.value )( - **yaml.safe_load( - setting_ocr_backend_arguments.value or '{}' + **yaml.load( + stream=setting_ocr_backend_arguments.value or '{}', Loader=SafeLoader ) ) diff --git a/mayan/apps/smart_settings/classes.py b/mayan/apps/smart_settings/classes.py index 32d1973b4c..96c559d6b5 100644 --- a/mayan/apps/smart_settings/classes.py +++ b/mayan/apps/smart_settings/classes.py @@ -8,6 +8,11 @@ import sys import yaml +try: + from yaml import CSafeLoader as SafeLoader, CDumper as Dumper +except ImportError: + from yaml import SafeLoader, Dumper + from django.apps import apps from django.conf import settings from django.utils.functional import Promise @@ -76,14 +81,14 @@ class Setting(object): @staticmethod def deserialize_value(value): - return yaml.safe_load(value) + return yaml.load(stream=value, Loader=SafeLoader) @staticmethod def serialize_value(value): if isinstance(value, Promise): value = force_text(value) - result = yaml.safe_dump(value, allow_unicode=True) + result = yaml.dump(data=value, allow_unicode=True, Dumper=Dumper) # safe_dump returns bytestrings # Disregard the last 3 dots that mark the end of the YAML document if force_text(result).endswith('...\n'): @@ -103,7 +108,9 @@ class Setting(object): else: dictionary[setting.global_name] = setting.value - return yaml.safe_dump(dictionary, default_flow_style=False) + return yaml.dump( + data=dictionary, default_flow_style=False, Dumper=Dumper + ) @classmethod def get(cls, global_name): diff --git a/mayan/apps/smart_settings/forms.py b/mayan/apps/smart_settings/forms.py index 3f008ff1ca..d72548316d 100644 --- a/mayan/apps/smart_settings/forms.py +++ b/mayan/apps/smart_settings/forms.py @@ -2,6 +2,11 @@ from __future__ import unicode_literals import yaml +try: + from yaml import CSafeLoader as SafeLoader +except ImportError: + from yaml import SafeLoader + from django import forms from django.core.exceptions import ValidationError from django.utils.translation import ugettext_lazy as _ @@ -33,7 +38,7 @@ class SettingForm(forms.Form): ) try: - yaml.safe_load(self.cleaned_data['value']) + yaml.load(stream=self.cleaned_data['value'], Loader=SafeLoader) except yaml.YAMLError: raise ValidationError( _( diff --git a/mayan/apps/sources/models/email_sources.py b/mayan/apps/sources/models/email_sources.py index 825ad4c55e..4227f9d4bb 100644 --- a/mayan/apps/sources/models/email_sources.py +++ b/mayan/apps/sources/models/email_sources.py @@ -5,6 +5,10 @@ import logging import poplib import yaml +try: + from yaml import CSafeLoader as SafeLoader +except ImportError: + from yaml import SafeLoader from django.core.exceptions import ValidationError from django.core.files.base import ContentFile @@ -129,8 +133,8 @@ class EmailBaseModel(IntervalBaseModel): label = message.detected_file_name or 'attachment-{}'.format(counter) with ContentFile(content=message.body, name=label) as file_object: if label == source.metadata_attachment_name: - metadata_dictionary = yaml.safe_load( - file_object.read() + metadata_dictionary = yaml.load( + stream=file_object.read(), Loader=SafeLoader ) logger.debug( 'Got metadata dictionary: %s', metadata_dictionary diff --git a/mayan/apps/sources/storages.py b/mayan/apps/sources/storages.py index e3bedf4311..30e23ab7c5 100644 --- a/mayan/apps/sources/storages.py +++ b/mayan/apps/sources/storages.py @@ -1,6 +1,10 @@ from __future__ import unicode_literals import yaml +try: + from yaml import CSafeLoader as SafeLoader +except ImportError: + from yaml import SafeLoader from django.utils.module_loading import import_string @@ -12,7 +16,8 @@ from .settings import ( storage_staging_file_image_cache = import_string( dotted_path=setting_staging_file_image_cache_storage.value )( - **yaml.safe_load( - setting_staging_file_image_cache_storage_arguments.value or '{}' + **yaml.load( + stream=setting_staging_file_image_cache_storage_arguments.value or '{}', + Loader=SafeLoader ) ) diff --git a/requirements/.base.txt.swp b/requirements/.base.txt.swp new file mode 100644 index 0000000000..94982803e2 Binary files /dev/null and b/requirements/.base.txt.swp differ diff --git a/requirements/base.txt b/requirements/base.txt index 80e9a31a31..84c9cea39f 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,5 +1,5 @@ Pillow==6.0.0 -PyYAML==3.13 +PyYAML==5.1 celery==3.1.24