diff --git a/mayan/apps/common/serialization.py b/mayan/apps/common/serialization.py new file mode 100644 index 0000000000..359500a622 --- /dev/null +++ b/mayan/apps/common/serialization.py @@ -0,0 +1,22 @@ +from __future__ import unicode_literals + +import yaml + +try: + from yaml import CSafeLoader as SafeLoader, CSafeDumper as SafeDumper +except ImportError: + from yaml import SafeLoader, SafeDumper + + +def yaml_dump(*args, **kwargs): + defaults = {'Dumper': SafeDumper} + defaults.update(kwargs) + + return yaml.dump(*args, **defaults) + + +def yaml_load(*args, **kwargs): + defaults = {'Loader': SafeLoader} + defaults.update(kwargs) + + return yaml.load(*args, **defaults) diff --git a/mayan/apps/common/storages.py b/mayan/apps/common/storages.py index ded073d4e8..44a97a08f6 100644 --- a/mayan/apps/common/storages.py +++ b/mayan/apps/common/storages.py @@ -1,14 +1,9 @@ from __future__ import unicode_literals -import yaml - -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader - from django.utils.module_loading import import_string +from mayan.apps.common.serialization import yaml_load + from .settings import ( setting_shared_storage, setting_shared_storage_arguments ) @@ -16,8 +11,7 @@ from .settings import ( storage_sharedupload = import_string( dotted_path=setting_shared_storage.value )( - **yaml.load( + **yaml_load( stream=setting_shared_storage_arguments.value or '{}', - Loader=SafeLoader ) ) diff --git a/mayan/apps/converter/backends/python.py b/mayan/apps/converter/backends/python.py index 672997f879..35cb0b31dc 100644 --- a/mayan/apps/converter/backends/python.py +++ b/mayan/apps/converter/backends/python.py @@ -7,15 +7,12 @@ import shutil from PIL import Image import PyPDF2 import sh -import yaml -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader + from django.utils.encoding import force_text from django.utils.translation import ugettext_lazy as _ +from mayan.apps.common.serialization import yaml_load from mayan.apps.storage.utils import NamedTemporaryFile from ..classes import ConverterBase @@ -27,8 +24,8 @@ from ..literals import ( DEFAULT_PDFINFO_PATH ) -pdftoppm_path = yaml.load( - stream=setting_graphics_backend_config.value, Loader=SafeLoader +pdftoppm_path = yaml_load( + stream=setting_graphics_backend_config.value ).get( 'pdftoppm_path', DEFAULT_PDFTOPPM_PATH ) @@ -39,16 +36,16 @@ except sh.CommandNotFound: pdftoppm = None else: pdftoppm_format = '-{}'.format( - yaml.load( - stream=setting_graphics_backend_config.value, Loader=SafeLoader + yaml_load( + stream=setting_graphics_backend_config.value ).get( 'pdftoppm_format', DEFAULT_PDFTOPPM_FORMAT ) ) pdftoppm_dpi = format( - yaml.load( - stream=setting_graphics_backend_config.value, Loader=SafeLoader + yaml_load( + stream=setting_graphics_backend_config.value ).get( 'pdftoppm_dpi', DEFAULT_PDFTOPPM_DPI ) @@ -56,8 +53,8 @@ else: pdftoppm = pdftoppm.bake(pdftoppm_format, '-r', pdftoppm_dpi) -pdfinfo_path = yaml.load( - stream=setting_graphics_backend_config.value, Loader=SafeLoader +pdfinfo_path = yaml_load( + stream=setting_graphics_backend_config.value ).get( 'pdfinfo_path', DEFAULT_PDFINFO_PATH ) diff --git a/mayan/apps/converter/classes.py b/mayan/apps/converter/classes.py index 517b04ac51..f248240561 100644 --- a/mayan/apps/converter/classes.py +++ b/mayan/apps/converter/classes.py @@ -7,15 +7,10 @@ import shutil from PIL import Image import sh -import yaml - -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader from django.utils.translation import ugettext_lazy as _ +from mayan.apps.common.serialization import yaml_load from mayan.apps.mimetype.api import get_mimetype from mayan.apps.storage.settings import setting_temporary_directory from mayan.apps.storage.utils import ( @@ -30,8 +25,8 @@ from .literals import ( from .settings import setting_graphics_backend_config logger = logging.getLogger(__name__) -BACKEND_CONFIG = yaml.load( - stream=setting_graphics_backend_config.value, Loader=SafeLoader +BACKEND_CONFIG = yaml_load( + stream=setting_graphics_backend_config.value ) libreoffice_path = BACKEND_CONFIG.get( 'libreoffice_path', DEFAULT_LIBREOFFICE_PATH @@ -62,8 +57,8 @@ class ConverterBase(object): pass def get_page(self, output_format=None): - output_format = output_format or yaml.load( - stream=setting_graphics_backend_config.value, Loader=SafeLoader + output_format = output_format or yaml_load( + stream=setting_graphics_backend_config.value ).get( 'pillow_format', DEFAULT_PILLOW_FORMAT ) diff --git a/mayan/apps/converter/forms.py b/mayan/apps/converter/forms.py index 73749d2897..3137f0dc80 100644 --- a/mayan/apps/converter/forms.py +++ b/mayan/apps/converter/forms.py @@ -2,15 +2,12 @@ from __future__ import unicode_literals import yaml -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader - from django import forms from django.core.exceptions import ValidationError from django.utils.translation import ugettext_lazy as _ +from mayan.apps.common.serialization import yaml_load + from .models import Transformation @@ -21,7 +18,7 @@ class TransformationForm(forms.ModelForm): def clean(self): try: - yaml.load(stream=self.cleaned_data['arguments'], Loader=SafeLoader) + yaml_load(stream=self.cleaned_data['arguments']) except yaml.YAMLError: raise ValidationError( _( diff --git a/mayan/apps/converter/managers.py b/mayan/apps/converter/managers.py index 5856231203..45c4d02a10 100644 --- a/mayan/apps/converter/managers.py +++ b/mayan/apps/converter/managers.py @@ -2,16 +2,11 @@ from __future__ import unicode_literals import logging -import yaml - -try: - from yaml import CSafeLoader as SafeLoader, CSafeDumper as SafeDumper -except ImportError: - from yaml import SafeLoader, SafeDumper - from django.contrib.contenttypes.models import ContentType from django.db import models, transaction +from mayan.apps.common.serialization import yaml_dump, yaml_load + from .transformations import BaseTransformation logger = logging.getLogger(__name__) @@ -23,8 +18,8 @@ class TransformationManager(models.Manager): self.create( content_type=content_type, object_id=obj.pk, - name=transformation.name, arguments=yaml.dump( - data=arguments, Dumper=SafeDumper + name=transformation.name, arguments=yaml_dump( + data=arguments ) ) @@ -96,9 +91,8 @@ class TransformationManager(models.Manager): # Some transformations don't require arguments # return an empty dictionary as ** doesn't allow None if transformation.arguments: - kwargs = yaml.load( + kwargs = yaml_load( stream=transformation.arguments, - Loader=SafeLoader ) else: kwargs = {} diff --git a/mayan/apps/converter/validators.py b/mayan/apps/converter/validators.py index 49c45d0cae..dd8562a945 100644 --- a/mayan/apps/converter/validators.py +++ b/mayan/apps/converter/validators.py @@ -2,15 +2,12 @@ from __future__ import unicode_literals import yaml -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader - from django.core.exceptions import ValidationError from django.utils.deconstruct import deconstructible from django.utils.translation import ugettext_lazy as _ +from mayan.apps.common.serialization import yaml_load + @deconstructible class YAMLValidator(object): @@ -20,7 +17,7 @@ class YAMLValidator(object): def __call__(self, value): value = value.strip() try: - yaml.load(stream=value, Loader=SafeLoader) + yaml_load(stream=value) except yaml.error.YAMLError: raise ValidationError( _('Enter a valid YAML value.'), diff --git a/mayan/apps/document_signatures/storages.py b/mayan/apps/document_signatures/storages.py index 45d5d63d5e..6ec00d087d 100644 --- a/mayan/apps/document_signatures/storages.py +++ b/mayan/apps/document_signatures/storages.py @@ -1,14 +1,9 @@ from __future__ import unicode_literals -import yaml - -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader - from django.utils.module_loading import import_string +from mayan.apps.common.serialization import yaml_load + from .settings import ( setting_storage_backend, setting_storage_backend_arguments ) @@ -16,8 +11,7 @@ from .settings import ( storage_detachedsignature = import_string( dotted_path=setting_storage_backend.value )( - **yaml.load( - stream=setting_storage_backend_arguments.value or '{}', - Loader=SafeLoader + **yaml_load( + stream=setting_storage_backend_arguments.value or '{}' ) ) diff --git a/mayan/apps/documents/storages.py b/mayan/apps/documents/storages.py index 95405e9242..5d2ca25472 100644 --- a/mayan/apps/documents/storages.py +++ b/mayan/apps/documents/storages.py @@ -1,14 +1,9 @@ from __future__ import unicode_literals -import yaml - -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader - from django.utils.module_loading import import_string +from mayan.apps.common.serialization import yaml_load + from .settings import ( setting_documentimagecache_storage, setting_documentimagecache_storage_arguments, @@ -18,17 +13,15 @@ from .settings import ( storage_documentversion = import_string( dotted_path=setting_storage_backend.value )( - **yaml.load( + **yaml_load( stream=setting_storage_backend_arguments.value or '{}', - Loader=SafeLoader ) ) storage_documentimagecache = import_string( dotted_path=setting_documentimagecache_storage.value )( - **yaml.load( + **yaml_load( stream=setting_documentimagecache_storage_arguments.value or '{}', - Loader=SafeLoader ) ) diff --git a/mayan/apps/file_metadata/drivers/exiftool.py b/mayan/apps/file_metadata/drivers/exiftool.py index 79a0993916..1c4df94ed8 100644 --- a/mayan/apps/file_metadata/drivers/exiftool.py +++ b/mayan/apps/file_metadata/drivers/exiftool.py @@ -4,15 +4,10 @@ import json import logging import sh -import yaml - -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader from django.utils.translation import ugettext_lazy as _ +from mayan.apps.common.serialization import yaml_load from mayan.apps.storage.utils import NamedTemporaryFile from ..literals import DEFAULT_EXIF_PATH @@ -57,8 +52,8 @@ class EXIFToolDriver(FileMetadataDriver): ) def read_settings(self): - driver_arguments = yaml.load( - stream=setting_drivers_arguments.value, Loader=SafeLoader + driver_arguments = yaml_load( + stream=setting_drivers_arguments.value ) self.exiftool_path = driver_arguments.get( diff --git a/mayan/apps/ocr/backends/tesseract.py b/mayan/apps/ocr/backends/tesseract.py index 3444198e4d..6a84510d43 100644 --- a/mayan/apps/ocr/backends/tesseract.py +++ b/mayan/apps/ocr/backends/tesseract.py @@ -4,15 +4,11 @@ import logging import shutil import sh -import yaml -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader from django.utils.encoding import force_text from django.utils.translation import ugettext_lazy as _ +from mayan.apps.common.serialization import yaml_load from mayan.apps.storage.utils import TemporaryFile from ..classes import OCRBackendBase @@ -115,8 +111,7 @@ class Tesseract(OCRBackendBase): logger.debug('Available languages: %s', ', '.join(self.languages)) def read_settings(self): - backend_arguments = yaml.load( - Loader=SafeLoader, + backend_arguments = yaml_load( stream=setting_ocr_backend_arguments.value or '{}', ) diff --git a/mayan/apps/ocr/runtime.py b/mayan/apps/ocr/runtime.py index 1d8643819b..6e2c3ed95c 100644 --- a/mayan/apps/ocr/runtime.py +++ b/mayan/apps/ocr/runtime.py @@ -1,20 +1,15 @@ from __future__ import unicode_literals -import yaml - -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader - from django.utils.module_loading import import_string +from mayan.apps.common.serialization import yaml_load + from .settings import setting_ocr_backend, setting_ocr_backend_arguments ocr_backend = import_string( dotted_path=setting_ocr_backend.value )( - **yaml.load( - stream=setting_ocr_backend_arguments.value or '{}', Loader=SafeLoader + **yaml_load( + stream=setting_ocr_backend_arguments.value or '{}' ) ) diff --git a/mayan/apps/platform/classes.py b/mayan/apps/platform/classes.py index 1d0152442f..e50cb0a477 100644 --- a/mayan/apps/platform/classes.py +++ b/mayan/apps/platform/classes.py @@ -2,16 +2,11 @@ from __future__ import absolute_import, unicode_literals import os -import yaml -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader - from django.template import loader from django.utils.encoding import force_text, python_2_unicode_compatible from django.utils.translation import ugettext_lazy as _ +from mayan.apps.common.serialization import yaml_load from mayan.apps.common.settings import ( setting_celery_broker_url, setting_celery_result_backend ) @@ -95,9 +90,7 @@ class PlatformTemplate(object): if context_string: context.update( - yaml.load( - stream=context_string, Loader=SafeLoader - ) + yaml_load(stream=context_string) ) return loader.render_to_string( template_name=self.get_template_name(), diff --git a/mayan/apps/smart_settings/classes.py b/mayan/apps/smart_settings/classes.py index 9f35323608..85b040feaa 100644 --- a/mayan/apps/smart_settings/classes.py +++ b/mayan/apps/smart_settings/classes.py @@ -9,11 +9,6 @@ import sys import yaml -try: - from yaml import CSafeLoader as SafeLoader, CSafeDumper as SafeDumper -except ImportError: - from yaml import SafeLoader, SafeDumper - from django.apps import apps from django.conf import settings from django.utils.functional import Promise @@ -21,6 +16,8 @@ from django.utils.encoding import ( force_bytes, force_text, python_2_unicode_compatible ) +from mayan.apps.common.serialization import yaml_dump, yaml_load + logger = logging.getLogger(__name__) @@ -85,7 +82,7 @@ class Setting(object): @staticmethod def deserialize_value(value): - return yaml.load(stream=value, Loader=SafeLoader) + return yaml_load(stream=value) @staticmethod def express_promises(value): @@ -101,9 +98,8 @@ class Setting(object): @staticmethod def serialize_value(value): - result = yaml.dump( + result = yaml_dump( data=Setting.express_promises(value), allow_unicode=True, - Dumper=SafeDumper ) # safe_dump returns bytestrings # Disregard the last 3 dots that mark the end of the YAML document @@ -128,8 +124,8 @@ class Setting(object): if (filter_term and filter_term.lower() in setting.global_name.lower()) or not filter_term: dictionary[setting.global_name] = Setting.express_promises(setting.value) - return yaml.dump( - data=dictionary, default_flow_style=False, Dumper=SafeDumper + return yaml_dump( + data=dictionary, default_flow_style=False ) @classmethod diff --git a/mayan/apps/smart_settings/forms.py b/mayan/apps/smart_settings/forms.py index d72548316d..d00aa2a6f2 100644 --- a/mayan/apps/smart_settings/forms.py +++ b/mayan/apps/smart_settings/forms.py @@ -2,15 +2,12 @@ from __future__ import unicode_literals import yaml -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader - from django import forms from django.core.exceptions import ValidationError from django.utils.translation import ugettext_lazy as _ +from mayan.apps.common.serialization import yaml_load + class SettingForm(forms.Form): value = forms.CharField( @@ -38,7 +35,7 @@ class SettingForm(forms.Form): ) try: - yaml.load(stream=self.cleaned_data['value'], Loader=SafeLoader) + yaml_load(stream=self.cleaned_data['value']) except yaml.YAMLError: raise ValidationError( _( diff --git a/mayan/apps/sources/models/email_sources.py b/mayan/apps/sources/models/email_sources.py index 4c07219088..fa8ba913f8 100644 --- a/mayan/apps/sources/models/email_sources.py +++ b/mayan/apps/sources/models/email_sources.py @@ -4,18 +4,13 @@ import imaplib import logging import poplib -import yaml -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader - from django.core.exceptions import ValidationError from django.core.files.base import ContentFile from django.db import models from django.utils.encoding import force_bytes from django.utils.translation import ugettext_lazy as _ +from mayan.apps.common.serialization import yaml_load from mayan.apps.documents.models import Document from mayan.apps.metadata.api import set_bulk_metadata from mayan.apps.metadata.models import MetadataType @@ -142,8 +137,8 @@ class EmailBaseModel(IntervalBaseModel): with ContentFile(content=message.body, name=label) as file_object: if label == source.metadata_attachment_name: - metadata_dictionary = yaml.load( - stream=file_object.read(), Loader=SafeLoader + metadata_dictionary = yaml_load( + stream=file_object.read() ) logger.debug( 'Got metadata dictionary: %s', diff --git a/mayan/apps/sources/storages.py b/mayan/apps/sources/storages.py index 30e23ab7c5..791d4b12bc 100644 --- a/mayan/apps/sources/storages.py +++ b/mayan/apps/sources/storages.py @@ -1,13 +1,9 @@ from __future__ import unicode_literals -import yaml -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader - from django.utils.module_loading import import_string +from mayan.apps.common.serialization import yaml_load + from .settings import ( setting_staging_file_image_cache_storage, setting_staging_file_image_cache_storage_arguments, @@ -16,8 +12,7 @@ from .settings import ( storage_staging_file_image_cache = import_string( dotted_path=setting_staging_file_image_cache_storage.value )( - **yaml.load( + **yaml_load( stream=setting_staging_file_image_cache_storage_arguments.value or '{}', - Loader=SafeLoader ) ) diff --git a/mayan/apps/sources/tests/test_models.py b/mayan/apps/sources/tests/test_models.py index 9288ff57be..a49b00ce42 100644 --- a/mayan/apps/sources/tests/test_models.py +++ b/mayan/apps/sources/tests/test_models.py @@ -6,15 +6,11 @@ import shutil import mock from pathlib2 import Path -import yaml -try: - from yaml import CSafeDumper as SafeDumper -except ImportError: - from yaml import SafeDumper from django.core import mail from django.utils.encoding import force_text +from mayan.apps.common.serialization import yaml_dump from mayan.apps.documents.models import Document from mayan.apps.documents.tests import ( GenericDocumentTestCase, TEST_COMPRESSED_DOCUMENT_PATH, @@ -213,8 +209,8 @@ class EmailBaseTestCase(GenericDocumentTestCase): metadata_type=test_metadata_type_2 ) - test_metadata_yaml = yaml.dump( - Dumper=SafeDumper, data={ + test_metadata_yaml = yaml_dump( + data={ test_metadata_type_1.name: TEST_METADATA_VALUE_1, test_metadata_type_2.name: TEST_METADATA_VALUE_2, }