Move YAML code to its own module

Code now resides in common.serialization in the form
of two new functions: yaml_load and yaml_dump.

Signed-off-by: Roberto Rosario <roberto.rosario@mayan-edms.com>

mayan/apps/common/serialization.py

@@ -0,0 +1,22 @@
+from __future__ import unicode_literals
+  
+import yaml
+
+try:
+    from yaml import CSafeLoader as SafeLoader, CSafeDumper as SafeDumper
+except ImportError:
+    from yaml import SafeLoader, SafeDumper
+
+
+def yaml_dump(*args, **kwargs):
+    defaults = {'Dumper': SafeDumper}
+    defaults.update(kwargs)
+
+    return yaml.dump(*args, **defaults)
+
+
+def yaml_load(*args, **kwargs):
+    defaults = {'Loader': SafeLoader}
+    defaults.update(kwargs)
+
+    return yaml.load(*args, **defaults)

mayan/apps/common/storages.py

@@ -1,14 +1,9 @@
 from __future__ import unicode_literals
 
-import yaml
-
-try:
-    from yaml import CSafeLoader as SafeLoader
-except ImportError:
-    from yaml import SafeLoader
-
 from django.utils.module_loading import import_string
 
+from mayan.apps.common.serialization import yaml_load
+
 from .settings import (
     setting_shared_storage, setting_shared_storage_arguments
 )
@@ -16,8 +11,7 @@ from .settings import (
 storage_sharedupload = import_string(
     dotted_path=setting_shared_storage.value
 )(
-    **yaml.load(
+    **yaml_load(
         stream=setting_shared_storage_arguments.value or '{}',
-        Loader=SafeLoader
     )
 )

mayan/apps/converter/backends/python.py

@@ -7,15 +7,12 @@ import shutil
 from PIL import Image
 import PyPDF2
 import sh
-import yaml
+
-try:
-    from yaml import CSafeLoader as SafeLoader
-except ImportError:
-    from yaml import SafeLoader
 
 from django.utils.encoding import force_text
 from django.utils.translation import ugettext_lazy as _
 
+from mayan.apps.common.serialization import yaml_load
 from mayan.apps.storage.utils import NamedTemporaryFile
 
 from ..classes import ConverterBase
@@ -27,8 +24,8 @@ from ..literals import (
     DEFAULT_PDFINFO_PATH
 )
 
-pdftoppm_path = yaml.load(
+pdftoppm_path = yaml_load(
-    stream=setting_graphics_backend_config.value, Loader=SafeLoader
+    stream=setting_graphics_backend_config.value
 ).get(
     'pdftoppm_path', DEFAULT_PDFTOPPM_PATH
 )
@@ -39,16 +36,16 @@ except sh.CommandNotFound:
     pdftoppm = None
 else:
     pdftoppm_format = '-{}'.format(
-        yaml.load(
+        yaml_load(
-            stream=setting_graphics_backend_config.value, Loader=SafeLoader
+            stream=setting_graphics_backend_config.value
         ).get(
             'pdftoppm_format', DEFAULT_PDFTOPPM_FORMAT
         )
     )
 
     pdftoppm_dpi = format(
-        yaml.load(
+        yaml_load(
-            stream=setting_graphics_backend_config.value, Loader=SafeLoader
+            stream=setting_graphics_backend_config.value
         ).get(
             'pdftoppm_dpi', DEFAULT_PDFTOPPM_DPI
         )
@@ -56,8 +53,8 @@ else:
 
     pdftoppm = pdftoppm.bake(pdftoppm_format, '-r', pdftoppm_dpi)
 
-pdfinfo_path = yaml.load(
+pdfinfo_path = yaml_load(
-    stream=setting_graphics_backend_config.value, Loader=SafeLoader
+    stream=setting_graphics_backend_config.value
 ).get(
     'pdfinfo_path', DEFAULT_PDFINFO_PATH
 )

mayan/apps/converter/classes.py

@@ -7,15 +7,10 @@ import shutil
 
 from PIL import Image
 import sh
-import yaml
-
-try:
-    from yaml import CSafeLoader as SafeLoader
-except ImportError:
-    from yaml import SafeLoader
 
 from django.utils.translation import ugettext_lazy as _
 
+from mayan.apps.common.serialization import yaml_load
 from mayan.apps.mimetype.api import get_mimetype
 from mayan.apps.storage.settings import setting_temporary_directory
 from mayan.apps.storage.utils import (
@@ -30,8 +25,8 @@ from .literals import (
 from .settings import setting_graphics_backend_config
 
 logger = logging.getLogger(__name__)
-BACKEND_CONFIG = yaml.load(
+BACKEND_CONFIG = yaml_load(
-    stream=setting_graphics_backend_config.value, Loader=SafeLoader
+    stream=setting_graphics_backend_config.value
 )
 libreoffice_path = BACKEND_CONFIG.get(
     'libreoffice_path', DEFAULT_LIBREOFFICE_PATH
@@ -62,8 +57,8 @@ class ConverterBase(object):
         pass
 
     def get_page(self, output_format=None):
-        output_format = output_format or yaml.load(
+        output_format = output_format or yaml_load(
-            stream=setting_graphics_backend_config.value, Loader=SafeLoader
+            stream=setting_graphics_backend_config.value
         ).get(
             'pillow_format', DEFAULT_PILLOW_FORMAT
         )

mayan/apps/converter/forms.py

@@ -2,15 +2,12 @@ from __future__ import unicode_literals
 
 import yaml
 
-try:
-    from yaml import CSafeLoader as SafeLoader
-except ImportError:
-    from yaml import SafeLoader
-
 from django import forms
 from django.core.exceptions import ValidationError
 from django.utils.translation import ugettext_lazy as _
 
+from mayan.apps.common.serialization import yaml_load
+
 from .models import Transformation
 
 
@@ -21,7 +18,7 @@ class TransformationForm(forms.ModelForm):
 
     def clean(self):
         try:
-            yaml.load(stream=self.cleaned_data['arguments'], Loader=SafeLoader)
+            yaml_load(stream=self.cleaned_data['arguments'])
         except yaml.YAMLError:
             raise ValidationError(
                 _(

mayan/apps/converter/managers.py

@@ -2,16 +2,11 @@ from __future__ import unicode_literals
 
 import logging
 
-import yaml
-
-try:
-    from yaml import CSafeLoader as SafeLoader, CSafeDumper as SafeDumper
-except ImportError:
-    from yaml import SafeLoader, SafeDumper
-
 from django.contrib.contenttypes.models import ContentType
 from django.db import models, transaction
 
+from mayan.apps.common.serialization import yaml_dump, yaml_load
+
 from .transformations import BaseTransformation
 
 logger = logging.getLogger(__name__)
@@ -23,8 +18,8 @@ class TransformationManager(models.Manager):
 
         self.create(
             content_type=content_type, object_id=obj.pk,
-            name=transformation.name, arguments=yaml.dump(
+            name=transformation.name, arguments=yaml_dump(
-                data=arguments, Dumper=SafeDumper
+                data=arguments
             )
         )
 
@@ -96,9 +91,8 @@ class TransformationManager(models.Manager):
                         # Some transformations don't require arguments
                         # return an empty dictionary as ** doesn't allow None
                         if transformation.arguments:
-                            kwargs = yaml.load(
+                            kwargs = yaml_load(
                                 stream=transformation.arguments,
-                                Loader=SafeLoader
                             )
                         else:
                             kwargs = {}

mayan/apps/converter/validators.py

@@ -2,15 +2,12 @@ from __future__ import unicode_literals
 
 import yaml
 
-try:
-    from yaml import CSafeLoader as SafeLoader
-except ImportError:
-    from yaml import SafeLoader
-
 from django.core.exceptions import ValidationError
 from django.utils.deconstruct import deconstructible
 from django.utils.translation import ugettext_lazy as _
 
+from mayan.apps.common.serialization import yaml_load
+
 
 @deconstructible
 class YAMLValidator(object):
@@ -20,7 +17,7 @@ class YAMLValidator(object):
     def __call__(self, value):
         value = value.strip()
         try:
-            yaml.load(stream=value, Loader=SafeLoader)
+            yaml_load(stream=value)
         except yaml.error.YAMLError:
             raise ValidationError(
                 _('Enter a valid YAML value.'),

mayan/apps/document_signatures/storages.py

@@ -1,14 +1,9 @@
 from __future__ import unicode_literals
 
-import yaml
-
-try:
-    from yaml import CSafeLoader as SafeLoader
-except ImportError:
-    from yaml import SafeLoader
-
 from django.utils.module_loading import import_string
 
+from mayan.apps.common.serialization import yaml_load
+
 from .settings import (
     setting_storage_backend, setting_storage_backend_arguments
 )
@@ -16,8 +11,7 @@ from .settings import (
 storage_detachedsignature = import_string(
     dotted_path=setting_storage_backend.value
 )(
-    **yaml.load(
+    **yaml_load(
-        stream=setting_storage_backend_arguments.value or '{}',
+        stream=setting_storage_backend_arguments.value or '{}'
-        Loader=SafeLoader
     )
 )

mayan/apps/documents/storages.py

@@ -1,14 +1,9 @@
 from __future__ import unicode_literals
 
-import yaml
-
-try:
-    from yaml import CSafeLoader as SafeLoader
-except ImportError:
-    from yaml import SafeLoader
-
 from django.utils.module_loading import import_string
 
+from mayan.apps.common.serialization import yaml_load
+
 from .settings import (
     setting_documentimagecache_storage,
     setting_documentimagecache_storage_arguments,
@@ -18,17 +13,15 @@ from .settings import (
 storage_documentversion = import_string(
     dotted_path=setting_storage_backend.value
 )(
-    **yaml.load(
+    **yaml_load(
         stream=setting_storage_backend_arguments.value or '{}',
-        Loader=SafeLoader
     )
 )
 
 storage_documentimagecache = import_string(
     dotted_path=setting_documentimagecache_storage.value
 )(
-    **yaml.load(
+    **yaml_load(
         stream=setting_documentimagecache_storage_arguments.value or '{}',
-        Loader=SafeLoader
     )
 )

mayan/apps/file_metadata/drivers/exiftool.py

@@ -4,15 +4,10 @@ import json
 import logging
 
 import sh
-import yaml
-
-try:
-    from yaml import CSafeLoader as SafeLoader
-except ImportError:
-    from yaml import SafeLoader
 
 from django.utils.translation import ugettext_lazy as _
 
+from mayan.apps.common.serialization import yaml_load
 from mayan.apps.storage.utils import NamedTemporaryFile
 
 from ..literals import DEFAULT_EXIF_PATH
@@ -57,8 +52,8 @@ class EXIFToolDriver(FileMetadataDriver):
             )
 
     def read_settings(self):
-        driver_arguments = yaml.load(
+        driver_arguments = yaml_load(
-            stream=setting_drivers_arguments.value, Loader=SafeLoader
+            stream=setting_drivers_arguments.value
         )
 
         self.exiftool_path = driver_arguments.get(

mayan/apps/ocr/backends/tesseract.py

@@ -4,15 +4,11 @@ import logging
 import shutil
 
 import sh
-import yaml
-try:
-    from yaml import CSafeLoader as SafeLoader
-except ImportError:
-    from yaml import SafeLoader
 
 from django.utils.encoding import force_text
 from django.utils.translation import ugettext_lazy as _
 
+from mayan.apps.common.serialization import yaml_load
 from mayan.apps.storage.utils import TemporaryFile
 
 from ..classes import OCRBackendBase
@@ -115,8 +111,7 @@ class Tesseract(OCRBackendBase):
             logger.debug('Available languages: %s', ', '.join(self.languages))
 
     def read_settings(self):
-        backend_arguments = yaml.load(
+        backend_arguments = yaml_load(
-            Loader=SafeLoader,
             stream=setting_ocr_backend_arguments.value or '{}',
         )
 

mayan/apps/ocr/runtime.py

@@ -1,20 +1,15 @@
 from __future__ import unicode_literals
 
-import yaml
-
-try:
-    from yaml import CSafeLoader as SafeLoader
-except ImportError:
-    from yaml import SafeLoader
-
 from django.utils.module_loading import import_string
 
+from mayan.apps.common.serialization import yaml_load
+
 from .settings import setting_ocr_backend, setting_ocr_backend_arguments
 
 ocr_backend = import_string(
     dotted_path=setting_ocr_backend.value
 )(
-    **yaml.load(
+    **yaml_load(
-        stream=setting_ocr_backend_arguments.value or '{}', Loader=SafeLoader
+        stream=setting_ocr_backend_arguments.value or '{}'
     )
 )

mayan/apps/platform/classes.py

@@ -2,16 +2,11 @@ from __future__ import absolute_import, unicode_literals
 
 import os
 
-import yaml
-try:
-    from yaml import CSafeLoader as SafeLoader
-except ImportError:
-    from yaml import SafeLoader
-
 from django.template import loader
 from django.utils.encoding import force_text, python_2_unicode_compatible
 from django.utils.translation import ugettext_lazy as _
 
+from mayan.apps.common.serialization import yaml_load
 from mayan.apps.common.settings import (
     setting_celery_broker_url, setting_celery_result_backend
 )
@@ -95,9 +90,7 @@ class PlatformTemplate(object):
 
         if context_string:
             context.update(
-                yaml.load(
+                yaml_load(stream=context_string)
-                    stream=context_string, Loader=SafeLoader
-                )
             )
         return loader.render_to_string(
             template_name=self.get_template_name(),

mayan/apps/smart_settings/classes.py

@@ -9,11 +9,6 @@ import sys
 
 import yaml
 
-try:
-    from yaml import CSafeLoader as SafeLoader, CSafeDumper as SafeDumper
-except ImportError:
-    from yaml import SafeLoader, SafeDumper
-
 from django.apps import apps
 from django.conf import settings
 from django.utils.functional import Promise
@@ -21,6 +16,8 @@ from django.utils.encoding import (
     force_bytes, force_text, python_2_unicode_compatible
 )
 
+from mayan.apps.common.serialization import yaml_dump, yaml_load
+
 logger = logging.getLogger(__name__)
 
 
@@ -85,7 +82,7 @@ class Setting(object):
 
     @staticmethod
     def deserialize_value(value):
-        return yaml.load(stream=value, Loader=SafeLoader)
+        return yaml_load(stream=value)
 
     @staticmethod
     def express_promises(value):
@@ -101,9 +98,8 @@ class Setting(object):
 
     @staticmethod
     def serialize_value(value):
-        result = yaml.dump(
+        result = yaml_dump(
             data=Setting.express_promises(value), allow_unicode=True,
-            Dumper=SafeDumper
         )
         # safe_dump returns bytestrings
         # Disregard the last 3 dots that mark the end of the YAML document
@@ -128,8 +124,8 @@ class Setting(object):
                 if (filter_term and filter_term.lower() in setting.global_name.lower()) or not filter_term:
                     dictionary[setting.global_name] = Setting.express_promises(setting.value)
 
-        return yaml.dump(
+        return yaml_dump(
-            data=dictionary, default_flow_style=False, Dumper=SafeDumper
+            data=dictionary, default_flow_style=False
         )
 
     @classmethod

mayan/apps/smart_settings/forms.py

@@ -2,15 +2,12 @@ from __future__ import unicode_literals
 
 import yaml
 
-try:
-    from yaml import CSafeLoader as SafeLoader
-except ImportError:
-    from yaml import SafeLoader
-
 from django import forms
 from django.core.exceptions import ValidationError
 from django.utils.translation import ugettext_lazy as _
 
+from mayan.apps.common.serialization import yaml_load
+
 
 class SettingForm(forms.Form):
     value = forms.CharField(
@@ -38,7 +35,7 @@ class SettingForm(forms.Form):
                 )
 
         try:
-            yaml.load(stream=self.cleaned_data['value'], Loader=SafeLoader)
+            yaml_load(stream=self.cleaned_data['value'])
         except yaml.YAMLError:
             raise ValidationError(
                 _(

mayan/apps/sources/models/email_sources.py

@@ -4,18 +4,13 @@ import imaplib
 import logging
 import poplib
 
-import yaml
-try:
-    from yaml import CSafeLoader as SafeLoader
-except ImportError:
-    from yaml import SafeLoader
-
 from django.core.exceptions import ValidationError
 from django.core.files.base import ContentFile
 from django.db import models
 from django.utils.encoding import force_bytes
 from django.utils.translation import ugettext_lazy as _
 
+from mayan.apps.common.serialization import yaml_load
 from mayan.apps.documents.models import Document
 from mayan.apps.metadata.api import set_bulk_metadata
 from mayan.apps.metadata.models import MetadataType
@@ -142,8 +137,8 @@ class EmailBaseModel(IntervalBaseModel):
 
                 with ContentFile(content=message.body, name=label) as file_object:
                     if label == source.metadata_attachment_name:
-                        metadata_dictionary = yaml.load(
+                        metadata_dictionary = yaml_load(
-                            stream=file_object.read(), Loader=SafeLoader
+                            stream=file_object.read()
                         )
                         logger.debug(
                             'Got metadata dictionary: %s',

mayan/apps/sources/storages.py

@@ -1,13 +1,9 @@
 from __future__ import unicode_literals
 
-import yaml
-try:
-    from yaml import CSafeLoader as SafeLoader
-except ImportError:
-    from yaml import SafeLoader
-
 from django.utils.module_loading import import_string
 
+from mayan.apps.common.serialization import yaml_load
+
 from .settings import (
     setting_staging_file_image_cache_storage,
     setting_staging_file_image_cache_storage_arguments,
@@ -16,8 +12,7 @@ from .settings import (
 storage_staging_file_image_cache = import_string(
     dotted_path=setting_staging_file_image_cache_storage.value
 )(
-    **yaml.load(
+    **yaml_load(
         stream=setting_staging_file_image_cache_storage_arguments.value or '{}',
-        Loader=SafeLoader
     )
 )

mayan/apps/sources/tests/test_models.py

@@ -6,15 +6,11 @@ import shutil
 
 import mock
 from pathlib2 import Path
-import yaml
-try:
-    from yaml import CSafeDumper as SafeDumper
-except ImportError:
-    from yaml import SafeDumper
 
 from django.core import mail
 from django.utils.encoding import force_text
 
+from mayan.apps.common.serialization import yaml_dump
 from mayan.apps.documents.models import Document
 from mayan.apps.documents.tests import (
     GenericDocumentTestCase, TEST_COMPRESSED_DOCUMENT_PATH,
@@ -213,8 +209,8 @@ class EmailBaseTestCase(GenericDocumentTestCase):
             metadata_type=test_metadata_type_2
         )
 
-        test_metadata_yaml = yaml.dump(
+        test_metadata_yaml = yaml_dump(
-            Dumper=SafeDumper, data={
+            data={
                 test_metadata_type_1.name: TEST_METADATA_VALUE_1,
                 test_metadata_type_2.name: TEST_METADATA_VALUE_2,
             }