From d5224d93a79a9d43ba4844d39368800854147aed Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Mon, 26 Nov 2018 17:27:57 -0400 Subject: [PATCH] Settings: Remove support for quoted settings Instead of passing strings as arguments to backends, all settings must be formatted according to YAML specifications. This is to remove the need to add separate YAML parsing to each backend argument in each app that needs it. Argument passing to backends is not fully uniform. Users need to update their config files. Example: DOCUMENTS_STORAGE_BACKEND_ARGUMENTS: '{location: /home/rosarior/development/mayan-edms/mayan/media/document_storage}' must be changed to: DOCUMENTS_STORAGE_BACKEND_ARGUMENTS: location: /home/rosarior/development/mayan-edms/mayan/media/document_storage Example 2: CONVERTER_GRAPHICS_BACKEND_CONFIG: ' { libreoffice_path: /usr/bin/libreoffice, pdftoppm_dpi: 300, pdftoppm_format: jpeg, pdftoppm_path: /usr/bin/pdftoppm, pdfinfo_path: /usr/bin/pdfinfo, pillow_format: JPEG } ' must be changed to: CONVERTER_GRAPHICS_BACKEND_CONFIG: libreoffice_path: /usr/bin/libreoffice pdftoppm_dpi: 300 pdftoppm_format: jpeg pdftoppm_path: /usr/bin/pdftoppm pdfinfo_path: /usr/bin/pdfinfo pillow_format: JPEG Example 3: OCR_BACKEND_ARGUMENTS: '' must be changed to: OCR_BACKEND_ARGUMENTS: {} Settings that need to be updated are: - COMMON_SHARED_STORAGE_ARGUMENTS - CONVERTER_GRAPHICS_BACKEND_CONFIG - DOCUMENTS_CACHE_STORAGE_BACKEND_ARGUMENTS - DOCUMENTS_STORAGE_BACKEND_ARGUMENTS - OCR_BACKEND_ARGUMENTS - SIGNATURES_STORAGE_BACKEND_ARGUMENTS - SOURCES_STAGING_FILE_CACHE_STORAGE_BACKEND_ARGUMENTS The following error will appear in the console if a setting is not yet updated to this new format:: TypeError: type object argument after ** must be a mapping, not str Signed-off-by: Roberto Rosario --- HISTORY.rst | 52 +++++++++++++++++++++- mayan/apps/common/settings.py | 2 +- mayan/apps/common/storages.py | 8 +--- mayan/apps/converter/backends/python.py | 9 ++-- mayan/apps/converter/classes.py | 7 +-- mayan/apps/converter/settings.py | 2 +- mayan/apps/document_signatures/settings.py | 2 +- mayan/apps/document_signatures/storages.py | 8 +--- mayan/apps/documents/settings.py | 4 +- mayan/apps/documents/storages.py | 19 +++----- mayan/apps/mailer/settings.py | 8 ++-- mayan/apps/ocr/runtime.py | 8 +--- mayan/apps/ocr/settings.py | 2 +- mayan/apps/smart_settings/classes.py | 3 +- mayan/apps/smart_settings/forms.py | 10 ----- mayan/apps/sources/settings.py | 4 +- mayan/apps/sources/storages.py | 8 +--- 17 files changed, 79 insertions(+), 77 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 7f2afdeaa7..90b6ab68ec 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -13,9 +13,59 @@ - Appearance: Remove unused form_empty_label flag. - Appearance: Allow subclassing the text area widget. - Documents: Add transformation support to document image API with serialized - transformations. + transformations. - Documents: Add icons to the document page image and document page reset views. +- Remove support for quoted settings. Instead all settings must be formatted + according to YAML specifications. Users need to update their config files. + Example: + + DOCUMENTS_STORAGE_BACKEND_ARGUMENTS: '{location: /home/rosarior/development/mayan-edms/mayan/media/document_storage}' + + must be changed to: + + DOCUMENTS_STORAGE_BACKEND_ARGUMENTS: + location: /home/rosarior/development/mayan-edms/mayan/media/document_storage + + Example 2: + + CONVERTER_GRAPHICS_BACKEND_CONFIG: ' { libreoffice_path: /usr/bin/libreoffice, pdftoppm_dpi: + 300, pdftoppm_format: jpeg, pdftoppm_path: /usr/bin/pdftoppm, pdfinfo_path: + /usr/bin/pdfinfo, pillow_format: JPEG } ' + + must be changed to: + + CONVERTER_GRAPHICS_BACKEND_CONFIG: + libreoffice_path: /usr/bin/libreoffice + pdftoppm_dpi: 300 + pdftoppm_format: jpeg + pdftoppm_path: /usr/bin/pdftoppm + pdfinfo_path: /usr/bin/pdfinfo + pillow_format: JPEG + + Example 3: + + OCR_BACKEND_ARGUMENTS: '' + + must be changed to: + + OCR_BACKEND_ARGUMENTS: {} + + Settings that need to be updated are: + + - COMMON_SHARED_STORAGE_ARGUMENTS + - CONVERTER_GRAPHICS_BACKEND_CONFIG + - DOCUMENTS_CACHE_STORAGE_BACKEND_ARGUMENTS + - DOCUMENTS_STORAGE_BACKEND_ARGUMENTS + - OCR_BACKEND_ARGUMENTS + - SIGNATURES_STORAGE_BACKEND_ARGUMENTS + - SOURCES_STAGING_FILE_CACHE_STORAGE_BACKEND_ARGUMENTS + + The following error will appear in the console if a setting is not yet + updated to this new format:: + + TypeError: type object argument after ** must be a mapping, not str + 3.1.9 (2018-11-01) ================== diff --git a/mayan/apps/common/settings.py b/mayan/apps/common/settings.py index f28ce9cb45..d79adf1b19 100644 --- a/mayan/apps/common/settings.py +++ b/mayan/apps/common/settings.py @@ -67,7 +67,7 @@ setting_shared_storage_arguments = namespace.add_setting( global_name='COMMON_SHARED_STORAGE_ARGUMENTS', default='{{location: {}}}'.format( os.path.join(settings.MEDIA_ROOT, 'shared_files') - ), quoted=True + ) ) setting_temporary_directory = namespace.add_setting( global_name='COMMON_TEMPORARY_DIRECTORY', default=tempfile.gettempdir(), diff --git a/mayan/apps/common/storages.py b/mayan/apps/common/storages.py index 371ce84d76..f1acfdb7ca 100644 --- a/mayan/apps/common/storages.py +++ b/mayan/apps/common/storages.py @@ -1,7 +1,5 @@ from __future__ import unicode_literals -import yaml - from django.utils.module_loading import import_string from .settings import ( @@ -10,8 +8,4 @@ from .settings import ( storage_sharedupload = import_string( dotted_path=setting_shared_storage.value -)( - **yaml.safe_load( - setting_shared_storage_arguments.value or '{}' - ) -) +)(**setting_shared_storage_arguments.value) diff --git a/mayan/apps/converter/backends/python.py b/mayan/apps/converter/backends/python.py index 706c8a744d..e58f2fac6c 100644 --- a/mayan/apps/converter/backends/python.py +++ b/mayan/apps/converter/backends/python.py @@ -7,7 +7,6 @@ import os from PIL import Image import PyPDF2 import sh -import yaml from django.utils.encoding import force_text from django.utils.translation import ugettext_lazy as _ @@ -25,7 +24,7 @@ from ..literals import ( try: pdftoppm = sh.Command( - yaml.load(setting_graphics_backend_config.value).get( + setting_graphics_backend_config.value.get( 'pdftoppm_path', DEFAULT_PDFTOPPM_PATH ) ) @@ -33,13 +32,13 @@ except sh.CommandNotFound: pdftoppm = None else: pdftoppm_format = '-{}'.format( - yaml.load(setting_graphics_backend_config.value).get( + setting_graphics_backend_config.value.get( 'pdftoppm_format', DEFAULT_PDFTOPPM_FORMAT ) ) pdftoppm_dpi = format( - yaml.load(setting_graphics_backend_config.value).get( + setting_graphics_backend_config.value.get( 'pdftoppm_dpi', DEFAULT_PDFTOPPM_DPI ) ) @@ -48,7 +47,7 @@ else: try: pdfinfo = sh.Command( - yaml.load(setting_graphics_backend_config.value).get( + setting_graphics_backend_config.value.get( 'pdfinfo_path', DEFAULT_PDFINFO_PATH ) ) diff --git a/mayan/apps/converter/classes.py b/mayan/apps/converter/classes.py index afdef6d93c..7464d29573 100644 --- a/mayan/apps/converter/classes.py +++ b/mayan/apps/converter/classes.py @@ -7,7 +7,6 @@ import os from PIL import Image import sh -import yaml from django.utils.translation import ugettext_lazy as _ @@ -26,7 +25,7 @@ logger = logging.getLogger(__name__) try: LIBREOFFICE = sh.Command( - yaml.load(setting_graphics_backend_config.value).get( + setting_graphics_backend_config.value.get( 'libreoffice_path', DEFAULT_LIBREOFFICE_PATH ) ).bake('--headless', '--convert-to', 'pdf:writer_pdf_Export') @@ -180,9 +179,7 @@ class ConverterBase(object): fs_cleanup(converted_output) def get_page(self, output_format=None, as_base64=False): - output_format = output_format or yaml.load( - setting_graphics_backend_config.value - ).get( + output_format = output_format or setting_graphics_backend_config.value.get( 'pillow_format', DEFAULT_PILLOW_FORMAT ) diff --git a/mayan/apps/converter/settings.py b/mayan/apps/converter/settings.py index 678b5de8b0..797e7bb6ea 100644 --- a/mayan/apps/converter/settings.py +++ b/mayan/apps/converter/settings.py @@ -32,5 +32,5 @@ setting_graphics_backend_config = namespace.add_setting( DEFAULT_PILLOW_FORMAT ), help_text=_( 'Configuration options for the graphics conversion backend.' - ), global_name='CONVERTER_GRAPHICS_BACKEND_CONFIG', quoted=True + ), global_name='CONVERTER_GRAPHICS_BACKEND_CONFIG' ) diff --git a/mayan/apps/document_signatures/settings.py b/mayan/apps/document_signatures/settings.py index 7b537a3dcd..5dc52a99cf 100644 --- a/mayan/apps/document_signatures/settings.py +++ b/mayan/apps/document_signatures/settings.py @@ -19,7 +19,7 @@ setting_storage_backend_arguments = namespace.add_setting( global_name='SIGNATURES_STORAGE_BACKEND_ARGUMENTS', default='{{location: {}}}'.format( os.path.join(settings.MEDIA_ROOT, 'document_signatures') - ), quoted=True, help_text=_( + ), help_text=_( 'Arguments to pass to the SIGNATURE_STORAGE_BACKEND. ' ) ) diff --git a/mayan/apps/document_signatures/storages.py b/mayan/apps/document_signatures/storages.py index 0393501def..2a06b3c913 100644 --- a/mayan/apps/document_signatures/storages.py +++ b/mayan/apps/document_signatures/storages.py @@ -1,7 +1,5 @@ from __future__ import unicode_literals -import yaml - from django.utils.module_loading import import_string from .settings import ( @@ -10,8 +8,4 @@ from .settings import ( storage_detachedsignature = import_string( dotted_path=setting_storage_backend.value -)( - **yaml.safe_load( - setting_storage_backend_arguments.value or '{}' - ) -) +)(**setting_storage_backend_arguments.value) diff --git a/mayan/apps/documents/settings.py b/mayan/apps/documents/settings.py index 794c433a5e..51b963c049 100644 --- a/mayan/apps/documents/settings.py +++ b/mayan/apps/documents/settings.py @@ -16,7 +16,7 @@ setting_documentimagecache_storage = namespace.add_setting( default='django.core.files.storage.FileSystemStorage', help_text=_( 'Path to the Storage subclass to use when storing the cached ' 'document image files.' - ), quoted=True + ) ) setting_documentimagecache_storage_arguments = namespace.add_setting( global_name='DOCUMENTS_CACHE_STORAGE_BACKEND_ARGUMENTS', @@ -24,7 +24,7 @@ setting_documentimagecache_storage_arguments = namespace.add_setting( os.path.join(settings.MEDIA_ROOT, 'document_cache') ), help_text=_( 'Arguments to pass to the DOCUMENT_CACHE_STORAGE_BACKEND.' - ), quoted=True, + ) ) setting_disable_base_image_cache = namespace.add_setting( global_name='DOCUMENTS_DISABLE_BASE_IMAGE_CACHE', default=False, diff --git a/mayan/apps/documents/storages.py b/mayan/apps/documents/storages.py index ae61eed877..de4399f623 100644 --- a/mayan/apps/documents/storages.py +++ b/mayan/apps/documents/storages.py @@ -1,26 +1,17 @@ from __future__ import unicode_literals -import yaml - from django.utils.module_loading import import_string from .settings import ( - setting_documentimagecache_storage, setting_documentimagecache_storage_arguments, - setting_storage_backend, setting_storage_backend_arguments + setting_documentimagecache_storage, + setting_documentimagecache_storage_arguments, setting_storage_backend, + setting_storage_backend_arguments ) storage_documentversion = import_string( dotted_path=setting_storage_backend.value -)( - **yaml.safe_load( - setting_storage_backend_arguments.value or '{}' - ) -) +)(**setting_storage_backend_arguments.value) storage_documentimagecache = import_string( dotted_path=setting_documentimagecache_storage.value -)( - **yaml.safe_load( - setting_documentimagecache_storage_arguments.value or '{}' - ) -) +)(**setting_documentimagecache_storage_arguments.value) diff --git a/mayan/apps/mailer/settings.py b/mayan/apps/mailer/settings.py index 6ba4fb5f97..60874e4cc1 100644 --- a/mayan/apps/mailer/settings.py +++ b/mayan/apps/mailer/settings.py @@ -13,20 +13,20 @@ namespace = Namespace(name='mailer', label=_('Mailing')) setting_link_subject_template = namespace.add_setting( default=_('Link for document: {{ document }}'), help_text=_('Template for the document link email form subject line.'), - global_name='MAILER_LINK_SUBJECT_TEMPLATE', quoted=True + global_name='MAILER_LINK_SUBJECT_TEMPLATE' ) setting_link_body_template = namespace.add_setting( default=DEFAULT_LINK_BODY_TEMPLATE, help_text=_('Template for the document link email form body text. Can include HTML.'), - global_name='MAILER_LINK_BODY_TEMPLATE', quoted=True + global_name='MAILER_LINK_BODY_TEMPLATE' ) setting_document_subject_template = namespace.add_setting( default=_('Document: {{ document }}'), help_text=_('Template for the document email form subject line.'), - global_name='MAILER_DOCUMENT_SUBJECT_TEMPLATE', quoted=True + global_name='MAILER_DOCUMENT_SUBJECT_TEMPLATE' ) setting_document_body_template = namespace.add_setting( default=DEFAULT_DOCUMENT_BODY_TEMPLATE, help_text=_('Template for the document email form body text. Can include HTML.'), - global_name='MAILER_DOCUMENT_BODY_TEMPLATE', quoted=True + global_name='MAILER_DOCUMENT_BODY_TEMPLATE' ) diff --git a/mayan/apps/ocr/runtime.py b/mayan/apps/ocr/runtime.py index 69819029e7..36192e4156 100644 --- a/mayan/apps/ocr/runtime.py +++ b/mayan/apps/ocr/runtime.py @@ -1,15 +1,9 @@ from __future__ import unicode_literals -import yaml - from django.utils.module_loading import import_string from .settings import setting_ocr_backend, setting_ocr_backend_arguments ocr_backend = import_string( setting_ocr_backend.value -)( - **yaml.safe_load( - setting_ocr_backend_arguments.value or '{}' - ) -) +)(**setting_ocr_backend_arguments.value) diff --git a/mayan/apps/ocr/settings.py b/mayan/apps/ocr/settings.py index 260f561078..05e3c5cd7a 100644 --- a/mayan/apps/ocr/settings.py +++ b/mayan/apps/ocr/settings.py @@ -12,7 +12,7 @@ setting_ocr_backend = namespace.add_setting( ) setting_ocr_backend_arguments = namespace.add_setting( global_name='OCR_BACKEND_ARGUMENTS', - default='' + default='{}' ) setting_auto_ocr = namespace.add_setting( global_name='OCR_AUTO_OCR', default=True, diff --git a/mayan/apps/smart_settings/classes.py b/mayan/apps/smart_settings/classes.py index 32d1973b4c..32d4a8cc08 100644 --- a/mayan/apps/smart_settings/classes.py +++ b/mayan/apps/smart_settings/classes.py @@ -134,13 +134,12 @@ class Setting(object): path=settings.CONFIGURATION_LAST_GOOD_FILEPATH ) - def __init__(self, namespace, global_name, default, help_text=None, is_path=False, quoted=False): + def __init__(self, namespace, global_name, default, help_text=None, is_path=False): self.global_name = global_name self.default = default self.help_text = help_text self.loaded = False self.namespace = namespace - self.quoted = quoted self.environment_variable = False namespace._settings.append(self) self.__class__._registry[global_name] = self diff --git a/mayan/apps/smart_settings/forms.py b/mayan/apps/smart_settings/forms.py index 3f008ff1ca..d48aff5bfe 100644 --- a/mayan/apps/smart_settings/forms.py +++ b/mayan/apps/smart_settings/forms.py @@ -22,16 +22,6 @@ class SettingForm(forms.Form): def clean(self): quotes = ['"', "'"] - if self.setting.quoted: - stripped = self.cleaned_data['value'].strip() - - if stripped[0] not in quotes or stripped[-1] not in quotes: - raise ValidationError( - _( - 'Value must be properly quoted.' - ) - ) - try: yaml.safe_load(self.cleaned_data['value']) except yaml.YAMLError: diff --git a/mayan/apps/sources/settings.py b/mayan/apps/sources/settings.py index dcefcd8c88..33d180f046 100644 --- a/mayan/apps/sources/settings.py +++ b/mayan/apps/sources/settings.py @@ -21,7 +21,7 @@ setting_staging_file_image_cache_storage = namespace.add_setting( default='django.core.files.storage.FileSystemStorage', help_text=_( 'Path to the Storage subclass to use when storing the cached ' 'staging_file image files.' - ), quoted=True + ) ) setting_staging_file_image_cache_storage_arguments = namespace.add_setting( global_name='SOURCES_STAGING_FILE_CACHE_STORAGE_BACKEND_ARGUMENTS', @@ -29,5 +29,5 @@ setting_staging_file_image_cache_storage_arguments = namespace.add_setting( os.path.join(settings.MEDIA_ROOT, 'staging_file_cache') ), help_text=_( 'Arguments to pass to the SOURCES_STAGING_FILE_CACHE_STORAGE_BACKEND.' - ), quoted=True, + ) ) diff --git a/mayan/apps/sources/storages.py b/mayan/apps/sources/storages.py index e3bedf4311..2849d8e915 100644 --- a/mayan/apps/sources/storages.py +++ b/mayan/apps/sources/storages.py @@ -1,7 +1,5 @@ from __future__ import unicode_literals -import yaml - from django.utils.module_loading import import_string from .settings import ( @@ -11,8 +9,4 @@ from .settings import ( storage_staging_file_image_cache = import_string( dotted_path=setting_staging_file_image_cache_storage.value -)( - **yaml.safe_load( - setting_staging_file_image_cache_storage_arguments.value or '{}' - ) -) +)(**setting_staging_file_image_cache_storage_arguments.value)