From 9564db398fed0134dc97c82f4f9a28d42b9a0e22 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 9 Jul 2019 15:40:20 -0400 Subject: [PATCH] Backport configuration file improvements Remove support for quoted entried. Support unquoted entries. Support custom location for the config files. Signed-off-by: Roberto Rosario --- HISTORY.rst | 8 ++ docs/releases/3.3.rst | 38 ++++++--- mayan/apps/common/settings.py | 6 +- mayan/apps/common/storages.py | 14 +-- mayan/apps/converter/backends/python.py | 23 ++--- mayan/apps/converter/classes.py | 20 ++--- mayan/apps/converter/settings.py | 29 +++---- mayan/apps/document_signatures/settings.py | 8 +- mayan/apps/document_signatures/storages.py | 14 +-- mayan/apps/documents/settings.py | 16 ++-- mayan/apps/documents/storages.py | 21 +---- mayan/apps/file_metadata/drivers/exiftool.py | 12 +-- mayan/apps/file_metadata/settings.py | 11 +-- mayan/apps/metadata/settings.py | 2 +- mayan/apps/mirroring/settings.py | 2 +- mayan/apps/ocr/backends/tesseract.py | 14 +-- mayan/apps/ocr/runtime.py | 13 +-- mayan/apps/ocr/settings.py | 2 +- mayan/apps/smart_settings/classes.py | 26 +++++- mayan/apps/smart_settings/forms.py | 12 --- mayan/apps/smart_settings/literals.py | 36 ++++++++ mayan/apps/smart_settings/utils.py | 73 ++++++++++++++++ mayan/apps/sources/settings.py | 12 +-- mayan/apps/sources/storages.py | 13 +-- mayan/settings/base.py | 90 +++++++++----------- mayan/settings/utils.py | 39 --------- 26 files changed, 258 insertions(+), 296 deletions(-) create mode 100644 mayan/apps/smart_settings/literals.py create mode 100644 mayan/apps/smart_settings/utils.py delete mode 100644 mayan/settings/utils.py diff --git a/HISTORY.rst b/HISTORY.rst index c4be3eb90d..84551e6da1 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -14,6 +14,14 @@ - Backport individual index rebuild support. - Rename the installjavascript command to installdependencies. - Remove database conversion command. +- Remove support for quoted configuration entries. Support unquoted, + nested dictionaries in the configuration. Requires manual + update of existing config.yml files. +- Support user specified locations for the configuration file with the + CONFIGURATION_FILEPATH (MAYAN_CONFIGURATION_FILEPATH environment variable), and + CONFIGURATION_LAST_GOOD_FILEPATH + (MAYAN_CONFIGURATION_LAST_GOOD_FILEPATH environment variable) settings. +- Move bootstrapped settings code to their own module in the smart_settings apps. 3.2.5 (2019-07-05) ================== diff --git a/docs/releases/3.3.rst b/docs/releases/3.3.rst index e2098af3f6..1c8fd00d20 100644 --- a/docs/releases/3.3.rst +++ b/docs/releases/3.3.rst @@ -26,6 +26,14 @@ Changes - Backport individual index rebuild support. - Rename the installjavascript command to installdependencies. - Remove database conversion command. +- Remove support for quoted configuration entries. Support unquoted, + nested dictionaries in the configuration. Requires manual + update of existing config.yml files. +- Support user specified locations for the configuration file with the + CONFIGURATION_FILEPATH (MAYAN_CONFIGURATION_FILEPATH environment variable), and + CONFIGURATION_LAST_GOOD_FILEPATH + (MAYAN_CONFIGURATION_LAST_GOOD_FILEPATH environment variable) settings. +- Move bootstrapped settings code to their own module in the smart_settings apps. Removals -------- @@ -41,11 +49,11 @@ If installed via Python's PIP Remove deprecated requirements:: - $ curl https://gitlab.com/mayan-edms/mayan-edms/raw/master/removals.txt | pip uninstall -r /dev/stdin + sudo -u mayan curl https://gitlab.com/mayan-edms/mayan-edms/raw/master/removals.txt -o /tmp/removals.txt && sudo -u mayan /opt/mayan-edms/bin/pip uninstall -y -r /tmp/removals.txt Type in the console:: - $ pip install mayan-edms==3.3 + /opt/mayan-edms/bin/pip install mayan-edms==3.3 the requirements will also be updated automatically. @@ -55,19 +63,19 @@ Using Git If you installed Mayan EDMS by cloning the Git repository issue the commands:: - $ git reset --hard HEAD - $ git pull + git reset --hard HEAD + git pull otherwise download the compressed archived and uncompress it overriding the existing installation. Remove deprecated requirements:: - $ pip uninstall -y -r removals.txt + pip uninstall -y -r removals.txt Next upgrade/add the new requirements:: - $ pip install --upgrade -r requirements.txt + pip install --upgrade -r requirements.txt Common steps @@ -84,9 +92,8 @@ variables values show here with your respective settings. This step will refresh the supervisord configuration file with the new queues and the latest recommended layout:: - sudo MAYAN_DATABASE_ENGINE=django.db.backends.postgresql MAYAN_DATABASE_NAME=mayan \ - MAYAN_DATABASE_PASSWORD=mayanuserpass MAYAN_DATABASE_USER=mayan \ - MAYAN_DATABASE_HOST=127.0.0.1 MAYAN_MEDIA_ROOT=/opt/mayan-edms/media \ + sudo MAYAN_DATABASES="{'default':{'ENGINE':'django.db.backends.postgresql','NAME':'mayan','PASSWORD':'mayanuserpass','USER':'mayan','HOST':'127.0.0.1'}}" \ + MAYAN_MEDIA_ROOT=/opt/mayan-edms/media \ /opt/mayan-edms/bin/mayan-edms.py platformtemplate supervisord > /etc/supervisor/conf.d/mayan.conf Edit the supervisord configuration file and update any setting the template @@ -96,11 +103,11 @@ generator missed:: Migrate existing database schema with:: - $ mayan-edms.py performupgrade + sudo -u mayan MAYAN_MEDIA_ROOT=/opt/mayan-edms/media /opt/mayan-edms/bin/mayan-edms.py performupgrade Add new static media:: - $ mayan-edms.py preparestatic --noinput + sudo -u mayan MAYAN_MEDIA_ROOT=/opt/mayan-edms/media /opt/mayan-edms/bin/mayan-edms.py preparestatic --noinput The upgrade procedure is now complete. @@ -108,7 +115,14 @@ The upgrade procedure is now complete. Backward incompatible changes ----------------------------- -- None +- Update quoted settings to be unquoted: + + - COMMON_SHARED_STORAGE_ARGUMENTS + - CONVERTER_GRAPHICS_BACKEND_ARGUMENTS + - DOCUMENTS_CACHE_STORAGE_BACKEND_ARGUMENTS + - DOCUMENTS_STORAGE_BACKEND_ARGUMENTS + - FILE_METADATA_DRIVERS_ARGUMENTS + - SIGNATURES_STORAGE_BACKEND_ARGUMENTS Bugs fixed or issues closed diff --git a/mayan/apps/common/settings.py b/mayan/apps/common/settings.py index 841549b38f..056e12764a 100644 --- a/mayan/apps/common/settings.py +++ b/mayan/apps/common/settings.py @@ -6,7 +6,7 @@ from django.conf import settings from django.utils.translation import ugettext_lazy as _ import mayan -from mayan.apps.smart_settings import Namespace +from mayan.apps.smart_settings.classes import Namespace from .literals import DEFAULT_COMMON_HOME_VIEW @@ -95,9 +95,7 @@ setting_shared_storage = namespace.add_setting( ) setting_shared_storage_arguments = namespace.add_setting( global_name='COMMON_SHARED_STORAGE_ARGUMENTS', - default='{{location: {}}}'.format( - os.path.join(settings.MEDIA_ROOT, 'shared_files') - ), quoted=True + default={'location': os.path.join(settings.MEDIA_ROOT, 'shared_files')} ) namespace = Namespace(label=_('Django'), name='django') diff --git a/mayan/apps/common/storages.py b/mayan/apps/common/storages.py index ded073d4e8..f1acfdb7ca 100644 --- a/mayan/apps/common/storages.py +++ b/mayan/apps/common/storages.py @@ -1,12 +1,5 @@ from __future__ import unicode_literals -import yaml - -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader - from django.utils.module_loading import import_string from .settings import ( @@ -15,9 +8,4 @@ from .settings import ( storage_sharedupload = import_string( dotted_path=setting_shared_storage.value -)( - **yaml.load( - stream=setting_shared_storage_arguments.value or '{}', - Loader=SafeLoader - ) -) +)(**setting_shared_storage_arguments.value) diff --git a/mayan/apps/converter/backends/python.py b/mayan/apps/converter/backends/python.py index 672997f879..700ad2a381 100644 --- a/mayan/apps/converter/backends/python.py +++ b/mayan/apps/converter/backends/python.py @@ -7,11 +7,6 @@ import shutil from PIL import Image import PyPDF2 import sh -import yaml -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader from django.utils.encoding import force_text from django.utils.translation import ugettext_lazy as _ @@ -20,16 +15,14 @@ from mayan.apps.storage.utils import NamedTemporaryFile from ..classes import ConverterBase from ..exceptions import PageCountError -from ..settings import setting_graphics_backend_config +from ..settings import setting_graphics_backend_arguments from ..literals import ( DEFAULT_PDFTOPPM_DPI, DEFAULT_PDFTOPPM_FORMAT, DEFAULT_PDFTOPPM_PATH, DEFAULT_PDFINFO_PATH ) -pdftoppm_path = yaml.load( - stream=setting_graphics_backend_config.value, Loader=SafeLoader -).get( +pdftoppm_path = setting_graphics_backend_arguments.value.get( 'pdftoppm_path', DEFAULT_PDFTOPPM_PATH ) @@ -39,26 +32,20 @@ except sh.CommandNotFound: pdftoppm = None else: pdftoppm_format = '-{}'.format( - yaml.load( - stream=setting_graphics_backend_config.value, Loader=SafeLoader - ).get( + setting_graphics_backend_arguments.value.get( 'pdftoppm_format', DEFAULT_PDFTOPPM_FORMAT ) ) pdftoppm_dpi = format( - yaml.load( - stream=setting_graphics_backend_config.value, Loader=SafeLoader - ).get( + setting_graphics_backend_arguments.value.get( 'pdftoppm_dpi', DEFAULT_PDFTOPPM_DPI ) ) pdftoppm = pdftoppm.bake(pdftoppm_format, '-r', pdftoppm_dpi) -pdfinfo_path = yaml.load( - stream=setting_graphics_backend_config.value, Loader=SafeLoader -).get( +pdfinfo_path = setting_graphics_backend_arguments.value.get( 'pdfinfo_path', DEFAULT_PDFINFO_PATH ) diff --git a/mayan/apps/converter/classes.py b/mayan/apps/converter/classes.py index 517b04ac51..30c94d663b 100644 --- a/mayan/apps/converter/classes.py +++ b/mayan/apps/converter/classes.py @@ -7,12 +7,6 @@ import shutil from PIL import Image import sh -import yaml - -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader from django.utils.translation import ugettext_lazy as _ @@ -27,16 +21,14 @@ from .literals import ( CONVERTER_OFFICE_FILE_MIMETYPES, DEFAULT_LIBREOFFICE_PATH, DEFAULT_PAGE_NUMBER, DEFAULT_PILLOW_FORMAT ) -from .settings import setting_graphics_backend_config +from .settings import setting_graphics_backend_arguments -logger = logging.getLogger(__name__) -BACKEND_CONFIG = yaml.load( - stream=setting_graphics_backend_config.value, Loader=SafeLoader -) -libreoffice_path = BACKEND_CONFIG.get( +libreoffice_path = setting_graphics_backend_arguments.value.get( 'libreoffice_path', DEFAULT_LIBREOFFICE_PATH ) +logger = logging.getLogger(__name__) + class ConverterBase(object): def __init__(self, file_object, mime_type=None): @@ -62,9 +54,7 @@ class ConverterBase(object): pass def get_page(self, output_format=None): - output_format = output_format or yaml.load( - stream=setting_graphics_backend_config.value, Loader=SafeLoader - ).get( + output_format = output_format or setting_graphics_backend_arguments.value.get( 'pillow_format', DEFAULT_PILLOW_FORMAT ) diff --git a/mayan/apps/converter/settings.py b/mayan/apps/converter/settings.py index 14ce07a295..0d30648fc2 100644 --- a/mayan/apps/converter/settings.py +++ b/mayan/apps/converter/settings.py @@ -2,7 +2,7 @@ from __future__ import unicode_literals from django.utils.translation import ugettext_lazy as _ -from mayan.apps.smart_settings import Namespace +from mayan.apps.smart_settings.classes import Namespace from .literals import ( DEFAULT_LIBREOFFICE_PATH, DEFAULT_PDFTOPPM_DPI, DEFAULT_PDFTOPPM_FORMAT, @@ -16,22 +16,15 @@ setting_graphics_backend = namespace.add_setting( help_text=_('Graphics conversion backend to use.'), global_name='CONVERTER_GRAPHICS_BACKEND', ) -setting_graphics_backend_config = namespace.add_setting( - default=''' - {{ - libreoffice_path: {}, - pdftoppm_dpi: {}, - pdftoppm_format: {}, - pdftoppm_path: {}, - pdfinfo_path: {}, - pillow_format: {} - - }} - '''.replace('\n', '').format( - DEFAULT_LIBREOFFICE_PATH, DEFAULT_PDFTOPPM_DPI, - DEFAULT_PDFTOPPM_FORMAT, DEFAULT_PDFTOPPM_PATH, DEFAULT_PDFINFO_PATH, - DEFAULT_PILLOW_FORMAT - ), help_text=_( +setting_graphics_backend_arguments = namespace.add_setting( + default={ + 'libreoffice_path': DEFAULT_LIBREOFFICE_PATH, + 'pdftoppm_dpi': DEFAULT_PDFTOPPM_DPI, + 'pdftoppm_format': DEFAULT_PDFTOPPM_FORMAT, + 'pdftoppm_path': DEFAULT_PDFTOPPM_PATH, + 'pdfinfo_path': DEFAULT_PDFINFO_PATH, + 'pillow_format': DEFAULT_PILLOW_FORMAT, + }, help_text=_( 'Configuration options for the graphics conversion backend.' - ), global_name='CONVERTER_GRAPHICS_BACKEND_CONFIG', quoted=True + ), global_name='CONVERTER_GRAPHICS_BACKEND_ARGUMENTS' ) diff --git a/mayan/apps/document_signatures/settings.py b/mayan/apps/document_signatures/settings.py index 6e0b75ec23..7a5e9dcfe1 100644 --- a/mayan/apps/document_signatures/settings.py +++ b/mayan/apps/document_signatures/settings.py @@ -5,7 +5,7 @@ import os from django.conf import settings from django.utils.translation import ugettext_lazy as _ -from mayan.apps.smart_settings import Namespace +from mayan.apps.smart_settings.classes import Namespace namespace = Namespace(label=_('Document signatures'), name='signatures') @@ -18,9 +18,9 @@ setting_storage_backend = namespace.add_setting( ) setting_storage_backend_arguments = namespace.add_setting( global_name='SIGNATURES_STORAGE_BACKEND_ARGUMENTS', - default='{{location: {}}}'.format( - os.path.join(settings.MEDIA_ROOT, 'document_signatures') - ), quoted=True, help_text=_( + default={ + 'location': os.path.join(settings.MEDIA_ROOT, 'document_signatures') + }, help_text=_( 'Arguments to pass to the SIGNATURE_STORAGE_BACKEND. ' ) ) diff --git a/mayan/apps/document_signatures/storages.py b/mayan/apps/document_signatures/storages.py index 45d5d63d5e..2a06b3c913 100644 --- a/mayan/apps/document_signatures/storages.py +++ b/mayan/apps/document_signatures/storages.py @@ -1,12 +1,5 @@ from __future__ import unicode_literals -import yaml - -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader - from django.utils.module_loading import import_string from .settings import ( @@ -15,9 +8,4 @@ from .settings import ( storage_detachedsignature = import_string( dotted_path=setting_storage_backend.value -)( - **yaml.load( - stream=setting_storage_backend_arguments.value or '{}', - Loader=SafeLoader - ) -) +)(**setting_storage_backend_arguments.value) diff --git a/mayan/apps/documents/settings.py b/mayan/apps/documents/settings.py index 19c0162ca8..a4d5014e91 100644 --- a/mayan/apps/documents/settings.py +++ b/mayan/apps/documents/settings.py @@ -5,7 +5,7 @@ import os from django.conf import settings from django.utils.translation import ugettext_lazy as _ -from mayan.apps.smart_settings import Namespace +from mayan.apps.smart_settings.classes import Namespace from .literals import ( DEFAULT_DOCUMENTS_HASH_BLOCK_SIZE, DEFAULT_LANGUAGE, DEFAULT_LANGUAGE_CODES @@ -18,15 +18,14 @@ setting_documentimagecache_storage = namespace.add_setting( default='django.core.files.storage.FileSystemStorage', help_text=_( 'Path to the Storage subclass to use when storing the cached ' 'document image files.' - ), quoted=True + ) ) setting_documentimagecache_storage_arguments = namespace.add_setting( global_name='DOCUMENTS_CACHE_STORAGE_BACKEND_ARGUMENTS', - default='{{location: {}}}'.format( - os.path.join(settings.MEDIA_ROOT, 'document_cache') - ), help_text=_( + default={'location': os.path.join(settings.MEDIA_ROOT, 'document_cache')}, + help_text=_( 'Arguments to pass to the DOCUMENT_CACHE_STORAGE_BACKEND.' - ), quoted=True, + ), ) setting_disable_base_image_cache = namespace.add_setting( global_name='DOCUMENTS_DISABLE_BASE_IMAGE_CACHE', default=False, @@ -127,9 +126,8 @@ setting_storage_backend = namespace.add_setting( ) setting_storage_backend_arguments = namespace.add_setting( global_name='DOCUMENTS_STORAGE_BACKEND_ARGUMENTS', - default='{{location: {}}}'.format( - os.path.join(settings.MEDIA_ROOT, 'document_storage') - ), help_text=_('Arguments to pass to the DOCUMENT_STORAGE_BACKEND.') + default={'location': os.path.join(settings.MEDIA_ROOT, 'document_storage')}, + help_text=_('Arguments to pass to the DOCUMENT_STORAGE_BACKEND.') ) setting_thumbnail_height = namespace.add_setting( global_name='DOCUMENTS_THUMBNAIL_HEIGHT', default='', help_text=_( diff --git a/mayan/apps/documents/storages.py b/mayan/apps/documents/storages.py index 95405e9242..a6cec362d6 100644 --- a/mayan/apps/documents/storages.py +++ b/mayan/apps/documents/storages.py @@ -1,12 +1,5 @@ from __future__ import unicode_literals -import yaml - -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader - from django.utils.module_loading import import_string from .settings import ( @@ -17,18 +10,8 @@ from .settings import ( storage_documentversion = import_string( dotted_path=setting_storage_backend.value -)( - **yaml.load( - stream=setting_storage_backend_arguments.value or '{}', - Loader=SafeLoader - ) -) +)(**setting_storage_backend_arguments.value) storage_documentimagecache = import_string( dotted_path=setting_documentimagecache_storage.value -)( - **yaml.load( - stream=setting_documentimagecache_storage_arguments.value or '{}', - Loader=SafeLoader - ) -) +)(**setting_documentimagecache_storage_arguments.value) diff --git a/mayan/apps/file_metadata/drivers/exiftool.py b/mayan/apps/file_metadata/drivers/exiftool.py index 79a0993916..cff210335e 100644 --- a/mayan/apps/file_metadata/drivers/exiftool.py +++ b/mayan/apps/file_metadata/drivers/exiftool.py @@ -4,12 +4,6 @@ import json import logging import sh -import yaml - -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader from django.utils.translation import ugettext_lazy as _ @@ -57,11 +51,7 @@ class EXIFToolDriver(FileMetadataDriver): ) def read_settings(self): - driver_arguments = yaml.load( - stream=setting_drivers_arguments.value, Loader=SafeLoader - ) - - self.exiftool_path = driver_arguments.get( + self.exiftool_path = setting_drivers_arguments.value.get( 'exif_driver', {} ).get('exiftool_path', DEFAULT_EXIF_PATH) diff --git a/mayan/apps/file_metadata/settings.py b/mayan/apps/file_metadata/settings.py index 247f0b72ec..b0f8c19064 100644 --- a/mayan/apps/file_metadata/settings.py +++ b/mayan/apps/file_metadata/settings.py @@ -2,7 +2,7 @@ from __future__ import unicode_literals from django.utils.translation import ugettext_lazy as _ -from mayan.apps.smart_settings import Namespace +from mayan.apps.smart_settings.classes import Namespace from .literals import DEFAULT_EXIF_PATH @@ -16,12 +16,7 @@ setting_auto_process = namespace.add_setting( ) ) setting_drivers_arguments = namespace.add_setting( - default=''' - {{ - exif_driver: {{exiftool_path: {}}}, - - }} - '''.replace('\n', '').format(DEFAULT_EXIF_PATH), help_text=_( + default={'exif_driver': {'exiftool_path': DEFAULT_EXIF_PATH}}, help_text=_( 'Arguments to pass to the drivers.' - ), global_name='FILE_METADATA_DRIVERS_ARGUMENTS', quoted=True + ), global_name='FILE_METADATA_DRIVERS_ARGUMENTS' ) diff --git a/mayan/apps/metadata/settings.py b/mayan/apps/metadata/settings.py index ce7aa6de35..0387f45309 100644 --- a/mayan/apps/metadata/settings.py +++ b/mayan/apps/metadata/settings.py @@ -2,7 +2,7 @@ from __future__ import unicode_literals from django.utils.translation import ugettext_lazy as _ -from mayan.apps.smart_settings import Namespace +from mayan.apps.smart_settings.classes import Namespace from .parsers import MetadataParser from .validators import MetadataValidator diff --git a/mayan/apps/mirroring/settings.py b/mayan/apps/mirroring/settings.py index aa16aa1335..41f9b372a5 100644 --- a/mayan/apps/mirroring/settings.py +++ b/mayan/apps/mirroring/settings.py @@ -2,7 +2,7 @@ from __future__ import unicode_literals from django.utils.translation import ugettext_lazy as _ -from mayan.apps.smart_settings import Namespace +from mayan.apps.smart_settings.classes import Namespace namespace = Namespace(label=_('Mirroring'), name='mirroring') diff --git a/mayan/apps/ocr/backends/tesseract.py b/mayan/apps/ocr/backends/tesseract.py index 3444198e4d..0a8f87cf2e 100644 --- a/mayan/apps/ocr/backends/tesseract.py +++ b/mayan/apps/ocr/backends/tesseract.py @@ -4,11 +4,6 @@ import logging import shutil import sh -import yaml -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader from django.utils.encoding import force_text from django.utils.translation import ugettext_lazy as _ @@ -115,15 +110,10 @@ class Tesseract(OCRBackendBase): logger.debug('Available languages: %s', ', '.join(self.languages)) def read_settings(self): - backend_arguments = yaml.load( - Loader=SafeLoader, - stream=setting_ocr_backend_arguments.value or '{}', - ) - - self.tesseract_binary_path = backend_arguments.get( + self.tesseract_binary_path = setting_ocr_backend_arguments.value.get( 'tesseract_path', DEFAULT_TESSERACT_BINARY_PATH ) - self.command_timeout = backend_arguments.get( + self.command_timeout = setting_ocr_backend_arguments.value.get( 'timeout', DEFAULT_TESSERACT_TIMEOUT ) diff --git a/mayan/apps/ocr/runtime.py b/mayan/apps/ocr/runtime.py index 1d8643819b..55e6a36a35 100644 --- a/mayan/apps/ocr/runtime.py +++ b/mayan/apps/ocr/runtime.py @@ -1,20 +1,9 @@ from __future__ import unicode_literals -import yaml - -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader - from django.utils.module_loading import import_string from .settings import setting_ocr_backend, setting_ocr_backend_arguments ocr_backend = import_string( dotted_path=setting_ocr_backend.value -)( - **yaml.load( - stream=setting_ocr_backend_arguments.value or '{}', Loader=SafeLoader - ) -) +)(**setting_ocr_backend_arguments.value) diff --git a/mayan/apps/ocr/settings.py b/mayan/apps/ocr/settings.py index f2aa1052a3..4293c3ac24 100644 --- a/mayan/apps/ocr/settings.py +++ b/mayan/apps/ocr/settings.py @@ -2,7 +2,7 @@ from __future__ import unicode_literals from django.utils.translation import ugettext_lazy as _ -from mayan.apps.smart_settings import Namespace +from mayan.apps.smart_settings.classes import Namespace namespace = Namespace(label=_('OCR'), name='ocr') diff --git a/mayan/apps/smart_settings/classes.py b/mayan/apps/smart_settings/classes.py index 9f35323608..8a79ba8757 100644 --- a/mayan/apps/smart_settings/classes.py +++ b/mayan/apps/smart_settings/classes.py @@ -21,6 +21,8 @@ from django.utils.encoding import ( force_bytes, force_text, python_2_unicode_compatible ) +from .utils import read_configuration_file + logger = logging.getLogger(__name__) @@ -82,6 +84,7 @@ class Namespace(object): class Setting(object): _registry = {} _cache_hash = None + _config_file_cache = None @staticmethod def deserialize_value(value): @@ -103,6 +106,7 @@ class Setting(object): def serialize_value(value): result = yaml.dump( data=Setting.express_promises(value), allow_unicode=True, + default_flow_style=False, Dumper=SafeDumper ) # safe_dump returns bytestrings @@ -140,6 +144,16 @@ class Setting(object): def get_all(cls): return sorted(cls._registry.values(), key=lambda x: x.global_name) + @classmethod + def get_config_file_content(cls): + # Cache content of config file to speed up initial boot up + if not cls._config_file_cache: + cls._config_file_cache = read_configuration_file( + path=settings.CONFIGURATION_FILEPATH + ) + + return cls._config_file_cache + @classmethod def get_hash(cls): return force_text( @@ -167,13 +181,12 @@ class Setting(object): path=settings.CONFIGURATION_LAST_GOOD_FILEPATH ) - def __init__(self, namespace, global_name, default, help_text=None, is_path=False, quoted=False): + def __init__(self, namespace, global_name, default, help_text=None, is_path=False): self.global_name = global_name self.default = default self.help_text = help_text self.loaded = False self.namespace = namespace - self.quoted = quoted self.environment_variable = False namespace._settings.append(self) self.__class__._registry[global_name] = self @@ -186,7 +199,7 @@ class Setting(object): if environment_value: self.environment_variable = True try: - self.raw_value = environment_value + self.raw_value = yaml.load(stream=environment_value, Loader=SafeLoader) except yaml.YAMLError as exception: raise type(exception)( 'Error interpreting environment variable: {} with ' @@ -195,7 +208,12 @@ class Setting(object): ) ) else: - self.raw_value = getattr(settings, self.global_name, self.default) + self.raw_value = self.get_config_file_content().get( + self.global_name, getattr( + settings, self.global_name, self.default + ) + ) + self.yaml = Setting.serialize_value(self.raw_value) self.loaded = True diff --git a/mayan/apps/smart_settings/forms.py b/mayan/apps/smart_settings/forms.py index d72548316d..25f8687597 100644 --- a/mayan/apps/smart_settings/forms.py +++ b/mayan/apps/smart_settings/forms.py @@ -25,18 +25,6 @@ class SettingForm(forms.Form): self.fields['value'].initial = self.setting.serialized_value def clean(self): - quotes = ['"', "'"] - - if self.setting.quoted: - stripped = self.cleaned_data['value'].strip() - - if stripped[0] not in quotes or stripped[-1] not in quotes: - raise ValidationError( - _( - 'Value must be properly quoted.' - ) - ) - try: yaml.load(stream=self.cleaned_data['value'], Loader=SafeLoader) except yaml.YAMLError: diff --git a/mayan/apps/smart_settings/literals.py b/mayan/apps/smart_settings/literals.py new file mode 100644 index 0000000000..6db3f2ee9c --- /dev/null +++ b/mayan/apps/smart_settings/literals.py @@ -0,0 +1,36 @@ +from __future__ import unicode_literals + +# Default in YAML format +BOOTSTRAP_SETTING_LIST = ( + {'name': 'ALLOWED_HOSTS', 'default': "['127.0.0.1', 'localhost', '[::1]']"}, + {'name': 'APPEND_SLASH'}, + {'name': 'AUTH_PASSWORD_VALIDATORS'}, + {'name': 'COMMON_DISABLED_APPS'}, + {'name': 'COMMON_EXTRA_APPS'}, + {'name': 'DATA_UPLOAD_MAX_MEMORY_SIZE'}, + {'name': 'DATABASES'}, + {'name': 'DEBUG', 'default': 'false'}, + {'name': 'DEFAULT_FROM_EMAIL'}, + {'name': 'DISALLOWED_USER_AGENTS'}, + {'name': 'EMAIL_BACKEND'}, + {'name': 'EMAIL_HOST'}, + {'name': 'EMAIL_HOST_PASSWORD'}, + {'name': 'EMAIL_HOST_USER'}, + {'name': 'EMAIL_PORT'}, + {'name': 'EMAIL_TIMEOUT'}, + {'name': 'EMAIL_USE_SSL'}, + {'name': 'EMAIL_USE_TLS'}, + {'name': 'FILE_UPLOAD_MAX_MEMORY_SIZE'}, + {'name': 'HOME_VIEW'}, + {'name': 'INSTALLED_APPS'}, + {'name': 'INTERNAL_IPS', 'default': "['127.0.0.1']"}, + {'name': 'LANGUAGES'}, + {'name': 'LANGUAGE_CODE'}, + {'name': 'LOGIN_REDIRECT_URL', 'default': 'common:home'}, + {'name': 'LOGIN_URL', 'default': 'authentication:login_view'}, + {'name': 'LOGOUT_REDIRECT_URL', 'default': 'authentication:login_view'}, + {'name': 'STATIC_URL'}, + {'name': 'STATICFILES_STORAGE'}, + {'name': 'TIME_ZONE'}, + {'name': 'WSGI_APPLICATION'} +) diff --git a/mayan/apps/smart_settings/utils.py b/mayan/apps/smart_settings/utils.py new file mode 100644 index 0000000000..4282d2c38d --- /dev/null +++ b/mayan/apps/smart_settings/utils.py @@ -0,0 +1,73 @@ +from __future__ import unicode_literals + +import errno +import os + +import yaml + +try: + from yaml import CSafeLoader as SafeLoader +except ImportError: + from yaml import SafeLoader + +from .literals import BOOTSTRAP_SETTING_LIST + + +def get_default(name, fallback_default=None): + for item in BOOTSTRAP_SETTING_LIST: + if item['name'] == name: + return item.get('default', fallback_default) + + return fallback_default + + +def get_environment_variables(): + result = {} + + for setting in BOOTSTRAP_SETTING_LIST: + environment_value = os.environ.get('MAYAN_{}'.format(setting['name'])) + if environment_value: + environment_value = yaml.load(stream=environment_value, Loader=SafeLoader) + result[setting['name']] = environment_value + + return result + + +def get_environment_setting(name, fallback_default=None): + value = os.environ.get('MAYAN_{}'.format(name), get_default(name=name, fallback_default=fallback_default)) + + if value: + return yaml.load(stream=value, Loader=SafeLoader) + + +def read_configuration_file(path): + try: + with open(path) as file_object: + file_object.seek(0, os.SEEK_END) + if file_object.tell(): + file_object.seek(0) + try: + return yaml.load(stream=file_object, Loader=SafeLoader) + except yaml.YAMLError as exception: + exit( + 'Error loading configuration file: {}; {}'.format( + path, exception + ) + ) + except IOError as exception: + if exception.errno == errno.ENOENT: + pass + else: + raise + + +def yaml_loads(data, error_message=None): + if not error_message: + error_message = 'Error loading: {}; {}' + + try: + return yaml.load(stream=data, Loader=SafeLoader) + except yaml.YAMLError as exception: + exit( + error_message.format(data, exception) + ) diff --git a/mayan/apps/sources/settings.py b/mayan/apps/sources/settings.py index 8f0156d427..aef20ad07d 100644 --- a/mayan/apps/sources/settings.py +++ b/mayan/apps/sources/settings.py @@ -5,7 +5,7 @@ import os from django.conf import settings from django.utils.translation import ugettext_lazy as _ -from mayan.apps.smart_settings import Namespace +from mayan.apps.smart_settings.classes import Namespace namespace = Namespace(label=_('Sources'), name='sources') @@ -21,13 +21,13 @@ setting_staging_file_image_cache_storage = namespace.add_setting( default='django.core.files.storage.FileSystemStorage', help_text=_( 'Path to the Storage subclass to use when storing the cached ' 'staging_file image files.' - ), quoted=True + ) ) setting_staging_file_image_cache_storage_arguments = namespace.add_setting( global_name='SOURCES_STAGING_FILE_CACHE_STORAGE_BACKEND_ARGUMENTS', - default='{{location: {}}}'.format( - os.path.join(settings.MEDIA_ROOT, 'staging_file_cache') - ), help_text=_( + default={ + 'location': os.path.join(settings.MEDIA_ROOT, 'staging_file_cache') + }, help_text=_( 'Arguments to pass to the SOURCES_STAGING_FILE_CACHE_STORAGE_BACKEND.' - ), quoted=True, + ) ) diff --git a/mayan/apps/sources/storages.py b/mayan/apps/sources/storages.py index 30e23ab7c5..2849d8e915 100644 --- a/mayan/apps/sources/storages.py +++ b/mayan/apps/sources/storages.py @@ -1,11 +1,5 @@ from __future__ import unicode_literals -import yaml -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader - from django.utils.module_loading import import_string from .settings import ( @@ -15,9 +9,4 @@ from .settings import ( storage_staging_file_image_cache = import_string( dotted_path=setting_staging_file_image_cache_storage.value -)( - **yaml.load( - stream=setting_staging_file_image_cache_storage_arguments.value or '{}', - Loader=SafeLoader - ) -) +)(**setting_staging_file_image_cache_storage_arguments.value) diff --git a/mayan/settings/base.py b/mayan/settings/base.py index 3ec88a5322..6292ecd168 100644 --- a/mayan/settings/base.py +++ b/mayan/settings/base.py @@ -17,15 +17,15 @@ import sys from django.core.exceptions import ImproperlyConfigured from django.utils.translation import ugettext_lazy as _ -import environ +from mayan.apps.smart_settings.literals import BOOTSTRAP_SETTING_LIST +from mayan.apps.smart_settings.utils import ( + get_environment_setting, get_environment_variables, read_configuration_file +) from .literals import ( CONFIGURATION_FILENAME, CONFIGURATION_LAST_GOOD_FILENAME, DEFAULT_SECRET_KEY, SECRET_KEY_FILENAME, SYSTEM_DIR ) -from .utils import yaml_loads, read_configuration_file - -env = environ.Env() # Build paths inside the project like this: os.path.join(BASE_DIR, ...) @@ -34,8 +34,8 @@ BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # Quick-start development settings - unsuitable for production # See https://docs.djangoproject.com/en/1.10/howto/deployment/checklist/ -MEDIA_ROOT = os.environ.get( - 'MAYAN_MEDIA_ROOT', os.path.join(BASE_DIR, 'media') +MEDIA_ROOT = get_environment_setting( + name='MAYAN_MEDIA_ROOT', fallback_default=os.path.join(BASE_DIR, 'media') ) # SECURITY WARNING: keep the secret key used in production secret! @@ -50,13 +50,9 @@ else: SECRET_KEY = DEFAULT_SECRET_KEY # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = env.bool('MAYAN_DEBUG', default=False) +DEBUG = get_environment_setting(name='DEBUG') -ALLOWED_HOSTS = yaml_loads( - env( - 'MAYAN_ALLOWED_HOSTS', default="['127.0.0.1', 'localhost', '[::1]']" - ) -) +ALLOWED_HOSTS = get_environment_setting(name='ALLOWED_HOSTS') # Application definition @@ -257,12 +253,10 @@ TEST_RUNNER = 'mayan.apps.common.tests.runner.MayanTestRunner' # --------- Django ------------------- -LOGIN_URL = env('MAYAN_LOGIN_URL', default='authentication:login_view') -LOGIN_REDIRECT_URL = env('MAYAN_LOGIN_REDIRECT_URL', default='common:root') -LOGOUT_REDIRECT_URL = env( - 'MAYAN_LOGOUT_REDIRECT_URL', default='authentication:login_view' -) -INTERNAL_IPS = ('127.0.0.1',) +LOGIN_URL = get_environment_setting(name='LOGIN_URL') +LOGIN_REDIRECT_URL = get_environment_setting(name='LOGIN_REDIRECT_URL') +LOGOUT_REDIRECT_URL = get_environment_setting(name='LOGOUT_REDIRECT_URL') +INTERNAL_IPS = get_environment_setting(name='INTERNAL_IPS') # ---------- Django REST framework ----------- @@ -325,51 +319,43 @@ AJAX_REDIRECT_CODE = 278 # ----- Celery ----- -BROKER_URL = os.environ.get('MAYAN_BROKER_URL') -CELERY_ALWAYS_EAGER = env.bool('MAYAN_CELERY_ALWAYS_EAGER', default=True) -CELERY_RESULT_BACKEND = os.environ.get('MAYAN_CELERY_RESULT_BACKEND') +BROKER_URL = get_environment_setting(name='BROKER_URL') +CELERY_ALWAYS_EAGER = get_environment_setting(name='CELERY_ALWAYS_EAGER') +CELERY_RESULT_BACKEND = get_environment_setting(name='CELERY_RESULT_BACKEND') # ----- Database ----- -environment_database_engine = os.environ.get('MAYAN_DATABASE_ENGINE') - -if environment_database_engine: - environment_database_conn_max_age = os.environ.get('MAYAN_DATABASE_CONN_MAX_AGE', 0) - if environment_database_conn_max_age: - environment_database_conn_max_age = int(environment_database_conn_max_age) - - DATABASES = { - 'default': { - 'ENGINE': environment_database_engine, - 'NAME': os.environ['MAYAN_DATABASE_NAME'], - 'USER': os.environ['MAYAN_DATABASE_USER'], - 'PASSWORD': os.environ['MAYAN_DATABASE_PASSWORD'], - 'HOST': os.environ.get('MAYAN_DATABASE_HOST', None), - 'PORT': os.environ.get('MAYAN_DATABASE_PORT', None), - 'CONN_MAX_AGE': environment_database_conn_max_age, - } +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': os.path.join(MEDIA_ROOT, 'db.sqlite3'), } -else: - DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': os.path.join(MEDIA_ROOT, 'db.sqlite3'), - } - } - +} BASE_INSTALLED_APPS = INSTALLED_APPS COMMON_EXTRA_APPS = () COMMON_DISABLED_APPS = () -CONFIGURATION_FILEPATH = os.path.join(MEDIA_ROOT, CONFIGURATION_FILENAME) -CONFIGURATION_LAST_GOOD_FILEPATH = os.path.join( - MEDIA_ROOT, CONFIGURATION_LAST_GOOD_FILENAME +CONFIGURATION_FILEPATH = get_environment_setting( + name='CONFIGURATION_FILEPATH', fallback_default=os.path.join( + MEDIA_ROOT, CONFIGURATION_FILENAME + ) +) + +CONFIGURATION_LAST_GOOD_FILEPATH = get_environment_setting( + name='CONFIGURATION_LAST_GOOD_FILEPATH', fallback_default=os.path.join( + MEDIA_ROOT, CONFIGURATION_LAST_GOOD_FILENAME + ) ) if 'revertsettings' not in sys.argv: - result = read_configuration_file(CONFIGURATION_FILEPATH) - if result: - globals().update(result) + configuration_result = read_configuration_file(CONFIGURATION_FILEPATH) + environment_result = get_environment_variables() + + for setting in BOOTSTRAP_SETTING_LIST: + if setting['name'] in configuration_result: + globals().update({setting['name']: configuration_result[setting['name']]}) + elif setting['name'] in environment_result: + globals().update({setting['name']: environment_result[setting['name']]}) for app in INSTALLED_APPS: diff --git a/mayan/settings/utils.py b/mayan/settings/utils.py deleted file mode 100644 index bb2a8da489..0000000000 --- a/mayan/settings/utils.py +++ /dev/null @@ -1,39 +0,0 @@ -from __future__ import unicode_literals - -import errno -import os - -import yaml - - -def read_configuration_file(path): - try: - with open(path) as file_object: - file_object.seek(0, os.SEEK_END) - if file_object.tell(): - file_object.seek(0) - try: - return yaml.safe_load(file_object) - except yaml.YAMLError as exception: - exit( - 'Error loading configuration file: {}; {}'.format( - path, exception - ) - ) - except IOError as exception: - if exception.errno == errno.ENOENT: - pass - else: - raise - - -def yaml_loads(data, error_message=None): - if not error_message: - error_message = 'Error loading: {}; {}' - - try: - return yaml.safe_load(data) - except yaml.YAMLError as exception: - exit( - error_message.format(data, exception) - )