Backport configuration file improvements

Remove support for quoted entried. Support unquoted entries. Support
custom location for the config files.

Signed-off-by: Roberto Rosario <roberto.rosario.gonzalez@gmail.com>

HISTORY.rst

@@ -14,6 +14,14 @@
 - Backport individual index rebuild support.
 - Rename the installjavascript command to installdependencies.
 - Remove database conversion command.
+- Remove support for quoted configuration entries. Support unquoted,
+  nested dictionaries in the configuration. Requires manual
+  update of existing config.yml files.
+- Support user specified locations for the configuration file with the
+  CONFIGURATION_FILEPATH (MAYAN_CONFIGURATION_FILEPATH environment variable), and
+  CONFIGURATION_LAST_GOOD_FILEPATH
+  (MAYAN_CONFIGURATION_LAST_GOOD_FILEPATH environment variable) settings.
+- Move bootstrapped settings code to their own module in the smart_settings apps.
 
 3.2.5 (2019-07-05)
 ==================

docs/releases/3.3.rst

@@ -26,6 +26,14 @@ Changes
 - Backport individual index rebuild support.
 - Rename the installjavascript command to installdependencies.
 - Remove database conversion command.
+- Remove support for quoted configuration entries. Support unquoted,
+  nested dictionaries in the configuration. Requires manual
+  update of existing config.yml files.
+- Support user specified locations for the configuration file with the
+  CONFIGURATION_FILEPATH (MAYAN_CONFIGURATION_FILEPATH environment variable), and
+  CONFIGURATION_LAST_GOOD_FILEPATH
+  (MAYAN_CONFIGURATION_LAST_GOOD_FILEPATH environment variable) settings.
+- Move bootstrapped settings code to their own module in the smart_settings apps.
 
 Removals
 --------
@@ -41,11 +49,11 @@ If installed via Python's PIP
 
 Remove deprecated requirements::
 
-    $ curl https://gitlab.com/mayan-edms/mayan-edms/raw/master/removals.txt | pip uninstall -r /dev/stdin
+    sudo -u mayan curl https://gitlab.com/mayan-edms/mayan-edms/raw/master/removals.txt -o /tmp/removals.txt && sudo -u mayan /opt/mayan-edms/bin/pip uninstall -y -r /tmp/removals.txt
 
 Type in the console::
 
-    $ pip install mayan-edms==3.3
+    /opt/mayan-edms/bin/pip install mayan-edms==3.3
 
 the requirements will also be updated automatically.
 
@@ -55,19 +63,19 @@ Using Git
 
 If you installed Mayan EDMS by cloning the Git repository issue the commands::
 
-    $ git reset --hard HEAD
+    git reset --hard HEAD
-    $ git pull
+    git pull
 
 otherwise download the compressed archived and uncompress it overriding the
 existing installation.
 
 Remove deprecated requirements::
 
-    $ pip uninstall -y -r removals.txt
+    pip uninstall -y -r removals.txt
 
 Next upgrade/add the new requirements::
 
-    $ pip install --upgrade -r requirements.txt
+    pip install --upgrade -r requirements.txt
 
 
 Common steps
@@ -84,9 +92,8 @@ variables values show here with your respective settings. This step will refresh
 the supervisord configuration file with the new queues and the latest
 recommended layout::
 
-    sudo MAYAN_DATABASE_ENGINE=django.db.backends.postgresql MAYAN_DATABASE_NAME=mayan \
+    sudo MAYAN_DATABASES="{'default':{'ENGINE':'django.db.backends.postgresql','NAME':'mayan','PASSWORD':'mayanuserpass','USER':'mayan','HOST':'127.0.0.1'}}" \
-    MAYAN_DATABASE_PASSWORD=mayanuserpass MAYAN_DATABASE_USER=mayan \
+    MAYAN_MEDIA_ROOT=/opt/mayan-edms/media \
-    MAYAN_DATABASE_HOST=127.0.0.1 MAYAN_MEDIA_ROOT=/opt/mayan-edms/media \
     /opt/mayan-edms/bin/mayan-edms.py platformtemplate supervisord > /etc/supervisor/conf.d/mayan.conf
 
 Edit the supervisord configuration file and update any setting the template
@@ -96,11 +103,11 @@ generator missed::
 
 Migrate existing database schema with::
 
-    $ mayan-edms.py performupgrade
+    sudo -u mayan MAYAN_MEDIA_ROOT=/opt/mayan-edms/media /opt/mayan-edms/bin/mayan-edms.py performupgrade
 
 Add new static media::
 
-    $ mayan-edms.py preparestatic --noinput
+    sudo -u mayan MAYAN_MEDIA_ROOT=/opt/mayan-edms/media /opt/mayan-edms/bin/mayan-edms.py preparestatic --noinput
 
 The upgrade procedure is now complete.
 
@@ -108,7 +115,14 @@ The upgrade procedure is now complete.
 Backward incompatible changes
 -----------------------------
 
-- None
+- Update quoted settings to be unquoted:
+
+  - COMMON_SHARED_STORAGE_ARGUMENTS
+  - CONVERTER_GRAPHICS_BACKEND_ARGUMENTS
+  - DOCUMENTS_CACHE_STORAGE_BACKEND_ARGUMENTS
+  - DOCUMENTS_STORAGE_BACKEND_ARGUMENTS
+  - FILE_METADATA_DRIVERS_ARGUMENTS
+  - SIGNATURES_STORAGE_BACKEND_ARGUMENTS
 
 
 Bugs fixed or issues closed

mayan/apps/common/settings.py

@@ -6,7 +6,7 @@ from django.conf import settings
 from django.utils.translation import ugettext_lazy as _
 
 import mayan
-from mayan.apps.smart_settings import Namespace
+from mayan.apps.smart_settings.classes import Namespace
 
 from .literals import DEFAULT_COMMON_HOME_VIEW
 
@@ -95,9 +95,7 @@ setting_shared_storage = namespace.add_setting(
 )
 setting_shared_storage_arguments = namespace.add_setting(
     global_name='COMMON_SHARED_STORAGE_ARGUMENTS',
-    default='{{location: {}}}'.format(
+    default={'location': os.path.join(settings.MEDIA_ROOT, 'shared_files')}
-        os.path.join(settings.MEDIA_ROOT, 'shared_files')
-    ), quoted=True
 )
 
 namespace = Namespace(label=_('Django'), name='django')

mayan/apps/common/storages.py

@@ -1,12 +1,5 @@
 from __future__ import unicode_literals
 
-import yaml
-
-try:
-    from yaml import CSafeLoader as SafeLoader
-except ImportError:
-    from yaml import SafeLoader
-
 from django.utils.module_loading import import_string
 
 from .settings import (
@@ -15,9 +8,4 @@ from .settings import (
 
 storage_sharedupload = import_string(
     dotted_path=setting_shared_storage.value
-)(
+)(**setting_shared_storage_arguments.value)
-    **yaml.load(
-        stream=setting_shared_storage_arguments.value or '{}',
-        Loader=SafeLoader
-    )
-)

mayan/apps/converter/backends/python.py

@@ -7,11 +7,6 @@ import shutil
 from PIL import Image
 import PyPDF2
 import sh
-import yaml
-try:
-    from yaml import CSafeLoader as SafeLoader
-except ImportError:
-    from yaml import SafeLoader
 
 from django.utils.encoding import force_text
 from django.utils.translation import ugettext_lazy as _
@@ -20,16 +15,14 @@ from mayan.apps.storage.utils import NamedTemporaryFile
 
 from ..classes import ConverterBase
 from ..exceptions import PageCountError
-from ..settings import setting_graphics_backend_config
+from ..settings import setting_graphics_backend_arguments
 
 from ..literals import (
     DEFAULT_PDFTOPPM_DPI, DEFAULT_PDFTOPPM_FORMAT, DEFAULT_PDFTOPPM_PATH,
     DEFAULT_PDFINFO_PATH
 )
 
-pdftoppm_path = yaml.load(
+pdftoppm_path = setting_graphics_backend_arguments.value.get(
-    stream=setting_graphics_backend_config.value, Loader=SafeLoader
-).get(
     'pdftoppm_path', DEFAULT_PDFTOPPM_PATH
 )
 
@@ -39,26 +32,20 @@ except sh.CommandNotFound:
     pdftoppm = None
 else:
     pdftoppm_format = '-{}'.format(
-        yaml.load(
+        setting_graphics_backend_arguments.value.get(
-            stream=setting_graphics_backend_config.value, Loader=SafeLoader
-        ).get(
             'pdftoppm_format', DEFAULT_PDFTOPPM_FORMAT
         )
     )
 
     pdftoppm_dpi = format(
-        yaml.load(
+        setting_graphics_backend_arguments.value.get(
-            stream=setting_graphics_backend_config.value, Loader=SafeLoader
-        ).get(
             'pdftoppm_dpi', DEFAULT_PDFTOPPM_DPI
         )
     )
 
     pdftoppm = pdftoppm.bake(pdftoppm_format, '-r', pdftoppm_dpi)
 
-pdfinfo_path = yaml.load(
+pdfinfo_path = setting_graphics_backend_arguments.value.get(
-    stream=setting_graphics_backend_config.value, Loader=SafeLoader
-).get(
     'pdfinfo_path', DEFAULT_PDFINFO_PATH
 )
 

mayan/apps/converter/classes.py

@@ -7,12 +7,6 @@ import shutil
 
 from PIL import Image
 import sh
-import yaml
-
-try:
-    from yaml import CSafeLoader as SafeLoader
-except ImportError:
-    from yaml import SafeLoader
 
 from django.utils.translation import ugettext_lazy as _
 
@@ -27,16 +21,14 @@ from .literals import (
     CONVERTER_OFFICE_FILE_MIMETYPES, DEFAULT_LIBREOFFICE_PATH,
     DEFAULT_PAGE_NUMBER, DEFAULT_PILLOW_FORMAT
 )
-from .settings import setting_graphics_backend_config
+from .settings import setting_graphics_backend_arguments
 
-logger = logging.getLogger(__name__)
+libreoffice_path = setting_graphics_backend_arguments.value.get(
-BACKEND_CONFIG = yaml.load(
-    stream=setting_graphics_backend_config.value, Loader=SafeLoader
-)
-libreoffice_path = BACKEND_CONFIG.get(
     'libreoffice_path', DEFAULT_LIBREOFFICE_PATH
 )
 
+logger = logging.getLogger(__name__)
+
 
 class ConverterBase(object):
     def __init__(self, file_object, mime_type=None):
@@ -62,9 +54,7 @@ class ConverterBase(object):
         pass
 
     def get_page(self, output_format=None):
-        output_format = output_format or yaml.load(
+        output_format = output_format or setting_graphics_backend_arguments.value.get(
-            stream=setting_graphics_backend_config.value, Loader=SafeLoader
-        ).get(
             'pillow_format', DEFAULT_PILLOW_FORMAT
         )
 

mayan/apps/converter/settings.py

@@ -2,7 +2,7 @@ from __future__ import unicode_literals
 
 from django.utils.translation import ugettext_lazy as _
 
-from mayan.apps.smart_settings import Namespace
+from mayan.apps.smart_settings.classes import Namespace
 
 from .literals import (
     DEFAULT_LIBREOFFICE_PATH, DEFAULT_PDFTOPPM_DPI, DEFAULT_PDFTOPPM_FORMAT,
@@ -16,22 +16,15 @@ setting_graphics_backend = namespace.add_setting(
     help_text=_('Graphics conversion backend to use.'),
     global_name='CONVERTER_GRAPHICS_BACKEND',
 )
-setting_graphics_backend_config = namespace.add_setting(
+setting_graphics_backend_arguments = namespace.add_setting(
-    default='''
+    default={
-        {{
+        'libreoffice_path': DEFAULT_LIBREOFFICE_PATH,
-            libreoffice_path: {},
+        'pdftoppm_dpi': DEFAULT_PDFTOPPM_DPI,
-            pdftoppm_dpi: {},
+        'pdftoppm_format': DEFAULT_PDFTOPPM_FORMAT,
-            pdftoppm_format: {},
+        'pdftoppm_path': DEFAULT_PDFTOPPM_PATH,
-            pdftoppm_path: {},
+        'pdfinfo_path': DEFAULT_PDFINFO_PATH,
-            pdfinfo_path: {},
+        'pillow_format': DEFAULT_PILLOW_FORMAT,
-            pillow_format: {}
+    }, help_text=_(
-
-        }}
-    '''.replace('\n', '').format(
-        DEFAULT_LIBREOFFICE_PATH, DEFAULT_PDFTOPPM_DPI,
-        DEFAULT_PDFTOPPM_FORMAT, DEFAULT_PDFTOPPM_PATH, DEFAULT_PDFINFO_PATH,
-        DEFAULT_PILLOW_FORMAT
-    ), help_text=_(
         'Configuration options for the graphics conversion backend.'
-    ), global_name='CONVERTER_GRAPHICS_BACKEND_CONFIG', quoted=True
+    ), global_name='CONVERTER_GRAPHICS_BACKEND_ARGUMENTS'
 )

mayan/apps/document_signatures/settings.py

@@ -5,7 +5,7 @@ import os
 from django.conf import settings
 from django.utils.translation import ugettext_lazy as _
 
-from mayan.apps.smart_settings import Namespace
+from mayan.apps.smart_settings.classes import Namespace
 
 namespace = Namespace(label=_('Document signatures'), name='signatures')
 
@@ -18,9 +18,9 @@ setting_storage_backend = namespace.add_setting(
 )
 setting_storage_backend_arguments = namespace.add_setting(
     global_name='SIGNATURES_STORAGE_BACKEND_ARGUMENTS',
-    default='{{location: {}}}'.format(
+    default={
-        os.path.join(settings.MEDIA_ROOT, 'document_signatures')
+        'location': os.path.join(settings.MEDIA_ROOT, 'document_signatures')
-    ), quoted=True, help_text=_(
+    }, help_text=_(
         'Arguments to pass to the SIGNATURE_STORAGE_BACKEND. '
     )
 )

mayan/apps/document_signatures/storages.py

@@ -1,12 +1,5 @@
 from __future__ import unicode_literals
 
-import yaml
-
-try:
-    from yaml import CSafeLoader as SafeLoader
-except ImportError:
-    from yaml import SafeLoader
-
 from django.utils.module_loading import import_string
 
 from .settings import (
@@ -15,9 +8,4 @@ from .settings import (
 
 storage_detachedsignature = import_string(
     dotted_path=setting_storage_backend.value
-)(
+)(**setting_storage_backend_arguments.value)
-    **yaml.load(
-        stream=setting_storage_backend_arguments.value or '{}',
-        Loader=SafeLoader
-    )
-)

mayan/apps/documents/settings.py

@@ -5,7 +5,7 @@ import os
 from django.conf import settings
 from django.utils.translation import ugettext_lazy as _
 
-from mayan.apps.smart_settings import Namespace
+from mayan.apps.smart_settings.classes import Namespace
 
 from .literals import (
     DEFAULT_DOCUMENTS_HASH_BLOCK_SIZE, DEFAULT_LANGUAGE, DEFAULT_LANGUAGE_CODES
@@ -18,15 +18,14 @@ setting_documentimagecache_storage = namespace.add_setting(
     default='django.core.files.storage.FileSystemStorage', help_text=_(
         'Path to the Storage subclass to use when storing the cached '
         'document image files.'
-    ), quoted=True
+    )
 )
 setting_documentimagecache_storage_arguments = namespace.add_setting(
     global_name='DOCUMENTS_CACHE_STORAGE_BACKEND_ARGUMENTS',
-    default='{{location: {}}}'.format(
+    default={'location': os.path.join(settings.MEDIA_ROOT, 'document_cache')},
-        os.path.join(settings.MEDIA_ROOT, 'document_cache')
+    help_text=_(
-    ), help_text=_(
         'Arguments to pass to the DOCUMENT_CACHE_STORAGE_BACKEND.'
-    ), quoted=True,
+    ),
 )
 setting_disable_base_image_cache = namespace.add_setting(
     global_name='DOCUMENTS_DISABLE_BASE_IMAGE_CACHE', default=False,
@@ -127,9 +126,8 @@ setting_storage_backend = namespace.add_setting(
 )
 setting_storage_backend_arguments = namespace.add_setting(
     global_name='DOCUMENTS_STORAGE_BACKEND_ARGUMENTS',
-    default='{{location: {}}}'.format(
+    default={'location': os.path.join(settings.MEDIA_ROOT, 'document_storage')},
-        os.path.join(settings.MEDIA_ROOT, 'document_storage')
+    help_text=_('Arguments to pass to the DOCUMENT_STORAGE_BACKEND.')
-    ), help_text=_('Arguments to pass to the DOCUMENT_STORAGE_BACKEND.')
 )
 setting_thumbnail_height = namespace.add_setting(
     global_name='DOCUMENTS_THUMBNAIL_HEIGHT', default='', help_text=_(

mayan/apps/documents/storages.py

@@ -1,12 +1,5 @@
 from __future__ import unicode_literals
 
-import yaml
-
-try:
-    from yaml import CSafeLoader as SafeLoader
-except ImportError:
-    from yaml import SafeLoader
-
 from django.utils.module_loading import import_string
 
 from .settings import (
@@ -17,18 +10,8 @@ from .settings import (
 
 storage_documentversion = import_string(
     dotted_path=setting_storage_backend.value
-)(
+)(**setting_storage_backend_arguments.value)
-    **yaml.load(
-        stream=setting_storage_backend_arguments.value or '{}',
-        Loader=SafeLoader
-    )
-)
 
 storage_documentimagecache = import_string(
     dotted_path=setting_documentimagecache_storage.value
-)(
+)(**setting_documentimagecache_storage_arguments.value)
-    **yaml.load(
-        stream=setting_documentimagecache_storage_arguments.value or '{}',
-        Loader=SafeLoader
-    )
-)

mayan/apps/file_metadata/drivers/exiftool.py

@@ -4,12 +4,6 @@ import json
 import logging
 
 import sh
-import yaml
-
-try:
-    from yaml import CSafeLoader as SafeLoader
-except ImportError:
-    from yaml import SafeLoader
 
 from django.utils.translation import ugettext_lazy as _
 
@@ -57,11 +51,7 @@ class EXIFToolDriver(FileMetadataDriver):
             )
 
     def read_settings(self):
-        driver_arguments = yaml.load(
+        self.exiftool_path = setting_drivers_arguments.value.get(
-            stream=setting_drivers_arguments.value, Loader=SafeLoader
-        )
-
-        self.exiftool_path = driver_arguments.get(
             'exif_driver', {}
         ).get('exiftool_path', DEFAULT_EXIF_PATH)
 

mayan/apps/file_metadata/settings.py

@@ -2,7 +2,7 @@ from __future__ import unicode_literals
 
 from django.utils.translation import ugettext_lazy as _
 
-from mayan.apps.smart_settings import Namespace
+from mayan.apps.smart_settings.classes import Namespace
 
 from .literals import DEFAULT_EXIF_PATH
 
@@ -16,12 +16,7 @@ setting_auto_process = namespace.add_setting(
     )
 )
 setting_drivers_arguments = namespace.add_setting(
-    default='''
+    default={'exif_driver': {'exiftool_path': DEFAULT_EXIF_PATH}}, help_text=_(
-        {{
-            exif_driver: {{exiftool_path: {}}},
-
-        }}
-    '''.replace('\n', '').format(DEFAULT_EXIF_PATH), help_text=_(
         'Arguments to pass to the drivers.'
-    ), global_name='FILE_METADATA_DRIVERS_ARGUMENTS', quoted=True
+    ), global_name='FILE_METADATA_DRIVERS_ARGUMENTS'
 )

mayan/apps/metadata/settings.py

@@ -2,7 +2,7 @@ from __future__ import unicode_literals
 
 from django.utils.translation import ugettext_lazy as _
 
-from mayan.apps.smart_settings import Namespace
+from mayan.apps.smart_settings.classes import Namespace
 
 from .parsers import MetadataParser
 from .validators import MetadataValidator

mayan/apps/mirroring/settings.py

@@ -2,7 +2,7 @@ from __future__ import unicode_literals
 
 from django.utils.translation import ugettext_lazy as _
 
-from mayan.apps.smart_settings import Namespace
+from mayan.apps.smart_settings.classes import Namespace
 
 namespace = Namespace(label=_('Mirroring'), name='mirroring')
 

mayan/apps/ocr/backends/tesseract.py

@@ -4,11 +4,6 @@ import logging
 import shutil
 
 import sh
-import yaml
-try:
-    from yaml import CSafeLoader as SafeLoader
-except ImportError:
-    from yaml import SafeLoader
 
 from django.utils.encoding import force_text
 from django.utils.translation import ugettext_lazy as _
@@ -115,15 +110,10 @@ class Tesseract(OCRBackendBase):
             logger.debug('Available languages: %s', ', '.join(self.languages))
 
     def read_settings(self):
-        backend_arguments = yaml.load(
+        self.tesseract_binary_path = setting_ocr_backend_arguments.value.get(
-            Loader=SafeLoader,
-            stream=setting_ocr_backend_arguments.value or '{}',
-        )
-
-        self.tesseract_binary_path = backend_arguments.get(
             'tesseract_path', DEFAULT_TESSERACT_BINARY_PATH
         )
 
-        self.command_timeout = backend_arguments.get(
+        self.command_timeout = setting_ocr_backend_arguments.value.get(
             'timeout', DEFAULT_TESSERACT_TIMEOUT
         )

mayan/apps/ocr/runtime.py

@@ -1,20 +1,9 @@
 from __future__ import unicode_literals
 
-import yaml
-
-try:
-    from yaml import CSafeLoader as SafeLoader
-except ImportError:
-    from yaml import SafeLoader
-
 from django.utils.module_loading import import_string
 
 from .settings import setting_ocr_backend, setting_ocr_backend_arguments
 
 ocr_backend = import_string(
     dotted_path=setting_ocr_backend.value
-)(
+)(**setting_ocr_backend_arguments.value)
-    **yaml.load(
-        stream=setting_ocr_backend_arguments.value or '{}', Loader=SafeLoader
-    )
-)

mayan/apps/ocr/settings.py

@@ -2,7 +2,7 @@ from __future__ import unicode_literals
 
 from django.utils.translation import ugettext_lazy as _
 
-from mayan.apps.smart_settings import Namespace
+from mayan.apps.smart_settings.classes import Namespace
 
 namespace = Namespace(label=_('OCR'), name='ocr')
 

mayan/apps/smart_settings/classes.py

@@ -21,6 +21,8 @@ from django.utils.encoding import (
     force_bytes, force_text, python_2_unicode_compatible
 )
 
+from .utils import read_configuration_file
+
 logger = logging.getLogger(__name__)
 
 
@@ -82,6 +84,7 @@ class Namespace(object):
 class Setting(object):
     _registry = {}
     _cache_hash = None
+    _config_file_cache = None
 
     @staticmethod
     def deserialize_value(value):
@@ -103,6 +106,7 @@ class Setting(object):
     def serialize_value(value):
         result = yaml.dump(
             data=Setting.express_promises(value), allow_unicode=True,
+            default_flow_style=False,
             Dumper=SafeDumper
         )
         # safe_dump returns bytestrings
@@ -140,6 +144,16 @@ class Setting(object):
     def get_all(cls):
         return sorted(cls._registry.values(), key=lambda x: x.global_name)
 
+    @classmethod
+    def get_config_file_content(cls):
+        # Cache content of config file to speed up initial boot up
+        if not cls._config_file_cache:
+            cls._config_file_cache = read_configuration_file(
+                path=settings.CONFIGURATION_FILEPATH
+            )
+
+        return cls._config_file_cache
+
     @classmethod
     def get_hash(cls):
         return force_text(
@@ -167,13 +181,12 @@ class Setting(object):
                 path=settings.CONFIGURATION_LAST_GOOD_FILEPATH
             )
 
-    def __init__(self, namespace, global_name, default, help_text=None, is_path=False, quoted=False):
+    def __init__(self, namespace, global_name, default, help_text=None, is_path=False):
         self.global_name = global_name
         self.default = default
         self.help_text = help_text
         self.loaded = False
         self.namespace = namespace
-        self.quoted = quoted
         self.environment_variable = False
         namespace._settings.append(self)
         self.__class__._registry[global_name] = self
@@ -186,7 +199,7 @@ class Setting(object):
         if environment_value:
             self.environment_variable = True
             try:
-                self.raw_value = environment_value
+                self.raw_value = yaml.load(stream=environment_value, Loader=SafeLoader)
             except yaml.YAMLError as exception:
                 raise type(exception)(
                     'Error interpreting environment variable: {} with '
@@ -195,7 +208,12 @@ class Setting(object):
                     )
                 )
         else:
-            self.raw_value = getattr(settings, self.global_name, self.default)
+            self.raw_value = self.get_config_file_content().get(
+                self.global_name, getattr(
+                    settings, self.global_name, self.default
+                )
+            )
+
         self.yaml = Setting.serialize_value(self.raw_value)
         self.loaded = True
 

mayan/apps/smart_settings/forms.py

@@ -25,18 +25,6 @@ class SettingForm(forms.Form):
         self.fields['value'].initial = self.setting.serialized_value
 
     def clean(self):
-        quotes = ['"', "'"]
-
-        if self.setting.quoted:
-            stripped = self.cleaned_data['value'].strip()
-
-            if stripped[0] not in quotes or stripped[-1] not in quotes:
-                raise ValidationError(
-                    _(
-                        'Value must be properly quoted.'
-                    )
-                )
-
         try:
             yaml.load(stream=self.cleaned_data['value'], Loader=SafeLoader)
         except yaml.YAMLError:

mayan/apps/smart_settings/literals.py

@@ -0,0 +1,36 @@
+from __future__ import unicode_literals
+
+# Default in YAML format
+BOOTSTRAP_SETTING_LIST = (
+    {'name': 'ALLOWED_HOSTS', 'default': "['127.0.0.1', 'localhost', '[::1]']"},
+    {'name': 'APPEND_SLASH'},
+    {'name': 'AUTH_PASSWORD_VALIDATORS'},
+    {'name': 'COMMON_DISABLED_APPS'},
+    {'name': 'COMMON_EXTRA_APPS'},
+    {'name': 'DATA_UPLOAD_MAX_MEMORY_SIZE'},
+    {'name': 'DATABASES'},
+    {'name': 'DEBUG', 'default': 'false'},
+    {'name': 'DEFAULT_FROM_EMAIL'},
+    {'name': 'DISALLOWED_USER_AGENTS'},
+    {'name': 'EMAIL_BACKEND'},
+    {'name': 'EMAIL_HOST'},
+    {'name': 'EMAIL_HOST_PASSWORD'},
+    {'name': 'EMAIL_HOST_USER'},
+    {'name': 'EMAIL_PORT'},
+    {'name': 'EMAIL_TIMEOUT'},
+    {'name': 'EMAIL_USE_SSL'},
+    {'name': 'EMAIL_USE_TLS'},
+    {'name': 'FILE_UPLOAD_MAX_MEMORY_SIZE'},
+    {'name': 'HOME_VIEW'},
+    {'name': 'INSTALLED_APPS'},
+    {'name': 'INTERNAL_IPS', 'default': "['127.0.0.1']"},
+    {'name': 'LANGUAGES'},
+    {'name': 'LANGUAGE_CODE'},
+    {'name': 'LOGIN_REDIRECT_URL', 'default': 'common:home'},
+    {'name': 'LOGIN_URL', 'default': 'authentication:login_view'},
+    {'name': 'LOGOUT_REDIRECT_URL', 'default': 'authentication:login_view'},
+    {'name': 'STATIC_URL'},
+    {'name': 'STATICFILES_STORAGE'},
+    {'name': 'TIME_ZONE'},
+    {'name': 'WSGI_APPLICATION'}
+)

mayan/apps/smart_settings/utils.py

@@ -0,0 +1,73 @@
+from __future__ import unicode_literals
+
+import errno
+import os
+
+import yaml
+
+try:
+    from yaml import CSafeLoader as SafeLoader
+except ImportError:
+    from yaml import SafeLoader
+
+from .literals import BOOTSTRAP_SETTING_LIST
+
+
+def get_default(name, fallback_default=None):
+    for item in BOOTSTRAP_SETTING_LIST:
+        if item['name'] == name:
+            return item.get('default', fallback_default)
+
+    return fallback_default
+
+
+def get_environment_variables():
+    result = {}
+
+    for setting in BOOTSTRAP_SETTING_LIST:
+        environment_value = os.environ.get('MAYAN_{}'.format(setting['name']))
+        if environment_value:
+            environment_value = yaml.load(stream=environment_value, Loader=SafeLoader)
+            result[setting['name']] = environment_value
+
+    return result
+
+
+def get_environment_setting(name, fallback_default=None):
+    value = os.environ.get('MAYAN_{}'.format(name), get_default(name=name, fallback_default=fallback_default))
+
+    if value:
+        return yaml.load(stream=value, Loader=SafeLoader)
+
+
+def read_configuration_file(path):
+    try:
+        with open(path) as file_object:
+            file_object.seek(0, os.SEEK_END)
+            if file_object.tell():
+                file_object.seek(0)
+                try:
+                    return yaml.load(stream=file_object, Loader=SafeLoader)
+                except yaml.YAMLError as exception:
+                    exit(
+                        'Error loading configuration file: {}; {}'.format(
+                            path, exception
+                        )
+                    )
+    except IOError as exception:
+        if exception.errno == errno.ENOENT:
+            pass
+        else:
+            raise
+
+
+def yaml_loads(data, error_message=None):
+    if not error_message:
+        error_message = 'Error loading: {}; {}'
+
+    try:
+        return yaml.load(stream=data, Loader=SafeLoader)
+    except yaml.YAMLError as exception:
+        exit(
+            error_message.format(data, exception)
+        )

mayan/apps/sources/settings.py

@@ -5,7 +5,7 @@ import os
 from django.conf import settings
 from django.utils.translation import ugettext_lazy as _
 
-from mayan.apps.smart_settings import Namespace
+from mayan.apps.smart_settings.classes import Namespace
 
 namespace = Namespace(label=_('Sources'), name='sources')
 
@@ -21,13 +21,13 @@ setting_staging_file_image_cache_storage = namespace.add_setting(
     default='django.core.files.storage.FileSystemStorage', help_text=_(
         'Path to the Storage subclass to use when storing the cached '
         'staging_file image files.'
-    ), quoted=True
+    )
 )
 setting_staging_file_image_cache_storage_arguments = namespace.add_setting(
     global_name='SOURCES_STAGING_FILE_CACHE_STORAGE_BACKEND_ARGUMENTS',
-    default='{{location: {}}}'.format(
+    default={
-        os.path.join(settings.MEDIA_ROOT, 'staging_file_cache')
+        'location': os.path.join(settings.MEDIA_ROOT, 'staging_file_cache')
-    ), help_text=_(
+    }, help_text=_(
         'Arguments to pass to the SOURCES_STAGING_FILE_CACHE_STORAGE_BACKEND.'
-    ), quoted=True,
+    )
 )

mayan/apps/sources/storages.py

@@ -1,11 +1,5 @@
 from __future__ import unicode_literals
 
-import yaml
-try:
-    from yaml import CSafeLoader as SafeLoader
-except ImportError:
-    from yaml import SafeLoader
-
 from django.utils.module_loading import import_string
 
 from .settings import (
@@ -15,9 +9,4 @@ from .settings import (
 
 storage_staging_file_image_cache = import_string(
     dotted_path=setting_staging_file_image_cache_storage.value
-)(
+)(**setting_staging_file_image_cache_storage_arguments.value)
-    **yaml.load(
-        stream=setting_staging_file_image_cache_storage_arguments.value or '{}',
-        Loader=SafeLoader
-    )
-)

mayan/settings/base.py

@@ -17,15 +17,15 @@ import sys
 from django.core.exceptions import ImproperlyConfigured
 from django.utils.translation import ugettext_lazy as _
 
-import environ
+from mayan.apps.smart_settings.literals import BOOTSTRAP_SETTING_LIST
+from mayan.apps.smart_settings.utils import (
+    get_environment_setting, get_environment_variables, read_configuration_file
+)
 
 from .literals import (
     CONFIGURATION_FILENAME, CONFIGURATION_LAST_GOOD_FILENAME,
     DEFAULT_SECRET_KEY, SECRET_KEY_FILENAME, SYSTEM_DIR
 )
-from .utils import yaml_loads, read_configuration_file
-
-env = environ.Env()
 
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
 
@@ -34,8 +34,8 @@ BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 # Quick-start development settings - unsuitable for production
 # See https://docs.djangoproject.com/en/1.10/howto/deployment/checklist/
 
-MEDIA_ROOT = os.environ.get(
+MEDIA_ROOT = get_environment_setting(
-    'MAYAN_MEDIA_ROOT', os.path.join(BASE_DIR, 'media')
+    name='MAYAN_MEDIA_ROOT', fallback_default=os.path.join(BASE_DIR, 'media')
 )
 
 # SECURITY WARNING: keep the secret key used in production secret!
@@ -50,13 +50,9 @@ else:
         SECRET_KEY = DEFAULT_SECRET_KEY
 
 # SECURITY WARNING: don't run with debug turned on in production!
-DEBUG = env.bool('MAYAN_DEBUG', default=False)
+DEBUG = get_environment_setting(name='DEBUG')
 
-ALLOWED_HOSTS = yaml_loads(
+ALLOWED_HOSTS = get_environment_setting(name='ALLOWED_HOSTS')
-    env(
-        'MAYAN_ALLOWED_HOSTS', default="['127.0.0.1', 'localhost', '[::1]']"
-    )
-)
 
 # Application definition
 
@@ -257,12 +253,10 @@ TEST_RUNNER = 'mayan.apps.common.tests.runner.MayanTestRunner'
 
 # --------- Django -------------------
 
-LOGIN_URL = env('MAYAN_LOGIN_URL', default='authentication:login_view')
+LOGIN_URL = get_environment_setting(name='LOGIN_URL')
-LOGIN_REDIRECT_URL = env('MAYAN_LOGIN_REDIRECT_URL', default='common:root')
+LOGIN_REDIRECT_URL = get_environment_setting(name='LOGIN_REDIRECT_URL')
-LOGOUT_REDIRECT_URL = env(
+LOGOUT_REDIRECT_URL = get_environment_setting(name='LOGOUT_REDIRECT_URL')
-    'MAYAN_LOGOUT_REDIRECT_URL', default='authentication:login_view'
+INTERNAL_IPS = get_environment_setting(name='INTERNAL_IPS')
-)
-INTERNAL_IPS = ('127.0.0.1',)
 
 # ---------- Django REST framework -----------
 
@@ -325,51 +319,43 @@ AJAX_REDIRECT_CODE = 278
 
 # ----- Celery -----
 
-BROKER_URL = os.environ.get('MAYAN_BROKER_URL')
+BROKER_URL = get_environment_setting(name='BROKER_URL')
-CELERY_ALWAYS_EAGER = env.bool('MAYAN_CELERY_ALWAYS_EAGER', default=True)
+CELERY_ALWAYS_EAGER = get_environment_setting(name='CELERY_ALWAYS_EAGER')
-CELERY_RESULT_BACKEND = os.environ.get('MAYAN_CELERY_RESULT_BACKEND')
+CELERY_RESULT_BACKEND = get_environment_setting(name='CELERY_RESULT_BACKEND')
 
 # ----- Database -----
-environment_database_engine = os.environ.get('MAYAN_DATABASE_ENGINE')
+DATABASES = {
-
+    'default': {
-if environment_database_engine:
+        'ENGINE': 'django.db.backends.sqlite3',
-    environment_database_conn_max_age = os.environ.get('MAYAN_DATABASE_CONN_MAX_AGE', 0)
+        'NAME': os.path.join(MEDIA_ROOT, 'db.sqlite3'),
-    if environment_database_conn_max_age:
-        environment_database_conn_max_age = int(environment_database_conn_max_age)
-
-    DATABASES = {
-        'default': {
-            'ENGINE': environment_database_engine,
-            'NAME': os.environ['MAYAN_DATABASE_NAME'],
-            'USER': os.environ['MAYAN_DATABASE_USER'],
-            'PASSWORD': os.environ['MAYAN_DATABASE_PASSWORD'],
-            'HOST': os.environ.get('MAYAN_DATABASE_HOST', None),
-            'PORT': os.environ.get('MAYAN_DATABASE_PORT', None),
-            'CONN_MAX_AGE': environment_database_conn_max_age,
-        }
     }
-else:
+}
-    DATABASES = {
-        'default': {
-            'ENGINE': 'django.db.backends.sqlite3',
-            'NAME': os.path.join(MEDIA_ROOT, 'db.sqlite3'),
-        }
-    }
-
 
 BASE_INSTALLED_APPS = INSTALLED_APPS
 COMMON_EXTRA_APPS = ()
 COMMON_DISABLED_APPS = ()
 
-CONFIGURATION_FILEPATH = os.path.join(MEDIA_ROOT, CONFIGURATION_FILENAME)
+CONFIGURATION_FILEPATH = get_environment_setting(
-CONFIGURATION_LAST_GOOD_FILEPATH = os.path.join(
+    name='CONFIGURATION_FILEPATH', fallback_default=os.path.join(
-    MEDIA_ROOT, CONFIGURATION_LAST_GOOD_FILENAME
+        MEDIA_ROOT, CONFIGURATION_FILENAME
+    )
+)
+
+CONFIGURATION_LAST_GOOD_FILEPATH = get_environment_setting(
+    name='CONFIGURATION_LAST_GOOD_FILEPATH', fallback_default=os.path.join(
+        MEDIA_ROOT, CONFIGURATION_LAST_GOOD_FILENAME
+    )
 )
 
 if 'revertsettings' not in sys.argv:
-    result = read_configuration_file(CONFIGURATION_FILEPATH)
+    configuration_result = read_configuration_file(CONFIGURATION_FILEPATH)
-    if result:
+    environment_result = get_environment_variables()
-        globals().update(result)
+
+    for setting in BOOTSTRAP_SETTING_LIST:
+        if setting['name'] in configuration_result:
+            globals().update({setting['name']: configuration_result[setting['name']]})
+        elif setting['name'] in environment_result:
+            globals().update({setting['name']: environment_result[setting['name']]})
 
 
 for app in INSTALLED_APPS:

mayan/settings/utils.py

@@ -1,39 +0,0 @@
-from __future__ import unicode_literals
-
-import errno
-import os
-
-import yaml
-
-
-def read_configuration_file(path):
-    try:
-        with open(path) as file_object:
-            file_object.seek(0, os.SEEK_END)
-            if file_object.tell():
-                file_object.seek(0)
-                try:
-                    return yaml.safe_load(file_object)
-                except yaml.YAMLError as exception:
-                    exit(
-                        'Error loading configuration file: {}; {}'.format(
-                            path, exception
-                        )
-                    )
-    except IOError as exception:
-        if exception.errno == errno.ENOENT:
-            pass
-        else:
-            raise
-
-
-def yaml_loads(data, error_message=None):
-    if not error_message:
-        error_message = 'Error loading: {}; {}'
-
-    try:
-        return yaml.safe_load(data)
-    except yaml.YAMLError as exception:
-        exit(
-            error_message.format(data, exception)
-        )