Initial settings: Refactor setting bootstrapping

Refactor the initial environment settings and configuration file loading
fixing some issues loading Django settings.

Consolidate all database settings into a new single setting
called "DATABASES". This mirrors Django database setting
structure. This changes makes it possible to use configure
multiple databases and database routers from the environment
variables or configuration file.

Remove usage of django-environ. Only a small set of the
features provided by django-environ were being used.
Variable typecasting is now only YAML. YAML parsing
is implemented in code.

Previously the initial setting code added all settings
it found into the global symbol table. Now the settings
found are matched to a explicit list of allowed settings.

Signed-off-by: Roberto Rosario <roberto.rosario.gonzalez@gmail.com>

HISTORY.rst

@@ -66,7 +66,16 @@
 
       TypeError: type object argument after ** must be a mapping, not str
 
-- Rename the CONVERTER_GRAPHICS_BACKEND_CONFIG to CONVERTER_GRAPHICS_BACKEND_ARGUMENTS.
+- Rename the CONVERTER_GRAPHICS_BACKEND_CONFIG to
+  CONVERTER_GRAPHICS_BACKEND_ARGUMENTS.
+- Refactor the initial environment settings and configuration file loading.
+- Consolidate all database settings into a new single setting called
+  "DATABASES".
+- Remove usage of django-environ.
+- Reduce attach surface by only loading a restricted list of
+  settings during the initial configuration loading.
+
+
 
 3.1.9 (2018-11-01)
 ==================

mayan/settings/base.py

@@ -16,15 +16,13 @@ import sys
 
 from django.utils.translation import ugettext_lazy as _
 
-import environ
-
 from .literals import (
     CONFIGURATION_FILENAME, CONFIGURATION_LAST_GOOD_FILENAME,
-    DEFAULT_SECRET_KEY, SECRET_KEY_FILENAME, SYSTEM_DIR
+    DEFAULT_SECRET_KEY, DJANGO_SETTINGS_LIST, SECRET_KEY_FILENAME, SYSTEM_DIR
+)
+from .utils import (
+    get_environment_variables, read_configuration_file, yaml_loads
 )
-from .utils import yaml_loads, read_configuration_file
-
-env = environ.Env()
 
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
 
@@ -49,13 +47,10 @@ else:
         SECRET_KEY = DEFAULT_SECRET_KEY
 
 # SECURITY WARNING: don't run with debug turned on in production!
-DEBUG = env.bool('MAYAN_DEBUG', default=False)
 
-ALLOWED_HOSTS = yaml_loads(
-    env(
-        'MAYAN_ALLOWED_HOSTS', default="['127.0.0.1', 'localhost', '[::1]']"
-    )
-)
+DEBUG = yaml_loads(os.environ.get('MAYAN_DEBUG', 'false'))
+
+ALLOWED_HOSTS = ['127.0.0.1', 'localhost', '[::1]']
 
 # Application definition
 
@@ -249,9 +244,13 @@ TEST_RUNNER = 'common.tests.runner.MayanTestRunner'
 
 # --------- Django -------------------
 
-HOME_VIEW = env('MAYAN_HOME_VIEW', default='common:home')
-LOGIN_URL = env('MAYAN_LOGIN_URL', default='authentication:login_view')
-LOGIN_REDIRECT_URL = env('MAYAN_LOGIN_REDIRECT_URL', default='common:root')
+HOME_VIEW = yaml_loads(os.environ.get('MAYAN_HOME_VIEW', 'common:home'))
+LOGIN_URL = yaml_loads(
+    os.environ.get('MAYAN_LOGIN_URL', 'authentication:login_view')
+)
+LOGIN_REDIRECT_URL = yaml_loads(
+    os.environ.get('MAYAN_LOGIN_REDIRECT_URL', 'common:root')
+)
 INTERNAL_IPS = ('127.0.0.1',)
 
 # ---------- Django REST framework -----------
@@ -316,35 +315,19 @@ AJAX_REDIRECT_CODE = 278
 # ----- Celery -----
 
 BROKER_URL = os.environ.get('MAYAN_BROKER_URL')
-CELERY_ALWAYS_EAGER = env.bool('MAYAN_CELERY_ALWAYS_EAGER', default=True)
+CELERY_ALWAYS_EAGER = yaml_loads(
+    os.environ.get('MAYAN_CELERY_ALWAYS_EAGER', 'true')
+)
 CELERY_RESULT_BACKEND = os.environ.get('MAYAN_CELERY_RESULT_BACKEND')
 
 # ----- Database -----
-environment_database_engine = os.environ.get('MAYAN_DATABASE_ENGINE')
 
-if environment_database_engine:
-    environment_database_conn_max_age = os.environ.get('MAYAN_DATABASE_CONN_MAX_AGE', None)
-    if environment_database_conn_max_age:
-        environment_database_conn_max_age = int(environment_database_conn_max_age)
-
-    DATABASES = {
-        'default': {
-            'ENGINE': environment_database_engine,
-            'NAME': os.environ['MAYAN_DATABASE_NAME'],
-            'USER': os.environ['MAYAN_DATABASE_USER'],
-            'PASSWORD': os.environ['MAYAN_DATABASE_PASSWORD'],
-            'HOST': os.environ.get('MAYAN_DATABASE_HOST', None),
-            'PORT': os.environ.get('MAYAN_DATABASE_PORT', None),
-            'CONN_MAX_AGE': environment_database_conn_max_age,
-        }
-    }
-else:
-    DATABASES = {
-        'default': {
-            'ENGINE': 'django.db.backends.sqlite3',
-            'NAME': os.path.join(MEDIA_ROOT, 'db.sqlite3'),
-        }
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.sqlite3',
+        'NAME': os.path.join(MEDIA_ROOT, 'db.sqlite3'),
     }
+}
 
 CONFIGURATION_FILEPATH = os.path.join(MEDIA_ROOT, CONFIGURATION_FILENAME)
 CONFIGURATION_LAST_GOOD_FILEPATH = os.path.join(
@@ -352,6 +335,11 @@ CONFIGURATION_LAST_GOOD_FILEPATH = os.path.join(
 )
 
 if 'revertsettings' not in sys.argv:
-    result = read_configuration_file(CONFIGURATION_FILEPATH)
-    if result:
-        globals().update(result)
+    configuration_result = read_configuration_file(CONFIGURATION_FILEPATH)
+    environment_result = get_environment_variables()
+
+    for setting in DJANGO_SETTINGS_LIST:
+        if setting in configuration_result:
+            globals().update({setting: configuration_result[setting]})
+        elif setting in environment_result:
+            globals().update({setting: environment_result[setting]})

mayan/settings/literals.py

@@ -1,5 +1,34 @@
 from __future__ import unicode_literals
 
+DJANGO_SETTINGS_LIST = (
+    'ALLOWED_HOSTS', 'APPEND_SLASH', 'AUTH_PASSWORD_VALIDATORS',
+    'DATA_UPLOAD_MAX_MEMORY_SIZE',
+    'DATABASES',
+    'DEBUG',
+    'DEFAULT_FROM_EMAIL',
+    'DISALLOWED_USER_AGENTS',
+    'EMAIL_BACKEND',
+    'EMAIL_HOST',
+    'EMAIL_HOST_PASSWORD',
+    'EMAIL_HOST_USER',
+    'EMAIL_PORT',
+    'EMAIL_TIMEOUT',
+    'EMAIL_USE_SSL',
+    'EMAIL_USE_TLS',
+    'FILE_UPLOAD_MAX_MEMORY_SIZE',
+    'HOME_VIEW',
+    'INSTALLED_APPS',
+    'INTERNAL_IPS',
+    'LANGUAGES',
+    'LANGUAGE_CODE',
+    'LOGIN_REDIRECT_URL',
+    'LOGIN_URL',
+    'STATIC_URL',
+    'STATICFILES_STORAGE',
+    'TIME_ZONE',
+    'WSGI_APPLICATION',
+)
+
 DEFAULT_SECRET_KEY = 'secret-key-missing!'
 SECRET_KEY_FILENAME = 'SECRET_KEY'
 SYSTEM_DIR = 'system'

mayan/settings/utils.py

@@ -5,6 +5,20 @@ import os
 
 import yaml
 
+from .literals import DJANGO_SETTINGS_LIST
+
+
+def get_environment_variables():
+    result = {}
+
+    for setting in DJANGO_SETTINGS_LIST:
+        environment_value = os.environ.get('MAYAN_{}'.format(setting))
+        if environment_value:
+            environment_value = yaml_loads(environment_value)
+            result[setting] = environment_value
+
+    return result
+
 
 def read_configuration_file(path):
     try:

removals.txt

@@ -2,6 +2,7 @@
 cssmin
 django-celery
 django-compressor
+django-environ
 django-filetransfers
 django-rest-swagger
 django-suit

requirements/base.txt

@@ -9,7 +9,6 @@ django-celery==3.2.2
 django-colorful==1.3
 django-cors-headers==2.4.0
 django-downloadview==1.9
-django-environ==0.4.5
 django-formtools==2.1
 django-pure-pagination==0.3.0
 django-mathfilters==0.4.0