From ba4858e77fb3ecef2127bbe7b42a5cf8113f7598 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 27 Nov 2018 04:06:21 -0400 Subject: [PATCH] Initial settings: Refactor setting bootstrapping Refactor the initial environment settings and configuration file loading fixing some issues loading Django settings. Consolidate all database settings into a new single setting called "DATABASES". This mirrors Django database setting structure. This changes makes it possible to use configure multiple databases and database routers from the environment variables or configuration file. Remove usage of django-environ. Only a small set of the features provided by django-environ were being used. Variable typecasting is now only YAML. YAML parsing is implemented in code. Previously the initial setting code added all settings it found into the global symbol table. Now the settings found are matched to a explicit list of allowed settings. Signed-off-by: Roberto Rosario --- HISTORY.rst | 11 +++++- mayan/settings/base.py | 72 ++++++++++++++++---------------------- mayan/settings/literals.py | 29 +++++++++++++++ mayan/settings/utils.py | 14 ++++++++ removals.txt | 1 + requirements/base.txt | 1 - 6 files changed, 84 insertions(+), 44 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index de0e2e9e6f..ba26366dfc 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -66,7 +66,16 @@ TypeError: type object argument after ** must be a mapping, not str -- Rename the CONVERTER_GRAPHICS_BACKEND_CONFIG to CONVERTER_GRAPHICS_BACKEND_ARGUMENTS. +- Rename the CONVERTER_GRAPHICS_BACKEND_CONFIG to + CONVERTER_GRAPHICS_BACKEND_ARGUMENTS. +- Refactor the initial environment settings and configuration file loading. +- Consolidate all database settings into a new single setting called + "DATABASES". +- Remove usage of django-environ. +- Reduce attach surface by only loading a restricted list of + settings during the initial configuration loading. + + 3.1.9 (2018-11-01) ================== diff --git a/mayan/settings/base.py b/mayan/settings/base.py index 736d64a90a..3b3a6ed15e 100644 --- a/mayan/settings/base.py +++ b/mayan/settings/base.py @@ -16,15 +16,13 @@ import sys from django.utils.translation import ugettext_lazy as _ -import environ - from .literals import ( CONFIGURATION_FILENAME, CONFIGURATION_LAST_GOOD_FILENAME, - DEFAULT_SECRET_KEY, SECRET_KEY_FILENAME, SYSTEM_DIR + DEFAULT_SECRET_KEY, DJANGO_SETTINGS_LIST, SECRET_KEY_FILENAME, SYSTEM_DIR +) +from .utils import ( + get_environment_variables, read_configuration_file, yaml_loads ) -from .utils import yaml_loads, read_configuration_file - -env = environ.Env() # Build paths inside the project like this: os.path.join(BASE_DIR, ...) @@ -49,13 +47,10 @@ else: SECRET_KEY = DEFAULT_SECRET_KEY # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = env.bool('MAYAN_DEBUG', default=False) -ALLOWED_HOSTS = yaml_loads( - env( - 'MAYAN_ALLOWED_HOSTS', default="['127.0.0.1', 'localhost', '[::1]']" - ) -) +DEBUG = yaml_loads(os.environ.get('MAYAN_DEBUG', 'false')) + +ALLOWED_HOSTS = ['127.0.0.1', 'localhost', '[::1]'] # Application definition @@ -249,9 +244,13 @@ TEST_RUNNER = 'common.tests.runner.MayanTestRunner' # --------- Django ------------------- -HOME_VIEW = env('MAYAN_HOME_VIEW', default='common:home') -LOGIN_URL = env('MAYAN_LOGIN_URL', default='authentication:login_view') -LOGIN_REDIRECT_URL = env('MAYAN_LOGIN_REDIRECT_URL', default='common:root') +HOME_VIEW = yaml_loads(os.environ.get('MAYAN_HOME_VIEW', 'common:home')) +LOGIN_URL = yaml_loads( + os.environ.get('MAYAN_LOGIN_URL', 'authentication:login_view') +) +LOGIN_REDIRECT_URL = yaml_loads( + os.environ.get('MAYAN_LOGIN_REDIRECT_URL', 'common:root') +) INTERNAL_IPS = ('127.0.0.1',) # ---------- Django REST framework ----------- @@ -316,35 +315,19 @@ AJAX_REDIRECT_CODE = 278 # ----- Celery ----- BROKER_URL = os.environ.get('MAYAN_BROKER_URL') -CELERY_ALWAYS_EAGER = env.bool('MAYAN_CELERY_ALWAYS_EAGER', default=True) +CELERY_ALWAYS_EAGER = yaml_loads( + os.environ.get('MAYAN_CELERY_ALWAYS_EAGER', 'true') +) CELERY_RESULT_BACKEND = os.environ.get('MAYAN_CELERY_RESULT_BACKEND') # ----- Database ----- -environment_database_engine = os.environ.get('MAYAN_DATABASE_ENGINE') -if environment_database_engine: - environment_database_conn_max_age = os.environ.get('MAYAN_DATABASE_CONN_MAX_AGE', None) - if environment_database_conn_max_age: - environment_database_conn_max_age = int(environment_database_conn_max_age) - - DATABASES = { - 'default': { - 'ENGINE': environment_database_engine, - 'NAME': os.environ['MAYAN_DATABASE_NAME'], - 'USER': os.environ['MAYAN_DATABASE_USER'], - 'PASSWORD': os.environ['MAYAN_DATABASE_PASSWORD'], - 'HOST': os.environ.get('MAYAN_DATABASE_HOST', None), - 'PORT': os.environ.get('MAYAN_DATABASE_PORT', None), - 'CONN_MAX_AGE': environment_database_conn_max_age, - } - } -else: - DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': os.path.join(MEDIA_ROOT, 'db.sqlite3'), - } +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': os.path.join(MEDIA_ROOT, 'db.sqlite3'), } +} CONFIGURATION_FILEPATH = os.path.join(MEDIA_ROOT, CONFIGURATION_FILENAME) CONFIGURATION_LAST_GOOD_FILEPATH = os.path.join( @@ -352,6 +335,11 @@ CONFIGURATION_LAST_GOOD_FILEPATH = os.path.join( ) if 'revertsettings' not in sys.argv: - result = read_configuration_file(CONFIGURATION_FILEPATH) - if result: - globals().update(result) + configuration_result = read_configuration_file(CONFIGURATION_FILEPATH) + environment_result = get_environment_variables() + + for setting in DJANGO_SETTINGS_LIST: + if setting in configuration_result: + globals().update({setting: configuration_result[setting]}) + elif setting in environment_result: + globals().update({setting: environment_result[setting]}) diff --git a/mayan/settings/literals.py b/mayan/settings/literals.py index b30b20544b..820a4c8169 100644 --- a/mayan/settings/literals.py +++ b/mayan/settings/literals.py @@ -1,5 +1,34 @@ from __future__ import unicode_literals +DJANGO_SETTINGS_LIST = ( + 'ALLOWED_HOSTS', 'APPEND_SLASH', 'AUTH_PASSWORD_VALIDATORS', + 'DATA_UPLOAD_MAX_MEMORY_SIZE', + 'DATABASES', + 'DEBUG', + 'DEFAULT_FROM_EMAIL', + 'DISALLOWED_USER_AGENTS', + 'EMAIL_BACKEND', + 'EMAIL_HOST', + 'EMAIL_HOST_PASSWORD', + 'EMAIL_HOST_USER', + 'EMAIL_PORT', + 'EMAIL_TIMEOUT', + 'EMAIL_USE_SSL', + 'EMAIL_USE_TLS', + 'FILE_UPLOAD_MAX_MEMORY_SIZE', + 'HOME_VIEW', + 'INSTALLED_APPS', + 'INTERNAL_IPS', + 'LANGUAGES', + 'LANGUAGE_CODE', + 'LOGIN_REDIRECT_URL', + 'LOGIN_URL', + 'STATIC_URL', + 'STATICFILES_STORAGE', + 'TIME_ZONE', + 'WSGI_APPLICATION', +) + DEFAULT_SECRET_KEY = 'secret-key-missing!' SECRET_KEY_FILENAME = 'SECRET_KEY' SYSTEM_DIR = 'system' diff --git a/mayan/settings/utils.py b/mayan/settings/utils.py index bb2a8da489..4d52cb7ac7 100644 --- a/mayan/settings/utils.py +++ b/mayan/settings/utils.py @@ -5,6 +5,20 @@ import os import yaml +from .literals import DJANGO_SETTINGS_LIST + + +def get_environment_variables(): + result = {} + + for setting in DJANGO_SETTINGS_LIST: + environment_value = os.environ.get('MAYAN_{}'.format(setting)) + if environment_value: + environment_value = yaml_loads(environment_value) + result[setting] = environment_value + + return result + def read_configuration_file(path): try: diff --git a/removals.txt b/removals.txt index 6d48bedcc6..35e0a35ecf 100644 --- a/removals.txt +++ b/removals.txt @@ -2,6 +2,7 @@ cssmin django-celery django-compressor +django-environ django-filetransfers django-rest-swagger django-suit diff --git a/requirements/base.txt b/requirements/base.txt index 83143955c1..d4f0a5eda3 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -9,7 +9,6 @@ django-celery==3.2.2 django-colorful==1.3 django-cors-headers==2.4.0 django-downloadview==1.9 -django-environ==0.4.5 django-formtools==2.1 django-pure-pagination==0.3.0 django-mathfilters==0.4.0