matthias/mayan-edms
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Add support for setting migrations

Browse Source 
Signed-off-by: Roberto Rosario <roberto.rosario@mayan-edms.com>
This commit is contained in:
Roberto Rosario
2019-11-18 00:27:33 -04:00
parent c75033f676
commit fa8fddb349
8 changed files with 224 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
HISTORY.rst
View File

@@ -44,13 +44,15 @@
Thanks to Jakob Haufe (@sur5r) and Jesaja Everling (@jeverling)
for much of the research and code updates.
- Support wildcard MIME type associations for the file metadata drivers.
- Rename MAYAN_GUID to MAYAN_GID
- Update Gunicorn to use sync workers.
- Include devpi-server as a development dependency.
- Include devpi-server as a development dependency. Used to speed up
local builds of the Docker image.
- Update default Docker stack file.
- Remove Redis from the Docker image.
- Remove Redis from the Docker image. A separate container must now
be deployed.
- Add Celery flower to the Docker image.
- Allow PIP proxying to the Docker image during build.
- Allow PIP proxying to the Docker image during build. Can be used
with the local devpi-server or other similar.
- Default Celery worker concurrency to 0 (auto).
- Set DJANGO_SETTINGS_MODULE environment variable to make it
available to sub processes.
@@ -187,6 +189,7 @@
Instead of throwing an error a sample label of
"Unknown action type" will be used and allow users to
delete the unknown state action.
- Add support for setting migrations.
3.2.9 (2019-11-03)
==================

12
docs/releases/3.3.rst
View File

@@ -61,20 +61,22 @@ Changes
Thanks to Jakob Haufe (@sur5r) and Jesaja Everling (@jeverling)
for much of the research and code updates.
- Support wildcard MIME type associations for the file metadata drivers.
- Rename MAYAN_GUID to MAYAN_GID
- Update Gunicorn to use sync workers.
- Include devpi-server as a development dependency.
- Include devpi-server as a development dependency. Used to speed up
local builds of the Docker image.
- Update default Docker stack file.
- Remove Redis from the Docker image.
- Remove Redis from the Docker image. A separate container must now
be deployed.
- Add Celery flower to the Docker image.
- Allow PIP proxying to the Docker image during build.
- Allow PIP proxying to the Docker image during build. Can be used
with the local devpi-server or other similar.
- Default Celery worker concurrency to 0 (auto).
- Set DJANGO_SETTINGS_MODULE environment variable to make it
available to sub processes.
- Add entrypoint commands to run single workers, single gunicorn
or single celery commands like "flower".
- Add platform template to return queues for a worker.
- Remove task inspection from task manager app.
- Remove task inspection from task manager app.
- Move pagination navigation inside the toolbar.
- Remove document image clear link and view.
This is now handled by the file caching app.

16
mayan/apps/common/tests/mixins.py
View File

@@ -141,6 +141,22 @@ class ContentTypeCheckTestCaseMixin(object):
self.client = CustomClient()
class EnvironmentTestCaseMixin(object):
def setUp(self):
super(EnvironmentTestCaseMixin, self).setUp()
self._test_environment_variables = []
def tearDown(self):
for name in self._test_environment_variables:
os.environ.pop(name)
super(EnvironmentTestCaseMixin, self).tearDown()
def _set_environment_variable(self, name, value):
self._test_environment_variables.append(name)
os.environ[name] = value
class ModelTestCaseMixin(object):
def _model_instance_to_dictionary(self, instance):
return instance._meta.model._default_manager.filter(

96
mayan/apps/smart_settings/classes.py
View File

@@ -19,6 +19,7 @@ from django.utils.encoding import (
from mayan.apps.common.serialization import yaml_dump, yaml_load
logger = logging.getLogger(__name__)
NAMESPACE_VERSION_INITIAL = '0001'
SMART_SETTINGS_NAMESPACES_NAME = 'SMART_SETTINGS_NAMESPACES'
@@ -81,11 +82,15 @@ class Namespace(object):
for namespace in cls.get_all():
namespace.invalidate_cache()
def __init__(self, name, label, version='0001'):
def __init__(
self, name, label, migration_class=None,
version=NAMESPACE_VERSION_INITIAL
):
if name in self.__class__._registry:
raise Exception(
'Namespace names must be unique; "%s" already exists.' % name
)
self.migration_class = migration_class
self.name = name
self.label = label
self.version = version
@@ -99,17 +104,75 @@ class Namespace(object):
return Setting(namespace=self, **kwargs)
def get_config_version(self):
return Namespace.get_namespace_config(name=self.name).get('version', None)
return Namespace.get_namespace_config(name=self.name).get(
'version', NAMESPACE_VERSION_INITIAL
)
def invalidate_cache(self):
for setting in self._settings:
setting.invalidate_cache()
def migrate(self, setting):
if self.migration_class:
self.migration_class(namespace=self).migrate(setting=setting)
@property
def settings(self):
return sorted(self._settings, key=lambda x: x.global_name)
class NamespaceMigration(object):
def __init__(self, namespace):
self.namespace = namespace
def get_method_name(self, setting):
return setting.global_name.lower()
def get_method_name_full(self, setting, version):
return '{}_{}'.format(
self.get_method_name(setting=setting),
version
)
def migrate(self, setting):
if self.namespace.get_config_version() != self.namespace.version:
method_name = self.get_method_name(setting=setting)
# Get methods for this setting
setting_methods = [
method for method in dir(self) if method.startswith(
method_name
)
]
# Get order of execution of setting methods
versions = [
method.replace(
'{}_'.format(method_name), ''
) for method in setting_methods
]
try:
start = versions.index(self.namespace.get_config_version())
except ValueError:
start = 0
try:
end = versions.index(self.namespace.version)
except ValueError:
end = None
value = setting.raw_value
for version in versions[start:end]:
method = getattr(
self, self.get_method_name_full(
setting=setting, version=version
), None
)
if method:
value = method(value=value)
setting.raw_value = value
@python_2_unicode_compatible
class Setting(object):
_registry = {}
@@ -191,7 +254,6 @@ class Setting(object):
cls._config_file_cache = read_configuration_file(
filepath=settings.CONFIGURATION_FILEPATH
)
return cls._config_file_cache
@classmethod
@@ -224,7 +286,10 @@ class Setting(object):
path=settings.CONFIGURATION_LAST_GOOD_FILEPATH
)
def __init__(self, namespace, global_name, default, help_text=None, is_path=False, post_edit_function=None):
def __init__(
self, namespace, global_name, default, help_text=None, is_path=False,
post_edit_function=None
):
self.global_name = global_name
self.default = default
self.help_text = help_text
@@ -252,11 +317,21 @@ class Setting(object):
)
)
else:
self.raw_value = self.get_config_file_content().get(
self.global_name, getattr(
settings, self.global_name, self.default
)
)
try:
# Try the config file
self.raw_value = self.get_config_file_content()[self.global_name]
except KeyError:
try:
# Try the Django settings variable
self.raw_value = getattr(
settings, self.global_name
)
except AttributeError:
# Finally set to the default value
self.raw_value = self.default
else:
# Found in the config file, try to migrate the value
self.migrate()
self.yaml = Setting.serialize_value(self.raw_value)
self.loaded = True
@@ -264,6 +339,9 @@ class Setting(object):
def invalidate_cache(self):
self.loaded = False
def migrate(self):
self.namespace.migrate(setting=self)
@property
def serialized_value(self):
"""

5
mayan/apps/smart_settings/tests/literals.py
View File

@@ -5,3 +5,8 @@ ENVIRONMENT_TEST_VALUE = '999999'
TEST_NAMESPACE_LABEL = 'test namespace label'
TEST_NAMESPACE_NAME = 'test_namespace_name'
TEST_SETTING_GLOBAL_NAME = 'SMART_SETTINGS_TEST_SETTING'
TEST_SETTING_INITIAL_VALUE = 'test_setting_initial_value'
TEST_SETTING_DEFAULT_VALUE = 'test_setting_default_value'
TEST_SETTING_VALUE = 'test_setting_value'

19
mayan/apps/smart_settings/tests/mixins.py
View File

@@ -2,7 +2,10 @@ from __future__ import absolute_import, unicode_literals
from ..classes import Namespace
from .literals import TEST_NAMESPACE_LABEL, TEST_NAMESPACE_NAME
from .literals import (
TEST_NAMESPACE_LABEL, TEST_NAMESPACE_NAME, TEST_SETTING_DEFAULT_VALUE,
TEST_SETTING_GLOBAL_NAME
)
class SmartSettingsTestCaseMixin(object):
@@ -12,16 +15,26 @@ class SmartSettingsTestCaseMixin(object):
class SmartSettingTestMixin(object):
def _create_test_settings_namespace(self):
def _create_test_settings_namespace(self, **kwargs):
try:
self.test_settings_namespace = Namespace.get(
name=TEST_NAMESPACE_NAME
)
self.test_settings_namespace.migration_class = None
self.test_settings_namespace.version = None
self.test_settings_namespace.__dict__.update(kwargs)
except KeyError:
self.test_settings_namespace = Namespace(
label=TEST_NAMESPACE_LABEL, name=TEST_NAMESPACE_NAME
label=TEST_NAMESPACE_LABEL, name=TEST_NAMESPACE_NAME,
**kwargs
)
def _create_test_setting(self):
self.test_setting = self.test_settings_namespace.add_setting(
global_name=TEST_SETTING_GLOBAL_NAME,
default=TEST_SETTING_DEFAULT_VALUE
)
class SmartSettingViewTestMixin(object):
def _request_namespace_list_view(self):

16
mayan/apps/smart_settings/tests/mocks.py Normal file
View File

@@ -0,0 +1,16 @@
from __future__ import absolute_import, unicode_literals
from ..classes import NamespaceMigration
class TestNamespaceMigrationOne(NamespaceMigration):
def smart_settings_test_setting_0001(self, value):
return '{}_0001'.format(value)
class TestNamespaceMigrationTwo(NamespaceMigration):
def smart_settings_test_setting_0001(self, value):
return '{}_0001'.format(value)
def smart_settings_test_setting_0002(self, value):
return '{}_0002'.format(value)

79
mayan/apps/smart_settings/tests/test_classes.py
View File

@@ -1,27 +1,32 @@
from __future__ import absolute_import, unicode_literals
import os
from pathlib2 import Path
from django.conf import settings
from django.utils.encoding import force_text
from django.utils.encoding import force_bytes, force_text
from mayan.apps.common.mixins import EnvironmentTestCaseMixin
from mayan.apps.common.settings import setting_paginate_by
from mayan.apps.common.tests.base import BaseTestCase
from mayan.apps.storage.utils import fs_cleanup
from mayan.apps.storage.utils import fs_cleanup, NamedTemporaryFile
from ..classes import Setting
from .literals import ENVIRONMENT_TEST_NAME, ENVIRONMENT_TEST_VALUE
from .literals import (
ENVIRONMENT_TEST_NAME, ENVIRONMENT_TEST_VALUE, TEST_SETTING_GLOBAL_NAME,
TEST_SETTING_INITIAL_VALUE, TEST_SETTING_VALUE
)
from .mixins import SmartSettingTestMixin
from .mocks import TestNamespaceMigrationOne, TestNamespaceMigrationTwo
class ClassesTestCase(SmartSettingTestMixin, BaseTestCase):
class ClassesTestCase(EnvironmentTestCaseMixin, SmartSettingTestMixin, BaseTestCase):
def test_environment_variable(self):
os.environ[
'MAYAN_{}'.format(ENVIRONMENT_TEST_NAME)
] = ENVIRONMENT_TEST_VALUE
self._set_environment_variable(
name='MAYAN_{}'.format(ENVIRONMENT_TEST_NAME),
value=ENVIRONMENT_TEST_VALUE
)
self.assertTrue(setting_paginate_by.value, ENVIRONMENT_TEST_VALUE)
def test_config_backup_creation(self):
@@ -53,3 +58,59 @@ class ClassesTestCase(SmartSettingTestMixin, BaseTestCase):
self.assertFalse(Setting.check_changed())
test_setting.value = 'test value edited'
self.assertTrue(Setting.check_changed())
class NamespaceMigrationTestCase(
EnvironmentTestCaseMixin, SmartSettingTestMixin, BaseTestCase
):
def test_environment_migration(self):
self._set_environment_variable(
name='MAYAN_{}'.format(TEST_SETTING_GLOBAL_NAME),
value=TEST_SETTING_INITIAL_VALUE
)
self._create_test_settings_namespace(
migration_class=TestNamespaceMigrationOne, version='0002'
)
self._create_test_setting()
self.assertEqual(
self.test_setting.value, TEST_SETTING_INITIAL_VALUE
)
def test_migration_0001_to_0002(self):
self._create_test_settings_namespace(
migration_class=TestNamespaceMigrationTwo, version='0002'
)
self._create_test_setting()
with NamedTemporaryFile() as file_object:
settings.CONFIGURATION_FILEPATH = file_object.name
file_object.write(
force_bytes(
'{}: {}'.format(TEST_SETTING_GLOBAL_NAME, TEST_SETTING_VALUE)
)
)
file_object.seek(0)
self.assertEqual(
self.test_setting.value, '{}_0001'.format(TEST_SETTING_VALUE)
)
def test_migration_0001_to_0003(self):
self._create_test_settings_namespace(
migration_class=TestNamespaceMigrationTwo, version='0003'
)
self._create_test_setting()
with NamedTemporaryFile() as file_object:
settings.CONFIGURATION_FILEPATH = file_object.name
file_object.write(
force_bytes(
'{}: {}'.format(TEST_SETTING_GLOBAL_NAME, TEST_SETTING_VALUE)
)
)
file_object.seek(0)
self.assertEqual(
self.test_setting.value, '{}_0001_0002'.format(TEST_SETTING_VALUE)
)