diff --git a/HISTORY.rst b/HISTORY.rst index ca59576c16..6dc3af6bcd 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -44,13 +44,15 @@ Thanks to Jakob Haufe (@sur5r) and Jesaja Everling (@jeverling) for much of the research and code updates. - Support wildcard MIME type associations for the file metadata drivers. -- Rename MAYAN_GUID to MAYAN_GID - Update Gunicorn to use sync workers. -- Include devpi-server as a development dependency. +- Include devpi-server as a development dependency. Used to speed up + local builds of the Docker image. - Update default Docker stack file. -- Remove Redis from the Docker image. +- Remove Redis from the Docker image. A separate container must now + be deployed. - Add Celery flower to the Docker image. -- Allow PIP proxying to the Docker image during build. +- Allow PIP proxying to the Docker image during build. Can be used + with the local devpi-server or other similar. - Default Celery worker concurrency to 0 (auto). - Set DJANGO_SETTINGS_MODULE environment variable to make it available to sub processes. @@ -187,6 +189,7 @@ Instead of throwing an error a sample label of "Unknown action type" will be used and allow users to delete the unknown state action. +- Add support for setting migrations. 3.2.9 (2019-11-03) ================== diff --git a/docs/releases/3.3.rst b/docs/releases/3.3.rst index ed45290273..2f51f4dbcb 100644 --- a/docs/releases/3.3.rst +++ b/docs/releases/3.3.rst @@ -61,20 +61,22 @@ Changes Thanks to Jakob Haufe (@sur5r) and Jesaja Everling (@jeverling) for much of the research and code updates. - Support wildcard MIME type associations for the file metadata drivers. -- Rename MAYAN_GUID to MAYAN_GID - Update Gunicorn to use sync workers. -- Include devpi-server as a development dependency. +- Include devpi-server as a development dependency. Used to speed up + local builds of the Docker image. - Update default Docker stack file. -- Remove Redis from the Docker image. +- Remove Redis from the Docker image. A separate container must now + be deployed. - Add Celery flower to the Docker image. -- Allow PIP proxying to the Docker image during build. +- Allow PIP proxying to the Docker image during build. Can be used + with the local devpi-server or other similar. - Default Celery worker concurrency to 0 (auto). - Set DJANGO_SETTINGS_MODULE environment variable to make it available to sub processes. - Add entrypoint commands to run single workers, single gunicorn or single celery commands like "flower". - Add platform template to return queues for a worker. -- Remove task inspection from task manager app. +- Remove task inspection from task manager app. - Move pagination navigation inside the toolbar. - Remove document image clear link and view. This is now handled by the file caching app. diff --git a/mayan/apps/common/tests/mixins.py b/mayan/apps/common/tests/mixins.py index 4d1a318b21..c6b2eb5527 100644 --- a/mayan/apps/common/tests/mixins.py +++ b/mayan/apps/common/tests/mixins.py @@ -141,6 +141,22 @@ class ContentTypeCheckTestCaseMixin(object): self.client = CustomClient() +class EnvironmentTestCaseMixin(object): + def setUp(self): + super(EnvironmentTestCaseMixin, self).setUp() + self._test_environment_variables = [] + + def tearDown(self): + for name in self._test_environment_variables: + os.environ.pop(name) + + super(EnvironmentTestCaseMixin, self).tearDown() + + def _set_environment_variable(self, name, value): + self._test_environment_variables.append(name) + os.environ[name] = value + + class ModelTestCaseMixin(object): def _model_instance_to_dictionary(self, instance): return instance._meta.model._default_manager.filter( diff --git a/mayan/apps/smart_settings/classes.py b/mayan/apps/smart_settings/classes.py index 43d16a1393..87637df544 100644 --- a/mayan/apps/smart_settings/classes.py +++ b/mayan/apps/smart_settings/classes.py @@ -19,6 +19,7 @@ from django.utils.encoding import ( from mayan.apps.common.serialization import yaml_dump, yaml_load logger = logging.getLogger(__name__) +NAMESPACE_VERSION_INITIAL = '0001' SMART_SETTINGS_NAMESPACES_NAME = 'SMART_SETTINGS_NAMESPACES' @@ -81,11 +82,15 @@ class Namespace(object): for namespace in cls.get_all(): namespace.invalidate_cache() - def __init__(self, name, label, version='0001'): + def __init__( + self, name, label, migration_class=None, + version=NAMESPACE_VERSION_INITIAL + ): if name in self.__class__._registry: raise Exception( 'Namespace names must be unique; "%s" already exists.' % name ) + self.migration_class = migration_class self.name = name self.label = label self.version = version @@ -99,17 +104,75 @@ class Namespace(object): return Setting(namespace=self, **kwargs) def get_config_version(self): - return Namespace.get_namespace_config(name=self.name).get('version', None) + return Namespace.get_namespace_config(name=self.name).get( + 'version', NAMESPACE_VERSION_INITIAL + ) def invalidate_cache(self): for setting in self._settings: setting.invalidate_cache() + def migrate(self, setting): + if self.migration_class: + self.migration_class(namespace=self).migrate(setting=setting) + @property def settings(self): return sorted(self._settings, key=lambda x: x.global_name) +class NamespaceMigration(object): + def __init__(self, namespace): + self.namespace = namespace + + def get_method_name(self, setting): + return setting.global_name.lower() + + def get_method_name_full(self, setting, version): + return '{}_{}'.format( + self.get_method_name(setting=setting), + version + ) + + def migrate(self, setting): + if self.namespace.get_config_version() != self.namespace.version: + method_name = self.get_method_name(setting=setting) + + # Get methods for this setting + setting_methods = [ + method for method in dir(self) if method.startswith( + method_name + ) + ] + # Get order of execution of setting methods + versions = [ + method.replace( + '{}_'.format(method_name), '' + ) for method in setting_methods + ] + try: + start = versions.index(self.namespace.get_config_version()) + except ValueError: + start = 0 + + try: + end = versions.index(self.namespace.version) + except ValueError: + end = None + + value = setting.raw_value + for version in versions[start:end]: + method = getattr( + self, self.get_method_name_full( + setting=setting, version=version + ), None + ) + if method: + value = method(value=value) + + setting.raw_value = value + + @python_2_unicode_compatible class Setting(object): _registry = {} @@ -191,7 +254,6 @@ class Setting(object): cls._config_file_cache = read_configuration_file( filepath=settings.CONFIGURATION_FILEPATH ) - return cls._config_file_cache @classmethod @@ -224,7 +286,10 @@ class Setting(object): path=settings.CONFIGURATION_LAST_GOOD_FILEPATH ) - def __init__(self, namespace, global_name, default, help_text=None, is_path=False, post_edit_function=None): + def __init__( + self, namespace, global_name, default, help_text=None, is_path=False, + post_edit_function=None + ): self.global_name = global_name self.default = default self.help_text = help_text @@ -252,11 +317,21 @@ class Setting(object): ) ) else: - self.raw_value = self.get_config_file_content().get( - self.global_name, getattr( - settings, self.global_name, self.default - ) - ) + try: + # Try the config file + self.raw_value = self.get_config_file_content()[self.global_name] + except KeyError: + try: + # Try the Django settings variable + self.raw_value = getattr( + settings, self.global_name + ) + except AttributeError: + # Finally set to the default value + self.raw_value = self.default + else: + # Found in the config file, try to migrate the value + self.migrate() self.yaml = Setting.serialize_value(self.raw_value) self.loaded = True @@ -264,6 +339,9 @@ class Setting(object): def invalidate_cache(self): self.loaded = False + def migrate(self): + self.namespace.migrate(setting=self) + @property def serialized_value(self): """ diff --git a/mayan/apps/smart_settings/tests/literals.py b/mayan/apps/smart_settings/tests/literals.py index 8db28403af..8f946eaa46 100644 --- a/mayan/apps/smart_settings/tests/literals.py +++ b/mayan/apps/smart_settings/tests/literals.py @@ -5,3 +5,8 @@ ENVIRONMENT_TEST_VALUE = '999999' TEST_NAMESPACE_LABEL = 'test namespace label' TEST_NAMESPACE_NAME = 'test_namespace_name' + +TEST_SETTING_GLOBAL_NAME = 'SMART_SETTINGS_TEST_SETTING' +TEST_SETTING_INITIAL_VALUE = 'test_setting_initial_value' +TEST_SETTING_DEFAULT_VALUE = 'test_setting_default_value' +TEST_SETTING_VALUE = 'test_setting_value' diff --git a/mayan/apps/smart_settings/tests/mixins.py b/mayan/apps/smart_settings/tests/mixins.py index aa177930dc..e926b1b4dc 100644 --- a/mayan/apps/smart_settings/tests/mixins.py +++ b/mayan/apps/smart_settings/tests/mixins.py @@ -2,7 +2,10 @@ from __future__ import absolute_import, unicode_literals from ..classes import Namespace -from .literals import TEST_NAMESPACE_LABEL, TEST_NAMESPACE_NAME +from .literals import ( + TEST_NAMESPACE_LABEL, TEST_NAMESPACE_NAME, TEST_SETTING_DEFAULT_VALUE, + TEST_SETTING_GLOBAL_NAME +) class SmartSettingsTestCaseMixin(object): @@ -12,16 +15,26 @@ class SmartSettingsTestCaseMixin(object): class SmartSettingTestMixin(object): - def _create_test_settings_namespace(self): + def _create_test_settings_namespace(self, **kwargs): try: self.test_settings_namespace = Namespace.get( name=TEST_NAMESPACE_NAME ) + self.test_settings_namespace.migration_class = None + self.test_settings_namespace.version = None + self.test_settings_namespace.__dict__.update(kwargs) except KeyError: self.test_settings_namespace = Namespace( - label=TEST_NAMESPACE_LABEL, name=TEST_NAMESPACE_NAME + label=TEST_NAMESPACE_LABEL, name=TEST_NAMESPACE_NAME, + **kwargs ) + def _create_test_setting(self): + self.test_setting = self.test_settings_namespace.add_setting( + global_name=TEST_SETTING_GLOBAL_NAME, + default=TEST_SETTING_DEFAULT_VALUE + ) + class SmartSettingViewTestMixin(object): def _request_namespace_list_view(self): diff --git a/mayan/apps/smart_settings/tests/mocks.py b/mayan/apps/smart_settings/tests/mocks.py new file mode 100644 index 0000000000..bf7f51af10 --- /dev/null +++ b/mayan/apps/smart_settings/tests/mocks.py @@ -0,0 +1,16 @@ +from __future__ import absolute_import, unicode_literals + +from ..classes import NamespaceMigration + + +class TestNamespaceMigrationOne(NamespaceMigration): + def smart_settings_test_setting_0001(self, value): + return '{}_0001'.format(value) + + +class TestNamespaceMigrationTwo(NamespaceMigration): + def smart_settings_test_setting_0001(self, value): + return '{}_0001'.format(value) + + def smart_settings_test_setting_0002(self, value): + return '{}_0002'.format(value) diff --git a/mayan/apps/smart_settings/tests/test_classes.py b/mayan/apps/smart_settings/tests/test_classes.py index 266e1e75f4..2d69e45ba9 100644 --- a/mayan/apps/smart_settings/tests/test_classes.py +++ b/mayan/apps/smart_settings/tests/test_classes.py @@ -1,27 +1,32 @@ from __future__ import absolute_import, unicode_literals -import os - from pathlib2 import Path from django.conf import settings -from django.utils.encoding import force_text +from django.utils.encoding import force_bytes, force_text +from mayan.apps.common.mixins import EnvironmentTestCaseMixin from mayan.apps.common.settings import setting_paginate_by from mayan.apps.common.tests.base import BaseTestCase -from mayan.apps.storage.utils import fs_cleanup +from mayan.apps.storage.utils import fs_cleanup, NamedTemporaryFile from ..classes import Setting -from .literals import ENVIRONMENT_TEST_NAME, ENVIRONMENT_TEST_VALUE +from .literals import ( + ENVIRONMENT_TEST_NAME, ENVIRONMENT_TEST_VALUE, TEST_SETTING_GLOBAL_NAME, + TEST_SETTING_INITIAL_VALUE, TEST_SETTING_VALUE +) from .mixins import SmartSettingTestMixin +from .mocks import TestNamespaceMigrationOne, TestNamespaceMigrationTwo -class ClassesTestCase(SmartSettingTestMixin, BaseTestCase): +class ClassesTestCase(EnvironmentTestCaseMixin, SmartSettingTestMixin, BaseTestCase): def test_environment_variable(self): - os.environ[ - 'MAYAN_{}'.format(ENVIRONMENT_TEST_NAME) - ] = ENVIRONMENT_TEST_VALUE + self._set_environment_variable( + name='MAYAN_{}'.format(ENVIRONMENT_TEST_NAME), + value=ENVIRONMENT_TEST_VALUE + ) + self.assertTrue(setting_paginate_by.value, ENVIRONMENT_TEST_VALUE) def test_config_backup_creation(self): @@ -53,3 +58,59 @@ class ClassesTestCase(SmartSettingTestMixin, BaseTestCase): self.assertFalse(Setting.check_changed()) test_setting.value = 'test value edited' self.assertTrue(Setting.check_changed()) + + +class NamespaceMigrationTestCase( + EnvironmentTestCaseMixin, SmartSettingTestMixin, BaseTestCase +): + def test_environment_migration(self): + self._set_environment_variable( + name='MAYAN_{}'.format(TEST_SETTING_GLOBAL_NAME), + value=TEST_SETTING_INITIAL_VALUE + ) + self._create_test_settings_namespace( + migration_class=TestNamespaceMigrationOne, version='0002' + ) + self._create_test_setting() + + self.assertEqual( + self.test_setting.value, TEST_SETTING_INITIAL_VALUE + ) + + def test_migration_0001_to_0002(self): + self._create_test_settings_namespace( + migration_class=TestNamespaceMigrationTwo, version='0002' + ) + self._create_test_setting() + + with NamedTemporaryFile() as file_object: + settings.CONFIGURATION_FILEPATH = file_object.name + file_object.write( + force_bytes( + '{}: {}'.format(TEST_SETTING_GLOBAL_NAME, TEST_SETTING_VALUE) + ) + ) + file_object.seek(0) + + self.assertEqual( + self.test_setting.value, '{}_0001'.format(TEST_SETTING_VALUE) + ) + + def test_migration_0001_to_0003(self): + self._create_test_settings_namespace( + migration_class=TestNamespaceMigrationTwo, version='0003' + ) + self._create_test_setting() + + with NamedTemporaryFile() as file_object: + settings.CONFIGURATION_FILEPATH = file_object.name + file_object.write( + force_bytes( + '{}: {}'.format(TEST_SETTING_GLOBAL_NAME, TEST_SETTING_VALUE) + ) + ) + file_object.seek(0) + + self.assertEqual( + self.test_setting.value, '{}_0001_0002'.format(TEST_SETTING_VALUE) + )