Add support for setting migrations

Signed-off-by: Roberto Rosario <roberto.rosario@mayan-edms.com>

HISTORY.rst

@@ -44,13 +44,15 @@
   Thanks to Jakob Haufe (@sur5r) and Jesaja Everling (@jeverling)
   for much of the research and code updates.
 - Support wildcard MIME type associations for the file metadata drivers.
-- Rename MAYAN_GUID to MAYAN_GID
 - Update Gunicorn to use sync workers.
-- Include devpi-server as a development dependency.
+- Include devpi-server as a development dependency. Used to speed up
+  local builds of the Docker image.
 - Update default Docker stack file.
-- Remove Redis from the Docker image.
+- Remove Redis from the Docker image. A separate container must now
+  be deployed.
 - Add Celery flower to the Docker image.
-- Allow PIP proxying to the Docker image during build.
+- Allow PIP proxying to the Docker image during build. Can be used
+  with the local devpi-server or other similar.
 - Default Celery worker concurrency to 0 (auto).
 - Set DJANGO_SETTINGS_MODULE environment variable to make it
   available to sub processes.
@@ -187,6 +189,7 @@
   Instead of throwing an error a sample label of
   "Unknown action type" will be used and allow users to
   delete the unknown state action.
+- Add support for setting migrations.
 
 3.2.9 (2019-11-03)
 ==================

docs/releases/3.3.rst

@@ -61,13 +61,15 @@ Changes
   Thanks to Jakob Haufe (@sur5r) and Jesaja Everling (@jeverling)
   for much of the research and code updates.
 - Support wildcard MIME type associations for the file metadata drivers.
-- Rename MAYAN_GUID to MAYAN_GID
 - Update Gunicorn to use sync workers.
-- Include devpi-server as a development dependency.
+- Include devpi-server as a development dependency. Used to speed up
+  local builds of the Docker image.
 - Update default Docker stack file.
-- Remove Redis from the Docker image.
+- Remove Redis from the Docker image. A separate container must now
+  be deployed.
 - Add Celery flower to the Docker image.
-- Allow PIP proxying to the Docker image during build.
+- Allow PIP proxying to the Docker image during build. Can be used
+  with the local devpi-server or other similar.
 - Default Celery worker concurrency to 0 (auto).
 - Set DJANGO_SETTINGS_MODULE environment variable to make it
   available to sub processes.

mayan/apps/common/tests/mixins.py

@@ -141,6 +141,22 @@ class ContentTypeCheckTestCaseMixin(object):
         self.client = CustomClient()
 
 
+class EnvironmentTestCaseMixin(object):
+    def setUp(self):
+        super(EnvironmentTestCaseMixin, self).setUp()
+        self._test_environment_variables = []
+
+    def tearDown(self):
+        for name in self._test_environment_variables:
+            os.environ.pop(name)
+
+        super(EnvironmentTestCaseMixin, self).tearDown()
+
+    def _set_environment_variable(self, name, value):
+        self._test_environment_variables.append(name)
+        os.environ[name] = value
+
+
 class ModelTestCaseMixin(object):
     def _model_instance_to_dictionary(self, instance):
         return instance._meta.model._default_manager.filter(

mayan/apps/smart_settings/classes.py

@@ -19,6 +19,7 @@ from django.utils.encoding import (
 from mayan.apps.common.serialization import yaml_dump, yaml_load
 
 logger = logging.getLogger(__name__)
+NAMESPACE_VERSION_INITIAL = '0001'
 SMART_SETTINGS_NAMESPACES_NAME = 'SMART_SETTINGS_NAMESPACES'
 
 
@@ -81,11 +82,15 @@ class Namespace(object):
         for namespace in cls.get_all():
             namespace.invalidate_cache()
 
-    def __init__(self, name, label, version='0001'):
+    def __init__(
+        self, name, label, migration_class=None,
+        version=NAMESPACE_VERSION_INITIAL
+    ):
         if name in self.__class__._registry:
             raise Exception(
                 'Namespace names must be unique; "%s" already exists.' % name
             )
+        self.migration_class = migration_class
         self.name = name
         self.label = label
         self.version = version
@@ -99,17 +104,75 @@ class Namespace(object):
         return Setting(namespace=self, **kwargs)
 
     def get_config_version(self):
-        return Namespace.get_namespace_config(name=self.name).get('version', None)
+        return Namespace.get_namespace_config(name=self.name).get(
+            'version', NAMESPACE_VERSION_INITIAL
+        )
 
     def invalidate_cache(self):
         for setting in self._settings:
             setting.invalidate_cache()
 
+    def migrate(self, setting):
+        if self.migration_class:
+            self.migration_class(namespace=self).migrate(setting=setting)
+
     @property
     def settings(self):
         return sorted(self._settings, key=lambda x: x.global_name)
 
 
+class NamespaceMigration(object):
+    def __init__(self, namespace):
+        self.namespace = namespace
+
+    def get_method_name(self, setting):
+        return setting.global_name.lower()
+
+    def get_method_name_full(self, setting, version):
+        return '{}_{}'.format(
+            self.get_method_name(setting=setting),
+            version
+        )
+
+    def migrate(self, setting):
+        if self.namespace.get_config_version() != self.namespace.version:
+            method_name = self.get_method_name(setting=setting)
+
+            # Get methods for this setting
+            setting_methods = [
+                method for method in dir(self) if method.startswith(
+                    method_name
+                )
+            ]
+            # Get order of execution of setting methods
+            versions = [
+                method.replace(
+                    '{}_'.format(method_name), ''
+                ) for method in setting_methods
+            ]
+            try:
+                start = versions.index(self.namespace.get_config_version())
+            except ValueError:
+                start = 0
+
+            try:
+                end = versions.index(self.namespace.version)
+            except ValueError:
+                end = None
+
+            value = setting.raw_value
+            for version in versions[start:end]:
+                method = getattr(
+                    self, self.get_method_name_full(
+                        setting=setting, version=version
+                    ), None
+                )
+                if method:
+                    value = method(value=value)
+
+            setting.raw_value = value
+
+
 @python_2_unicode_compatible
 class Setting(object):
     _registry = {}
@@ -191,7 +254,6 @@ class Setting(object):
             cls._config_file_cache = read_configuration_file(
                 filepath=settings.CONFIGURATION_FILEPATH
             )
-
         return cls._config_file_cache
 
     @classmethod
@@ -224,7 +286,10 @@ class Setting(object):
                 path=settings.CONFIGURATION_LAST_GOOD_FILEPATH
             )
 
-    def __init__(self, namespace, global_name, default, help_text=None, is_path=False, post_edit_function=None):
+    def __init__(
+        self, namespace, global_name, default, help_text=None, is_path=False,
+        post_edit_function=None
+    ):
         self.global_name = global_name
         self.default = default
         self.help_text = help_text
@@ -252,11 +317,21 @@ class Setting(object):
                     )
                 )
         else:
-            self.raw_value = self.get_config_file_content().get(
+            try:
-                self.global_name, getattr(
+                # Try the config file
-                    settings, self.global_name, self.default
+                self.raw_value = self.get_config_file_content()[self.global_name]
-                )
+            except KeyError:
+                try:
+                    # Try the Django settings variable
+                    self.raw_value = getattr(
+                        settings, self.global_name
                     )
+                except AttributeError:
+                    # Finally set to the default value
+                    self.raw_value = self.default
+            else:
+                # Found in the config file, try to migrate the value
+                self.migrate()
 
         self.yaml = Setting.serialize_value(self.raw_value)
         self.loaded = True
@@ -264,6 +339,9 @@ class Setting(object):
     def invalidate_cache(self):
         self.loaded = False
 
+    def migrate(self):
+        self.namespace.migrate(setting=self)
+
     @property
     def serialized_value(self):
         """

mayan/apps/smart_settings/tests/literals.py

@@ -5,3 +5,8 @@ ENVIRONMENT_TEST_VALUE = '999999'
 
 TEST_NAMESPACE_LABEL = 'test namespace label'
 TEST_NAMESPACE_NAME = 'test_namespace_name'
+
+TEST_SETTING_GLOBAL_NAME = 'SMART_SETTINGS_TEST_SETTING'
+TEST_SETTING_INITIAL_VALUE = 'test_setting_initial_value'
+TEST_SETTING_DEFAULT_VALUE = 'test_setting_default_value'
+TEST_SETTING_VALUE = 'test_setting_value'

mayan/apps/smart_settings/tests/mixins.py

@@ -2,7 +2,10 @@ from __future__ import absolute_import, unicode_literals
 
 from ..classes import Namespace
 
-from .literals import TEST_NAMESPACE_LABEL, TEST_NAMESPACE_NAME
+from .literals import (
+    TEST_NAMESPACE_LABEL, TEST_NAMESPACE_NAME, TEST_SETTING_DEFAULT_VALUE,
+    TEST_SETTING_GLOBAL_NAME
+)
 
 
 class SmartSettingsTestCaseMixin(object):
@@ -12,14 +15,24 @@ class SmartSettingsTestCaseMixin(object):
 
 
 class SmartSettingTestMixin(object):
-    def _create_test_settings_namespace(self):
+    def _create_test_settings_namespace(self, **kwargs):
         try:
             self.test_settings_namespace = Namespace.get(
                 name=TEST_NAMESPACE_NAME
             )
+            self.test_settings_namespace.migration_class = None
+            self.test_settings_namespace.version = None
+            self.test_settings_namespace.__dict__.update(kwargs)
         except KeyError:
             self.test_settings_namespace = Namespace(
-                label=TEST_NAMESPACE_LABEL, name=TEST_NAMESPACE_NAME
+                label=TEST_NAMESPACE_LABEL, name=TEST_NAMESPACE_NAME,
+                **kwargs
+            )
+
+    def _create_test_setting(self):
+        self.test_setting = self.test_settings_namespace.add_setting(
+            global_name=TEST_SETTING_GLOBAL_NAME,
+            default=TEST_SETTING_DEFAULT_VALUE
         )
 
 

mayan/apps/smart_settings/tests/mocks.py

@@ -0,0 +1,16 @@
+from __future__ import absolute_import, unicode_literals
+
+from ..classes import NamespaceMigration
+
+
+class TestNamespaceMigrationOne(NamespaceMigration):
+    def smart_settings_test_setting_0001(self, value):
+        return '{}_0001'.format(value)
+
+
+class TestNamespaceMigrationTwo(NamespaceMigration):
+    def smart_settings_test_setting_0001(self, value):
+        return '{}_0001'.format(value)
+
+    def smart_settings_test_setting_0002(self, value):
+        return '{}_0002'.format(value)

mayan/apps/smart_settings/tests/test_classes.py

@@ -1,27 +1,32 @@
 from __future__ import absolute_import, unicode_literals
 
-import os
-
 from pathlib2 import Path
 
 from django.conf import settings
-from django.utils.encoding import force_text
+from django.utils.encoding import force_bytes, force_text
 
+from mayan.apps.common.mixins import EnvironmentTestCaseMixin
 from mayan.apps.common.settings import setting_paginate_by
 from mayan.apps.common.tests.base import BaseTestCase
-from mayan.apps.storage.utils import fs_cleanup
+from mayan.apps.storage.utils import fs_cleanup, NamedTemporaryFile
 
 from ..classes import Setting
 
-from .literals import ENVIRONMENT_TEST_NAME, ENVIRONMENT_TEST_VALUE
+from .literals import (
+    ENVIRONMENT_TEST_NAME, ENVIRONMENT_TEST_VALUE, TEST_SETTING_GLOBAL_NAME,
+    TEST_SETTING_INITIAL_VALUE, TEST_SETTING_VALUE
+)
 from .mixins import SmartSettingTestMixin
+from .mocks import TestNamespaceMigrationOne, TestNamespaceMigrationTwo
 
 
-class ClassesTestCase(SmartSettingTestMixin, BaseTestCase):
+class ClassesTestCase(EnvironmentTestCaseMixin, SmartSettingTestMixin, BaseTestCase):
     def test_environment_variable(self):
-        os.environ[
+        self._set_environment_variable(
-            'MAYAN_{}'.format(ENVIRONMENT_TEST_NAME)
+            name='MAYAN_{}'.format(ENVIRONMENT_TEST_NAME),
-        ] = ENVIRONMENT_TEST_VALUE
+            value=ENVIRONMENT_TEST_VALUE
+        )
+
         self.assertTrue(setting_paginate_by.value, ENVIRONMENT_TEST_VALUE)
 
     def test_config_backup_creation(self):
@@ -53,3 +58,59 @@ class ClassesTestCase(SmartSettingTestMixin, BaseTestCase):
         self.assertFalse(Setting.check_changed())
         test_setting.value = 'test value edited'
         self.assertTrue(Setting.check_changed())
+
+
+class NamespaceMigrationTestCase(
+    EnvironmentTestCaseMixin, SmartSettingTestMixin, BaseTestCase
+):
+    def test_environment_migration(self):
+        self._set_environment_variable(
+            name='MAYAN_{}'.format(TEST_SETTING_GLOBAL_NAME),
+            value=TEST_SETTING_INITIAL_VALUE
+        )
+        self._create_test_settings_namespace(
+            migration_class=TestNamespaceMigrationOne, version='0002'
+        )
+        self._create_test_setting()
+
+        self.assertEqual(
+            self.test_setting.value, TEST_SETTING_INITIAL_VALUE
+        )
+
+    def test_migration_0001_to_0002(self):
+        self._create_test_settings_namespace(
+            migration_class=TestNamespaceMigrationTwo, version='0002'
+        )
+        self._create_test_setting()
+
+        with NamedTemporaryFile() as file_object:
+            settings.CONFIGURATION_FILEPATH = file_object.name
+            file_object.write(
+                force_bytes(
+                    '{}: {}'.format(TEST_SETTING_GLOBAL_NAME, TEST_SETTING_VALUE)
+                )
+            )
+            file_object.seek(0)
+
+            self.assertEqual(
+                self.test_setting.value, '{}_0001'.format(TEST_SETTING_VALUE)
+            )
+
+    def test_migration_0001_to_0003(self):
+        self._create_test_settings_namespace(
+            migration_class=TestNamespaceMigrationTwo, version='0003'
+        )
+        self._create_test_setting()
+
+        with NamedTemporaryFile() as file_object:
+            settings.CONFIGURATION_FILEPATH = file_object.name
+            file_object.write(
+                force_bytes(
+                    '{}: {}'.format(TEST_SETTING_GLOBAL_NAME, TEST_SETTING_VALUE)
+                )
+            )
+            file_object.seek(0)
+
+            self.assertEqual(
+                self.test_setting.value, '{}_0001_0002'.format(TEST_SETTING_VALUE)
+            )