Move Django and Celery settings

Django settings now reside in the smart settings app.
Celery settings now reside in the task manager app.

Signed-off-by: Roberto Rosario <roberto.rosario@mayan-edms.com>

HISTORY.rst

@@ -1,3 +1,13 @@
+3.2.6 (2019-07-10)
+==================
+* Remove the smart settings app * import.
+* Encode settings YAML before hashing.
+* Fix document icon used in the workflow runtime links.
+* Add trashed date time label.
+* Fix thumbnail generation issue. GitLab issue #637.
+  Thanks to Giacomo Cariello (@giacomocariello) for the report
+  and the merge request fixing the issue.
+
 3.2.5 (2019-07-05)
 ==================
 * Don't error out if the EXTRA_APPS or the DISABLED_APPS settings

docker/rootfs/version

@@ -1 +1 @@
-3.2.5
+3.2.6

docs/releases/3.2.6.rst

@@ -0,0 +1,111 @@
+Version 3.2.6
+=============
+
+Released: July 10, 2019
+
+
+Changes
+-------
+
+- Remove the smart settings app * import. Following MERC 0005.
+- Encode settings YAML before hashing. Avoids unicode issues with Python 3.
+- Fix document icon used in the workflow runtime links.
+- Add trashed date time label.
+- Fix thumbnail generation issue. GitLab issue #637.
+  Thanks to Giacomo Cariello (@giacomocariello) for the report
+  and the merge request fixing the issue.
+
+Removals
+--------
+
+- None
+
+
+Upgrading from a previous version
+---------------------------------
+
+If installed via Python's PIP
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Remove deprecated requirements::
+
+    sudo -u mayan curl https://gitlab.com/mayan-edms/mayan-edms/raw/master/removals.txt -o /tmp/removals.txt && sudo -u mayan /opt/mayan-edms/bin/pip uninstall -y -r /tmp/removals.txt
+
+Type in the console::
+
+    sudo -u mayan /opt/mayan-edms/bin/pip install mayan-edms==3.2.6
+
+the requirements will also be updated automatically.
+
+
+Using Git
+^^^^^^^^^
+
+If you installed Mayan EDMS by cloning the Git repository issue the commands::
+
+    $ git reset --hard HEAD
+    $ git pull
+
+otherwise download the compressed archived and uncompress it overriding the
+existing installation.
+
+Remove deprecated requirements::
+
+    $ pip uninstall -y -r removals.txt
+
+Next upgrade/add the new requirements::
+
+    $ pip install --upgrade -r requirements.txt
+
+
+Common steps
+^^^^^^^^^^^^
+
+Perform these steps after updating the code from either step above.
+
+Make a backup of your supervisord file::
+
+    sudo cp /etc/supervisor/conf.d/mayan.conf /etc/supervisor/conf.d/mayan.conf.bck
+
+Update the supervisord configuration file. Replace the environment
+variables values show here with your respective settings. This step will refresh
+the supervisord configuration file with the new queues and the latest
+recommended layout::
+
+    sudo MAYAN_DATABASE_ENGINE=django.db.backends.postgresql MAYAN_DATABASE_NAME=mayan \
+    MAYAN_DATABASE_PASSWORD=mayanuserpass MAYAN_DATABASE_USER=mayan \
+    MAYAN_DATABASE_HOST=127.0.0.1 MAYAN_MEDIA_ROOT=/opt/mayan-edms/media \
+    /opt/mayan-edms/bin/mayan-edms.py platformtemplate supervisord > /etc/supervisor/conf.d/mayan.conf
+
+Edit the supervisord configuration file and update any setting the template
+generator missed::
+
+    sudo vi /etc/supervisor/conf.d/mayan.conf
+
+Migrate existing database schema with::
+
+    sudo -u mayan MAYAN_DATABASE_ENGINE=django.db.backends.postgresql MAYAN_DATABASE_NAME=mayan \
+    MAYAN_DATABASE_PASSWORD=mayanuserpass MAYAN_DATABASE_USER=mayan \
+    MAYAN_DATABASE_HOST=127.0.0.1 MAYAN_MEDIA_ROOT=/opt/mayan-edms/media \
+    /opt/mayan-edms/bin/mayan-edms.py performupgrade
+
+Add new static media::
+
+    sudo -u mayan MAYAN_MEDIA_ROOT=/opt/mayan-edms/media \
+    /opt/mayan-edms/bin/mayan-edms.py preparestatic --noinput
+
+The upgrade procedure is now complete.
+
+
+Backward incompatible changes
+-----------------------------
+
+- None
+
+
+Bugs fixed or issues closed
+---------------------------
+
+- :gitlab-issue:`637` Thumbnail generation bug
+
+.. _PyPI: https://pypi.python.org/pypi/mayan-edms/

docs/releases/index.rst

@@ -20,6 +20,7 @@ versions of the documentation contain the release notes for any later releases.
 .. toctree::
    :maxdepth: 1
 
+   3.2.6
    3.2.5
    3.2.4
    3.2.3

mayan/__init__.py

@@ -1,9 +1,9 @@
 from __future__ import unicode_literals
 
 __title__ = 'Mayan EDMS'
-__version__ = '3.2.5'
-__build__ = 0x030205
-__build_string__ = 'v3.2.5_Fri Jul 5 16:39:17 2019 -0400'
+__version__ = '3.2.6'
+__build__ = 0x030206
+__build_string__ = 'v3.2.6_Wed Jul 10 03:18:15 2019 -0400'
 __django_version__ = '1.11'
 __author__ = 'Roberto Rosario'
 __author_email__ = 'roberto.rosario@mayan-edms.com'

mayan/apps/appearance/settings.py

@@ -2,7 +2,7 @@ from __future__ import unicode_literals
 
 from django.utils.translation import ugettext_lazy as _
 
-from mayan.apps.smart_settings import Namespace
+from mayan.apps.smart_settings.classes import Namespace
 
 from .literals import DEFAULT_MAXIMUM_TITLE_LENGTH
 

mayan/apps/authentication/settings.py

@@ -2,7 +2,7 @@ from __future__ import unicode_literals
 
 from django.utils.translation import ugettext_lazy as _
 
-from mayan.apps.smart_settings import Namespace
+from mayan.apps.smart_settings.classes import Namespace
 
 from .literals import DEFAULT_LOGIN_METHOD, DEFAULT_MAXIMUM_SESSION_LENGTH
 

mayan/apps/autoadmin/settings.py

@@ -2,7 +2,7 @@ from __future__ import unicode_literals
 
 from django.utils.translation import ugettext_lazy as _
 
-from mayan.apps.smart_settings import Namespace
+from mayan.apps.smart_settings.classes import Namespace
 
 from .literals import DEFAULT_EMAIL, DEFAULT_PASSWORD, DEFAULT_USERNAME
 

mayan/apps/common/settings.py

@@ -6,11 +6,10 @@ from django.conf import settings
 from django.utils.translation import ugettext_lazy as _
 
 import mayan
-from mayan.apps.smart_settings import Namespace
+from mayan.apps.smart_settings.classes import Namespace
 
 from .literals import DEFAULT_COMMON_HOME_VIEW
 
-
 namespace = Namespace(label=_('Common'), name='common')
 
 setting_auto_logging = namespace.add_setting(
@@ -99,318 +98,3 @@ setting_shared_storage_arguments = namespace.add_setting(
         os.path.join(settings.MEDIA_ROOT, 'shared_files')
     ), quoted=True
 )
-
-namespace = Namespace(label=_('Django'), name='django')
-
-setting_django_allowed_hosts = namespace.add_setting(
-    global_name='ALLOWED_HOSTS', default=settings.ALLOWED_HOSTS,
-    help_text=_(
-        'A list of strings representing the host/domain names that this site '
-        'can serve. This is a security measure to prevent HTTP Host header '
-        'attacks, which are possible even under many seemingly-safe web '
-        'server configurations. Values in this list can be '
-        'fully qualified names (e.g. \'www.example.com\'), in which case '
-        'they will be matched against the request\'s Host header exactly '
-        '(case-insensitive, not including port). A value beginning with a '
-        'period can be used as a subdomain wildcard: \'.example.com\' will '
-        'match example.com, www.example.com, and any other subdomain of '
-        'example.com. A value of \'*\' will match anything; in this case you '
-        'are responsible to provide your own validation of the Host header '
-        '(perhaps in a middleware; if so this middleware must be listed '
-        'first in MIDDLEWARE).'
-    ),
-)
-setting_django_append_slash = namespace.add_setting(
-    global_name='APPEND_SLASH', default=settings.APPEND_SLASH,
-    help_text=_(
-        'When set to True, if the request URL does not match any of the '
-        'patterns in the URLconf and it doesn\'t end in a slash, an HTTP '
-        'redirect is issued to the same URL with a slash appended. Note '
-        'that the redirect may cause any data submitted in a POST request '
-        'to be lost. The APPEND_SLASH setting is only used if '
-        'CommonMiddleware is installed (see Middleware). See also '
-        'PREPEND_WWW.'
-    )
-)
-setting_django_auth_password_validators = namespace.add_setting(
-    global_name='AUTH_PASSWORD_VALIDATORS',
-    default=settings.AUTH_PASSWORD_VALIDATORS,
-    help_text=_(
-        'The list of validators that are used to check the strength of '
-        'user\'s passwords.'
-    )
-)
-setting_django_databases = namespace.add_setting(
-    global_name='DATABASES', default=settings.DATABASES,
-    help_text=_(
-        'A dictionary containing the settings for all databases to be used '
-        'with Django. It is a nested dictionary whose contents map a '
-        'database alias to a dictionary containing the options for an '
-        'individual database. The DATABASES setting must configure a '
-        'default database; any number of additional databases may also '
-        'be specified.'
-    ),
-)
-setting_django_data_upload_max_memory_size = namespace.add_setting(
-    global_name='DATA_UPLOAD_MAX_MEMORY_SIZE',
-    default=settings.DATA_UPLOAD_MAX_MEMORY_SIZE,
-    help_text=_(
-        'Default: 2621440 (i.e. 2.5 MB). The maximum size in bytes that a '
-        'request body may be before a SuspiciousOperation '
-        '(RequestDataTooBig) is raised. The check is done when accessing '
-        'request.body or request.POST and is calculated against the total '
-        'request size excluding any file upload data. You can set this to '
-        'None to disable the check. Applications that are expected to '
-        'receive unusually large form posts should tune this setting. The '
-        'amount of request data is correlated to the amount of memory '
-        'needed to process the request and populate the GET and POST '
-        'dictionaries. Large requests could be used as a '
-        'denial-of-service attack vector if left unchecked. Since web '
-        'servers don\'t typically perform deep request inspection, it\'s '
-        'not possible to perform a similar check at that level. See also '
-        'FILE_UPLOAD_MAX_MEMORY_SIZE.'
-    ),
-)
-setting_django_default_from_email = namespace.add_setting(
-    global_name='DEFAULT_FROM_EMAIL',
-    default=settings.DEFAULT_FROM_EMAIL,
-    help_text=_(
-        'Default: \'webmaster@localhost\' '
-        'Default email address to use for various automated correspondence '
-        'from the site manager(s). This doesn\'t include error messages sent '
-        'to ADMINS and MANAGERS; for that, see SERVER_EMAIL.'
-    ),
-)
-setting_django_disallowed_user_agents = namespace.add_setting(
-    global_name='DISALLOWED_USER_AGENTS',
-    default=settings.DISALLOWED_USER_AGENTS,
-    help_text=_(
-        'Default: [] (Empty list). List of compiled regular expression '
-        'objects representing User-Agent strings that are not allowed to '
-        'visit any page, systemwide. Use this for bad robots/crawlers. '
-        'This is only used if CommonMiddleware is installed '
-        '(see Middleware).'
-    ),
-)
-setting_django_email_backend = namespace.add_setting(
-    global_name='EMAIL_BACKEND',
-    default=settings.EMAIL_BACKEND,
-    help_text=_(
-        'Default: \'django.core.mail.backends.smtp.EmailBackend\'. The '
-        'backend to use for sending emails.'
-    ),
-)
-setting_django_email_host = namespace.add_setting(
-    global_name='EMAIL_HOST',
-    default=settings.EMAIL_HOST,
-    help_text=_(
-        'Default: \'localhost\'. The host to use for sending email.'
-    ),
-)
-setting_django_email_host_password = namespace.add_setting(
-    global_name='EMAIL_HOST_PASSWORD',
-    default=settings.EMAIL_HOST_PASSWORD,
-    help_text=_(
-        'Default: \'\' (Empty string). Password to use for the SMTP '
-        'server defined in EMAIL_HOST. This setting is used in '
-        'conjunction with EMAIL_HOST_USER when authenticating to the '
-        'SMTP server. If either of these settings is empty, '
-        'Django won\'t attempt authentication.'
-    ),
-)
-setting_django_email_host_user = namespace.add_setting(
-    global_name='EMAIL_HOST_USER',
-    default=settings.EMAIL_HOST_USER,
-    help_text=_(
-        'Default: \'\' (Empty string). Username to use for the SMTP '
-        'server defined in EMAIL_HOST. If empty, Django won\'t attempt '
-        'authentication.'
-    ),
-)
-setting_django_email_port = namespace.add_setting(
-    global_name='EMAIL_PORT',
-    default=settings.EMAIL_PORT,
-    help_text=_(
-        'Default: 25. Port to use for the SMTP server defined in EMAIL_HOST.'
-    ),
-)
-setting_django_email_timeout = namespace.add_setting(
-    global_name='EMAIL_TIMEOUT',
-    default=settings.EMAIL_TIMEOUT,
-    help_text=_(
-        'Default: None. Specifies a timeout in seconds for blocking '
-        'operations like the connection attempt.'
-    ),
-)
-setting_django_email_user_tls = namespace.add_setting(
-    global_name='EMAIL_USE_TLS',
-    default=settings.EMAIL_USE_TLS,
-    help_text=_(
-        'Default: False. Whether to use a TLS (secure) connection when '
-        'talking to the SMTP server. This is used for explicit TLS '
-        'connections, generally on port 587. If you are experiencing '
-        'hanging connections, see the implicit TLS setting EMAIL_USE_SSL.'
-    ),
-)
-setting_django_email_user_ssl = namespace.add_setting(
-    global_name='EMAIL_USE_SSL',
-    default=settings.EMAIL_USE_SSL,
-    help_text=_(
-        'Default: False. Whether to use an implicit TLS (secure) connection '
-        'when talking to the SMTP server. In most email documentation this '
-        'type of TLS connection is referred to as SSL. It is generally used '
-        'on port 465. If you are experiencing problems, see the explicit '
-        'TLS setting EMAIL_USE_TLS. Note that EMAIL_USE_TLS/EMAIL_USE_SSL '
-        'are mutually exclusive, so only set one of those settings to True.'
-    ),
-)
-setting_django_file_upload_max_memory_size = namespace.add_setting(
-    global_name='FILE_UPLOAD_MAX_MEMORY_SIZE',
-    default=settings.FILE_UPLOAD_MAX_MEMORY_SIZE,
-    help_text=_(
-        'Default: 2621440 (i.e. 2.5 MB). The maximum size (in bytes) '
-        'that an upload will be before it gets streamed to the file '
-        'system. See Managing files for details. See also '
-        'DATA_UPLOAD_MAX_MEMORY_SIZE.'
-    ),
-)
-setting_django_login_url = namespace.add_setting(
-    global_name='LOGIN_URL',
-    default=settings.LOGIN_URL,
-    help_text=_(
-        'Default: \'/accounts/login/\' The URL where requests are '
-        'redirected for login, especially when using the login_required() '
-        'decorator. This setting also accepts named URL patterns which '
-        'can be used to reduce configuration duplication since you '
-        'don\'t have to define the URL in two places (settings '
-        'and URLconf).'
-    )
-)
-setting_django_login_redirect_url = namespace.add_setting(
-    global_name='LOGIN_REDIRECT_URL',
-    default=settings.LOGIN_REDIRECT_URL,
-    help_text=_(
-        'Default: \'/accounts/profile/\' The URL where requests are '
-        'redirected after login when the contrib.auth.login view gets no '
-        'next parameter. This is used by the login_required() decorator, '
-        'for example. This setting also accepts named URL patterns which '
-        'can be used to reduce configuration duplication since you don\'t '
-        'have to define the URL in two places (settings and URLconf).'
-    ),
-)
-setting_django_logout_redirect_url = namespace.add_setting(
-    global_name='LOGOUT_REDIRECT_URL',
-    default=settings.LOGOUT_REDIRECT_URL,
-    help_text=_(
-        'Default: None. The URL where requests are redirected after a user '
-        'logs out using LogoutView (if the view doesn\'t get a next_page '
-        'argument). If None, no redirect will be performed and the logout '
-        'view will be rendered. This setting also accepts named URL '
-        'patterns which can be used to reduce configuration duplication '
-        'since you don\'t have to define the URL in two places (settings '
-        'and URLconf).'
-    )
-)
-setting_django_internal_ips = namespace.add_setting(
-    global_name='INTERNAL_IPS',
-    default=settings.INTERNAL_IPS,
-    help_text=_(
-        'A list of IP addresses, as strings, that: Allow the debug() '
-        'context processor to add some variables to the template context. '
-        'Can use the admindocs bookmarklets even if not logged in as a '
-        'staff user. Are marked as "internal" (as opposed to "EXTERNAL") '
-        'in AdminEmailHandler emails.'
-    ),
-)
-setting_django_languages = namespace.add_setting(
-    global_name='LANGUAGES',
-    default=settings.LANGUAGES,
-    help_text=_(
-        'A list of all available languages. The list is a list of '
-        'two-tuples in the format (language code, language name) '
-        'for example, (\'ja\', \'Japanese\'). This specifies which '
-        'languages are available for language selection. '
-        'Generally, the default value should suffice. Only set this '
-        'setting if you want to restrict language selection to a '
-        'subset of the Django-provided languages. '
-    ),
-)
-setting_django_language_code = namespace.add_setting(
-    global_name='LANGUAGE_CODE',
-    default=settings.LANGUAGE_CODE,
-    help_text=_(
-        'A string representing the language code for this installation. '
-        'This should be in standard language ID format. For example, U.S. '
-        'English is "en-us". It serves two purposes: If the locale '
-        'middleware isn\'t in use, it decides which translation is served '
-        'to all users. If the locale middleware is active, it provides a '
-        'fallback language in case the user\'s preferred language can\'t '
-        'be determined or is not supported by the website. It also provides '
-        'the fallback translation when a translation for a given literal '
-        'doesn\'t exist for the user\'s preferred language.'
-    ),
-)
-setting_django_static_url = namespace.add_setting(
-    global_name='STATIC_URL',
-    default=settings.STATIC_URL,
-    help_text=_(
-        'URL to use when referring to static files located in STATIC_ROOT. '
-        'Example: "/static/" or "http://static.example.com/" '
-        'If not None, this will be used as the base path for asset '
-        'definitions (the Media class) and the staticfiles app. '
-        'It must end in a slash if set to a non-empty value.'
-    ),
-)
-setting_django_staticfiles_storage = namespace.add_setting(
-    global_name='STATICFILES_STORAGE',
-    default=settings.STATICFILES_STORAGE,
-    help_text=_(
-        'The file storage engine to use when collecting static files with '
-        'the collectstatic management command. A ready-to-use instance of '
-        'the storage backend defined in this setting can be found at '
-        'django.contrib.staticfiles.storage.staticfiles_storage.'
-    ),
-)
-setting_django_time_zone = namespace.add_setting(
-    global_name='TIME_ZONE',
-    default=settings.TIME_ZONE,
-    help_text=_(
-        'A string representing the time zone for this installation. '
-        'Note that this isn\'t necessarily the time zone of the server. '
-        'For example, one server may serve multiple Django-powered sites, '
-        'each with a separate time zone setting.'
-    ),
-)
-setting_django_wsgi_application = namespace.add_setting(
-    global_name='WSGI_APPLICATION',
-    default=settings.WSGI_APPLICATION,
-    help_text=_(
-        'The full Python path of the WSGI application object that Django\'s '
-        'built-in servers (e.g. runserver) will use. The django-admin '
-        'startproject management command will create a simple wsgi.py '
-        'file with an application callable in it, and point this setting '
-        'to that application.'
-    ),
-)
-
-namespace = Namespace(label=_('Celery'), name='celery')
-
-setting_celery_broker_url = namespace.add_setting(
-    global_name='BROKER_URL', default=settings.BROKER_URL,
-    help_text=_(
-        'Default: "amqp://". Default broker URL. This must be a URL in '
-        'the form of: transport://userid:password@hostname:port/virtual_host '
-        'Only the scheme part (transport://) is required, the rest is '
-        'optional, and defaults to the specific transports default values.'
-    ),
-)
-setting_celery_result_backend = namespace.add_setting(
-    global_name='CELERY_RESULT_BACKEND',
-    default=settings.CELERY_RESULT_BACKEND,
-    help_text=_(
-        'Default: No result backend enabled by default. The backend used '
-        'to store task results (tombstones). Refer to '
-        'http://docs.celeryproject.org/en/v4.1.0/userguide/configuration.'
-        'html#result-backend'
-    )
-)

mayan/apps/converter/settings.py

@@ -2,7 +2,7 @@ from __future__ import unicode_literals
 
 from django.utils.translation import ugettext_lazy as _
 
-from mayan.apps.smart_settings import Namespace
+from mayan.apps.smart_settings.classes import Namespace
 
 from .literals import (
     DEFAULT_LIBREOFFICE_PATH, DEFAULT_PDFTOPPM_DPI, DEFAULT_PDFTOPPM_FORMAT,

mayan/apps/django_gpg/settings.py

@@ -5,7 +5,7 @@ import os
 from django.conf import settings
 from django.utils.translation import ugettext_lazy as _
 
-from mayan.apps.smart_settings import Namespace
+from mayan.apps.smart_settings.classes import Namespace
 
 namespace = Namespace(label=_('Signatures'), name='django_gpg')
 

mayan/apps/document_indexing/tests/test_views.py

@@ -14,7 +14,11 @@ from .literals import TEST_INDEX_LABEL, TEST_INDEX_LABEL_EDITED
 from .mixins import IndexTestMixin, IndexViewTestMixin
 
 
-class IndexViewTestCase(IndexTestMixin, IndexViewTestMixin, GenericDocumentViewTestCase):
+class IndexViewTestCase(
+    IndexTestMixin, IndexViewTestMixin, GenericDocumentViewTestCase
+):
+    auto_upload_document = False
+
     def test_index_create_view_no_permission(self):
         response = self._request_test_index_create_view()
         self.assertEqual(response.status_code, 403)
@@ -72,6 +76,10 @@ class IndexViewTestCase(IndexTestMixin, IndexViewTestMixin, GenericDocumentViewT
         self.test_index.refresh_from_db()
         self.assertEqual(self.test_index.label, TEST_INDEX_LABEL_EDITED)
 
+
+class IndexInstanceViewTestCase(
+    IndexTestMixin, IndexViewTestMixin, GenericDocumentViewTestCase
+):
     def _request_index_instance_node_view(self, index_instance_node):
         return self.get(
             viewname='indexing:index_instance_node_view', kwargs={
@@ -100,27 +108,27 @@ class IndexViewTestCase(IndexTestMixin, IndexViewTestMixin, GenericDocumentViewT
         )
         self.assertContains(response, text=TEST_INDEX_LABEL, status_code=200)
 
-    def _request_index_rebuild_get_view(self):
+    def _request_indexes_rebuild_get_view(self):
         return self.get(
             viewname='indexing:rebuild_index_instances',
         )
 
-    def _request_index_rebuild_post_view(self):
+    def _request_indexes_rebuild_post_view(self):
         return self.post(
             viewname='indexing:rebuild_index_instances', data={
                 'index_templates': self.test_index.pk
             }
         )
 
-    def test_index_rebuild_no_permission(self):
+    def test_indexes_rebuild_no_permission(self):
         self._create_test_index(rebuild=False)
 
-        response = self._request_index_rebuild_get_view()
+        response = self._request_indexes_rebuild_get_view()
         self.assertNotContains(
             response=response, text=self.test_index.label, status_code=200
         )
 
-        response = self._request_index_rebuild_post_view()
+        response = self._request_indexes_rebuild_post_view()
         # No error since we just don't see the index
         self.assertEqual(response.status_code, 200)
 
@@ -128,7 +136,7 @@ class IndexViewTestCase(IndexTestMixin, IndexViewTestMixin, GenericDocumentViewT
             self.test_index.instance_root.get_children_count(), 0
         )
 
-    def test_index_rebuild_with_access(self):
+    def test_indexes_rebuild_with_access(self):
         self._create_test_index(rebuild=False)
 
         self.grant_access(
@@ -136,12 +144,12 @@ class IndexViewTestCase(IndexTestMixin, IndexViewTestMixin, GenericDocumentViewT
             permission=permission_document_indexing_rebuild
         )
 
-        response = self._request_index_rebuild_get_view()
+        response = self._request_indexes_rebuild_get_view()
         self.assertContains(
             response=response, text=self.test_index.label, status_code=200
         )
 
-        response = self._request_index_rebuild_post_view()
+        response = self._request_indexes_rebuild_post_view()
         self.assertEqual(response.status_code, 302)
 
         # An instance root exists

mayan/apps/document_parsing/settings.py

@@ -2,7 +2,7 @@ from __future__ import unicode_literals
 
 from django.utils.translation import ugettext_lazy as _
 
-from mayan.apps.smart_settings import Namespace
+from mayan.apps.smart_settings.classes import Namespace
 
 namespace = Namespace(label=_('Document parsing'), name='document_parsing')
 

mayan/apps/document_signatures/settings.py

@@ -5,7 +5,7 @@ import os
 from django.conf import settings
 from django.utils.translation import ugettext_lazy as _
 
-from mayan.apps.smart_settings import Namespace
+from mayan.apps.smart_settings.classes import Namespace
 
 namespace = Namespace(label=_('Document signatures'), name='signatures')
 

mayan/apps/document_states/icons.py

@@ -1,7 +1,7 @@
 from __future__ import absolute_import, unicode_literals
 
 from mayan.apps.appearance.classes import Icon
-from mayan.apps.documents.icons import icon_document_type
+from mayan.apps.documents.icons import icon_document, icon_document_type
 
 
 icon_workflow = Icon(driver_name='fontawesome', symbol='sitemap')
@@ -31,11 +31,11 @@ icon_workflow_instance_transition = Icon(
     driver_name='fontawesome', symbol='arrows-alt-h'
 )
 
-icon_workflow_runtime_proxy_document_list = icon_document_type
+icon_workflow_runtime_proxy_document_list = icon_document
 icon_workflow_runtime_proxy_list = Icon(
     driver_name='fontawesome', symbol='sitemap'
 )
-icon_workflow_runtime_proxy_state_document_list = icon_document_type
+icon_workflow_runtime_proxy_state_document_list = icon_document
 icon_workflow_runtime_proxy_state_list = Icon(
     driver_name='fontawesome', symbol='circle'
 )

mayan/apps/documents/apps.py

@@ -319,7 +319,8 @@ class DocumentsApp(MayanAppConfig):
             attribute='document_type', is_sortable=True, source=DeletedDocument
         )
         SourceColumn(
-            attribute='deleted_date_time', source=DeletedDocument
+            attribute='deleted_date_time', include_label=True, order=99,
+            source=DeletedDocument
         )
 
         # DocumentVersion

mayan/apps/documents/models/document_page_models.py

@@ -229,7 +229,7 @@ class DocumentPage(models.Model):
                     for transformation in transformations:
                         converter.transform(transformation=transformation)
 
-                    return page_image
+                    return converter.get_page()
             except Exception as exception:
                 # Cleanup in case of error
                 logger.error(

mayan/apps/documents/settings.py

@@ -5,7 +5,7 @@ import os
 from django.conf import settings
 from django.utils.translation import ugettext_lazy as _
 
-from mayan.apps.smart_settings import Namespace
+from mayan.apps.smart_settings.classes import Namespace
 
 from .literals import (
     DEFAULT_DOCUMENTS_HASH_BLOCK_SIZE, DEFAULT_LANGUAGE, DEFAULT_LANGUAGE_CODES

mayan/apps/file_metadata/settings.py

@@ -2,7 +2,7 @@ from __future__ import unicode_literals
 
 from django.utils.translation import ugettext_lazy as _
 
-from mayan.apps.smart_settings import Namespace
+from mayan.apps.smart_settings.classes import Namespace
 
 from .literals import DEFAULT_EXIF_PATH
 

mayan/apps/lock_manager/settings.py

@@ -2,7 +2,7 @@ from __future__ import unicode_literals
 
 from django.utils.translation import ugettext_lazy as _
 
-from mayan.apps.smart_settings import Namespace
+from mayan.apps.smart_settings.classes import Namespace
 
 from .literals import DEFAULT_BACKEND, DEFAULT_LOCK_TIMEOUT_VALUE
 

mayan/apps/mailer/settings.py

@@ -2,7 +2,7 @@ from __future__ import unicode_literals
 
 from django.utils.translation import ugettext_lazy as _
 
-from mayan.apps.smart_settings import Namespace
+from mayan.apps.smart_settings.classes import Namespace
 
 from .literals import (
     DEFAULT_DOCUMENT_BODY_TEMPLATE, DEFAULT_LINK_BODY_TEMPLATE

mayan/apps/metadata/settings.py

@@ -2,7 +2,7 @@ from __future__ import unicode_literals
 
 from django.utils.translation import ugettext_lazy as _
 
-from mayan.apps.smart_settings import Namespace
+from mayan.apps.smart_settings.classes import Namespace
 
 from .parsers import MetadataParser
 from .validators import MetadataValidator

mayan/apps/mirroring/settings.py

@@ -2,7 +2,7 @@ from __future__ import unicode_literals
 
 from django.utils.translation import ugettext_lazy as _
 
-from mayan.apps.smart_settings import Namespace
+from mayan.apps.smart_settings.classes import Namespace
 
 namespace = Namespace(label=_('Mirroring'), name='mirroring')
 

mayan/apps/ocr/settings.py

@@ -2,7 +2,7 @@ from __future__ import unicode_literals
 
 from django.utils.translation import ugettext_lazy as _
 
-from mayan.apps.smart_settings import Namespace
+from mayan.apps.smart_settings.classes import Namespace
 
 namespace = Namespace(label=_('OCR'), name='ocr')
 

mayan/apps/smart_settings/__init__.py

@@ -1,5 +1,3 @@
 from __future__ import unicode_literals
 
-from .classes import Namespace, Setting  # NOQA
-
 default_app_config = 'mayan.apps.smart_settings.apps.SmartSettingsApp'

mayan/apps/smart_settings/apps.py

@@ -11,6 +11,7 @@ from .links import (
     link_namespace_detail, link_namespace_list, link_namespace_root_list,
     link_setting_edit
 )
+from .settings import *  # NOQA
 from .widgets import setting_widget
 
 

mayan/apps/smart_settings/classes.py

@@ -17,7 +17,9 @@ except ImportError:
 from django.apps import apps
 from django.conf import settings
 from django.utils.functional import Promise
-from django.utils.encoding import force_text, python_2_unicode_compatible
+from django.utils.encoding import (
+    force_bytes, force_text, python_2_unicode_compatible
+)
 
 logger = logging.getLogger(__name__)
 
@@ -141,7 +143,7 @@ class Setting(object):
     @classmethod
     def get_hash(cls):
         return force_text(
-            hashlib.sha256(cls.dump_data()).hexdigest()
+            hashlib.sha256(force_bytes(cls.dump_data())).hexdigest()
         )
 
     @classmethod

mayan/apps/smart_settings/settings.py

@@ -0,0 +1,302 @@
+from __future__ import unicode_literals
+
+from django.conf import settings
+from django.utils.translation import ugettext_lazy as _
+
+from mayan.apps.smart_settings.classes import Namespace
+
+# Don't import anything on start import, we just want to make it easy
+# for apps.py to activate the settings in this module.
+__all__ = ()
+namespace = Namespace(label=_('Django'), name='django')
+
+setting_django_allowed_hosts = namespace.add_setting(
+    global_name='ALLOWED_HOSTS', default=settings.ALLOWED_HOSTS,
+    help_text=_(
+        'A list of strings representing the host/domain names that this site '
+        'can serve. This is a security measure to prevent HTTP Host header '
+        'attacks, which are possible even under many seemingly-safe web '
+        'server configurations. Values in this list can be '
+        'fully qualified names (e.g. \'www.example.com\'), in which case '
+        'they will be matched against the request\'s Host header exactly '
+        '(case-insensitive, not including port). A value beginning with a '
+        'period can be used as a subdomain wildcard: \'.example.com\' will '
+        'match example.com, www.example.com, and any other subdomain of '
+        'example.com. A value of \'*\' will match anything; in this case you '
+        'are responsible to provide your own validation of the Host header '
+        '(perhaps in a middleware; if so this middleware must be listed '
+        'first in MIDDLEWARE).'
+    ),
+)
+setting_django_append_slash = namespace.add_setting(
+    global_name='APPEND_SLASH', default=settings.APPEND_SLASH,
+    help_text=_(
+        'When set to True, if the request URL does not match any of the '
+        'patterns in the URLconf and it doesn\'t end in a slash, an HTTP '
+        'redirect is issued to the same URL with a slash appended. Note '
+        'that the redirect may cause any data submitted in a POST request '
+        'to be lost. The APPEND_SLASH setting is only used if '
+        'CommonMiddleware is installed (see Middleware). See also '
+        'PREPEND_WWW.'
+    )
+)
+setting_django_auth_password_validators = namespace.add_setting(
+    global_name='AUTH_PASSWORD_VALIDATORS',
+    default=settings.AUTH_PASSWORD_VALIDATORS,
+    help_text=_(
+        'The list of validators that are used to check the strength of '
+        'user\'s passwords.'
+    )
+)
+setting_django_databases = namespace.add_setting(
+    global_name='DATABASES', default=settings.DATABASES,
+    help_text=_(
+        'A dictionary containing the settings for all databases to be used '
+        'with Django. It is a nested dictionary whose contents map a '
+        'database alias to a dictionary containing the options for an '
+        'individual database. The DATABASES setting must configure a '
+        'default database; any number of additional databases may also '
+        'be specified.'
+    ),
+)
+setting_django_data_upload_max_memory_size = namespace.add_setting(
+    global_name='DATA_UPLOAD_MAX_MEMORY_SIZE',
+    default=settings.DATA_UPLOAD_MAX_MEMORY_SIZE,
+    help_text=_(
+        'Default: 2621440 (i.e. 2.5 MB). The maximum size in bytes that a '
+        'request body may be before a SuspiciousOperation '
+        '(RequestDataTooBig) is raised. The check is done when accessing '
+        'request.body or request.POST and is calculated against the total '
+        'request size excluding any file upload data. You can set this to '
+        'None to disable the check. Applications that are expected to '
+        'receive unusually large form posts should tune this setting. The '
+        'amount of request data is correlated to the amount of memory '
+        'needed to process the request and populate the GET and POST '
+        'dictionaries. Large requests could be used as a '
+        'denial-of-service attack vector if left unchecked. Since web '
+        'servers don\'t typically perform deep request inspection, it\'s '
+        'not possible to perform a similar check at that level. See also '
+        'FILE_UPLOAD_MAX_MEMORY_SIZE.'
+    ),
+)
+setting_django_default_from_email = namespace.add_setting(
+    global_name='DEFAULT_FROM_EMAIL',
+    default=settings.DEFAULT_FROM_EMAIL,
+    help_text=_(
+        'Default: \'webmaster@localhost\' '
+        'Default email address to use for various automated correspondence '
+        'from the site manager(s). This doesn\'t include error messages sent '
+        'to ADMINS and MANAGERS; for that, see SERVER_EMAIL.'
+    ),
+)
+setting_django_disallowed_user_agents = namespace.add_setting(
+    global_name='DISALLOWED_USER_AGENTS',
+    default=settings.DISALLOWED_USER_AGENTS,
+    help_text=_(
+        'Default: [] (Empty list). List of compiled regular expression '
+        'objects representing User-Agent strings that are not allowed to '
+        'visit any page, systemwide. Use this for bad robots/crawlers. '
+        'This is only used if CommonMiddleware is installed '
+        '(see Middleware).'
+    ),
+)
+setting_django_email_backend = namespace.add_setting(
+    global_name='EMAIL_BACKEND',
+    default=settings.EMAIL_BACKEND,
+    help_text=_(
+        'Default: \'django.core.mail.backends.smtp.EmailBackend\'. The '
+        'backend to use for sending emails.'
+    ),
+)
+setting_django_email_host = namespace.add_setting(
+    global_name='EMAIL_HOST',
+    default=settings.EMAIL_HOST,
+    help_text=_(
+        'Default: \'localhost\'. The host to use for sending email.'
+    ),
+)
+setting_django_email_host_password = namespace.add_setting(
+    global_name='EMAIL_HOST_PASSWORD',
+    default=settings.EMAIL_HOST_PASSWORD,
+    help_text=_(
+        'Default: \'\' (Empty string). Password to use for the SMTP '
+        'server defined in EMAIL_HOST. This setting is used in '
+        'conjunction with EMAIL_HOST_USER when authenticating to the '
+        'SMTP server. If either of these settings is empty, '
+        'Django won\'t attempt authentication.'
+    ),
+)
+setting_django_email_host_user = namespace.add_setting(
+    global_name='EMAIL_HOST_USER',
+    default=settings.EMAIL_HOST_USER,
+    help_text=_(
+        'Default: \'\' (Empty string). Username to use for the SMTP '
+        'server defined in EMAIL_HOST. If empty, Django won\'t attempt '
+        'authentication.'
+    ),
+)
+setting_django_email_port = namespace.add_setting(
+    global_name='EMAIL_PORT',
+    default=settings.EMAIL_PORT,
+    help_text=_(
+        'Default: 25. Port to use for the SMTP server defined in EMAIL_HOST.'
+    ),
+)
+setting_django_email_timeout = namespace.add_setting(
+    global_name='EMAIL_TIMEOUT',
+    default=settings.EMAIL_TIMEOUT,
+    help_text=_(
+        'Default: None. Specifies a timeout in seconds for blocking '
+        'operations like the connection attempt.'
+    ),
+)
+setting_django_email_user_tls = namespace.add_setting(
+    global_name='EMAIL_USE_TLS',
+    default=settings.EMAIL_USE_TLS,
+    help_text=_(
+        'Default: False. Whether to use a TLS (secure) connection when '
+        'talking to the SMTP server. This is used for explicit TLS '
+        'connections, generally on port 587. If you are experiencing '
+        'hanging connections, see the implicit TLS setting EMAIL_USE_SSL.'
+    ),
+)
+setting_django_email_user_ssl = namespace.add_setting(
+    global_name='EMAIL_USE_SSL',
+    default=settings.EMAIL_USE_SSL,
+    help_text=_(
+        'Default: False. Whether to use an implicit TLS (secure) connection '
+        'when talking to the SMTP server. In most email documentation this '
+        'type of TLS connection is referred to as SSL. It is generally used '
+        'on port 465. If you are experiencing problems, see the explicit '
+        'TLS setting EMAIL_USE_TLS. Note that EMAIL_USE_TLS/EMAIL_USE_SSL '
+        'are mutually exclusive, so only set one of those settings to True.'
+    ),
+)
+setting_django_file_upload_max_memory_size = namespace.add_setting(
+    global_name='FILE_UPLOAD_MAX_MEMORY_SIZE',
+    default=settings.FILE_UPLOAD_MAX_MEMORY_SIZE,
+    help_text=_(
+        'Default: 2621440 (i.e. 2.5 MB). The maximum size (in bytes) '
+        'that an upload will be before it gets streamed to the file '
+        'system. See Managing files for details. See also '
+        'DATA_UPLOAD_MAX_MEMORY_SIZE.'
+    ),
+)
+setting_django_login_url = namespace.add_setting(
+    global_name='LOGIN_URL',
+    default=settings.LOGIN_URL,
+    help_text=_(
+        'Default: \'/accounts/login/\' The URL where requests are '
+        'redirected for login, especially when using the login_required() '
+        'decorator. This setting also accepts named URL patterns which '
+        'can be used to reduce configuration duplication since you '
+        'don\'t have to define the URL in two places (settings '
+        'and URLconf).'
+    )
+)
+setting_django_login_redirect_url = namespace.add_setting(
+    global_name='LOGIN_REDIRECT_URL',
+    default=settings.LOGIN_REDIRECT_URL,
+    help_text=_(
+        'Default: \'/accounts/profile/\' The URL where requests are '
+        'redirected after login when the contrib.auth.login view gets no '
+        'next parameter. This is used by the login_required() decorator, '
+        'for example. This setting also accepts named URL patterns which '
+        'can be used to reduce configuration duplication since you don\'t '
+        'have to define the URL in two places (settings and URLconf).'
+    ),
+)
+setting_django_logout_redirect_url = namespace.add_setting(
+    global_name='LOGOUT_REDIRECT_URL',
+    default=settings.LOGOUT_REDIRECT_URL,
+    help_text=_(
+        'Default: None. The URL where requests are redirected after a user '
+        'logs out using LogoutView (if the view doesn\'t get a next_page '
+        'argument). If None, no redirect will be performed and the logout '
+        'view will be rendered. This setting also accepts named URL '
+        'patterns which can be used to reduce configuration duplication '
+        'since you don\'t have to define the URL in two places (settings '
+        'and URLconf).'
+    )
+)
+setting_django_internal_ips = namespace.add_setting(
+    global_name='INTERNAL_IPS',
+    default=settings.INTERNAL_IPS,
+    help_text=_(
+        'A list of IP addresses, as strings, that: Allow the debug() '
+        'context processor to add some variables to the template context. '
+        'Can use the admindocs bookmarklets even if not logged in as a '
+        'staff user. Are marked as "internal" (as opposed to "EXTERNAL") '
+        'in AdminEmailHandler emails.'
+    ),
+)
+setting_django_languages = namespace.add_setting(
+    global_name='LANGUAGES',
+    default=settings.LANGUAGES,
+    help_text=_(
+        'A list of all available languages. The list is a list of '
+        'two-tuples in the format (language code, language name) '
+        'for example, (\'ja\', \'Japanese\'). This specifies which '
+        'languages are available for language selection. '
+        'Generally, the default value should suffice. Only set this '
+        'setting if you want to restrict language selection to a '
+        'subset of the Django-provided languages. '
+    ),
+)
+setting_django_language_code = namespace.add_setting(
+    global_name='LANGUAGE_CODE',
+    default=settings.LANGUAGE_CODE,
+    help_text=_(
+        'A string representing the language code for this installation. '
+        'This should be in standard language ID format. For example, U.S. '
+        'English is "en-us". It serves two purposes: If the locale '
+        'middleware isn\'t in use, it decides which translation is served '
+        'to all users. If the locale middleware is active, it provides a '
+        'fallback language in case the user\'s preferred language can\'t '
+        'be determined or is not supported by the website. It also provides '
+        'the fallback translation when a translation for a given literal '
+        'doesn\'t exist for the user\'s preferred language.'
+    ),
+)
+setting_django_static_url = namespace.add_setting(
+    global_name='STATIC_URL',
+    default=settings.STATIC_URL,
+    help_text=_(
+        'URL to use when referring to static files located in STATIC_ROOT. '
+        'Example: "/static/" or "http://static.example.com/" '
+        'If not None, this will be used as the base path for asset '
+        'definitions (the Media class) and the staticfiles app. '
+        'It must end in a slash if set to a non-empty value.'
+    ),
+)
+setting_django_staticfiles_storage = namespace.add_setting(
+    global_name='STATICFILES_STORAGE',
+    default=settings.STATICFILES_STORAGE,
+    help_text=_(
+        'The file storage engine to use when collecting static files with '
+        'the collectstatic management command. A ready-to-use instance of '
+        'the storage backend defined in this setting can be found at '
+        'django.contrib.staticfiles.storage.staticfiles_storage.'
+    ),
+)
+setting_django_time_zone = namespace.add_setting(
+    global_name='TIME_ZONE',
+    default=settings.TIME_ZONE,
+    help_text=_(
+        'A string representing the time zone for this installation. '
+        'Note that this isn\'t necessarily the time zone of the server. '
+        'For example, one server may serve multiple Django-powered sites, '
+        'each with a separate time zone setting.'
+    ),
+)
+setting_django_wsgi_application = namespace.add_setting(
+    global_name='WSGI_APPLICATION',
+    default=settings.WSGI_APPLICATION,
+    help_text=_(
+        'The full Python path of the WSGI application object that Django\'s '
+        'built-in servers (e.g. runserver) will use. The django-admin '
+        'startproject management command will create a simple wsgi.py '
+        'file with an application callable in it, and point this setting '
+        'to that application.'
+    ),
+)

mayan/apps/smart_settings/tests/test_classes.py

@@ -14,9 +14,10 @@ from mayan.apps.storage.utils import fs_cleanup
 from ..classes import Setting
 
 from .literals import ENVIRONMENT_TEST_NAME, ENVIRONMENT_TEST_VALUE
+from .mixins import SmartSettingTestMixin
 
 
-class ClassesTestCase(BaseTestCase):
+class ClassesTestCase(SmartSettingTestMixin, BaseTestCase):
     def test_environment_variable(self):
         os.environ[
             'MAYAN_{}'.format(ENVIRONMENT_TEST_NAME)
@@ -39,3 +40,16 @@ class ClassesTestCase(BaseTestCase):
 
         with path_config_backup.open(mode='r') as file_object:
             self.assertFalse('!!python/' in file_object.read())
+
+    def test_setting_check_changed(self):
+        self._create_test_settings_namespace()
+        test_setting = self.test_settings_namespace.add_setting(
+            global_name='SMART_SETTINGS_TEST_SETTING',
+            default='test value'
+        )
+        # Initialize hash cache
+        Setting._cache_hash = None
+        Setting.check_changed()
+        self.assertFalse(Setting.check_changed())
+        test_setting.value = 'test value edited'
+        self.assertTrue(Setting.check_changed())

mayan/apps/sources/settings.py

@@ -5,7 +5,7 @@ import os
 from django.conf import settings
 from django.utils.translation import ugettext_lazy as _
 
-from mayan.apps.smart_settings import Namespace
+from mayan.apps.smart_settings.classes import Namespace
 
 namespace = Namespace(label=_('Sources'), name='sources')
 

mayan/apps/storage/settings.py

@@ -4,7 +4,7 @@ import tempfile
 
 from django.utils.translation import ugettext_lazy as _
 
-from mayan.apps.smart_settings import Namespace
+from mayan.apps.smart_settings.classes import Namespace
 
 namespace = Namespace(label=_('Storage'), name='storage')
 

mayan/apps/task_manager/apps.py

@@ -13,6 +13,7 @@ from .links import (
     link_queue_scheduled_task_list, link_queue_reserved_task_list,
     link_task_manager
 )
+from .settings import *  # NOQA
 
 
 class TaskManagerApp(MayanAppConfig):

mayan/apps/task_manager/settings.py

@@ -0,0 +1,29 @@
+from __future__ import unicode_literals
+
+from django.utils.translation import ugettext_lazy as _
+
+from mayan.apps.smart_settings.classes import Namespace
+
+# Don't import anything on start import, we just want to make it easy
+# for apps.py to activate the settings in this module.
+__all__ = ()
+namespace = Namespace(label=_('Celery'), name='celery')
+
+setting_celery_broker_url = namespace.add_setting(
+    global_name='BROKER_URL', default=None,
+    help_text=_(
+        'Default: "amqp://". Default broker URL. This must be a URL in '
+        'the form of: transport://userid:password@hostname:port/virtual_host '
+        'Only the scheme part (transport://) is required, the rest is '
+        'optional, and defaults to the specific transports default values.'
+    ),
+)
+setting_celery_result_backend = namespace.add_setting(
+    global_name='CELERY_RESULT_BACKEND', default=None,
+    help_text=_(
+        'Default: No result backend enabled by default. The backend used '
+        'to store task results (tombstones). Refer to '
+        'http://docs.celeryproject.org/en/v4.1.0/userguide/configuration.'
+        'html#result-backend'
+    )
+)

mayan/conf.py

@@ -6,7 +6,7 @@ from __future__ import unicode_literals
 
 from django.utils.translation import ugettext_lazy as _
 
-from mayan.apps.smart_settings import Namespace
+from mayan.apps.smart_settings.classes import Namespace
 
 namespace = Namespace(name='mayan', label=_('Mayan'))