Merge branch 'versions/minor' into bc_merge

Signed-off-by: Roberto Rosario <roberto.rosario.gonzalez@gmail.com>

HISTORY.rst

@@ -1,4 +1,17 @@
-=======
+3.2.5 (2019-07-XX)
+==================
+* Don't error out if the EXTRA_APPS or the DISABLED_APPS settings
+  are set to blank.
+* Update troubleshooting documentation topic.
+* Add data migration to the file metadata app. Synchronizes the
+  document type settings model of existing document types.
+* Fix cabinet and tags upload wizard steps missing some entries.
+  GitLab issue #632. Thanks to Matthias Urhahn (@d4rken) for the
+  report.
+* Add alert when settings are changed and util the installation
+  is restarted. GitLab issue #605. Thanks to
+  Vikas Kedia (@vikaskedia) to the report.
+
 3.2.4 (2019-06-29)
 ==================
 * Support configurable GUnicorn timeouts. Defaults to

docs/releases/3.2.5.rst

@@ -0,0 +1,113 @@
+Version 3.2.5
+=============
+
+Released: July XX, 2019
+
+
+Changes
+-------
+
+- Don't error out if the EXTRA_APPS or the DISABLED_APPS settings
+  are set to blank.
+- Update troubleshooting documentation topic.
+- Add data migration to the file metadata app. Synchronizes the
+  document type settings model of existing document types.
+- Fix cabinet and tags upload wizard steps missing some entries.
+  GitLab issue #632. Thanks to Matthias Urhahn (@d4rken) for the
+  report.
+- Add alert when settings are changed and util the installation
+  is restarted. GitLab issue #605. Thanks to
+  Vikas Kedia (@vikaskedia) to the report.
+
+Removals
+--------
+
+- None
+
+
+Upgrading from a previous version
+---------------------------------
+
+If installed via Python's PIP
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Remove deprecated requirements::
+
+    $ curl https://gitlab.com/mayan-edms/mayan-edms/raw/master/removals.txt | pip uninstall -r /dev/stdin
+
+Type in the console::
+
+    $ pip install mayan-edms==3.2.5
+
+the requirements will also be updated automatically.
+
+
+Using Git
+^^^^^^^^^
+
+If you installed Mayan EDMS by cloning the Git repository issue the commands::
+
+    $ git reset --hard HEAD
+    $ git pull
+
+otherwise download the compressed archived and uncompress it overriding the
+existing installation.
+
+Remove deprecated requirements::
+
+    $ pip uninstall -y -r removals.txt
+
+Next upgrade/add the new requirements::
+
+    $ pip install --upgrade -r requirements.txt
+
+
+Common steps
+^^^^^^^^^^^^
+
+Perform these steps after updating the code from either step above.
+
+Make a backup of your supervisord file::
+
+    sudo cp /etc/supervisor/conf.d/mayan.conf /etc/supervisor/conf.d/mayan.conf.bck
+
+Update the supervisord configuration file. Replace the environment
+variables values show here with your respective settings. This step will refresh
+the supervisord configuration file with the new queues and the latest
+recommended layout::
+
+    sudo MAYAN_DATABASE_ENGINE=django.db.backends.postgresql MAYAN_DATABASE_NAME=mayan \
+    MAYAN_DATABASE_PASSWORD=mayanuserpass MAYAN_DATABASE_USER=mayan \
+    MAYAN_DATABASE_HOST=127.0.0.1 MAYAN_MEDIA_ROOT=/opt/mayan-edms/media \
+    /opt/mayan-edms/bin/mayan-edms.py platformtemplate supervisord > /etc/supervisor/conf.d/mayan.conf
+
+Edit the supervisord configuration file and update any setting the template
+generator missed::
+
+    sudo vi /etc/supervisor/conf.d/mayan.conf
+
+Migrate existing database schema with::
+
+    $ mayan-edms.py performupgrade
+
+Add new static media::
+
+    $ mayan-edms.py preparestatic --noinput
+
+The upgrade procedure is now complete.
+
+
+Backward incompatible changes
+-----------------------------
+
+- None
+
+
+Bugs fixed or issues closed
+---------------------------
+
+- :gitlab-issue:`605` Project title fluctuates between default value and new value [Video]
+- :gitlab-issue:`629` Cannot Upgrade to 3.2.X Docker Image
+- :gitlab-issue:`632` Tags get lost when uploading through the webui
+
+.. _PyPI: https://pypi.python.org/pypi/mayan-edms/

docs/releases/index.rst

@@ -20,6 +20,7 @@ versions of the documentation contain the release notes for any later releases.
 .. toctree::
    :maxdepth: 1
 
+   3.2.5
    3.2.4
    3.2.3
    3.2.2

docs/topics/troubleshooting.rst

@@ -168,3 +168,16 @@ files to a temporary directory on the same partition as the watchfolder first.
 Then move the files to the watchfolder. The move will be executed as an atomic
 operation and will prevent the files to be uploaded in the middle of the
 copying process.
+
+************
+Dependencies
+************
+
+Error: ``unable to execute 'x86_64-linux-gnu-gcc': No such file or directory``
+==============================================================================
+
+This happens when using the ``MAYAN_APT_INSTALLS`` feature. It means that the
+``GCC`` package is required to compile the packages specified with
+``MAYAN_APT_INSTALLS``.
+
+Solution: Include ``gcc`` in the list of packages specified with ``MAYAN_APT_INSTALLS``.

mayan/apps/appearance/templates/appearance/base.html

@@ -34,6 +34,14 @@
                 {% endif %}
                 {% block messages %}
                 {% endblock %}
+
+                {% smart_settings_check_changed as settings_changed %}
+                {% if settings_changed %}
+                    <div class="alert alert-dismissible alert-warning">
+                        <button type="button" class="close" data-dismiss="alert">&times;</button>
+                        <p><strong>{% trans 'Warning' %}</strong> {% trans 'Settings updated, restart your installation for changes to take proper effect.' %}</p>
+                    </div>
+                {% endif %}
             </div>
         </div>
 

mayan/apps/cabinets/tests/test_wizard_steps.py

@@ -11,6 +11,7 @@ from mayan.apps.sources.tests.literals import (
 )
 from mayan.apps.sources.wizards import WizardStep
 
+from ..models import Cabinet
 from ..wizard_steps import WizardStepCabinets
 
 from .mixins import CabinetTestMixin
@@ -38,11 +39,12 @@ class CabinetDocumentUploadTestCase(CabinetTestMixin, GenericDocumentViewTestCas
                 }, data={
                     'document_type_id': self.test_document_type.pk,
                     'source-file': file_object,
-                    'cabinets': self.test_cabinet.pk
+                    'cabinets': Cabinet.objects.values_list('pk', flat=True)
                 }
             )
 
     def test_upload_interactive_view_with_access(self):
+        self._create_test_cabinet()
         self._create_test_cabinet()
         self.grant_access(
             obj=self.test_document_type, permission=permission_document_create
@@ -51,7 +53,10 @@ class CabinetDocumentUploadTestCase(CabinetTestMixin, GenericDocumentViewTestCas
 
         self.assertEqual(response.status_code, 302)
         self.assertTrue(
-            self.test_cabinet in Document.objects.first().cabinets.all()
+            self.test_cabinets[0] in Document.objects.first().cabinets.all()
+        )
+        self.assertTrue(
+            self.test_cabinets[1] in Document.objects.first().cabinets.all()
         )
 
     def _request_wizard_view(self):

mayan/apps/cabinets/wizard_steps.py

@@ -1,11 +1,10 @@
 from __future__ import unicode_literals
 
-from furl import furl
-
 from django.apps import apps
 from django.utils.encoding import force_text
 from django.utils.translation import ugettext_lazy as _
 
+from mayan.apps.common.http import URL
 from mayan.apps.sources.wizards import WizardStep
 
 from .forms import CabinetListForm
@@ -48,10 +47,10 @@ class WizardStepCabinets(WizardStep):
 
     @classmethod
     def step_post_upload_process(cls, document, querystring=None):
-        furl_instance = furl(querystring)
         Cabinet = apps.get_model(app_label='cabinets', model_name='Cabinet')
+        cabinet_id_list = URL(query_string=querystring).args.getlist('cabinets')
 
-        for cabinet in Cabinet.objects.filter(pk__in=furl_instance.args.getlist('cabinets')):
+        for cabinet in Cabinet.objects.filter(pk__in=cabinet_id_list):
             cabinet.documents.add(document)
 
 

mayan/apps/common/http.py

@@ -0,0 +1,36 @@
+from __future__ import unicode_literals
+
+from django.http import QueryDict
+from django.utils.encoding import force_bytes
+
+
+class URL(object):
+    def __init__(self, path=None, query_string=None):
+        self._path = path
+        self._query_string = query_string
+        kwargs = {'mutable': True}
+        if query_string:
+            kwargs['query_string'] = query_string.encode('utf-8')
+
+        self._args = QueryDict(**kwargs)
+
+    @property
+    def args(self):
+        return self._args
+
+    def to_string(self):
+        if self._args.keys():
+            query = force_bytes(
+                '?{}'.format(self._args.urlencode())
+            )
+        else:
+            query = ''
+
+        if self._path:
+            path = self._path
+        else:
+            path = ''
+
+        result = force_bytes('{}{}'.format(path, query))
+
+        return result

mayan/apps/common/utils.py

@@ -8,10 +8,6 @@ from django.core.exceptions import FieldDoesNotExist
 from django.db.models.constants import LOOKUP_SEP
 from django.urls import resolve as django_resolve
 from django.urls.base import get_script_prefix
-from django.utils.datastructures import MultiValueDict
-from django.utils.http import (
-    urlencode as django_urlencode, urlquote as django_urlquote
-)
 from django.utils.six.moves import reduce as reduce_function
 
 from mayan.apps.common.compat import dict_type, dictionary_type
@@ -150,50 +146,3 @@ def return_related(instance, related_field):
     using double underscore.
     """
     return reduce_function(getattr, related_field.split('__'), instance)
-
-
-def urlquote(link=None, get=None):
-    """
-    This method does both: urlquote() and urlencode()
-
-    urlqoute(): Quote special characters in 'link'
-
-    urlencode(): Map dictionary to query string key=value&...
-
-    HTML escaping is not done.
-
-    Example:
-
-    urlquote('/wiki/Python_(programming_language)')
-        --> '/wiki/Python_%28programming_language%29'
-    urlquote('/mypath/', {'key': 'value'})
-        --> '/mypath/?key=value'
-    urlquote('/mypath/', {'key': ['value1', 'value2']})
-        --> '/mypath/?key=value1&key=value2'
-    urlquote({'key': ['value1', 'value2']})
-        --> 'key=value1&key=value2'
-    """
-    if get is None:
-        get = []
-
-    assert link or get
-    if isinstance(link, dict):
-        # urlqoute({'key': 'value', 'key2': 'value2'}) -->
-        # key=value&key2=value2
-        assert not get, get
-        get = link
-        link = ''
-    assert isinstance(get, dict), 'wrong type "%s", dict required' % type(get)
-    # assert not (link.startswith('http://') or link.startswith('https://')),
-    #    'This method should only quote the url path.
-    #    It should not start with http(s)://  (%s)' % (
-    #    link)
-    if get:
-        # http://code.djangoproject.com/ticket/9089
-        if isinstance(get, MultiValueDict):
-            get = get.lists()
-        if link:
-            link = '%s?' % django_urlquote(link)
-        return '%s%s' % (link, django_urlencode(get, doseq=True))
-    else:
-        return django_urlquote(link)

mayan/apps/file_metadata/migrations/0002_documenttypesettings.py

@@ -0,0 +1,53 @@
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+def operation_create_file_metadata_setting_for_existing_document_types(apps, schema_editor):
+    DocumentType = apps.get_model(
+        app_label='documents', model_name='DocumentType'
+    )
+    DocumentTypeSettings = apps.get_model(
+        app_label='file_metadata', model_name='DocumentTypeSettings'
+    )
+
+    for document_type in DocumentType.objects.using(schema_editor.connection.alias).all():
+        try:
+            DocumentTypeSettings.objects.using(
+                schema_editor.connection.alias
+            ).get_or_create(document_type=document_type)
+        except DocumentTypeSettings.DoesNotExist:
+            pass
+
+
+def operation_delete_file_metadata_setting_for_existing_document_types(apps, schema_editor):
+    DocumentType = apps.get_model(
+        app_label='documents', model_name='DocumentType'
+    )
+    DocumentTypeSettings = apps.get_model(
+        app_label='file_metadata', model_name='DocumentTypeSettings'
+    )
+
+    for document_type in DocumentType.objects.using(schema_editor.connection.alias).all():
+        try:
+            DocumentTypeSettings.objects.using(
+                schema_editor.connection.alias
+            ).get(document_type=document_type).delete()
+        except DocumentTypeSettings.DoesNotExist:
+            pass
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('documents', '0047_auto_20180917_0737'),
+        ('file_metadata', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.RunPython(
+            code=operation_create_file_metadata_setting_for_existing_document_types,
+            reverse_code=operation_delete_file_metadata_setting_for_existing_document_types,
+        )
+    ]

mayan/apps/linking/apps.py

@@ -57,9 +57,7 @@ class LinkingApp(MayanAppConfig):
         SmartLinkCondition = self.get_model(model_name='SmartLinkCondition')
 
         ModelEventType.register(
-            event_types=(
-                event_smart_link_edited,
-            ), model=SmartLink
+            event_types=(event_smart_link_edited,), model=SmartLink
         )
 
         ModelPermission.register(

mayan/apps/metadata/api.py

@@ -1,9 +1,8 @@
 from __future__ import unicode_literals
 
-from furl import furl
-
 from django.shortcuts import get_object_or_404
-from django.utils.encoding import force_bytes
+
+from mayan.apps.common.http import URL
 
 from .models import DocumentMetadata, MetadataType
 
@@ -19,7 +18,7 @@ def decode_metadata_from_querystring(querystring=None):
     metadata_list = []
     if querystring:
         # Match out of order metadata_type ids with metadata values from request
-        for key, value in furl(force_bytes(querystring)).args.items():
+        for key, value in URL(query_string=querystring).args.items():
             if 'metadata' in key:
                 index, element = key[8:].split('_')
                 metadata_dict[element][index] = value
@@ -27,10 +26,12 @@ def decode_metadata_from_querystring(querystring=None):
         # Convert the nested dictionary into a list of id+values dictionaries
         for order, identifier in metadata_dict['id'].items():
             if order in metadata_dict['value'].keys():
-                metadata_list.append({
-                    'id': identifier,
-                    'value': metadata_dict['value'][order]
-                })
+                metadata_list.append(
+                    {
+                        'id': identifier,
+                        'value': metadata_dict['value'][order]
+                    }
+                )
 
     return metadata_list
 

mayan/apps/metadata/tests/test_wizard_steps.py

@@ -1,9 +1,8 @@
 from __future__ import unicode_literals
 
-from furl import furl
-
 from django.urls import reverse
 
+from mayan.apps.common.http import URL
 from mayan.apps.documents.models import Document
 from mayan.apps.documents.permissions import permission_document_create
 from mayan.apps.documents.tests import (
@@ -35,7 +34,9 @@ class DocumentUploadMetadataTestCase(MetadataTypeTestMixin, GenericDocumentViewT
         )
 
     def test_upload_interactive_with_unicode_metadata(self):
-        url = furl(reverse(viewname='sources:upload_interactive'))
+        url = URL(
+            path=reverse(viewname='sources:upload_interactive')
+        )
         url.args['metadata0_id'] = self.test_metadata_type.pk
         url.args['metadata0_value'] = TEST_METADATA_VALUE_UNICODE
 
@@ -46,7 +47,7 @@ class DocumentUploadMetadataTestCase(MetadataTypeTestMixin, GenericDocumentViewT
         # Upload the test document
         with open(TEST_SMALL_DOCUMENT_PATH, mode='rb') as file_descriptor:
             response = self.post(
-                path=url, data={
+                path=url.to_string(), data={
                     'document-language': 'eng', 'source-file': file_descriptor,
                     'document_type_id': self.test_document_type.pk,
                 }
@@ -60,7 +61,9 @@ class DocumentUploadMetadataTestCase(MetadataTypeTestMixin, GenericDocumentViewT
         )
 
     def test_upload_interactive_with_ampersand_metadata(self):
-        url = furl(reverse(viewname='sources:upload_interactive'))
+        url = URL(
+            path=reverse(viewname='sources:upload_interactive')
+        )
         url.args['metadata0_id'] = self.test_metadata_type.pk
         url.args['metadata0_value'] = TEST_METADATA_VALUE_WITH_AMPERSAND
 
@@ -70,7 +73,7 @@ class DocumentUploadMetadataTestCase(MetadataTypeTestMixin, GenericDocumentViewT
         # Upload the test document
         with open(TEST_SMALL_DOCUMENT_PATH, mode='rb') as file_descriptor:
             response = self.post(
-                path=url, data={
+                path=url.to_string(), data={
                     'document-language': 'eng', 'source-file': file_descriptor,
                     'document_type_id': self.test_document_type.pk,
                 }

mayan/apps/smart_settings/classes.py

@@ -1,6 +1,7 @@
 from __future__ import unicode_literals
 
 import errno
+import hashlib
 from importlib import import_module
 import logging
 import os
@@ -78,6 +79,7 @@ class Namespace(object):
 @python_2_unicode_compatible
 class Setting(object):
     _registry = {}
+    _cache_hash = None
 
     @staticmethod
     def deserialize_value(value):
@@ -108,6 +110,13 @@ class Setting(object):
 
         return result
 
+    @classmethod
+    def check_changed(cls):
+        if not cls._cache_hash:
+            cls._cache_hash = cls.get_hash()
+
+        return cls._cache_hash != cls.get_hash()
+
     @classmethod
     def dump_data(cls, filter_term=None, namespace=None):
         dictionary = {}
@@ -129,6 +138,12 @@ class Setting(object):
     def get_all(cls):
         return sorted(cls._registry.values(), key=lambda x: x.global_name)
 
+    @classmethod
+    def get_hash(cls):
+        return force_text(
+            hashlib.sha256(cls.dump_data()).hexdigest()
+        )
+
     @classmethod
     def save_configuration(cls, path=settings.CONFIGURATION_FILEPATH):
         try:

mayan/apps/smart_settings/templatetags/smart_settings_tags.py

@@ -10,3 +10,9 @@ register = Library()
 @register.simple_tag
 def smart_setting(global_name):
     return Setting.get(global_name=global_name).value
+
+
+@register.simple_tag
+def smart_settings_check_changed():
+    return Setting.check_changed()
+

mayan/apps/sources/views.py

@@ -2,8 +2,6 @@ from __future__ import absolute_import, unicode_literals
 
 import logging
 
-from furl import furl
-
 from django.contrib import messages
 from django.http import HttpResponseRedirect, JsonResponse
 from django.shortcuts import get_object_or_404
@@ -257,9 +255,8 @@ class UploadInteractiveView(UploadBaseView):
             except Exception as exception:
                 messages.error(message=exception, request=self.request)
 
-            querystring = furl()
-            querystring.args.update(self.request.GET)
-            querystring.args.update(self.request.POST)
+            querystring = self.request.GET.copy()
+            querystring.update(self.request.POST)
 
             try:
                 task_source_handle_upload.apply_async(
@@ -271,7 +268,7 @@ class UploadInteractiveView(UploadBaseView):
                             filename=force_text(shared_uploaded_file)
                         ),
                         language=forms['document_form'].cleaned_data.get('language'),
-                        querystring=querystring.tostr(),
+                        querystring=querystring.urlencode(),
                         shared_uploaded_file_id=shared_uploaded_file.pk,
                         source_id=self.source.pk,
                         user_id=user_id,

mayan/apps/tags/tests/test_wizard_steps.py

@@ -33,9 +33,7 @@ class TaggedDocumentUploadTestCase(TagTestMixin, GenericDocumentViewTestCase):
                 }, data={
                     'document_type_id': self.test_document_type.pk,
                     'source-file': file_object,
-                    'tags': ','.join(
-                        map(str, Tag.objects.values_list('pk', flat=True))
-                    )
+                    'tags': Tag.objects.values_list('pk', flat=True)
                 }
             )
 

mayan/apps/tags/wizard_steps.py

@@ -1,11 +1,10 @@
 from __future__ import unicode_literals
 
-from furl import furl
-
 from django.apps import apps
 from django.utils.encoding import force_text
 from django.utils.translation import ugettext_lazy as _
 
+from mayan.apps.common.http import URL
 from mayan.apps.sources.wizards import WizardStep
 
 from .forms import TagMultipleSelectionForm
@@ -46,13 +45,9 @@ class WizardStepTags(WizardStep):
 
     @classmethod
     def step_post_upload_process(cls, document, querystring=None):
-        furl_instance = furl(querystring)
         Tag = apps.get_model(app_label='tags', model_name='Tag')
 
-        tag_id_list = furl_instance.args.get('tags', '')
-
-        if tag_id_list:
-            tag_id_list = tag_id_list.split(',')
+        tag_id_list = URL(query_string=querystring).args.getlist('tags')
 
         for tag in Tag.objects.filter(pk__in=tag_id_list):
             tag.documents.add(document)