Fix cabinet and tags upload wizard steps

Steps were missing some entries. Closes GitLab issue #632.
Thanks to Matthias Urhahn (@d4rken) for the report.

Signed-off-by: Roberto Rosario <roberto.rosario.gonzalez@gmail.com>

HISTORY.rst

@@ -1,3 +1,14 @@
+3.2.5 (2019-07-XX)
+==================
+* Don't error out if the EXTRA_APPS or the DISABLED_APPS settings
+  are set to blank.
+* Update troubleshooting documentation topic.
+* Add data migration to the file metadata app. Synchronizes the
+  document type settings model of existing document types.
+* Fix cabinet and tags upload wizard steps missing some entries.
+  GitLab issue #632. Thanks to Matthias Urhahn (@d4rken) for the
+  report.
+
 3.2.4 (2019-06-29)
 ==================
 * Support configurable GUnicorn timeouts. Defaults to

docs/releases/3.2.5.rst

@@ -0,0 +1,99 @@
+Version 3.2.4
+=============
+
+Released: July XX, 2019
+
+
+Changes
+-------
+
+Removals
+--------
+
+- None
+
+
+Upgrading from a previous version
+---------------------------------
+
+If installed via Python's PIP
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Remove deprecated requirements::
+
+    $ curl https://gitlab.com/mayan-edms/mayan-edms/raw/master/removals.txt | pip uninstall -r /dev/stdin
+
+Type in the console::
+
+    $ pip install mayan-edms==3.2.55555
+
+the requirements will also be updated automatically.
+
+
+Using Git
+^^^^^^^^^
+
+If you installed Mayan EDMS by cloning the Git repository issue the commands::
+
+    $ git reset --hard HEAD
+    $ git pull
+
+otherwise download the compressed archived and uncompress it overriding the
+existing installation.
+
+Remove deprecated requirements::
+
+    $ pip uninstall -y -r removals.txt
+
+Next upgrade/add the new requirements::
+
+    $ pip install --upgrade -r requirements.txt
+
+
+Common steps
+^^^^^^^^^^^^
+
+Perform these steps after updating the code from either step above.
+
+Make a backup of your supervisord file::
+
+    sudo cp /etc/supervisor/conf.d/mayan.conf /etc/supervisor/conf.d/mayan.conf.bck
+
+Update the supervisord configuration file. Replace the environment
+variables values show here with your respective settings. This step will refresh
+the supervisord configuration file with the new queues and the latest
+recommended layout::
+
+    sudo MAYAN_DATABASE_ENGINE=django.db.backends.postgresql MAYAN_DATABASE_NAME=mayan \
+    MAYAN_DATABASE_PASSWORD=mayanuserpass MAYAN_DATABASE_USER=mayan \
+    MAYAN_DATABASE_HOST=127.0.0.1 MAYAN_MEDIA_ROOT=/opt/mayan-edms/media \
+    /opt/mayan-edms/bin/mayan-edms.py platformtemplate supervisord > /etc/supervisor/conf.d/mayan.conf
+
+Edit the supervisord configuration file and update any setting the template
+generator missed::
+
+    sudo vi /etc/supervisor/conf.d/mayan.conf
+
+Migrate existing database schema with::
+
+    $ mayan-edms.py performupgrade
+
+Add new static media::
+
+    $ mayan-edms.py preparestatic --noinput
+
+The upgrade procedure is now complete.
+
+
+Backward incompatible changes
+-----------------------------
+
+- None
+
+
+Bugs fixed or issues closed
+---------------------------
+
+- :gitlab-issue:`632` Tags get lost when uploading through the webui
+
+.. _PyPI: https://pypi.python.org/pypi/mayan-edms/

docs/releases/index.rst

@@ -20,6 +20,7 @@ versions of the documentation contain the release notes for any later releases.
 .. toctree::
    :maxdepth: 1
 
+   3.2.5
    3.2.4
    3.2.3
    3.2.2

mayan/apps/cabinets/tests/test_wizard_steps.py

@@ -1,7 +1,5 @@
 from __future__ import unicode_literals
 
-from django.utils.encoding import force_text
-
 from mayan.apps.documents.models import Document
 from mayan.apps.documents.permissions import permission_document_create
 from mayan.apps.documents.tests import (
@@ -41,9 +39,7 @@ class CabinetDocumentUploadTestCase(CabinetTestMixin, GenericDocumentViewTestCas
                 }, data={
                     'document_type_id': self.test_document_type.pk,
                     'source-file': file_object,
-                    'cabinets': ','.join(
-                        map(force_text, Cabinet.objects.values_list('pk', flat=True))
-                    )
+                    'cabinets': Cabinet.objects.values_list('pk', flat=True)
                 }
             )
 

mayan/apps/cabinets/wizard_steps.py

@@ -1,11 +1,10 @@
 from __future__ import unicode_literals
 
-from furl import furl
-
 from django.apps import apps
 from django.utils.encoding import force_text
 from django.utils.translation import ugettext_lazy as _
 
+from mayan.apps.common.http import URL
 from mayan.apps.sources.wizards import WizardStep
 
 from .forms import CabinetListForm
@@ -48,13 +47,8 @@ class WizardStepCabinets(WizardStep):
 
     @classmethod
     def step_post_upload_process(cls, document, querystring=None):
-        furl_instance = furl(querystring)
         Cabinet = apps.get_model(app_label='cabinets', model_name='Cabinet')
-
-        cabinet_id_list = furl_instance.args.get('cabinets', '')
-
-        if cabinet_id_list:
-            cabinet_id_list = cabinet_id_list.split(',')
+        cabinet_id_list = URL(query_string=querystring).args.getlist('cabinets')
 
         for cabinet in Cabinet.objects.filter(pk__in=cabinet_id_list):
             cabinet.documents.add(document)

mayan/apps/common/http.py

@@ -0,0 +1,36 @@
+from __future__ import unicode_literals
+
+from django.http import QueryDict
+from django.utils.encoding import force_bytes
+
+
+class URL(object):
+    def __init__(self, path=None, query_string=None):
+        self._path = path
+        self._query_string = query_string
+        kwargs = {'mutable': True}
+        if query_string:
+            kwargs['query_string'] = query_string.encode('utf-8')
+
+        self._args = QueryDict(**kwargs)
+
+    @property
+    def args(self):
+        return self._args
+
+    def to_string(self):
+        if self._args.keys():
+            query = force_bytes(
+                '?{}'.format(self._args.urlencode())
+            )
+        else:
+            query = ''
+
+        if self._path:
+            path = self._path
+        else:
+            path = ''
+
+        result = force_bytes('{}{}'.format(path, query))
+
+        return result

mayan/apps/common/utils.py

@@ -8,10 +8,6 @@ from django.core.exceptions import FieldDoesNotExist
 from django.db.models.constants import LOOKUP_SEP
 from django.urls import resolve as django_resolve
 from django.urls.base import get_script_prefix
-from django.utils.datastructures import MultiValueDict
-from django.utils.http import (
-    urlencode as django_urlencode, urlquote as django_urlquote
-)
 from django.utils.six.moves import reduce as reduce_function
 
 from mayan.apps.common.compat import dict_type, dictionary_type
@@ -150,50 +146,3 @@ def return_related(instance, related_field):
     using double underscore.
     """
     return reduce_function(getattr, related_field.split('__'), instance)
-
-
-def urlquote(link=None, get=None):
-    """
-    This method does both: urlquote() and urlencode()
-
-    urlqoute(): Quote special characters in 'link'
-
-    urlencode(): Map dictionary to query string key=value&...
-
-    HTML escaping is not done.
-
-    Example:
-
-    urlquote('/wiki/Python_(programming_language)')
-        --> '/wiki/Python_%28programming_language%29'
-    urlquote('/mypath/', {'key': 'value'})
-        --> '/mypath/?key=value'
-    urlquote('/mypath/', {'key': ['value1', 'value2']})
-        --> '/mypath/?key=value1&key=value2'
-    urlquote({'key': ['value1', 'value2']})
-        --> 'key=value1&key=value2'
-    """
-    if get is None:
-        get = []
-
-    assert link or get
-    if isinstance(link, dict):
-        # urlqoute({'key': 'value', 'key2': 'value2'}) -->
-        # key=value&key2=value2
-        assert not get, get
-        get = link
-        link = ''
-    assert isinstance(get, dict), 'wrong type "%s", dict required' % type(get)
-    # assert not (link.startswith('http://') or link.startswith('https://')),
-    #    'This method should only quote the url path.
-    #    It should not start with http(s)://  (%s)' % (
-    #    link)
-    if get:
-        # http://code.djangoproject.com/ticket/9089
-        if isinstance(get, MultiValueDict):
-            get = get.lists()
-        if link:
-            link = '%s?' % django_urlquote(link)
-        return '%s%s' % (link, django_urlencode(get, doseq=True))
-    else:
-        return django_urlquote(link)

mayan/apps/metadata/api.py

@@ -1,9 +1,8 @@
 from __future__ import unicode_literals
 
-from furl import furl
-
 from django.shortcuts import get_object_or_404
-from django.utils.encoding import force_bytes
+
+from mayan.apps.common.http import URL
 
 from .models import DocumentMetadata, MetadataType
 
@@ -19,7 +18,7 @@ def decode_metadata_from_querystring(querystring=None):
     metadata_list = []
     if querystring:
         # Match out of order metadata_type ids with metadata values from request
-        for key, value in furl(force_bytes(querystring)).args.items():
+        for key, value in URL(query_string=querystring).args.items():
             if 'metadata' in key:
                 index, element = key[8:].split('_')
                 metadata_dict[element][index] = value
@@ -27,10 +26,12 @@ def decode_metadata_from_querystring(querystring=None):
         # Convert the nested dictionary into a list of id+values dictionaries
         for order, identifier in metadata_dict['id'].items():
             if order in metadata_dict['value'].keys():
-                metadata_list.append({
-                    'id': identifier,
-                    'value': metadata_dict['value'][order]
-                })
+                metadata_list.append(
+                    {
+                        'id': identifier,
+                        'value': metadata_dict['value'][order]
+                    }
+                )
 
     return metadata_list
 

mayan/apps/metadata/tests/test_wizard_steps.py

@@ -1,9 +1,8 @@
 from __future__ import unicode_literals
 
-from furl import furl
-
 from django.urls import reverse
 
+from mayan.apps.common.http import URL
 from mayan.apps.documents.models import Document
 from mayan.apps.documents.permissions import permission_document_create
 from mayan.apps.documents.tests import (
@@ -35,7 +34,9 @@ class DocumentUploadMetadataTestCase(MetadataTypeTestMixin, GenericDocumentViewT
         )
 
     def test_upload_interactive_with_unicode_metadata(self):
-        url = furl(reverse(viewname='sources:upload_interactive'))
+        url = URL(
+            path=reverse(viewname='sources:upload_interactive')
+        )
         url.args['metadata0_id'] = self.test_metadata_type.pk
         url.args['metadata0_value'] = TEST_METADATA_VALUE_UNICODE
 
@@ -46,7 +47,7 @@ class DocumentUploadMetadataTestCase(MetadataTypeTestMixin, GenericDocumentViewT
         # Upload the test document
         with open(TEST_SMALL_DOCUMENT_PATH, mode='rb') as file_descriptor:
             response = self.post(
-                path=url, data={
+                path=url.to_string(), data={
                     'document-language': 'eng', 'source-file': file_descriptor,
                     'document_type_id': self.test_document_type.pk,
                 }
@@ -60,7 +61,9 @@ class DocumentUploadMetadataTestCase(MetadataTypeTestMixin, GenericDocumentViewT
         )
 
     def test_upload_interactive_with_ampersand_metadata(self):
-        url = furl(reverse(viewname='sources:upload_interactive'))
+        url = URL(
+            path=reverse(viewname='sources:upload_interactive')
+        )
         url.args['metadata0_id'] = self.test_metadata_type.pk
         url.args['metadata0_value'] = TEST_METADATA_VALUE_WITH_AMPERSAND
 
@@ -70,7 +73,7 @@ class DocumentUploadMetadataTestCase(MetadataTypeTestMixin, GenericDocumentViewT
         # Upload the test document
         with open(TEST_SMALL_DOCUMENT_PATH, mode='rb') as file_descriptor:
             response = self.post(
-                path=url, data={
+                path=url.to_string(), data={
                     'document-language': 'eng', 'source-file': file_descriptor,
                     'document_type_id': self.test_document_type.pk,
                 }

mayan/apps/sources/views.py

@@ -2,8 +2,6 @@ from __future__ import absolute_import, unicode_literals
 
 import logging
 
-from furl import furl
-
 from django.contrib import messages
 from django.http import HttpResponseRedirect, JsonResponse
 from django.shortcuts import get_object_or_404
@@ -257,9 +255,8 @@ class UploadInteractiveView(UploadBaseView):
             except Exception as exception:
                 messages.error(message=exception, request=self.request)
 
-            querystring = furl()
-            querystring.args.update(self.request.GET)
-            querystring.args.update(self.request.POST)
+            querystring = self.request.GET.copy()
+            querystring.update(self.request.POST)
 
             try:
                 task_source_handle_upload.apply_async(
@@ -271,7 +268,7 @@ class UploadInteractiveView(UploadBaseView):
                             filename=force_text(shared_uploaded_file)
                         ),
                         language=forms['document_form'].cleaned_data.get('language'),
-                        querystring=querystring.tostr(),
+                        querystring=querystring.urlencode(),
                         shared_uploaded_file_id=shared_uploaded_file.pk,
                         source_id=self.source.pk,
                         user_id=user_id,

mayan/apps/tags/tests/test_wizard_steps.py

@@ -33,9 +33,7 @@ class TaggedDocumentUploadTestCase(TagTestMixin, GenericDocumentViewTestCase):
                 }, data={
                     'document_type_id': self.test_document_type.pk,
                     'source-file': file_object,
-                    'tags': ','.join(
-                        map(str, Tag.objects.values_list('pk', flat=True))
-                    )
+                    'tags': Tag.objects.values_list('pk', flat=True)
                 }
             )
 

mayan/apps/tags/wizard_steps.py

@@ -1,11 +1,10 @@
 from __future__ import unicode_literals
 
-from furl import furl
-
 from django.apps import apps
 from django.utils.encoding import force_text
 from django.utils.translation import ugettext_lazy as _
 
+from mayan.apps.common.http import URL
 from mayan.apps.sources.wizards import WizardStep
 
 from .forms import TagMultipleSelectionForm
@@ -46,13 +45,9 @@ class WizardStepTags(WizardStep):
 
     @classmethod
     def step_post_upload_process(cls, document, querystring=None):
-        furl_instance = furl(querystring)
         Tag = apps.get_model(app_label='tags', model_name='Tag')
 
-        tag_id_list = furl_instance.args.get('tags', '')
-
-        if tag_id_list:
-            tag_id_list = tag_id_list.split(',')
+        tag_id_list = URL(query_string=querystring).args.getlist('tags')
 
         for tag in Tag.objects.filter(pk__in=tag_id_list):
             tag.documents.add(document)