diff --git a/HISTORY.rst b/HISTORY.rst index b62da7e331..1b77e729c7 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,3 +1,14 @@ +3.2.5 (2019-07-XX) +================== +* Don't error out if the EXTRA_APPS or the DISABLED_APPS settings + are set to blank. +* Update troubleshooting documentation topic. +* Add data migration to the file metadata app. Synchronizes the + document type settings model of existing document types. +* Fix cabinet and tags upload wizard steps missing some entries. + GitLab issue #632. Thanks to Matthias Urhahn (@d4rken) for the + report. + 3.2.4 (2019-06-29) ================== * Support configurable GUnicorn timeouts. Defaults to diff --git a/docs/releases/3.2.5.rst b/docs/releases/3.2.5.rst new file mode 100644 index 0000000000..0fd13140b5 --- /dev/null +++ b/docs/releases/3.2.5.rst @@ -0,0 +1,99 @@ +Version 3.2.4 +============= + +Released: July XX, 2019 + + +Changes +------- + +Removals +-------- + +- None + + +Upgrading from a previous version +--------------------------------- + +If installed via Python's PIP +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Remove deprecated requirements:: + + $ curl https://gitlab.com/mayan-edms/mayan-edms/raw/master/removals.txt | pip uninstall -r /dev/stdin + +Type in the console:: + + $ pip install mayan-edms==3.2.55555 + +the requirements will also be updated automatically. + + +Using Git +^^^^^^^^^ + +If you installed Mayan EDMS by cloning the Git repository issue the commands:: + + $ git reset --hard HEAD + $ git pull + +otherwise download the compressed archived and uncompress it overriding the +existing installation. + +Remove deprecated requirements:: + + $ pip uninstall -y -r removals.txt + +Next upgrade/add the new requirements:: + + $ pip install --upgrade -r requirements.txt + + +Common steps +^^^^^^^^^^^^ + +Perform these steps after updating the code from either step above. + +Make a backup of your supervisord file:: + + sudo cp /etc/supervisor/conf.d/mayan.conf /etc/supervisor/conf.d/mayan.conf.bck + +Update the supervisord configuration file. Replace the environment +variables values show here with your respective settings. This step will refresh +the supervisord configuration file with the new queues and the latest +recommended layout:: + + sudo MAYAN_DATABASE_ENGINE=django.db.backends.postgresql MAYAN_DATABASE_NAME=mayan \ + MAYAN_DATABASE_PASSWORD=mayanuserpass MAYAN_DATABASE_USER=mayan \ + MAYAN_DATABASE_HOST=127.0.0.1 MAYAN_MEDIA_ROOT=/opt/mayan-edms/media \ + /opt/mayan-edms/bin/mayan-edms.py platformtemplate supervisord > /etc/supervisor/conf.d/mayan.conf + +Edit the supervisord configuration file and update any setting the template +generator missed:: + + sudo vi /etc/supervisor/conf.d/mayan.conf + +Migrate existing database schema with:: + + $ mayan-edms.py performupgrade + +Add new static media:: + + $ mayan-edms.py preparestatic --noinput + +The upgrade procedure is now complete. + + +Backward incompatible changes +----------------------------- + +- None + + +Bugs fixed or issues closed +--------------------------- + +- :gitlab-issue:`632` Tags get lost when uploading through the webui + +.. _PyPI: https://pypi.python.org/pypi/mayan-edms/ diff --git a/docs/releases/index.rst b/docs/releases/index.rst index f1a003eb4f..80bdb19c1e 100644 --- a/docs/releases/index.rst +++ b/docs/releases/index.rst @@ -20,6 +20,7 @@ versions of the documentation contain the release notes for any later releases. .. toctree:: :maxdepth: 1 + 3.2.5 3.2.4 3.2.3 3.2.2 diff --git a/mayan/apps/cabinets/tests/test_wizard_steps.py b/mayan/apps/cabinets/tests/test_wizard_steps.py index d9feb3502d..bcdc7f30ce 100644 --- a/mayan/apps/cabinets/tests/test_wizard_steps.py +++ b/mayan/apps/cabinets/tests/test_wizard_steps.py @@ -1,7 +1,5 @@ from __future__ import unicode_literals -from django.utils.encoding import force_text - from mayan.apps.documents.models import Document from mayan.apps.documents.permissions import permission_document_create from mayan.apps.documents.tests import ( @@ -41,9 +39,7 @@ class CabinetDocumentUploadTestCase(CabinetTestMixin, GenericDocumentViewTestCas }, data={ 'document_type_id': self.test_document_type.pk, 'source-file': file_object, - 'cabinets': ','.join( - map(force_text, Cabinet.objects.values_list('pk', flat=True)) - ) + 'cabinets': Cabinet.objects.values_list('pk', flat=True) } ) diff --git a/mayan/apps/cabinets/wizard_steps.py b/mayan/apps/cabinets/wizard_steps.py index eaa1f78965..502db7aeb2 100644 --- a/mayan/apps/cabinets/wizard_steps.py +++ b/mayan/apps/cabinets/wizard_steps.py @@ -1,11 +1,10 @@ from __future__ import unicode_literals -from furl import furl - from django.apps import apps from django.utils.encoding import force_text from django.utils.translation import ugettext_lazy as _ +from mayan.apps.common.http import URL from mayan.apps.sources.wizards import WizardStep from .forms import CabinetListForm @@ -48,13 +47,8 @@ class WizardStepCabinets(WizardStep): @classmethod def step_post_upload_process(cls, document, querystring=None): - furl_instance = furl(querystring) Cabinet = apps.get_model(app_label='cabinets', model_name='Cabinet') - - cabinet_id_list = furl_instance.args.get('cabinets', '') - - if cabinet_id_list: - cabinet_id_list = cabinet_id_list.split(',') + cabinet_id_list = URL(query_string=querystring).args.getlist('cabinets') for cabinet in Cabinet.objects.filter(pk__in=cabinet_id_list): cabinet.documents.add(document) diff --git a/mayan/apps/common/http.py b/mayan/apps/common/http.py new file mode 100644 index 0000000000..f353dfb99a --- /dev/null +++ b/mayan/apps/common/http.py @@ -0,0 +1,36 @@ +from __future__ import unicode_literals + +from django.http import QueryDict +from django.utils.encoding import force_bytes + + +class URL(object): + def __init__(self, path=None, query_string=None): + self._path = path + self._query_string = query_string + kwargs = {'mutable': True} + if query_string: + kwargs['query_string'] = query_string.encode('utf-8') + + self._args = QueryDict(**kwargs) + + @property + def args(self): + return self._args + + def to_string(self): + if self._args.keys(): + query = force_bytes( + '?{}'.format(self._args.urlencode()) + ) + else: + query = '' + + if self._path: + path = self._path + else: + path = '' + + result = force_bytes('{}{}'.format(path, query)) + + return result diff --git a/mayan/apps/common/utils.py b/mayan/apps/common/utils.py index 58683c662f..2d76f17d43 100644 --- a/mayan/apps/common/utils.py +++ b/mayan/apps/common/utils.py @@ -8,10 +8,6 @@ from django.core.exceptions import FieldDoesNotExist from django.db.models.constants import LOOKUP_SEP from django.urls import resolve as django_resolve from django.urls.base import get_script_prefix -from django.utils.datastructures import MultiValueDict -from django.utils.http import ( - urlencode as django_urlencode, urlquote as django_urlquote -) from django.utils.six.moves import reduce as reduce_function from mayan.apps.common.compat import dict_type, dictionary_type @@ -150,50 +146,3 @@ def return_related(instance, related_field): using double underscore. """ return reduce_function(getattr, related_field.split('__'), instance) - - -def urlquote(link=None, get=None): - """ - This method does both: urlquote() and urlencode() - - urlqoute(): Quote special characters in 'link' - - urlencode(): Map dictionary to query string key=value&... - - HTML escaping is not done. - - Example: - - urlquote('/wiki/Python_(programming_language)') - --> '/wiki/Python_%28programming_language%29' - urlquote('/mypath/', {'key': 'value'}) - --> '/mypath/?key=value' - urlquote('/mypath/', {'key': ['value1', 'value2']}) - --> '/mypath/?key=value1&key=value2' - urlquote({'key': ['value1', 'value2']}) - --> 'key=value1&key=value2' - """ - if get is None: - get = [] - - assert link or get - if isinstance(link, dict): - # urlqoute({'key': 'value', 'key2': 'value2'}) --> - # key=value&key2=value2 - assert not get, get - get = link - link = '' - assert isinstance(get, dict), 'wrong type "%s", dict required' % type(get) - # assert not (link.startswith('http://') or link.startswith('https://')), - # 'This method should only quote the url path. - # It should not start with http(s):// (%s)' % ( - # link) - if get: - # http://code.djangoproject.com/ticket/9089 - if isinstance(get, MultiValueDict): - get = get.lists() - if link: - link = '%s?' % django_urlquote(link) - return '%s%s' % (link, django_urlencode(get, doseq=True)) - else: - return django_urlquote(link) diff --git a/mayan/apps/metadata/api.py b/mayan/apps/metadata/api.py index c70c6acad8..b2fb34d385 100644 --- a/mayan/apps/metadata/api.py +++ b/mayan/apps/metadata/api.py @@ -1,9 +1,8 @@ from __future__ import unicode_literals -from furl import furl - from django.shortcuts import get_object_or_404 -from django.utils.encoding import force_bytes + +from mayan.apps.common.http import URL from .models import DocumentMetadata, MetadataType @@ -19,7 +18,7 @@ def decode_metadata_from_querystring(querystring=None): metadata_list = [] if querystring: # Match out of order metadata_type ids with metadata values from request - for key, value in furl(force_bytes(querystring)).args.items(): + for key, value in URL(query_string=querystring).args.items(): if 'metadata' in key: index, element = key[8:].split('_') metadata_dict[element][index] = value @@ -27,10 +26,12 @@ def decode_metadata_from_querystring(querystring=None): # Convert the nested dictionary into a list of id+values dictionaries for order, identifier in metadata_dict['id'].items(): if order in metadata_dict['value'].keys(): - metadata_list.append({ - 'id': identifier, - 'value': metadata_dict['value'][order] - }) + metadata_list.append( + { + 'id': identifier, + 'value': metadata_dict['value'][order] + } + ) return metadata_list diff --git a/mayan/apps/metadata/tests/test_wizard_steps.py b/mayan/apps/metadata/tests/test_wizard_steps.py index 396c2cf514..e6b7644c3e 100644 --- a/mayan/apps/metadata/tests/test_wizard_steps.py +++ b/mayan/apps/metadata/tests/test_wizard_steps.py @@ -1,9 +1,8 @@ from __future__ import unicode_literals -from furl import furl - from django.urls import reverse +from mayan.apps.common.http import URL from mayan.apps.documents.models import Document from mayan.apps.documents.permissions import permission_document_create from mayan.apps.documents.tests import ( @@ -35,7 +34,9 @@ class DocumentUploadMetadataTestCase(MetadataTypeTestMixin, GenericDocumentViewT ) def test_upload_interactive_with_unicode_metadata(self): - url = furl(reverse(viewname='sources:upload_interactive')) + url = URL( + path=reverse(viewname='sources:upload_interactive') + ) url.args['metadata0_id'] = self.test_metadata_type.pk url.args['metadata0_value'] = TEST_METADATA_VALUE_UNICODE @@ -46,7 +47,7 @@ class DocumentUploadMetadataTestCase(MetadataTypeTestMixin, GenericDocumentViewT # Upload the test document with open(TEST_SMALL_DOCUMENT_PATH, mode='rb') as file_descriptor: response = self.post( - path=url, data={ + path=url.to_string(), data={ 'document-language': 'eng', 'source-file': file_descriptor, 'document_type_id': self.test_document_type.pk, } @@ -60,7 +61,9 @@ class DocumentUploadMetadataTestCase(MetadataTypeTestMixin, GenericDocumentViewT ) def test_upload_interactive_with_ampersand_metadata(self): - url = furl(reverse(viewname='sources:upload_interactive')) + url = URL( + path=reverse(viewname='sources:upload_interactive') + ) url.args['metadata0_id'] = self.test_metadata_type.pk url.args['metadata0_value'] = TEST_METADATA_VALUE_WITH_AMPERSAND @@ -70,7 +73,7 @@ class DocumentUploadMetadataTestCase(MetadataTypeTestMixin, GenericDocumentViewT # Upload the test document with open(TEST_SMALL_DOCUMENT_PATH, mode='rb') as file_descriptor: response = self.post( - path=url, data={ + path=url.to_string(), data={ 'document-language': 'eng', 'source-file': file_descriptor, 'document_type_id': self.test_document_type.pk, } diff --git a/mayan/apps/sources/views.py b/mayan/apps/sources/views.py index 0e5395bfd0..167877354f 100644 --- a/mayan/apps/sources/views.py +++ b/mayan/apps/sources/views.py @@ -2,8 +2,6 @@ from __future__ import absolute_import, unicode_literals import logging -from furl import furl - from django.contrib import messages from django.http import HttpResponseRedirect, JsonResponse from django.shortcuts import get_object_or_404 @@ -257,9 +255,8 @@ class UploadInteractiveView(UploadBaseView): except Exception as exception: messages.error(message=exception, request=self.request) - querystring = furl() - querystring.args.update(self.request.GET) - querystring.args.update(self.request.POST) + querystring = self.request.GET.copy() + querystring.update(self.request.POST) try: task_source_handle_upload.apply_async( @@ -271,7 +268,7 @@ class UploadInteractiveView(UploadBaseView): filename=force_text(shared_uploaded_file) ), language=forms['document_form'].cleaned_data.get('language'), - querystring=querystring.tostr(), + querystring=querystring.urlencode(), shared_uploaded_file_id=shared_uploaded_file.pk, source_id=self.source.pk, user_id=user_id, diff --git a/mayan/apps/tags/tests/test_wizard_steps.py b/mayan/apps/tags/tests/test_wizard_steps.py index d844bc78d9..a241b9725d 100644 --- a/mayan/apps/tags/tests/test_wizard_steps.py +++ b/mayan/apps/tags/tests/test_wizard_steps.py @@ -33,9 +33,7 @@ class TaggedDocumentUploadTestCase(TagTestMixin, GenericDocumentViewTestCase): }, data={ 'document_type_id': self.test_document_type.pk, 'source-file': file_object, - 'tags': ','.join( - map(str, Tag.objects.values_list('pk', flat=True)) - ) + 'tags': Tag.objects.values_list('pk', flat=True) } ) diff --git a/mayan/apps/tags/wizard_steps.py b/mayan/apps/tags/wizard_steps.py index 8026a2a60b..440638c75c 100644 --- a/mayan/apps/tags/wizard_steps.py +++ b/mayan/apps/tags/wizard_steps.py @@ -1,11 +1,10 @@ from __future__ import unicode_literals -from furl import furl - from django.apps import apps from django.utils.encoding import force_text from django.utils.translation import ugettext_lazy as _ +from mayan.apps.common.http import URL from mayan.apps.sources.wizards import WizardStep from .forms import TagMultipleSelectionForm @@ -46,13 +45,9 @@ class WizardStepTags(WizardStep): @classmethod def step_post_upload_process(cls, document, querystring=None): - furl_instance = furl(querystring) Tag = apps.get_model(app_label='tags', model_name='Tag') - tag_id_list = furl_instance.args.get('tags', '') - - if tag_id_list: - tag_id_list = tag_id_list.split(',') + tag_id_list = URL(query_string=querystring).args.getlist('tags') for tag in Tag.objects.filter(pk__in=tag_id_list): tag.documents.add(document)