diff --git a/HISTORY.rst b/HISTORY.rst index a8c5ac651c..2b453abbd6 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,4 +1,17 @@ -======= +3.2.5 (2019-07-XX) +================== +* Don't error out if the EXTRA_APPS or the DISABLED_APPS settings + are set to blank. +* Update troubleshooting documentation topic. +* Add data migration to the file metadata app. Synchronizes the + document type settings model of existing document types. +* Fix cabinet and tags upload wizard steps missing some entries. + GitLab issue #632. Thanks to Matthias Urhahn (@d4rken) for the + report. +* Add alert when settings are changed and util the installation + is restarted. GitLab issue #605. Thanks to + Vikas Kedia (@vikaskedia) to the report. + 3.2.4 (2019-06-29) ================== * Support configurable GUnicorn timeouts. Defaults to diff --git a/docs/releases/3.2.5.rst b/docs/releases/3.2.5.rst new file mode 100644 index 0000000000..4bfc90146b --- /dev/null +++ b/docs/releases/3.2.5.rst @@ -0,0 +1,113 @@ +Version 3.2.5 +============= + +Released: July XX, 2019 + + +Changes +------- + +- Don't error out if the EXTRA_APPS or the DISABLED_APPS settings + are set to blank. +- Update troubleshooting documentation topic. +- Add data migration to the file metadata app. Synchronizes the + document type settings model of existing document types. +- Fix cabinet and tags upload wizard steps missing some entries. + GitLab issue #632. Thanks to Matthias Urhahn (@d4rken) for the + report. +- Add alert when settings are changed and util the installation + is restarted. GitLab issue #605. Thanks to + Vikas Kedia (@vikaskedia) to the report. + +Removals +-------- + +- None + + +Upgrading from a previous version +--------------------------------- + +If installed via Python's PIP +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Remove deprecated requirements:: + + $ curl https://gitlab.com/mayan-edms/mayan-edms/raw/master/removals.txt | pip uninstall -r /dev/stdin + +Type in the console:: + + $ pip install mayan-edms==3.2.5 + +the requirements will also be updated automatically. + + +Using Git +^^^^^^^^^ + +If you installed Mayan EDMS by cloning the Git repository issue the commands:: + + $ git reset --hard HEAD + $ git pull + +otherwise download the compressed archived and uncompress it overriding the +existing installation. + +Remove deprecated requirements:: + + $ pip uninstall -y -r removals.txt + +Next upgrade/add the new requirements:: + + $ pip install --upgrade -r requirements.txt + + +Common steps +^^^^^^^^^^^^ + +Perform these steps after updating the code from either step above. + +Make a backup of your supervisord file:: + + sudo cp /etc/supervisor/conf.d/mayan.conf /etc/supervisor/conf.d/mayan.conf.bck + +Update the supervisord configuration file. Replace the environment +variables values show here with your respective settings. This step will refresh +the supervisord configuration file with the new queues and the latest +recommended layout:: + + sudo MAYAN_DATABASE_ENGINE=django.db.backends.postgresql MAYAN_DATABASE_NAME=mayan \ + MAYAN_DATABASE_PASSWORD=mayanuserpass MAYAN_DATABASE_USER=mayan \ + MAYAN_DATABASE_HOST=127.0.0.1 MAYAN_MEDIA_ROOT=/opt/mayan-edms/media \ + /opt/mayan-edms/bin/mayan-edms.py platformtemplate supervisord > /etc/supervisor/conf.d/mayan.conf + +Edit the supervisord configuration file and update any setting the template +generator missed:: + + sudo vi /etc/supervisor/conf.d/mayan.conf + +Migrate existing database schema with:: + + $ mayan-edms.py performupgrade + +Add new static media:: + + $ mayan-edms.py preparestatic --noinput + +The upgrade procedure is now complete. + + +Backward incompatible changes +----------------------------- + +- None + + +Bugs fixed or issues closed +--------------------------- + +- :gitlab-issue:`605` Project title fluctuates between default value and new value [Video] +- :gitlab-issue:`629` Cannot Upgrade to 3.2.X Docker Image +- :gitlab-issue:`632` Tags get lost when uploading through the webui + +.. _PyPI: https://pypi.python.org/pypi/mayan-edms/ diff --git a/docs/releases/index.rst b/docs/releases/index.rst index f1a003eb4f..80bdb19c1e 100644 --- a/docs/releases/index.rst +++ b/docs/releases/index.rst @@ -20,6 +20,7 @@ versions of the documentation contain the release notes for any later releases. .. toctree:: :maxdepth: 1 + 3.2.5 3.2.4 3.2.3 3.2.2 diff --git a/docs/topics/troubleshooting.rst b/docs/topics/troubleshooting.rst index d3ea0880e3..c8498acddc 100644 --- a/docs/topics/troubleshooting.rst +++ b/docs/topics/troubleshooting.rst @@ -168,3 +168,16 @@ files to a temporary directory on the same partition as the watchfolder first. Then move the files to the watchfolder. The move will be executed as an atomic operation and will prevent the files to be uploaded in the middle of the copying process. + +************ +Dependencies +************ + +Error: ``unable to execute 'x86_64-linux-gnu-gcc': No such file or directory`` +============================================================================== + +This happens when using the ``MAYAN_APT_INSTALLS`` feature. It means that the +``GCC`` package is required to compile the packages specified with +``MAYAN_APT_INSTALLS``. + +Solution: Include ``gcc`` in the list of packages specified with ``MAYAN_APT_INSTALLS``. diff --git a/mayan/apps/appearance/templates/appearance/base.html b/mayan/apps/appearance/templates/appearance/base.html index 3e04061e2c..09228b5fb5 100644 --- a/mayan/apps/appearance/templates/appearance/base.html +++ b/mayan/apps/appearance/templates/appearance/base.html @@ -34,6 +34,14 @@ {% endif %} {% block messages %} {% endblock %} + + {% smart_settings_check_changed as settings_changed %} + {% if settings_changed %} +
+ +

{% trans 'Warning' %} {% trans 'Settings updated, restart your installation for changes to take proper effect.' %}

+
+ {% endif %} diff --git a/mayan/apps/cabinets/tests/test_wizard_steps.py b/mayan/apps/cabinets/tests/test_wizard_steps.py index 5fa5e471c1..bcdc7f30ce 100644 --- a/mayan/apps/cabinets/tests/test_wizard_steps.py +++ b/mayan/apps/cabinets/tests/test_wizard_steps.py @@ -11,6 +11,7 @@ from mayan.apps.sources.tests.literals import ( ) from mayan.apps.sources.wizards import WizardStep +from ..models import Cabinet from ..wizard_steps import WizardStepCabinets from .mixins import CabinetTestMixin @@ -38,11 +39,12 @@ class CabinetDocumentUploadTestCase(CabinetTestMixin, GenericDocumentViewTestCas }, data={ 'document_type_id': self.test_document_type.pk, 'source-file': file_object, - 'cabinets': self.test_cabinet.pk + 'cabinets': Cabinet.objects.values_list('pk', flat=True) } ) def test_upload_interactive_view_with_access(self): + self._create_test_cabinet() self._create_test_cabinet() self.grant_access( obj=self.test_document_type, permission=permission_document_create @@ -51,7 +53,10 @@ class CabinetDocumentUploadTestCase(CabinetTestMixin, GenericDocumentViewTestCas self.assertEqual(response.status_code, 302) self.assertTrue( - self.test_cabinet in Document.objects.first().cabinets.all() + self.test_cabinets[0] in Document.objects.first().cabinets.all() + ) + self.assertTrue( + self.test_cabinets[1] in Document.objects.first().cabinets.all() ) def _request_wizard_view(self): diff --git a/mayan/apps/cabinets/wizard_steps.py b/mayan/apps/cabinets/wizard_steps.py index 230ad96564..502db7aeb2 100644 --- a/mayan/apps/cabinets/wizard_steps.py +++ b/mayan/apps/cabinets/wizard_steps.py @@ -1,11 +1,10 @@ from __future__ import unicode_literals -from furl import furl - from django.apps import apps from django.utils.encoding import force_text from django.utils.translation import ugettext_lazy as _ +from mayan.apps.common.http import URL from mayan.apps.sources.wizards import WizardStep from .forms import CabinetListForm @@ -48,10 +47,10 @@ class WizardStepCabinets(WizardStep): @classmethod def step_post_upload_process(cls, document, querystring=None): - furl_instance = furl(querystring) Cabinet = apps.get_model(app_label='cabinets', model_name='Cabinet') + cabinet_id_list = URL(query_string=querystring).args.getlist('cabinets') - for cabinet in Cabinet.objects.filter(pk__in=furl_instance.args.getlist('cabinets')): + for cabinet in Cabinet.objects.filter(pk__in=cabinet_id_list): cabinet.documents.add(document) diff --git a/mayan/apps/common/http.py b/mayan/apps/common/http.py new file mode 100644 index 0000000000..f353dfb99a --- /dev/null +++ b/mayan/apps/common/http.py @@ -0,0 +1,36 @@ +from __future__ import unicode_literals + +from django.http import QueryDict +from django.utils.encoding import force_bytes + + +class URL(object): + def __init__(self, path=None, query_string=None): + self._path = path + self._query_string = query_string + kwargs = {'mutable': True} + if query_string: + kwargs['query_string'] = query_string.encode('utf-8') + + self._args = QueryDict(**kwargs) + + @property + def args(self): + return self._args + + def to_string(self): + if self._args.keys(): + query = force_bytes( + '?{}'.format(self._args.urlencode()) + ) + else: + query = '' + + if self._path: + path = self._path + else: + path = '' + + result = force_bytes('{}{}'.format(path, query)) + + return result diff --git a/mayan/apps/common/utils.py b/mayan/apps/common/utils.py index 58683c662f..2d76f17d43 100644 --- a/mayan/apps/common/utils.py +++ b/mayan/apps/common/utils.py @@ -8,10 +8,6 @@ from django.core.exceptions import FieldDoesNotExist from django.db.models.constants import LOOKUP_SEP from django.urls import resolve as django_resolve from django.urls.base import get_script_prefix -from django.utils.datastructures import MultiValueDict -from django.utils.http import ( - urlencode as django_urlencode, urlquote as django_urlquote -) from django.utils.six.moves import reduce as reduce_function from mayan.apps.common.compat import dict_type, dictionary_type @@ -150,50 +146,3 @@ def return_related(instance, related_field): using double underscore. """ return reduce_function(getattr, related_field.split('__'), instance) - - -def urlquote(link=None, get=None): - """ - This method does both: urlquote() and urlencode() - - urlqoute(): Quote special characters in 'link' - - urlencode(): Map dictionary to query string key=value&... - - HTML escaping is not done. - - Example: - - urlquote('/wiki/Python_(programming_language)') - --> '/wiki/Python_%28programming_language%29' - urlquote('/mypath/', {'key': 'value'}) - --> '/mypath/?key=value' - urlquote('/mypath/', {'key': ['value1', 'value2']}) - --> '/mypath/?key=value1&key=value2' - urlquote({'key': ['value1', 'value2']}) - --> 'key=value1&key=value2' - """ - if get is None: - get = [] - - assert link or get - if isinstance(link, dict): - # urlqoute({'key': 'value', 'key2': 'value2'}) --> - # key=value&key2=value2 - assert not get, get - get = link - link = '' - assert isinstance(get, dict), 'wrong type "%s", dict required' % type(get) - # assert not (link.startswith('http://') or link.startswith('https://')), - # 'This method should only quote the url path. - # It should not start with http(s):// (%s)' % ( - # link) - if get: - # http://code.djangoproject.com/ticket/9089 - if isinstance(get, MultiValueDict): - get = get.lists() - if link: - link = '%s?' % django_urlquote(link) - return '%s%s' % (link, django_urlencode(get, doseq=True)) - else: - return django_urlquote(link) diff --git a/mayan/apps/file_metadata/migrations/0002_documenttypesettings.py b/mayan/apps/file_metadata/migrations/0002_documenttypesettings.py new file mode 100644 index 0000000000..fd0918c330 --- /dev/null +++ b/mayan/apps/file_metadata/migrations/0002_documenttypesettings.py @@ -0,0 +1,53 @@ +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +def operation_create_file_metadata_setting_for_existing_document_types(apps, schema_editor): + DocumentType = apps.get_model( + app_label='documents', model_name='DocumentType' + ) + DocumentTypeSettings = apps.get_model( + app_label='file_metadata', model_name='DocumentTypeSettings' + ) + + for document_type in DocumentType.objects.using(schema_editor.connection.alias).all(): + try: + DocumentTypeSettings.objects.using( + schema_editor.connection.alias + ).get_or_create(document_type=document_type) + except DocumentTypeSettings.DoesNotExist: + pass + + +def operation_delete_file_metadata_setting_for_existing_document_types(apps, schema_editor): + DocumentType = apps.get_model( + app_label='documents', model_name='DocumentType' + ) + DocumentTypeSettings = apps.get_model( + app_label='file_metadata', model_name='DocumentTypeSettings' + ) + + for document_type in DocumentType.objects.using(schema_editor.connection.alias).all(): + try: + DocumentTypeSettings.objects.using( + schema_editor.connection.alias + ).get(document_type=document_type).delete() + except DocumentTypeSettings.DoesNotExist: + pass + + +class Migration(migrations.Migration): + + dependencies = [ + ('documents', '0047_auto_20180917_0737'), + ('file_metadata', '0001_initial'), + ] + + operations = [ + migrations.RunPython( + code=operation_create_file_metadata_setting_for_existing_document_types, + reverse_code=operation_delete_file_metadata_setting_for_existing_document_types, + ) + ] diff --git a/mayan/apps/linking/apps.py b/mayan/apps/linking/apps.py index aca8761959..426a4eccaa 100644 --- a/mayan/apps/linking/apps.py +++ b/mayan/apps/linking/apps.py @@ -57,9 +57,7 @@ class LinkingApp(MayanAppConfig): SmartLinkCondition = self.get_model(model_name='SmartLinkCondition') ModelEventType.register( - event_types=( - event_smart_link_edited, - ), model=SmartLink + event_types=(event_smart_link_edited,), model=SmartLink ) ModelPermission.register( diff --git a/mayan/apps/metadata/api.py b/mayan/apps/metadata/api.py index c70c6acad8..b2fb34d385 100644 --- a/mayan/apps/metadata/api.py +++ b/mayan/apps/metadata/api.py @@ -1,9 +1,8 @@ from __future__ import unicode_literals -from furl import furl - from django.shortcuts import get_object_or_404 -from django.utils.encoding import force_bytes + +from mayan.apps.common.http import URL from .models import DocumentMetadata, MetadataType @@ -19,7 +18,7 @@ def decode_metadata_from_querystring(querystring=None): metadata_list = [] if querystring: # Match out of order metadata_type ids with metadata values from request - for key, value in furl(force_bytes(querystring)).args.items(): + for key, value in URL(query_string=querystring).args.items(): if 'metadata' in key: index, element = key[8:].split('_') metadata_dict[element][index] = value @@ -27,10 +26,12 @@ def decode_metadata_from_querystring(querystring=None): # Convert the nested dictionary into a list of id+values dictionaries for order, identifier in metadata_dict['id'].items(): if order in metadata_dict['value'].keys(): - metadata_list.append({ - 'id': identifier, - 'value': metadata_dict['value'][order] - }) + metadata_list.append( + { + 'id': identifier, + 'value': metadata_dict['value'][order] + } + ) return metadata_list diff --git a/mayan/apps/metadata/tests/test_wizard_steps.py b/mayan/apps/metadata/tests/test_wizard_steps.py index 396c2cf514..e6b7644c3e 100644 --- a/mayan/apps/metadata/tests/test_wizard_steps.py +++ b/mayan/apps/metadata/tests/test_wizard_steps.py @@ -1,9 +1,8 @@ from __future__ import unicode_literals -from furl import furl - from django.urls import reverse +from mayan.apps.common.http import URL from mayan.apps.documents.models import Document from mayan.apps.documents.permissions import permission_document_create from mayan.apps.documents.tests import ( @@ -35,7 +34,9 @@ class DocumentUploadMetadataTestCase(MetadataTypeTestMixin, GenericDocumentViewT ) def test_upload_interactive_with_unicode_metadata(self): - url = furl(reverse(viewname='sources:upload_interactive')) + url = URL( + path=reverse(viewname='sources:upload_interactive') + ) url.args['metadata0_id'] = self.test_metadata_type.pk url.args['metadata0_value'] = TEST_METADATA_VALUE_UNICODE @@ -46,7 +47,7 @@ class DocumentUploadMetadataTestCase(MetadataTypeTestMixin, GenericDocumentViewT # Upload the test document with open(TEST_SMALL_DOCUMENT_PATH, mode='rb') as file_descriptor: response = self.post( - path=url, data={ + path=url.to_string(), data={ 'document-language': 'eng', 'source-file': file_descriptor, 'document_type_id': self.test_document_type.pk, } @@ -60,7 +61,9 @@ class DocumentUploadMetadataTestCase(MetadataTypeTestMixin, GenericDocumentViewT ) def test_upload_interactive_with_ampersand_metadata(self): - url = furl(reverse(viewname='sources:upload_interactive')) + url = URL( + path=reverse(viewname='sources:upload_interactive') + ) url.args['metadata0_id'] = self.test_metadata_type.pk url.args['metadata0_value'] = TEST_METADATA_VALUE_WITH_AMPERSAND @@ -70,7 +73,7 @@ class DocumentUploadMetadataTestCase(MetadataTypeTestMixin, GenericDocumentViewT # Upload the test document with open(TEST_SMALL_DOCUMENT_PATH, mode='rb') as file_descriptor: response = self.post( - path=url, data={ + path=url.to_string(), data={ 'document-language': 'eng', 'source-file': file_descriptor, 'document_type_id': self.test_document_type.pk, } diff --git a/mayan/apps/smart_settings/classes.py b/mayan/apps/smart_settings/classes.py index d5cc477095..ccedbd5083 100644 --- a/mayan/apps/smart_settings/classes.py +++ b/mayan/apps/smart_settings/classes.py @@ -1,6 +1,7 @@ from __future__ import unicode_literals import errno +import hashlib from importlib import import_module import logging import os @@ -78,6 +79,7 @@ class Namespace(object): @python_2_unicode_compatible class Setting(object): _registry = {} + _cache_hash = None @staticmethod def deserialize_value(value): @@ -108,6 +110,13 @@ class Setting(object): return result + @classmethod + def check_changed(cls): + if not cls._cache_hash: + cls._cache_hash = cls.get_hash() + + return cls._cache_hash != cls.get_hash() + @classmethod def dump_data(cls, filter_term=None, namespace=None): dictionary = {} @@ -129,6 +138,12 @@ class Setting(object): def get_all(cls): return sorted(cls._registry.values(), key=lambda x: x.global_name) + @classmethod + def get_hash(cls): + return force_text( + hashlib.sha256(cls.dump_data()).hexdigest() + ) + @classmethod def save_configuration(cls, path=settings.CONFIGURATION_FILEPATH): try: diff --git a/mayan/apps/smart_settings/templatetags/smart_settings_tags.py b/mayan/apps/smart_settings/templatetags/smart_settings_tags.py index 38f7743c0a..6ef94c12f3 100644 --- a/mayan/apps/smart_settings/templatetags/smart_settings_tags.py +++ b/mayan/apps/smart_settings/templatetags/smart_settings_tags.py @@ -10,3 +10,9 @@ register = Library() @register.simple_tag def smart_setting(global_name): return Setting.get(global_name=global_name).value + + +@register.simple_tag +def smart_settings_check_changed(): + return Setting.check_changed() + diff --git a/mayan/apps/sources/views.py b/mayan/apps/sources/views.py index 0e5395bfd0..167877354f 100644 --- a/mayan/apps/sources/views.py +++ b/mayan/apps/sources/views.py @@ -2,8 +2,6 @@ from __future__ import absolute_import, unicode_literals import logging -from furl import furl - from django.contrib import messages from django.http import HttpResponseRedirect, JsonResponse from django.shortcuts import get_object_or_404 @@ -257,9 +255,8 @@ class UploadInteractiveView(UploadBaseView): except Exception as exception: messages.error(message=exception, request=self.request) - querystring = furl() - querystring.args.update(self.request.GET) - querystring.args.update(self.request.POST) + querystring = self.request.GET.copy() + querystring.update(self.request.POST) try: task_source_handle_upload.apply_async( @@ -271,7 +268,7 @@ class UploadInteractiveView(UploadBaseView): filename=force_text(shared_uploaded_file) ), language=forms['document_form'].cleaned_data.get('language'), - querystring=querystring.tostr(), + querystring=querystring.urlencode(), shared_uploaded_file_id=shared_uploaded_file.pk, source_id=self.source.pk, user_id=user_id, diff --git a/mayan/apps/tags/tests/test_wizard_steps.py b/mayan/apps/tags/tests/test_wizard_steps.py index d844bc78d9..a241b9725d 100644 --- a/mayan/apps/tags/tests/test_wizard_steps.py +++ b/mayan/apps/tags/tests/test_wizard_steps.py @@ -33,9 +33,7 @@ class TaggedDocumentUploadTestCase(TagTestMixin, GenericDocumentViewTestCase): }, data={ 'document_type_id': self.test_document_type.pk, 'source-file': file_object, - 'tags': ','.join( - map(str, Tag.objects.values_list('pk', flat=True)) - ) + 'tags': Tag.objects.values_list('pk', flat=True) } ) diff --git a/mayan/apps/tags/wizard_steps.py b/mayan/apps/tags/wizard_steps.py index 8026a2a60b..440638c75c 100644 --- a/mayan/apps/tags/wizard_steps.py +++ b/mayan/apps/tags/wizard_steps.py @@ -1,11 +1,10 @@ from __future__ import unicode_literals -from furl import furl - from django.apps import apps from django.utils.encoding import force_text from django.utils.translation import ugettext_lazy as _ +from mayan.apps.common.http import URL from mayan.apps.sources.wizards import WizardStep from .forms import TagMultipleSelectionForm @@ -46,13 +45,9 @@ class WizardStepTags(WizardStep): @classmethod def step_post_upload_process(cls, document, querystring=None): - furl_instance = furl(querystring) Tag = apps.get_model(app_label='tags', model_name='Tag') - tag_id_list = furl_instance.args.get('tags', '') - - if tag_id_list: - tag_id_list = tag_id_list.split(',') + tag_id_list = URL(query_string=querystring).args.getlist('tags') for tag in Tag.objects.filter(pk__in=tag_id_list): tag.documents.add(document)