Move post document upload processing of metadata and tags from sources.model to each wizard step.

Signed-off-by: Michael Price <loneviking72@gmail.com>

mayan/apps/metadata/apps.py

@@ -58,6 +58,8 @@ class MetadataApp(MayanAppConfig):
     def ready(self):
         super(MetadataApp, self).ready()
 
+        from .wizard_steps import WizardStepMetadata  # NOQA
+
         Document = apps.get_model(
             app_label='documents', model_name='Document'
         )

mayan/apps/metadata/wizard_steps.py

@@ -0,0 +1,68 @@
+from __future__ import unicode_literals
+
+from django.utils.translation import ugettext_lazy as _
+
+from metadata.api import decode_metadata_from_url, save_metadata_list
+from metadata.forms import DocumentMetadataFormSet
+
+from sources.wizards import WizardStep, WizardStepDocumentType
+
+
+class WizardStepMetadata(WizardStep):
+    form_class = DocumentMetadataFormSet
+    label = _('Enter document metadata')
+    name = 'metadata_entry'
+    number = 1
+
+    @classmethod
+    def condition(cls, wizard):
+        """
+        Skip step if document type has no associated metadata
+        """
+        cleaned_data = wizard.get_cleaned_data_for_step(WizardStepDocumentType.name) or {}
+
+        document_type = cleaned_data.get('document_type')
+
+        if document_type:
+            return document_type.metadata.exists()
+
+    @classmethod
+    def get_form_initial(cls, wizard):
+        initial = []
+
+        step_data = wizard.get_cleaned_data_for_step(WizardStepDocumentType.name)
+        if step_data:
+            document_type = step_data['document_type']
+            for document_type_metadata_type in document_type.metadata.all():
+                initial.append(
+                    {
+                        'document_type': document_type,
+                        'metadata_type': document_type_metadata_type.metadata_type,
+                    }
+                )
+
+        return initial
+
+    @classmethod
+    def done(cls, wizard):
+        result = {}
+        cleaned_data = wizard.get_cleaned_data_for_step(cls.name)
+        if cleaned_data:
+            for identifier, metadata in enumerate(wizard.get_cleaned_data_for_step(cls.name)):
+                if metadata.get('update'):
+                    result['metadata%s_id' % identifier] = metadata['id']
+                    result['metadata%s_value' % identifier] = metadata['value']
+
+        return result
+
+    @classmethod
+    def step_post_upload_process(cls, document, request_data=None):
+        metadata_dict_list = decode_metadata_from_url(url_dict=request_data)
+        if metadata_dict_list:
+            save_metadata_list(
+                metadata_list=metadata_dict_list, document=document,
+                create=True
+            )
+
+
+WizardStep.register(WizardStepMetadata)

mayan/apps/sources/models.py

@@ -34,9 +34,8 @@ from converter.models import Transformation
 from djcelery.models import PeriodicTask, IntervalSchedule
 from documents.models import Document, DocumentType
 from documents.settings import setting_language
-from metadata.api import save_metadata_list, set_bulk_metadata
+from metadata.api import set_bulk_metadata
 from metadata.models import MetadataType
-from tags.models import Tag
 
 from .classes import Attachment, PseudoFile, SourceUploadedFile, StagingFile
 from .exceptions import SourceException
@@ -52,6 +51,7 @@ from .literals import (
     SOURCE_CHOICE_EMAIL_POP3, SOURCE_CHOICE_SANE_SCANNER,
 )
 from .settings import setting_scanimage_path
+from .wizards import WizardStep
 
 logger = logging.getLogger(__name__)
 
@@ -82,7 +82,7 @@ class Source(models.Model):
     def fullname(self):
         return ' '.join([self.class_fullname(), '"%s"' % self.label])
 
-    def handle_upload(self, file_object, description=None, document_type=None, expand=False, label=None, language=None, metadata_dict_list=None, metadata_dictionary=None, tag_ids=None, user=None):
+    def handle_upload(self, file_object, description=None, document_type=None, expand=False, label=None, language=None, user=None):
         """
         Handle an upload request from a file object which may be an individual
         document or a compressed file containing multiple documents.
@@ -93,8 +93,6 @@ class Source(models.Model):
         kwargs = {
             'description': description, 'document_type': document_type,
             'label': label, 'language': language,
-            'metadata_dict_list': metadata_dict_list,
-            'metadata_dictionary': metadata_dictionary, 'tag_ids': tag_ids,
             'user': user
         }
 
@@ -103,22 +101,22 @@ class Source(models.Model):
                 compressed_file = CompressedFile(file_object)
                 for compressed_file_child in compressed_file.children():
                     kwargs.update({'label': force_text(compressed_file_child)})
-                    self.upload_document(
+                    return self.upload_document(
                         file_object=File(compressed_file_child), **kwargs
                     )
                     compressed_file_child.close()
 
             except NotACompressedFile:
                 logging.debug('Exception: NotACompressedFile')
-                self.upload_document(file_object=file_object, **kwargs)
+                return self.upload_document(file_object=file_object, **kwargs)
         else:
-            self.upload_document(file_object=file_object, **kwargs)
+            return self.upload_document(file_object=file_object, **kwargs)
 
     def get_upload_file_object(self, form_data):
         pass
         # TODO: Should raise NotImplementedError?
 
-    def upload_document(self, file_object, document_type, description=None, label=None, language=None, metadata_dict_list=None, metadata_dictionary=None, tag_ids=None, user=None):
+    def upload_document(self, file_object, document_type, description=None, label=None, language=None, request_data=None, user=None):
         """
         Upload an individual document
         """
@@ -150,20 +148,6 @@ class Source(models.Model):
                     source=self, targets=document_version.pages.all()
                 )
 
-                if metadata_dict_list:
-                    save_metadata_list(
-                        metadata_dict_list, document, create=True
-                    )
-
-                if metadata_dictionary:
-                    set_bulk_metadata(
-                        document=document,
-                        metadata_dictionary=metadata_dictionary
-                    )
-
-                if tag_ids:
-                    for tag in Tag.objects.filter(pk__in=tag_ids):
-                        tag.documents.add(document)
             except Exception as exception:
                 logger.critical(
                     'Unexpected exception while trying to create version for '
@@ -172,6 +156,11 @@ class Source(models.Model):
                 )
                 document.delete(to_trash=False)
                 raise
+            else:
+                WizardStep.post_upload_process(
+                    document=document, request_data=request_data
+                )
+                return document
 
 
 class InteractiveSource(Source):
@@ -629,13 +618,18 @@ class EmailBaseModel(IntervalBaseModel):
                             'Got metadata dictionary: %s', metadata_dictionary
                         )
                     else:
-                        source.handle_upload(
+                        document = source.handle_upload(
                             document_type=source.document_type,
                             file_object=file_object, label=filename,
                             expand=(
                                 source.uncompress == SOURCE_UNCOMPRESS_CHOICE_Y
-                            ), metadata_dictionary=metadata_dictionary
+                            )
                         )
+                        if metadata_dictionary:
+                            set_bulk_metadata(
+                                document=document,
+                                metadata_dictionary=metadata_dictionary
+                            )
             else:
                 logger.debug('No Content-Disposition')
 
@@ -646,12 +640,16 @@ class EmailBaseModel(IntervalBaseModel):
                 if content_type == 'text/plain' and source.store_body:
                     content = part.get_payload(decode=True).decode(part.get_content_charset())
                     with ContentFile(content=content, name='email_body.txt') as file_object:
-                        source.handle_upload(
+                        document = source.handle_upload(
                             document_type=source.document_type,
                             file_object=file_object,
                             expand=SOURCE_UNCOMPRESS_CHOICE_N, label='email_body.txt',
-                            metadata_dictionary=metadata_dictionary
                         )
+                        if metadata_dictionary:
+                            set_bulk_metadata(
+                                document=document,
+                                metadata_dictionary=metadata_dictionary
+                            )
 
 
 class POP3Email(EmailBaseModel):

mayan/apps/sources/tasks.py

@@ -36,7 +36,7 @@ def task_check_interval_source(source_id):
 
 
 @app.task(bind=True, default_retry_delay=DEFAULT_SOURCE_TASK_RETRY_DELAY, ignore_result=True)
-def task_upload_document(self, source_id, document_type_id, shared_uploaded_file_id, description=None, label=None, language=None, metadata_dict_list=None, tag_ids=None, user_id=None):
+def task_upload_document(self, source_id, document_type_id, shared_uploaded_file_id, description=None, label=None, language=None, request_data=None, user_id=None):
     SharedUploadedFile = apps.get_model(
         app_label='common', model_name='SharedUploadedFile'
     )
@@ -65,8 +65,7 @@ def task_upload_document(self, source_id, document_type_id, shared_uploaded_file
             source.upload_document(
                 file_object=file_object, document_type=document_type,
                 description=description, label=label, language=language,
-                metadata_dict_list=metadata_dict_list, user=user,
+                request_data=request_data, user=user,
-                tag_ids=tag_ids
             )
 
     except OperationalError as exception:
@@ -87,7 +86,7 @@ def task_upload_document(self, source_id, document_type_id, shared_uploaded_file
 
 
 @app.task(bind=True, default_retry_delay=DEFAULT_SOURCE_TASK_RETRY_DELAY, ignore_result=True)
-def task_source_handle_upload(self, document_type_id, shared_uploaded_file_id, source_id, description=None, expand=False, label=None, language=None, metadata_dict_list=None, skip_list=None, tag_ids=None, user_id=None):
+def task_source_handle_upload(self, document_type_id, shared_uploaded_file_id, source_id, description=None, expand=False, label=None, language=None, skip_list=None, request_data=None, user_id=None):
     SharedUploadedFile = apps.get_model(
         app_label='common', model_name='SharedUploadedFile'
     )
@@ -115,8 +114,7 @@ def task_source_handle_upload(self, document_type_id, shared_uploaded_file_id, s
     kwargs = {
         'description': description, 'document_type_id': document_type.pk,
         'label': label, 'language': language,
-        'metadata_dict_list': metadata_dict_list,
+        'source_id': source_id, 'request_data': request_data, 'user_id': user_id
-        'source_id': source_id, 'tag_ids': tag_ids, 'user_id': user_id
     }
 
     if not skip_list:
@@ -144,14 +142,15 @@ def task_source_handle_upload(self, document_type_id, shared_uploaded_file_id, s
                                 'child document: %s. Rescheduling.', exception
                             )
 
+                            # TODO: Don't call the task itself again
+                            # Update to use celery's retry feature
                             task_source_handle_upload.delay(
                                 document_type_id=document_type_id,
                                 shared_uploaded_file_id=shared_uploaded_file_id,
                                 source_id=source_id, description=description,
                                 expand=expand, label=label,
                                 language=language,
-                                metadata_dict_list=metadata_dict_list,
+                                skip_list=skip_list, request_data=request_data,
-                                skip_list=skip_list, tag_ids=tag_ids,
                                 user_id=user_id
                             )
                             return

mayan/apps/sources/views.py

@@ -22,7 +22,6 @@ from documents.permissions import (
     permission_document_create, permission_document_new_version
 )
 from documents.tasks import task_upload_new_version
-from metadata.api import decode_metadata_from_url
 from navigation import Link
 
 from .exceptions import SourceException
@@ -249,10 +248,9 @@ class UploadInteractiveView(UploadBaseView):
                     expand=expand,
                     label=label,
                     language=forms['document_form'].cleaned_data.get('language'),
-                    metadata_dict_list=decode_metadata_from_url(self.request.GET),
+                    request_data=self.request.GET,
                     shared_uploaded_file_id=shared_uploaded_file.pk,
                     source_id=self.source.pk,
-                    tag_ids=self.request.GET.getlist('tags'),
                     user_id=user_id,
                 )
             )

mayan/apps/sources/wizards.py

@@ -1,22 +1,16 @@
 from __future__ import unicode_literals
 
+from django.apps import apps
 from django.contrib import messages
 from django.http import HttpResponseRedirect
 from django.urls import reverse
 from django.utils.decorators import classonlymethod
-from django.utils.encoding import force_text
 from django.utils.http import urlencode
 from django.utils.translation import ugettext_lazy as _
 
 from formtools.wizard.views import SessionWizardView
 
-from common.mixins import ViewPermissionCheckMixin
 from documents.forms import DocumentTypeSelectForm
-from metadata.forms import DocumentMetadataFormSet
-from tags.forms import TagMultipleSelectionForm
-from tags.models import Tag
-
-from .models import InteractiveSource
 
 
 class WizardStep(object):
@@ -52,10 +46,19 @@ class WizardStep(object):
     def get_form_kwargs(cls, wizard):
         return {}
 
+    @classmethod
+    def post_upload_process(cls, document, request_data=None):
+        for step in cls.get_all():
+            step.step_post_upload_process(document=document, request_data=request_data)
+
     @classmethod
     def register(cls, step):
         cls._registry[step.name] = step
 
+    @classmethod
+    def step_post_upload_process(cls, document, request_data=None):
+        pass
+
 
 class WizardStepDocumentType(WizardStep):
     form_class = DocumentTypeSelectForm
@@ -80,89 +83,10 @@ class WizardStepDocumentType(WizardStep):
         return {'user': wizard.request.user}
 
 
-class WizardStepMetadata(WizardStep):
-    form_class = DocumentMetadataFormSet
-    label = _('Enter document metadata')
-    name = 'metadata_entry'
-    number = 1
-
-    @classmethod
-    def condition(cls, wizard):
-        """
-        Skip step if document type has no associated metadata
-        """
-        cleaned_data = wizard.get_cleaned_data_for_step(WizardStepDocumentType.name) or {}
-
-        document_type = cleaned_data.get('document_type')
-
-        if document_type:
-            return document_type.metadata.exists()
-
-    @classmethod
-    def get_form_initial(cls, wizard):
-        initial = []
-
-        step_data = wizard.get_cleaned_data_for_step(WizardStepDocumentType.name)
-        if step_data:
-            document_type = step_data['document_type']
-            for document_type_metadata_type in document_type.metadata.all():
-                initial.append(
-                    {
-                        'document_type': document_type,
-                        'metadata_type': document_type_metadata_type.metadata_type,
-                    }
-                )
-
-        return initial
-
-    @classmethod
-    def done(cls, wizard):
-        result = {}
-        cleaned_data = wizard.get_cleaned_data_for_step(cls.name)
-        if cleaned_data:
-            for identifier, metadata in enumerate(wizard.get_cleaned_data_for_step(cls.name)):
-                if metadata.get('update'):
-                    result['metadata%s_id' % identifier] = metadata['id']
-                    result['metadata%s_value' % identifier] = metadata['value']
-
-        return result
-
-
-class WizardStepTags(WizardStep):
-    form_class = TagMultipleSelectionForm
-    label = _('Select tags')
-    name = 'tag_selection'
-    number = 2
-
-    @classmethod
-    def condition(cls, wizard):
-        return Tag.objects.exists()
-
-    @classmethod
-    def get_form_kwargs(self, wizard):
-        return {
-            'help_text': _('Tags to be attached.'),
-            'user': wizard.request.user
-        }
-
-    @classmethod
-    def done(cls, wizard):
-        result = {}
-        cleaned_data = wizard.get_cleaned_data_for_step(cls.name)
-        if cleaned_data:
-            result['tags'] = [
-                force_text(tag.pk) for tag in cleaned_data['tags']
-            ]
-
-        return result
-
-
 WizardStep.register(WizardStepDocumentType)
-WizardStep.register(WizardStepMetadata)
-WizardStep.register(WizardStepTags)
 
 
-class DocumentCreateWizard(ViewPermissionCheckMixin, SessionWizardView):
+class DocumentCreateWizard(SessionWizardView):
     template_name = 'appearance/generic_wizard.html'
 
     @classonlymethod
@@ -172,6 +96,16 @@ class DocumentCreateWizard(ViewPermissionCheckMixin, SessionWizardView):
         return super(DocumentCreateWizard, cls).as_view(*args, **kwargs)
 
     def dispatch(self, request, *args, **kwargs):
+        InteractiveSource = apps.get_model(
+            app_label='sources', model_name='InteractiveSource'
+        )
+
+        form_list = WizardStep.get_choices(attribute_name='form_class')
+        condition_dict = dict(WizardStep.get_choices(attribute_name='condition'))
+        result = self.__class__.get_initkwargs(form_list=form_list, condition_dict=condition_dict)
+        self.form_list = result['form_list']
+        self.condition_dict = result['condition_dict']
+
         if not InteractiveSource.objects.filter(enabled=True).exists():
             messages.error(
                 request,

mayan/apps/tags/apps.py

@@ -38,6 +38,8 @@ class TagsApp(MayanAppConfig):
         super(TagsApp, self).ready()
         from actstream import registry
 
+        from .wizard_steps import WizardStepTags  # NOQA
+
         Document = apps.get_model(
             app_label='documents', model_name='Document'
         )

mayan/apps/tags/wizard_steps.py

@@ -0,0 +1,54 @@
+from __future__ import unicode_literals
+
+from furl import furl
+
+from django.apps import apps
+from django.utils.encoding import force_text
+from django.utils.translation import ugettext_lazy as _
+
+from sources.wizards import WizardStep
+
+from .forms import TagMultipleSelectionForm
+
+
+class WizardStepTags(WizardStep):
+    form_class = TagMultipleSelectionForm
+    label = _('Select tags')
+    name = 'tag_selection'
+    number = 2
+
+    @classmethod
+    def condition(cls, wizard):
+        Tag = apps.get_model(app_label='tags', model_name='Tag')
+        return Tag.objects.exists()
+
+    @classmethod
+    def get_form_kwargs(self, wizard):
+        return {
+            'help_text': _('Tags to be attached.'),
+            'user': wizard.request.user
+        }
+
+    @classmethod
+    def done(cls, wizard):
+        result = {}
+        cleaned_data = wizard.get_cleaned_data_for_step(cls.name)
+        if cleaned_data:
+            result['tags'] = [
+                force_text(tag.pk) for tag in cleaned_data['tags']
+            ]
+
+        return result
+
+    @classmethod
+    def step_post_upload_process(cls, document, request_data=None):
+        f = furl(request_data)
+        f.args.getlist('tags')
+
+        Tag = apps.get_model(app_label='tags', model_name='Tag')
+
+        for tag in Tag.objects.filter(pk__in=request_data.getlist('tags')):
+            tag.documents.add(document)
+
+
+WizardStep.register(WizardStepTags)