Assign tasks to specific queues. Add support for transient queues. gh-issue #222, gh-issue #230.

mayan/apps/sources/apps.py

@@ -2,6 +2,8 @@ from __future__ import absolute_import, unicode_literals
 
 from django.utils.translation import ugettext_lazy as _
 
+from kombu import Exchange, Queue
+
 from common import (
     MayanAppConfig, MissingItem, menu_front_page, menu_object, menu_secondary,
     menu_sidebar, menu_setup
@@ -11,6 +13,7 @@ from common.utils import encapsulate
 from converter.links import link_transformation_list
 from documents.models import Document
 from documents.signals import post_version_upload
+from mayan.celery import app
 from navigation import SourceColumn
 from rest_api.classes import APIEndPoint
 
@@ -44,6 +47,24 @@ class SourcesApp(MayanAppConfig):
 
         SourceColumn(source=StagingFile, label=_('Thumbnail'), attribute=encapsulate(lambda staging_file: staging_file_thumbnail(staging_file, gallery_name='sources:staging_list', title=staging_file.filename, size='100')))
 
+        app.conf.CELERY_QUEUES.extend(
+            (
+                Queue('sources', Exchange('sources'), routing_key='sources'),
+                Queue('sources_periodic', Exchange('sources_periodic'), routing_key='sources_periodic', delivery_mode=1),
+            )
+        )
+
+        app.conf.CELERY_ROUTES.update(
+            {
+                'sources.tasks.task_check_interval_source': {
+                    'queue': 'sources_periodic'
+                },
+                'sources.tasks.task_source_upload_document': {
+                    'queue': 'sources'
+                },
+            }
+        )
+
         menu_front_page.bind_links(links=[link_document_create_multiple])
         menu_object.bind_links(links=[link_document_create_siblings], sources=[Document])
         menu_object.bind_links(links=[link_setup_source_edit, link_setup_source_delete, link_transformation_list, link_setup_source_logs], sources=[Source])

mayan/apps/sources/models.py

@@ -206,7 +206,6 @@ class IntervalBaseModel(OutOfProcessSource):
             name=self._get_periodic_task_name(),
             interval=interval_instance,
             task='sources.tasks.task_check_interval_source',
-            queue='uploads',
             kwargs=json.dumps({'source_id': self.pk})
         )
 

mayan/apps/sources/views.py

@@ -245,7 +245,7 @@ class UploadInteractiveView(UploadBaseView):
             shared_uploaded_file_id=shared_uploaded_file.pk,
             source_id=self.source.pk,
             user_id=user_id,
-        ), queue='uploads')
+        ))
         messages.success(self.request, _('New document queued for uploaded and will be available shortly.'))
         return HttpResponseRedirect(self.request.get_full_path())
 
@@ -310,7 +310,7 @@ class UploadInteractiveVersionView(UploadBaseView):
             document_id=self.document.pk,
             user_id=user_id,
             comment=forms['document_form'].cleaned_data.get('comment')
-        ), queue='uploads')
+        ))
 
         messages.success(self.request, _('New document version queued for uploaded and will be available shortly.'))
         return HttpResponseRedirect(reverse('documents:document_version_list', args=[self.document.pk]))