Assign tasks to specific queues. Add support for transient queues. gh-issue #222, gh-issue #230.

mayan/apps/documents/api_views.py

@@ -194,7 +194,7 @@ class APIDocumentImageView(generics.GenericAPIView):
         document_page = document.pages.get(page_number=page)
 
         try:
-            task = task_get_document_page_image.apply_async(kwargs=dict(document_page_id=document_page.pk, size=size, zoom=zoom, rotation=rotation, as_base64=True, version=version), queue='converter')
+            task = task_get_document_page_image.apply_async(kwargs=dict(document_page_id=document_page.pk, size=size, zoom=zoom, rotation=rotation, as_base64=True, version=version))
             # TODO: prepend 'data:%s;base64,%s' based on format specified in
             # async call
             return Response({

mayan/apps/documents/apps.py

@@ -2,6 +2,8 @@ from __future__ import absolute_import, unicode_literals
 
 from datetime import timedelta
 
+from kombu import Exchange, Queue
+
 from django.utils.translation import ugettext_lazy as _
 
 from actstream import registry
@@ -111,21 +113,49 @@ class DocumentsApp(MayanAppConfig):
         SourceColumn(source=DeletedDocument, label=_('Type'), attribute='document_type')
         SourceColumn(source=DeletedDocument, label=_('Date time trashed'), attribute='deleted_date_time')
 
-        app.conf.CELERYBEAT_SCHEDULE.update({
-            'task_check_trash_periods': {
-                'task': 'documents.tasks.task_check_trash_periods',
-                'schedule': timedelta(seconds=CHECK_TRASH_PERIOD_INTERVAL),
-                'options': {'queue': 'documents'}
-            },
-        })
+        app.conf.CELERYBEAT_SCHEDULE.update(
+            {
+                'task_check_delete_periods': {
+                    'task': 'documents.tasks.task_check_delete_periods',
+                    'schedule': timedelta(seconds=CHECK_DELETE_PERIOD_INTERVAL),
+                },
+                'task_check_trash_periods': {
+                    'task': 'documents.tasks.task_check_trash_periods',
+                    'schedule': timedelta(seconds=CHECK_TRASH_PERIOD_INTERVAL),
+                },
+            }
+        )
 
-        app.conf.CELERYBEAT_SCHEDULE.update({
-            'task_check_delete_periods': {
-                'task': 'documents.tasks.task_check_delete_periods',
-                'schedule': timedelta(seconds=CHECK_DELETE_PERIOD_INTERVAL),
-                'options': {'queue': 'documents'}
-            },
-        })
+        app.conf.CELERY_QUEUES.extend(
+            (
+                Queue('converter', Exchange('converter'), routing_key='converter', delivery_mode=1),
+                Queue('documents_periodic', Exchange('documents_periodic'), routing_key='documents_periodic', delivery_mode=1),
+                Queue('uploads', Exchange('uploads'), routing_key='uploads'),
+            )
+        )
+
+        app.conf.CELERY_ROUTES.update(
+            {
+                'documents.tasks.task_check_delete_periods': {
+                    'queue': 'documents'
+                },
+                'documents.tasks.task_check_trash_periods': {
+                    'queue': 'documents'
+                },
+                'documents.tasks.task_clear_image_cache': {
+                    'queue': 'tools'
+                },
+                'documents.tasks.task_get_document_page_image': {
+                    'queue': 'converter'
+                },
+                'documents.tasks.task_update_page_count': {
+                    'queue': 'tools'
+                },
+                'documents.tasks.task_upload_new_version': {
+                    'queue': 'uploads'
+                },
+            }
+        )
 
         menu_front_page.bind_links(links=[link_document_list_recent, link_document_list, link_document_list_deleted])
         menu_setup.bind_links(links=[link_document_type_setup])

mayan/apps/documents/models.py

@@ -189,7 +189,7 @@ class Document(models.Model):
         task_upload_new_version.apply_async(kwargs=dict(
             shared_uploaded_file_id=shared_uploaded_file.pk,
             document_id=self.pk, user_id=user_id,
-        ), queue='uploads')
+        ))
 
         logger.info('New document version queued for document: %s', self)
 

mayan/apps/documents/tasks.py

@@ -17,10 +17,42 @@ from .models import (
 logger = logging.getLogger(__name__)
 
 
-@app.task(compression='zlib')
-def task_get_document_page_image(document_page_id, *args, **kwargs):
-    document_page = DocumentPage.objects.get(pk=document_page_id)
-    return document_page.get_image(*args, **kwargs)
+@app.task(ignore_result=True)
+def task_check_delete_periods():
+    logger.info('Executing')
+
+    for document_type in DocumentType.objects.all():
+        logger.info('Checking deletion period of document type: %s', document_type)
+        if document_type.delete_time_period and document_type.delete_time_unit:
+            delta = timedelta(**{document_type.delete_time_unit: document_type.delete_time_period})
+            logger.info('Document type: %s, has a deletion period delta of: %s', document_type, delta)
+            for document in DeletedDocument.objects.filter(document_type=document_type):
+                if now() > document.deleted_date_time + delta:
+                    logger.info('Document "%s" with id: %d, trashed on: %s, exceded delete period', document, document.pk, document.deleted_date_time)
+                    document.delete()
+        else:
+            logger.info('Document type: %s, has a no retention delta', document_type)
+
+    logger.info('Finshed')
+
+
+@app.task(ignore_result=True)
+def task_check_trash_periods():
+    logger.info('Executing')
+
+    for document_type in DocumentType.objects.all():
+        logger.info('Checking trash period of document type: %s', document_type)
+        if document_type.trash_time_period and document_type.trash_time_unit:
+            delta = timedelta(**{document_type.trash_time_unit: document_type.trash_time_period})
+            logger.info('Document type: %s, has a trash period delta of: %s', document_type, delta)
+            for document in Document.objects.filter(document_type=document_type):
+                if now() > document.date_added + delta:
+                    logger.info('Document "%s" with id: %d, added on: %s, exceded trash period', document, document.pk, document.date_added)
+                    document.delete()
+        else:
+            logger.info('Document type: %s, has a no retention delta', document_type)
+
+    logger.info('Finshed')
 
 
 @app.task(ignore_result=True)
@@ -31,6 +63,12 @@ def task_clear_image_cache():
     logger.info('Finished document cache invalidation')
 
 
+@app.task(compression='zlib')
+def task_get_document_page_image(document_page_id, *args, **kwargs):
+    document_page = DocumentPage.objects.get(pk=document_page_id)
+    return document_page.get_image(*args, **kwargs)
+
+
 @app.task(ignore_result=True)
 def task_update_page_count(version_id):
     document_version = DocumentVersion.objects.get(pk=version_id)
@@ -56,41 +94,3 @@ def task_upload_new_version(document_id, shared_uploaded_file_id, user_id, comme
             logger.info('Warning during attempt to create new document version for document: %s ; %s', document, warning)
         finally:
             shared_file.delete()
-
-
-@app.task(ignore_result=True)
-def task_check_trash_periods():
-    logger.info('Executing')
-
-    for document_type in DocumentType.objects.all():
-        logger.info('Checking trash period of document type: %s', document_type)
-        if document_type.trash_time_period and document_type.trash_time_unit:
-            delta = timedelta(**{document_type.trash_time_unit: document_type.trash_time_period})
-            logger.info('Document type: %s, has a trash period delta of: %s', document_type, delta)
-            for document in Document.objects.filter(document_type=document_type):
-                if now() > document.date_added + delta:
-                    logger.info('Document "%s" with id: %d, added on: %s, exceded trash period', document, document.pk, document.date_added)
-                    document.delete()
-        else:
-            logger.info('Document type: %s, has a no retention delta', document_type)
-
-    logger.info('Finshed')
-
-
-@app.task(ignore_result=True)
-def task_check_delete_periods():
-    logger.info('Executing')
-
-    for document_type in DocumentType.objects.all():
-        logger.info('Checking deletion period of document type: %s', document_type)
-        if document_type.delete_time_period and document_type.delete_time_unit:
-            delta = timedelta(**{document_type.delete_time_unit: document_type.delete_time_period})
-            logger.info('Document type: %s, has a deletion period delta of: %s', document_type, delta)
-            for document in DeletedDocument.objects.filter(document_type=document_type):
-                if now() > document.deleted_date_time + delta:
-                    logger.info('Document "%s" with id: %d, trashed on: %s, exceded delete period', document, document.pk, document.deleted_date_time)
-                    document.delete()
-        else:
-            logger.info('Document type: %s, has a no retention delta', document_type)
-
-    logger.info('Finshed')

mayan/apps/documents/views.py

@@ -455,7 +455,7 @@ def get_document_image(request, document_id, size=setting_preview_size.value):
 
     document_page = document.pages.get(page_number=page)
 
-    task = task_get_document_page_image.apply_async(kwargs=dict(document_page_id=document_page.pk, size=size, zoom=zoom, rotation=rotation, as_base64=True, version=version), queue='converter')
+    task = task_get_document_page_image.apply_async(kwargs=dict(document_page_id=document_page.pk, size=size, zoom=zoom, rotation=rotation, as_base64=True, version=version))
     data = task.get(timeout=DOCUMENT_IMAGE_TASK_TIMEOUT)
     return HttpResponse(base64.b64decode(data[21:]), content_type='image')
 
@@ -587,7 +587,7 @@ def document_update_page_count(request, document_id=None, document_id_list=None)
 
     if request.method == 'POST':
         for document in documents:
-            task_update_page_count.apply_async(kwargs={'version_id': document.latest_version.pk}, queue='tools')
+            task_update_page_count.apply_async(kwargs={'version_id': document.latest_version.pk})
 
         messages.success(
             request,
@@ -1102,7 +1102,7 @@ def document_clear_image_cache(request):
     previous = request.POST.get('previous', request.GET.get('previous', request.META.get('HTTP_REFERER', reverse(settings.LOGIN_REDIRECT_URL))))
 
     if request.method == 'POST':
-        task_clear_image_cache.apply_async(queue='tools')
+        task_clear_image_cache.apply_async()
         messages.success(request, _('Document image cache clearing queued successfully.'))
 
         return HttpResponseRedirect(previous)