From e4e6c2f5fb7b616e7d679164d00e01e9e704a417 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Fri, 3 Oct 2014 14:58:55 -0400 Subject: [PATCH] Move document image queueing up in process to avoid queueing serialization overhead --- mayan/apps/documents/api_views.py | 4 +++- mayan/apps/documents/models.py | 8 ++------ mayan/apps/documents/tasks.py | 13 +++++++++++++ 3 files changed, 18 insertions(+), 7 deletions(-) create mode 100644 mayan/apps/documents/tasks.py diff --git a/mayan/apps/documents/api_views.py b/mayan/apps/documents/api_views.py index 2d52752e49..df20610d1f 100644 --- a/mayan/apps/documents/api_views.py +++ b/mayan/apps/documents/api_views.py @@ -28,6 +28,7 @@ from .serializers import (DocumentImageSerializer, DocumentPageSerializer, DocumentSerializer, DocumentTypeSerializer, DocumentVersionSerializer) from .settings import DISPLAY_SIZE, ZOOM_MAX_LEVEL, ZOOM_MIN_LEVEL +from .tasks import task_get_document_image class APIDocumentListView(generics.ListCreateAPIView): @@ -161,9 +162,10 @@ class APIDocumentImageView(generics.GenericAPIView): rotation = int(request.GET.get('rotation', DEFAULT_ROTATION)) % 360 try: + task = task_get_document_image.apply_async(kwargs=dict(document_id=document.pk, size=size, page=page, zoom=zoom, rotation=rotation, as_base64=True, version=version), queue='converter') return Response({ 'status': 'success', - 'data': document.get_image(size=size, page=page, zoom=zoom, rotation=rotation, as_base64=True, version=version) + 'data': task.get(timeout=1) }) except UnknownFileFormat as exception: return Response({'status': 'error', 'detail': 'unknown_file_format', 'message': unicode(exception)}) diff --git a/mayan/apps/documents/models.py b/mayan/apps/documents/models.py index 5cc78405bc..74443370b8 100644 --- a/mayan/apps/documents/models.py +++ b/mayan/apps/documents/models.py @@ -24,7 +24,6 @@ from converter.api import (convert, get_page_count, from converter.exceptions import UnknownFileFormat from converter.literals import (DEFAULT_ZOOM_LEVEL, DEFAULT_ROTATION, DEFAULT_PAGE_NUMBER) -from converter.tasks import task_convert from mimetype.api import get_mimetype from .exceptions import NewDocumentVersionNotAllowed @@ -38,7 +37,6 @@ from .settings import (CACHE_PATH, CHECKSUM_FUNCTION, DISPLAY_SIZE, UUID_FUNCTION, ZOOM_MAX_LEVEL, ZOOM_MIN_LEVEL) from .utils import document_save_to_temp_dir -CONVERTER_TASK_TIMEOUT = 2 # In seconds HASH_FUNCTION = lambda x: hashlib.sha256(x).hexdigest() # document image cache name hash function logger = logging.getLogger(__name__) @@ -125,8 +123,7 @@ class Document(models.Model): else: document_version = DocumentVersion.objects.get(pk=version) document_file = document_save_to_temp_dir(document_version, document_version.checksum) - task = task_convert.apply_async(kwargs=dict(input_filepath=document_file, output_filepath=cache_file_path, page=page, transformations=transformations, mimetype=self.file_mimetype), queue='converter') - return task.get(timeout=CONVERTER_TASK_TIMEOUT) + return convert(input_filepath=document_file, output_filepath=cache_file_path, page=page, transformations=transformations, mimetype=self.file_mimetype) def get_valid_image(self, size=DISPLAY_SIZE, page=DEFAULT_PAGE_NUMBER, zoom=DEFAULT_ZOOM_LEVEL, rotation=DEFAULT_ROTATION, version=None): if not version: @@ -135,8 +132,7 @@ class Document(models.Model): logger.debug('image_cache_name: %s' % image_cache_name) - task = task_convert.apply_async(kwargs=dict(input_filepath=image_cache_name, cleanup_files=False, size=size, zoom=zoom, rotation=rotation), queue='converter') - return task.get(timeout=CONVERTER_TASK_TIMEOUT) + return convert(input_filepath=image_cache_name, cleanup_files=False, size=size, zoom=zoom, rotation=rotation) def get_image(self, size=DISPLAY_SIZE, page=DEFAULT_PAGE_NUMBER, zoom=DEFAULT_ZOOM_LEVEL, rotation=DEFAULT_ROTATION, as_base64=False, version=None): if zoom < ZOOM_MIN_LEVEL: diff --git a/mayan/apps/documents/tasks.py b/mayan/apps/documents/tasks.py new file mode 100644 index 0000000000..179ed23826 --- /dev/null +++ b/mayan/apps/documents/tasks.py @@ -0,0 +1,13 @@ +import logging + +from mayan.celery import app + +from .models import Document + +logger = logging.getLogger(__name__) + + +@app.task +def task_get_document_image(document_id, *args, **kwargs): + document = Document.objects.get(pk=document_id) + return document.get_image(*args, **kwargs)