Improvements to the queued upload code to support large files

mayan/apps/sources/tasks.py

@@ -1,17 +1,19 @@
 import logging
 
+from django.core.files import File
+
 from mayan.celery import app
 
 from documents.exceptions import NewDocumentVersionNotAllowed
 from documents.models import DocumentType
 
-from .models import Source, StagingFolderSource
+from .models import Source
 
 logger = logging.getLogger(__name__)
 
 
 @app.task
-def task_upload_document(source_id, file_object, filename=None, use_file_name=False, document_type_id=None, expand=False, metadata_dict_list=None, user=None, document_id=None, new_version_data=None, command_line=False, description=None, staging_file=None):
+def task_upload_document(source_id, file_path, filename=None, use_file_name=False, document_type_id=None, expand=False, metadata_dict_list=None, user=None, document_id=None, new_version_data=None, command_line=False, description=None):
     source = Source.objects.get_subclass(pk=source_id)
 
     if document_type_id:
@@ -24,10 +26,13 @@ def task_upload_document(source_id, file_object, filename=None, use_file_name=Fa
     else:
         document = None
 
-    #try:
-    result = source.upload_file(file_object, filename, use_file_name, document_type, expand, metadata_dict_list, user, document, new_version_data, command_line, description)
-    #except NewDocumentVersionNotAllowed:
-    #    messages.error(request, _(u'New version uploads are not allowed for this document.'))
+    with File(file=open(file_path, mode='rb')) as file_object:
+        #try:
+        result = source.upload_file(file_object, filename, use_file_name, document_type, expand, metadata_dict_list, user, document, new_version_data, command_line, description)
+        #except NewDocumentVersionNotAllowed:
+        #    messages.error(request, _(u'New version uploads are not allowed for this document.'))
+
+    # TODO: delete temporary_file
 
     #if not document:
         #    if result['is_compressed'] is None:
@@ -39,6 +44,4 @@ def task_upload_document(source_id, file_object, filename=None, use_file_name=Fa
         #    if result['is_compressed'] is False:
         #        messages.warning(request, _(u'File was not a compressed file, uploaded as it was.'))
 
-    if isinstance(source, StagingFolderSource):
-        if source.delete_after_upload:
-            staging_file.delete()
+

mayan/apps/sources/views.py

@@ -1,5 +1,7 @@
 from __future__ import absolute_import
 
+import tempfile
+
 from django.conf import settings
 from django.contrib import messages
 from django.core.exceptions import PermissionDenied
@@ -186,9 +188,19 @@ def upload_interactive(request, source_id=None, document_pk=None):
                     else:
                         document_id = None
 
+                    temporary_file = tempfile.NamedTemporaryFile(delete=False)
+                    for chunk in file_object.chunks():
+                        temporary_file.write(chunk)
+
+                    temporary_file.close()
+
+                    if isinstance(source, StagingFolderSource):
+                        if source.delete_after_upload:
+                            staging_file.delete()
+
                     task_upload_document.apply_async(kwargs=dict(
                         source_id=source.pk,
-                        file_object=file_object, filename=new_filename,
+                        file_path=temporary_file.name, filename=new_filename,
                         use_file_name=form.cleaned_data.get('use_file_name', False),
                         document_type_id=document_type_id,
                         expand=expand,
@@ -197,7 +209,6 @@ def upload_interactive(request, source_id=None, document_pk=None):
                         document_id=document_id,
                         new_version_data=form.cleaned_data.get('new_version_data'),
                         description=form.cleaned_data.get('description'),
-                        staging_file=None
                     ), queue='uploads')
 
                     # TODO: Notify user