Merged document upload code and moved it to the model, improved method of handling compressed file

apps/documents/models.py

@@ -257,6 +257,9 @@ class Document(models.Model):
             os.unlink(self.get_cached_image_name(page)[0])
         except OSError:
             pass
+
+    def add_as_recent_document_for_user(self, user):
+        RecentDocument.objects.add_document_for_user(user, self)
             
 
 class DocumentTypeFilename(models.Model):

apps/sources/admin.py

@@ -2,7 +2,6 @@ from django.contrib import admin
 
 from sources.models import StagingFolder, WebForm, SourceTransformation
 
-
 admin.site.register(StagingFolder)
 admin.site.register(WebForm)
 admin.site.register(SourceTransformation)

apps/sources/compressed_file.py

@@ -0,0 +1,25 @@
+import os
+import zipfile
+
+from django.core.files.uploadedfile import SimpleUploadedFile
+
+
+class NotACompressedFile(Exception):
+    pass
+
+
+class CompressedFile(object):
+    def __init__(self, file_object):
+        self.file_object = file_object
+
+    def children(self):        
+        try:
+            # Try for a ZIP file
+            zfobj = zipfile.ZipFile(self.file_object)
+            filenames = [filename for filename in zfobj.namelist() if not filename.endswith('/')]
+            return (SimpleUploadedFile(name=filename, content=zfobj.read(filename)) for filename in filenames)
+        except zipfile.BadZipfile:
+            raise NotACompressedFile
+        
+    #def close(self):
+    #    self.file_object.close()

apps/sources/models.py

@@ -6,17 +6,23 @@ from django.contrib.contenttypes.models import ContentType
 from django.contrib.contenttypes import generic
 from django.core.exceptions import ValidationError
 
-from documents.models import DocumentType
-from metadata.models import MetadataType
 from converter.api import get_available_transformations_choices
 from converter.literals import DIMENSION_SEPARATOR    
+from documents.models import DocumentType, Document#, RecentDocument
+from documents.literals import HISTORY_DOCUMENT_CREATED
+from document_indexing.api import update_indexes
+from history.api import create_history
+from metadata.models import MetadataType
+from metadata.api import save_metadata_list
 from scheduler.api import register_interval_job, remove_job
 
 from sources.managers import SourceTransformationManager
 from sources.literals import SOURCE_CHOICES, SOURCE_CHOICES_PLURAL, \
     SOURCE_INTERACTIVE_UNCOMPRESS_CHOICES, SOURCE_CHOICE_WEB_FORM, \
     SOURCE_CHOICE_STAGING, SOURCE_ICON_DISK, SOURCE_ICON_DRIVE, \
-    SOURCE_ICON_CHOICES, SOURCE_CHOICE_WATCH, SOURCE_UNCOMPRESS_CHOICES
+    SOURCE_ICON_CHOICES, SOURCE_CHOICE_WATCH, SOURCE_UNCOMPRESS_CHOICES, \
+    SOURCE_UNCOMPRESS_CHOICE_Y
+from sources.compressed_file import CompressedFile, NotACompressedFile
 
 
 class BaseModel(models.Model):
@@ -25,13 +31,7 @@ class BaseModel(models.Model):
     whitelist = models.TextField(blank=True, verbose_name=_(u'whitelist'))
     blacklist = models.TextField(blank=True, verbose_name=_(u'blacklist'))
     document_type = models.ForeignKey(DocumentType, blank=True, null=True, verbose_name=_(u'document type'), help_text=(u'Optional document type to be applied to documents uploaded from this source.'))
-
+    
-    def __unicode__(self):
-        return u'%s' % self.title
-        
-    def fullname(self):
-        return u' '.join([self.class_fullname(), '"%s"' % self.title])
-
     @classmethod
     def class_fullname(cls):
         return unicode(dict(SOURCE_CHOICES).get(cls.source_type))
@@ -39,6 +39,55 @@ class BaseModel(models.Model):
     @classmethod
     def class_fullname_plural(cls):
         return unicode(dict(SOURCE_CHOICES_PLURAL).get(cls.source_type))
+
+    def __unicode__(self):
+        return u'%s' % self.title
+        
+    def fullname(self):
+        return u' '.join([self.class_fullname(), '"%s"' % self.title])
+        
+    def internal_name(self):
+        return u'%s_%d' % (self.source_type, self.pk)
+
+    def get_transformation_list(self):
+        return SourceTransformation.transformations.get_for_object_as_list(self)
+
+    def upload_file(self, file_object, filename=None, document_type=None, expand=False, metadata_dict_list=None, user=None):
+        if expand:
+            try:
+                cf = CompressedFile(file_object)
+                for fp in cf.children():
+                    self.upload_single_file(fp, None, document_type, metadata_dict_list, user)
+                    fp.close()
+
+            except NotACompressedFile:
+                self.upload_single_file(file_object, filename, document_type, metadata_dict_list, user)
+        else:
+            self.upload_single_file(file_object, filename, document_type, metadata_dict_list, user)
+           
+        file_object.close()
+            
+    def upload_single_file(self, file_object, filename=None, document_type=None, metadata_dict_list=None, user=None):
+        transformations, errors = self.get_transformation_list()
+        document = Document(file=file_object)
+        if document_type:
+            document.document_type = document_type
+        document.save()
+        if filename:
+            document.file_filename = filename
+            document.save()    
+
+        document.apply_default_transformations(transformations)
+
+        if metadata_dict_list:
+            save_metadata_list(metadata_dict_list, document, create=True)
+        warnings = update_indexes(document)
+
+        if user:
+            document.add_as_recent_document_for_user(user)
+            create_history(HISTORY_DOCUMENT_CREATED, document, {'user': user})
+        else:
+            create_history(HISTORY_DOCUMENT_CREATED, document)
         
     class Meta:
         ordering = ('title',)
@@ -108,8 +157,6 @@ class WebForm(InteractiveBaseModel):
         verbose_name = _(u'web form')
         verbose_name_plural = _(u'web forms')
 
-def test():
-    print 'WatchFolder'
     
 class WatchFolder(BaseModel):
     is_interactive = False
@@ -122,11 +169,25 @@ class WatchFolder(BaseModel):
     
     def save(self, *args, **kwargs):
         if self.pk:
-            remove_job('watch_folder_%d' % self.pk)
+            remove_job(self.internal_name())
         super(WatchFolder, self).save(*args, **kwargs)
-        if self.enabled:
+        self.schedule()
-            register_interval_job('watch_folder_%d' % self.pk, self.fullname(), test, seconds=self.interval)
 
+    def schedule(self):
+        if self.enabled:
+            register_interval_job(self.internal_name(), 
+                title=self.fullname(), func=self.execute, 
+                kwargs={'source_id': self.pk}, seconds=self.interval
+            )
+                        
+    def execute(self, source_id):
+        source = WatchFolder.objects.get(pk=source_id)
+        if source.uncompress == SOURCE_UNCOMPRESS_CHOICE_Y:
+            expand = True
+        else:
+            expand = False
+        print 'execute: %s' % self.internal_name()
+            
     class Meta(BaseModel.Meta):
         verbose_name = _(u'watch folder')
         verbose_name_plural = _(u'watch folders')

apps/sources/utils.py

@@ -3,6 +3,7 @@ import re
 from django.core.exceptions import ValidationError
 from django.utils.translation import ugettext
 
+
 # From http://www.peterbe.com/plog/whitelist-blacklist-logic
 def accept_item(value, whitelist, blacklist, default_accept=True): 
     """ return true if this item is either whitelisted or  

apps/sources/views.py

@@ -17,7 +17,7 @@ from documents.literals import PICTURE_ERROR_SMALL, PICTURE_ERROR_MEDIUM, \
     PICTURE_UNKNOWN_SMALL, PICTURE_UNKNOWN_MEDIUM
 from documents.literals import PERMISSION_DOCUMENT_CREATE
 from documents.literals import HISTORY_DOCUMENT_CREATED
-from documents.models import RecentDocument, Document, DocumentType
+from documents.models import Document, DocumentType
 from document_indexing.api import update_indexes
 from documents.conf.settings import THUMBNAIL_SIZE
 from history.api import create_history
@@ -137,16 +137,14 @@ def upload_interactive(request, source_type=None, source_id=None):
                             else:
                                 expand = False
 
-                        transformations, errors = SourceTransformation.transformations.get_for_object_as_list(web_form)
+                        new_filename = get_form_filename(form)
-                               
+                        web_form.upload_file(request.FILES['file'],
-                        if (not expand) or (expand and not _handle_zip_file(request, request.FILES['file'], document_type=document_type, transformations=transformations)):
+                            new_filename, document_type=document_type,
-                            instance = form.save()
+                            expand=expand,
-                            instance.save()
+                            metadata_dict_list=decode_metadata_from_url(request.GET),
-                            instance.apply_default_transformations(transformations)
+                            user=request.user
-                            if document_type:
+                        )
-                                instance.document_type = document_type
+                        messages.success(request, _(u'Document uploaded successfully.'))
-                            _handle_save_document(request, instance, form)
-                            messages.success(request, _(u'Document uploaded successfully.'))
                     except Exception, e:
                         messages.error(request, e)
 
@@ -185,15 +183,14 @@ def upload_interactive(request, source_type=None, source_id=None):
                                 expand = True
                             else:
                                 expand = False                        
-                        transformations, errors = SourceTransformation.transformations.get_for_object_as_list(staging_folder)
+                        new_filename = get_form_filename(form)
-                        if (not expand) or (expand and not _handle_zip_file(request, staging_file.upload(), document_type=document_type, transformations=transformations)):
+                        staging_folder.upload_file(staging_file.upload(),
-                            document = Document(file=staging_file.upload())
+                            new_filename, document_type=document_type,
-                            if document_type:
+                            expand=expand,
-                                document.document_type = document_type
+                            metadata_dict_list=decode_metadata_from_url(request.GET),
-                            document.save()
+                            user=request.user
-                            document.apply_default_transformations(transformations)
+                        )                        
-                            _handle_save_document(request, document, form)
+                        messages.success(request, _(u'Staging file: %s, uploaded successfully.') % staging_file.filename)
-                            messages.success(request, _(u'Staging file: %s, uploaded successfully.') % staging_file.filename)
 
                         if staging_folder.delete_after_upload:
                             staging_file.delete(preview_size=staging_folder.get_preview_size(), transformations=transformations)
@@ -250,49 +247,19 @@ def upload_interactive(request, source_type=None, source_id=None):
         context_instance=RequestContext(request))    
 
 
-def _handle_save_document(request, document, form=None):
+def get_form_filename(form):
-    RecentDocument.objects.add_document_for_user(request.user, document)
+    filename = None
-    
     if form:
         if form.cleaned_data['new_filename']:
-            document.file_filename = form.cleaned_data['new_filename']
+            return form.cleaned_data['new_filename']
-            document.save()
 
     if form and 'document_type_available_filenames' in form.cleaned_data:
         if form.cleaned_data['document_type_available_filenames']:
-            document.file_filename = form.cleaned_data['document_type_available_filenames'].filename
+            return form.cleaned_data['document_type_available_filenames'].filename
-            document.save()
+            
-
+    return filename
-    save_metadata_list(decode_metadata_from_url(request.GET), document, create=True)
+    
-
+       
-    warnings = update_indexes(document)
-    if request.user.is_staff or request.user.is_superuser:
-        for warning in warnings:
-            messages.warning(request, warning)
-
-    create_history(HISTORY_DOCUMENT_CREATED, document, {'user': request.user})
-
-
-def _handle_zip_file(request, uploaded_file, document_type=None, transformations=None):
-    filename = getattr(uploaded_file, 'filename', getattr(uploaded_file, 'name', ''))
-    if filename.lower().endswith('zip'):
-        zfobj = zipfile.ZipFile(uploaded_file)
-        for filename in zfobj.namelist():
-            if not filename.endswith('/'):
-                zip_document = Document(file=SimpleUploadedFile(
-                    name=filename, content=zfobj.read(filename)))
-                if document_type:
-                    zip_document.document_type = document_type
-                zip_document.save()
-                _handle_save_document(request, zip_document)
-                messages.success(request, _(u'Extracted file: %s, uploaded successfully.') % filename)
-        #Signal that uploaded file was a zip file
-        return True
-    else:
-        #Otherwise tell parent to handle file
-        return False
-        
-        
 def staging_file_preview(request, source_type, source_id, staging_file_id):
     check_permissions(request.user, [PERMISSION_DOCUMENT_CREATE])
     staging_folder = get_object_or_404(StagingFolder, pk=source_id)

apps/sources/watch_folders.py

@@ -1,21 +0,0 @@
-
-
-                            if staging_folder.uncompress == SOURCE_UNCOMPRESS_CHOICE_Y:
-                                expand = True
-                            else:
-                                expand = False                        
-                        transformations, errors = SourceTransformation.transformations.get_for_object_as_list(staging_folder)
-                        if (not expand) or (expand and not _handle_zip_file(request, staging_file.upload(), document_type=document_type, transformations=transformations)):
-                            document = Document(file=staging_file.upload())
-                            if document_type:
-                                document.document_type = document_type
-                            document.save()
-                            document.apply_default_transformations(transformations)
-                            _handle_save_document(request, document, form)
-                            messages.success(request, _(u'Staging file: %s, uploaded successfully.') % staging_file.filename)
-
-                        if staging_folder.delete_after_upload:
-                            staging_file.delete(preview_size=staging_folder.get_preview_size(), transformations=transformations)
-                            messages.success(request, _(u'Staging file: %s, deleted successfully.') % staging_file.filename)
-                    except Exception, e:
-                        messages.error(request, e)