Merge branch 'feature/bulk_upload' into development

apps/metadata/api.py

@@ -107,3 +107,15 @@ def get_metadata_string(document):
     Return a formated representation of a document's metadata values
     """
     return u', '.join([u'%s - %s' % (metadata.metadata_type, metadata.value) for metadata in DocumentMetadata.objects.filter(document=document).select_related('metadata_type')])
+
+
+def convert_dict_to_dict_list(dictionary):
+    result = []
+    for key, value in dictionary.items():
+        try:
+            metadata_type = MetadataType.objects.get(name=key)
+        except MetadataType.DoesNotExist:
+            raise ValueError('Unknown metadata type name')
+        result.append({'id': metadata_type.pk, 'value': value})
+    
+    return result

apps/sources/management/commands/bulk_upload.py

@@ -0,0 +1,80 @@
+from __future__ import absolute_import
+
+import os, sys
+from optparse import make_option
+
+from django.core.management.base import BaseCommand, CommandError, LabelCommand
+from django.utils.simplejson import loads, dumps
+
+from metadata.api import convert_dict_to_dict_list
+from documents.models import DocumentType
+
+from ...models import OutOfProcess
+from ...compressed_file import CompressedFile, NotACompressedFile
+
+
+class Command(LabelCommand):
+    args = '<filename>'
+    help = 'Upload documents from a compressed file in to the database.'
+    option_list = LabelCommand.option_list + (
+        make_option('--noinput', action='store_false', dest='interactive',
+            default=True, help='Do not ask the user for confirmation before '
+                'starting.'),
+        make_option('--metadata', action='store', dest='metadata',
+            help='A metadata dictionary list to apply to the documents.'),
+        make_option('--document_type', action='store', dest='document_type_name',
+            help='The document type to apply to the uploaded documents.'),
+    )
+        
+    def handle_label(self, label, **options):
+        if not os.access(label, os.R_OK):
+            raise CommandError("File '%s' is not readable." % label)
+
+        if options['metadata']:
+            try:
+                metadata_dict = loads(options['metadata'])
+                metadata_dict_list = convert_dict_to_dict_list(metadata_dict)
+            except Exception, e:
+                sys.exit('Metadata error: %s' % e)
+        else:
+            metadata_dict_list = None
+
+        if options['document_type_name']:
+            try:
+                document_type = DocumentType.objects.get(name=options['document_type_name'])
+            except DocumentType.DoesNotExist:
+                sys.exit('Unknown document type')
+        else:
+            document_type = None
+
+        if _confirm(options['interactive']) == 'yes':
+            print 'Beginning upload...'
+            if metadata_dict_list:
+                print 'Using the metadata values:'
+                for key, value in metadata_dict.items():
+                    print '%s: %s' % (key, value)
+                    
+            if document_type:
+                print 'Uploaded document will be of type: %s' % options['document_type_name']
+                    
+            source = OutOfProcess()
+            fd = open(label)
+            try:
+                result = source.upload_file(fd, filename=None, use_file_name=False, document_type=document_type, expand=True, metadata_dict_list=metadata_dict_list, user=None, document=None, new_version_data=None, verbose=True)
+                pass
+            except NotACompressedFile:
+                print '%s is not a compressed file.'
+            else:
+                print 'Finished.'
+
+            fd.close()
+        else:
+            print 'Cancelled.'
+
+    
+def _confirm(interactive):
+    if not interactive:
+        return 'yes'
+    return raw_input('You have requested to bulk upload a number of documents from a compressed file.\n' 
+            'Are you sure you want to do this?\n'
+            'Type \'yes\' to continue, or any other value to cancel: ')

apps/sources/models.py

@@ -59,15 +59,19 @@ class BaseModel(models.Model):
     def get_transformation_list(self):
         return SourceTransformation.transformations.get_for_object_as_list(self)
 
-    def upload_file(self, file_object, filename=None, use_file_name=False, document_type=None, expand=False, metadata_dict_list=None, user=None, document=None, new_version_data=None):
+    def upload_file(self, file_object, filename=None, use_file_name=False, document_type=None, expand=False, metadata_dict_list=None, user=None, document=None, new_version_data=None, verbose=False):
         is_compressed = None
 
         if expand:
             try:
                 cf = CompressedFile(file_object)
+                count = 1
                 for fp in cf.children():
+                    if verbose:
+                        print 'Uploading file #%d: %s' % (count, fp)
                     self.upload_single_file(file_object=fp, filename=None, document_type=document_type, metadata_dict_list=metadata_dict_list, user=user)
                     fp.close()
+                    count += 1
 
             except NotACompressedFile:
                 is_compressed = False
@@ -90,7 +94,7 @@ class BaseModel(models.Model):
             document.save()
 
             apply_default_acls(document, user)
-
+ 
             if metadata_dict_list:
                 save_metadata_list(metadata_dict_list, document, create=True)
             warnings = update_indexes(document)
@@ -159,7 +163,7 @@ class StagingFolder(InteractiveBaseModel):
         verbose_name = _(u'staging folder')
         verbose_name_plural = _(u'staging folders')
 
-'''
+"""
 class SourceMetadata(models.Model):
     content_type = models.ForeignKey(ContentType)
     object_id = models.PositiveIntegerField()
@@ -173,7 +177,7 @@ class SourceMetadata(models.Model):
     class Meta:
         verbose_name = _(u'source metadata')
         verbose_name_plural = _(u'sources metadata')
-'''
+"""
 
 
 class WebForm(InteractiveBaseModel):
@@ -235,9 +239,9 @@ class ArgumentsValidator(object):
             self.code = code
 
     def __call__(self, value):
-        '''
+        """
         Validates that the input evaluates correctly.
-        '''
+        """
         value = value.strip()
         try:
             literal_eval(value)
@@ -246,10 +250,10 @@ class ArgumentsValidator(object):
 
 
 class SourceTransformation(models.Model):
-    '''
+    """
     Model that stores the transformation and transformation arguments
     for a given document source
-    '''
+    """
     content_type = models.ForeignKey(ContentType)
     object_id = models.PositiveIntegerField()
     content_object = generic.GenericForeignKey('content_type', 'object_id')
@@ -268,3 +272,10 @@ class SourceTransformation(models.Model):
         ordering = ('order',)
         verbose_name = _(u'document source transformation')
         verbose_name_plural = _(u'document source transformations')
+
+
+class OutOfProcess(BaseModel):
+    is_interactive = False
+    class Meta(BaseModel.Meta):
+        verbose_name = _(u'out of process')
+        verbose_name_plural = _(u'out of process')