diff --git a/apps/documents/models.py b/apps/documents/models.py index 51acc8815f..ea5faf1ba6 100644 --- a/apps/documents/models.py +++ b/apps/documents/models.py @@ -257,6 +257,9 @@ class Document(models.Model): os.unlink(self.get_cached_image_name(page)[0]) except OSError: pass + + def add_as_recent_document_for_user(self, user): + RecentDocument.objects.add_document_for_user(user, self) class DocumentTypeFilename(models.Model): diff --git a/apps/sources/admin.py b/apps/sources/admin.py index e0522e62ab..ac03e7c9f8 100644 --- a/apps/sources/admin.py +++ b/apps/sources/admin.py @@ -2,7 +2,6 @@ from django.contrib import admin from sources.models import StagingFolder, WebForm, SourceTransformation - admin.site.register(StagingFolder) admin.site.register(WebForm) admin.site.register(SourceTransformation) diff --git a/apps/sources/compressed_file.py b/apps/sources/compressed_file.py new file mode 100644 index 0000000000..977e786160 --- /dev/null +++ b/apps/sources/compressed_file.py @@ -0,0 +1,25 @@ +import os +import zipfile + +from django.core.files.uploadedfile import SimpleUploadedFile + + +class NotACompressedFile(Exception): + pass + + +class CompressedFile(object): + def __init__(self, file_object): + self.file_object = file_object + + def children(self): + try: + # Try for a ZIP file + zfobj = zipfile.ZipFile(self.file_object) + filenames = [filename for filename in zfobj.namelist() if not filename.endswith('/')] + return (SimpleUploadedFile(name=filename, content=zfobj.read(filename)) for filename in filenames) + except zipfile.BadZipfile: + raise NotACompressedFile + + #def close(self): + # self.file_object.close() diff --git a/apps/sources/models.py b/apps/sources/models.py index 092fe94ee2..6172292178 100644 --- a/apps/sources/models.py +++ b/apps/sources/models.py @@ -6,17 +6,23 @@ from django.contrib.contenttypes.models import ContentType from django.contrib.contenttypes import generic from django.core.exceptions import ValidationError -from documents.models import DocumentType -from metadata.models import MetadataType from converter.api import get_available_transformations_choices from converter.literals import DIMENSION_SEPARATOR +from documents.models import DocumentType, Document#, RecentDocument +from documents.literals import HISTORY_DOCUMENT_CREATED +from document_indexing.api import update_indexes +from history.api import create_history +from metadata.models import MetadataType +from metadata.api import save_metadata_list from scheduler.api import register_interval_job, remove_job from sources.managers import SourceTransformationManager from sources.literals import SOURCE_CHOICES, SOURCE_CHOICES_PLURAL, \ SOURCE_INTERACTIVE_UNCOMPRESS_CHOICES, SOURCE_CHOICE_WEB_FORM, \ SOURCE_CHOICE_STAGING, SOURCE_ICON_DISK, SOURCE_ICON_DRIVE, \ - SOURCE_ICON_CHOICES, SOURCE_CHOICE_WATCH, SOURCE_UNCOMPRESS_CHOICES + SOURCE_ICON_CHOICES, SOURCE_CHOICE_WATCH, SOURCE_UNCOMPRESS_CHOICES, \ + SOURCE_UNCOMPRESS_CHOICE_Y +from sources.compressed_file import CompressedFile, NotACompressedFile class BaseModel(models.Model): @@ -25,13 +31,7 @@ class BaseModel(models.Model): whitelist = models.TextField(blank=True, verbose_name=_(u'whitelist')) blacklist = models.TextField(blank=True, verbose_name=_(u'blacklist')) document_type = models.ForeignKey(DocumentType, blank=True, null=True, verbose_name=_(u'document type'), help_text=(u'Optional document type to be applied to documents uploaded from this source.')) - - def __unicode__(self): - return u'%s' % self.title - - def fullname(self): - return u' '.join([self.class_fullname(), '"%s"' % self.title]) - + @classmethod def class_fullname(cls): return unicode(dict(SOURCE_CHOICES).get(cls.source_type)) @@ -39,6 +39,55 @@ class BaseModel(models.Model): @classmethod def class_fullname_plural(cls): return unicode(dict(SOURCE_CHOICES_PLURAL).get(cls.source_type)) + + def __unicode__(self): + return u'%s' % self.title + + def fullname(self): + return u' '.join([self.class_fullname(), '"%s"' % self.title]) + + def internal_name(self): + return u'%s_%d' % (self.source_type, self.pk) + + def get_transformation_list(self): + return SourceTransformation.transformations.get_for_object_as_list(self) + + def upload_file(self, file_object, filename=None, document_type=None, expand=False, metadata_dict_list=None, user=None): + if expand: + try: + cf = CompressedFile(file_object) + for fp in cf.children(): + self.upload_single_file(fp, None, document_type, metadata_dict_list, user) + fp.close() + + except NotACompressedFile: + self.upload_single_file(file_object, filename, document_type, metadata_dict_list, user) + else: + self.upload_single_file(file_object, filename, document_type, metadata_dict_list, user) + + file_object.close() + + def upload_single_file(self, file_object, filename=None, document_type=None, metadata_dict_list=None, user=None): + transformations, errors = self.get_transformation_list() + document = Document(file=file_object) + if document_type: + document.document_type = document_type + document.save() + if filename: + document.file_filename = filename + document.save() + + document.apply_default_transformations(transformations) + + if metadata_dict_list: + save_metadata_list(metadata_dict_list, document, create=True) + warnings = update_indexes(document) + + if user: + document.add_as_recent_document_for_user(user) + create_history(HISTORY_DOCUMENT_CREATED, document, {'user': user}) + else: + create_history(HISTORY_DOCUMENT_CREATED, document) class Meta: ordering = ('title',) @@ -108,8 +157,6 @@ class WebForm(InteractiveBaseModel): verbose_name = _(u'web form') verbose_name_plural = _(u'web forms') -def test(): - print 'WatchFolder' class WatchFolder(BaseModel): is_interactive = False @@ -122,11 +169,25 @@ class WatchFolder(BaseModel): def save(self, *args, **kwargs): if self.pk: - remove_job('watch_folder_%d' % self.pk) + remove_job(self.internal_name()) super(WatchFolder, self).save(*args, **kwargs) - if self.enabled: - register_interval_job('watch_folder_%d' % self.pk, self.fullname(), test, seconds=self.interval) + self.schedule() + def schedule(self): + if self.enabled: + register_interval_job(self.internal_name(), + title=self.fullname(), func=self.execute, + kwargs={'source_id': self.pk}, seconds=self.interval + ) + + def execute(self, source_id): + source = WatchFolder.objects.get(pk=source_id) + if source.uncompress == SOURCE_UNCOMPRESS_CHOICE_Y: + expand = True + else: + expand = False + print 'execute: %s' % self.internal_name() + class Meta(BaseModel.Meta): verbose_name = _(u'watch folder') verbose_name_plural = _(u'watch folders') diff --git a/apps/sources/utils.py b/apps/sources/utils.py index 574d81771f..eca2f3235f 100644 --- a/apps/sources/utils.py +++ b/apps/sources/utils.py @@ -3,6 +3,7 @@ import re from django.core.exceptions import ValidationError from django.utils.translation import ugettext + # From http://www.peterbe.com/plog/whitelist-blacklist-logic def accept_item(value, whitelist, blacklist, default_accept=True): """ return true if this item is either whitelisted or diff --git a/apps/sources/views.py b/apps/sources/views.py index daa1e82d56..2dc5002129 100644 --- a/apps/sources/views.py +++ b/apps/sources/views.py @@ -17,7 +17,7 @@ from documents.literals import PICTURE_ERROR_SMALL, PICTURE_ERROR_MEDIUM, \ PICTURE_UNKNOWN_SMALL, PICTURE_UNKNOWN_MEDIUM from documents.literals import PERMISSION_DOCUMENT_CREATE from documents.literals import HISTORY_DOCUMENT_CREATED -from documents.models import RecentDocument, Document, DocumentType +from documents.models import Document, DocumentType from document_indexing.api import update_indexes from documents.conf.settings import THUMBNAIL_SIZE from history.api import create_history @@ -137,16 +137,14 @@ def upload_interactive(request, source_type=None, source_id=None): else: expand = False - transformations, errors = SourceTransformation.transformations.get_for_object_as_list(web_form) - - if (not expand) or (expand and not _handle_zip_file(request, request.FILES['file'], document_type=document_type, transformations=transformations)): - instance = form.save() - instance.save() - instance.apply_default_transformations(transformations) - if document_type: - instance.document_type = document_type - _handle_save_document(request, instance, form) - messages.success(request, _(u'Document uploaded successfully.')) + new_filename = get_form_filename(form) + web_form.upload_file(request.FILES['file'], + new_filename, document_type=document_type, + expand=expand, + metadata_dict_list=decode_metadata_from_url(request.GET), + user=request.user + ) + messages.success(request, _(u'Document uploaded successfully.')) except Exception, e: messages.error(request, e) @@ -185,15 +183,14 @@ def upload_interactive(request, source_type=None, source_id=None): expand = True else: expand = False - transformations, errors = SourceTransformation.transformations.get_for_object_as_list(staging_folder) - if (not expand) or (expand and not _handle_zip_file(request, staging_file.upload(), document_type=document_type, transformations=transformations)): - document = Document(file=staging_file.upload()) - if document_type: - document.document_type = document_type - document.save() - document.apply_default_transformations(transformations) - _handle_save_document(request, document, form) - messages.success(request, _(u'Staging file: %s, uploaded successfully.') % staging_file.filename) + new_filename = get_form_filename(form) + staging_folder.upload_file(staging_file.upload(), + new_filename, document_type=document_type, + expand=expand, + metadata_dict_list=decode_metadata_from_url(request.GET), + user=request.user + ) + messages.success(request, _(u'Staging file: %s, uploaded successfully.') % staging_file.filename) if staging_folder.delete_after_upload: staging_file.delete(preview_size=staging_folder.get_preview_size(), transformations=transformations) @@ -250,49 +247,19 @@ def upload_interactive(request, source_type=None, source_id=None): context_instance=RequestContext(request)) -def _handle_save_document(request, document, form=None): - RecentDocument.objects.add_document_for_user(request.user, document) - +def get_form_filename(form): + filename = None if form: if form.cleaned_data['new_filename']: - document.file_filename = form.cleaned_data['new_filename'] - document.save() + return form.cleaned_data['new_filename'] if form and 'document_type_available_filenames' in form.cleaned_data: if form.cleaned_data['document_type_available_filenames']: - document.file_filename = form.cleaned_data['document_type_available_filenames'].filename - document.save() - - save_metadata_list(decode_metadata_from_url(request.GET), document, create=True) - - warnings = update_indexes(document) - if request.user.is_staff or request.user.is_superuser: - for warning in warnings: - messages.warning(request, warning) - - create_history(HISTORY_DOCUMENT_CREATED, document, {'user': request.user}) - - -def _handle_zip_file(request, uploaded_file, document_type=None, transformations=None): - filename = getattr(uploaded_file, 'filename', getattr(uploaded_file, 'name', '')) - if filename.lower().endswith('zip'): - zfobj = zipfile.ZipFile(uploaded_file) - for filename in zfobj.namelist(): - if not filename.endswith('/'): - zip_document = Document(file=SimpleUploadedFile( - name=filename, content=zfobj.read(filename))) - if document_type: - zip_document.document_type = document_type - zip_document.save() - _handle_save_document(request, zip_document) - messages.success(request, _(u'Extracted file: %s, uploaded successfully.') % filename) - #Signal that uploaded file was a zip file - return True - else: - #Otherwise tell parent to handle file - return False - - + return form.cleaned_data['document_type_available_filenames'].filename + + return filename + + def staging_file_preview(request, source_type, source_id, staging_file_id): check_permissions(request.user, [PERMISSION_DOCUMENT_CREATE]) staging_folder = get_object_or_404(StagingFolder, pk=source_id) diff --git a/apps/sources/watch_folders.py b/apps/sources/watch_folders.py deleted file mode 100644 index 205bd30655..0000000000 --- a/apps/sources/watch_folders.py +++ /dev/null @@ -1,21 +0,0 @@ - - - if staging_folder.uncompress == SOURCE_UNCOMPRESS_CHOICE_Y: - expand = True - else: - expand = False - transformations, errors = SourceTransformation.transformations.get_for_object_as_list(staging_folder) - if (not expand) or (expand and not _handle_zip_file(request, staging_file.upload(), document_type=document_type, transformations=transformations)): - document = Document(file=staging_file.upload()) - if document_type: - document.document_type = document_type - document.save() - document.apply_default_transformations(transformations) - _handle_save_document(request, document, form) - messages.success(request, _(u'Staging file: %s, uploaded successfully.') % staging_file.filename) - - if staging_folder.delete_after_upload: - staging_file.delete(preview_size=staging_folder.get_preview_size(), transformations=transformations) - messages.success(request, _(u'Staging file: %s, deleted successfully.') % staging_file.filename) - except Exception, e: - messages.error(request, e)