diff --git a/mayan/apps/sources/api_views.py b/mayan/apps/sources/api_views.py index 773a4b7919..64f07635ab 100644 --- a/mayan/apps/sources/api_views.py +++ b/mayan/apps/sources/api_views.py @@ -20,7 +20,7 @@ from .serializers import (NewDocumentSerializer, StagingFolderFileSerializer, StagingFolderSerializer, StagingSourceFileImageSerializer, WebFormSourceSerializer) -from .tasks import task_upload_document +from .tasks import task_source_upload_document class APIStagingSourceFileView(generics.GenericAPIView): @@ -88,56 +88,3 @@ class APIStagingSourceFileImageView(generics.GenericAPIView): return Response({'status': 'error', 'detail': 'unknown_file_format', 'message': unicode(exception)}) except UnkownConvertError as exception: return Response({'status': 'error', 'detail': 'converter_error', 'message': unicode(exception)}) - - -class APIDocumentCreateView(generics.CreateAPIView): - """ - Create a new document from an uploaded file. - """ - - serializer_class = NewDocumentSerializer - - permission_classes = (MayanPermission,) - mayan_view_permissions = {'POST': [PERMISSION_DOCUMENT_CREATE]} - - def get_serializer_class(self): - return NewDocumentSerializer - - def create(self, request): - # TODO: use serializer instance instead of raw request - request = self.request.POST - - if self.request.FILES: - temporary_file = tempfile.NamedTemporaryFile(delete=False) - for chunk in self.request.FILES['file'].chunks(): - temporary_file.write(chunk) - - temporary_file.close() - self.request.FILES['file'].close() - else: - return Response({ - 'status': 'error', - 'message': 'No file provided.' - }) - - if not self.request.user.is_anonymous(): - user_id = self.request.user.pk - else: - user_id = None - - task_upload_document.apply_async(kwargs=dict( - source_id=int(request.get('source')), - file_path=temporary_file.name, - filename=request.get('filename', self.request.FILES['file'].name), - use_file_name=request.get('use_file_name', False), - document_type_id=int(request.get('document_type', 0)) or None, - expand=request.get('expand', False), - metadata_dict_list={}, - user_id=user_id, - description=request.get('description', None), - ), queue='uploads') - - return Response({ - 'status': 'success', - 'message': 'New document creation queued.' - }) diff --git a/mayan/apps/sources/models.py b/mayan/apps/sources/models.py index 8b1e58b247..ea6dd88cbd 100644 --- a/mayan/apps/sources/models.py +++ b/mayan/apps/sources/models.py @@ -18,7 +18,6 @@ from django.utils.translation import ugettext_lazy as _ from model_utils.managers import InheritanceManager -from common.compressed_files import CompressedFile, NotACompressedFile from converter.api import get_available_transformations_choices from converter.literals import DIMENSION_SEPARATOR from djcelery.models import PeriodicTask, IntervalSchedule @@ -57,43 +56,25 @@ class Source(models.Model): def get_transformation_list(self): return SourceTransformation.transformations.get_for_object_as_list(self) - def upload_document(self, file_object, document_type, label, expand=False, metadata_dict_list=None, user=None, document=None, command_line=False, description=None, language=None): - is_compressed = None - - if expand: - try: - cf = CompressedFile(file_object) - count = 1 - for fp in cf.children(): - if command_line: - print 'Uploading file #%d: %s' % (count, fp) - self.upload_single_document(file_object=fp, label=unicode(fp), document_type=document_type, metadata_dict_list=metadata_dict_list, user=user, description=description, language=language) - fp.close() - count += 1 - - except NotACompressedFile: - is_compressed = False - logging.debug('Exception: NotACompressedFile') - if command_line: - raise - self.upload_single_document(file_object=file_object, label=label, document_type=document_type, metadata_dict_list=metadata_dict_list, user=user, description=description, language=language) - else: - is_compressed = True - else: - self.upload_single_document(file_object=file_object, label=label, document_type=document_type, metadata_dict_list=metadata_dict_list, user=user, description=description, language=language) - - file_object.close() - - def upload_single_document(self, file_object, label, document_type, metadata_dict_list=None, user=None, description=None, language=None): - new_version = Document.objects.new_document(file_object=file_object, document_type=document_type, label=label, description=description, language=language, user=user) + def upload_document(self, file_object, label, description=None, document_type=None, expand=False, language=None, metadata_dict_list=None, user=None): + new_versions = Document.objects.new_document( + description=description, + document_type=document_type or self.document_type, + expand=expand, + file_object=file_object, + label=label, + language=language, + user=user + ) transformations, errors = self.get_transformation_list() - new_version.apply_default_transformations(transformations) + for new_version in new_versions: + new_version.apply_default_transformations(transformations) - if metadata_dict_list: - save_metadata_list(metadata_dict_list, new_version.document, create=True) + if metadata_dict_list: + save_metadata_list(metadata_dict_list, new_version.document, create=True) - def get_upload_file_object(self, request, form): + def get_upload_file_object(self, form_data): pass def clean_up_upload_file(self, upload_file_object): @@ -141,8 +122,8 @@ class StagingFolderSource(InteractiveSource): except OSError as exception: raise Exception(_(u'Unable get list of staging files: %s') % exception) - def get_upload_file_object(self, request, form): - staging_file = self.get_file(encoded_filename=form.cleaned_data['staging_file_id']) + def get_upload_file_object(self, form_data): + staging_file = self.get_file(encoded_filename=form_data['staging_file_id']) return SourceUploadedFile(source=self, file=staging_file.as_file(), extra_data=staging_file) def clean_up_upload_file(self, upload_file_object): @@ -165,8 +146,8 @@ class WebFormSource(InteractiveSource): uncompress = models.CharField(max_length=1, choices=SOURCE_INTERACTIVE_UNCOMPRESS_CHOICES, verbose_name=_(u'Uncompress'), help_text=_(u'Whether to expand or not compressed archives.')) # Default path - def get_upload_file_object(self, request, form): - return SourceUploadedFile(source=self, file=form.cleaned_data['file']) + def get_upload_file_object(self, form_data): + return SourceUploadedFile(source=self, file=form_data['file']) class Meta: verbose_name = _(u'Web form') @@ -256,8 +237,8 @@ class EmailBaseModel(IntervalBaseModel): logger.debug('filename: %s' % filename) - document_file = Attachment(part, name=filename) - source.upload_file(document_file, expand=(source.uncompress == SOURCE_UNCOMPRESS_CHOICE_Y), document_type=source.document_type) + file_object = Attachment(part, name=filename) + source.upload_file(file_object, expand=(source.uncompress == SOURCE_UNCOMPRESS_CHOICE_Y), document_type=source.document_type) class Meta: verbose_name = _('Email source') @@ -358,15 +339,12 @@ class WatchFolderSource(IntervalBaseModel): folder_path = models.CharField(max_length=255, verbose_name=_(u'Folder path'), help_text=_(u'Server side filesystem path.')) def check_source(self): - # TEMP: until default document language problem is fixed - from documents.settings import LANGUAGE - for file_name in os.listdir(self.folder_path): full_path = os.path.join(self.folder_path, file_name) if os.path.isfile(full_path): with File(file=open(full_path, mode='rb')) as file_object: - self.upload_document(file_object, label=file_name, document_type=self.document_type, expand=(self.uncompress == SOURCE_UNCOMPRESS_CHOICE_Y), language=LANGUAGE) + self.upload_document(file_object, label=file_name, expand=(self.uncompress == SOURCE_UNCOMPRESS_CHOICE_Y)) os.unlink(full_path) class Meta: diff --git a/mayan/apps/sources/tasks.py b/mayan/apps/sources/tasks.py index 8c43d4e897..0e6a776e1e 100644 --- a/mayan/apps/sources/tasks.py +++ b/mayan/apps/sources/tasks.py @@ -5,6 +5,7 @@ from django.core.files import File from mayan.celery import app +from common.models import SharedUploadedFile from documents.models import Document, DocumentType from .models import Source @@ -20,7 +21,8 @@ def task_check_interval_source(source_id): @app.task(ignore_result=True) -def task_upload_document(source_id, file_path, label, document_type_id, expand=False, metadata_dict_list=None, user_id=None, command_line=False, description=None, language=None): +def task_source_upload_document(label, document_type_id, shared_uploaded_file_id, source_id, description=None, expand=False, language=None, metadata_dict_list=None, user_id=None): + shared_uploaded_file = SharedUploadedFile.objects.get(pk=shared_uploaded_file_id) source = Source.objects.get_subclass(pk=source_id) document_type = DocumentType.objects.get(pk=document_type_id) @@ -29,10 +31,10 @@ def task_upload_document(source_id, file_path, label, document_type_id, expand=F else: user = None - with File(file=open(file_path, mode='rb')) as file_object: - source.upload_document(file_object, label=label, document_type=document_type, expand=expand, metadata_dict_list=metadata_dict_list, user=user, command_line=command_line, description=description, language=language) + with File(file=shared_uploaded_file.file) as file_object: + source.upload_document(description=description, document_type=document_type, expand=expand, file_object=file_object, label=label, language=language, metadata_dict_list=metadata_dict_list, user=user) - # TODO: delete temporary_file + shared_uploaded_file.delete() # TODO: Report/record how was file uploaded # if result['is_compressed'] is None: @@ -43,20 +45,3 @@ def task_upload_document(source_id, file_path, label, document_type_id, expand=F # if result['is_compressed'] is False: # messages.warning(request, _(u'File was not a compressed file, uploaded as it was.')) - - -@app.task(ignore_result=True) -def task_upload_new_version(source_id, file_path, document_id, user_id, version_update=None, comment=None): - source = Source.objects.get_subclass(pk=source_id) - document = Document.objects.get(pk=document_id) - - if user_id: - user = User.objects.get(pk=user_id) - else: - user = None - - # Use File class otherwise we get error: - # 'file' object has no attribute '_committed' - - with File(file=open(file_path, mode='rb')) as file_object: - document.new_version(file=file_object, user=user, version_update=version_update, comment=comment) diff --git a/mayan/apps/sources/urls.py b/mayan/apps/sources/urls.py index ac225c9393..da99680c50 100644 --- a/mayan/apps/sources/urls.py +++ b/mayan/apps/sources/urls.py @@ -2,9 +2,8 @@ from __future__ import absolute_import from django.conf.urls import patterns, url -from .api_views import (APIDocumentCreateView, APIStagingSourceFileView, - APIStagingSourceFileImageView, APIStagingSourceListView, - APIStagingSourceView) +from .api_views import (APIStagingSourceFileView, APIStagingSourceFileImageView, + APIStagingSourceListView, APIStagingSourceView) from .views import UploadInteractiveVersionView, UploadInteractiveView from .wizards import DocumentCreateWizard @@ -36,7 +35,6 @@ urlpatterns = patterns('sources.views', ) api_urls = patterns('', - url(r'^document/create/$', APIDocumentCreateView.as_view(), name='document-create-view'), url(r'^staging_folders/file/(?P[0-9]+)/(?P.+)/image/$', APIStagingSourceFileImageView.as_view(), name='stagingfolderfile-image-view'), url(r'^staging_folders/file/(?P[0-9]+)/(?P.+)/$', APIStagingSourceFileView.as_view(), name='stagingfolderfile-detail'), url(r'^staging_folders/$', APIStagingSourceListView.as_view(), name='stagingfolder-list'), diff --git a/mayan/apps/sources/views.py b/mayan/apps/sources/views.py index f5deab25d7..3a5f4434ec 100644 --- a/mayan/apps/sources/views.py +++ b/mayan/apps/sources/views.py @@ -14,11 +14,13 @@ from django.utils.safestring import mark_safe from django.utils.translation import ugettext_lazy as _ from acls.models import AccessEntry +from common.models import SharedUploadedFile from common.utils import encapsulate from common.views import MultiFormView from documents.models import DocumentType, Document from documents.permissions import (PERMISSION_DOCUMENT_CREATE, PERMISSION_DOCUMENT_NEW_VERSION) +from documents.tasks import task_new_document, task_upload_new_version from metadata.api import decode_metadata_from_url, metadata_repr_as_list from permissions.models import Permission @@ -32,7 +34,7 @@ from .permissions import (PERMISSION_SOURCES_SETUP_CREATE, PERMISSION_SOURCES_SETUP_DELETE, PERMISSION_SOURCES_SETUP_EDIT, PERMISSION_SOURCES_SETUP_VIEW) -from .tasks import task_upload_document, task_upload_new_version +from .tasks import task_source_upload_document from .utils import get_class, get_form_class, get_upload_form_class @@ -191,41 +193,35 @@ class UploadInteractiveView(UploadBaseView): else: expand = False - uploaded_file = self.source.get_upload_file_object(self.request, forms['source_form']) + uploaded_file = self.source.get_upload_file_object(forms['source_form'].cleaned_data) - file_object = uploaded_file.file - temporary_file = tempfile.NamedTemporaryFile(delete=False) - for chunk in file_object.chunks(): - temporary_file.write(chunk) + shared_uploaded_file = SharedUploadedFile.objects.create(file=uploaded_file.file) - temporary_file.close() - file_object.close() - - try: - self.source.clean_up_upload_file(uploaded_file) - except Exception as exception: - messages.error(self.request, exception) + label = shared_uploaded_file.filename + if 'document_type_available_filenames' in forms['document_form'].cleaned_data: + if forms['document_form'].cleaned_data['document_type_available_filenames']: + label = forms['document_form'].cleaned_data['document_type_available_filenames'].filename if not self.request.user.is_anonymous(): user_id = self.request.user.pk else: user_id = None - label = file_object.name - if 'document_type_available_filenames' in forms['document_form'].cleaned_data: - if forms['document_form'].cleaned_data['document_type_available_filenames']: - label = forms['document_form'].cleaned_data['document_type_available_filenames'].filename + try: + self.source.clean_up_upload_file(uploaded_file) + except Exception as exception: + messages.error(self.request, exception) - task_upload_document.apply_async(kwargs=dict( - source_id=self.source.pk, - file_path=temporary_file.name, - label=label, + task_source_upload_document.apply_async(kwargs=dict( + description=forms['document_form'].cleaned_data.get('description'), document_type_id=self.document_type.pk, expand=expand, + label=label, + language=forms['document_form'].cleaned_data.get('language'), metadata_dict_list=decode_metadata_from_url(self.request.GET), + shared_uploaded_file_id=shared_uploaded_file.pk, + source_id=self.source.pk, user_id=user_id, - description=forms['document_form'].cleaned_data.get('description'), - language=forms['document_form'].cleaned_data.get('language') ), queue='uploads') messages.success(self.request, _(u'New document queued for uploaded and will be available shortly.')) @@ -296,15 +292,9 @@ class UploadInteractiveVersionView(UploadBaseView): return super(UploadInteractiveVersionView, self).dispatch(request, *args, **kwargs) def forms_valid(self, forms): - uploaded_file = self.source.get_upload_file_object(self.request, forms['source_form']) + uploaded_file = self.source.get_upload_file_object(forms['source_form'].cleaned_data) - file_object = uploaded_file.file - temporary_file = tempfile.NamedTemporaryFile(delete=False) - for chunk in file_object.chunks(): - temporary_file.write(chunk) - - temporary_file.close() - file_object.close() + shared_uploaded_file = SharedUploadedFile.objects.create(file=uploaded_file.file) try: self.source.clean_up_upload_file(uploaded_file) @@ -317,8 +307,7 @@ class UploadInteractiveVersionView(UploadBaseView): user_id = None task_upload_new_version.apply_async(kwargs=dict( - source_id=self.source.pk, - file_path=temporary_file.name, + shared_uploaded_file_id=shared_uploaded_file.pk, document_id=self.document.pk, user_id=user_id, version_update=forms['document_form'].cleaned_data.get('version_update'), @@ -398,6 +387,10 @@ def setup_source_list(request): 'name': _('Type'), 'attribute': encapsulate(lambda x: x.class_fullname()) }, + { + 'name': _('Enabled'), + 'attribute': encapsulate(lambda x: _('Yes') if x.enabled else _('No')) + }, ] }