From d8d8c75a2800b361edde146cbd619bcbbe650ccf Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Sat, 5 Feb 2011 21:00:23 -0400 Subject: [PATCH] Complete refactoring to allow out of order document, document file or metadata creation --- apps/common/templatetags/navigation.py | 4 +- apps/documents/__init__.py | 6 +++ apps/documents/admin.py | 17 ++++-- apps/documents/conf/settings.py | 3 ++ apps/documents/forms.py | 13 +++-- apps/documents/models.py | 71 ++++++++++++++++---------- apps/documents/staging.py | 67 ++++++++++++++++++++++++ apps/documents/urls.py | 7 ++- apps/documents/views.py | 62 ++++++++++++++-------- 9 files changed, 193 insertions(+), 57 deletions(-) create mode 100644 apps/documents/staging.py diff --git a/apps/common/templatetags/navigation.py b/apps/common/templatetags/navigation.py index 980941596e..8b778662c6 100644 --- a/apps/common/templatetags/navigation.py +++ b/apps/common/templatetags/navigation.py @@ -195,7 +195,9 @@ def resolve_template_variable(context, name): try: return unescape_string_literal(name) except ValueError: - return Variable(name).resolve(context) + #return Variable(name).resolve(context) + #Research if should return always as a str + return str(Variable(name).resolve(context)) except TypeError: return name diff --git a/apps/documents/__init__.py b/apps/documents/__init__.py index 9ecf086794..4c667ef63e 100644 --- a/apps/documents/__init__.py +++ b/apps/documents/__init__.py @@ -3,6 +3,7 @@ from django.utils.translation import ugettext_lazy as _ from common.api import register_links, register_menu from models import Document +from staging import StagingFile document_list = {'text':_(u'documents list'), 'view':'document_list', 'famfam':'page'} document_create = {'text':_('upload a document'), 'view':'document_create', 'famfam':'page_add'} @@ -11,10 +12,15 @@ document_view = {'text':_('details'), 'view':'document_view', 'args':'object.id' document_delete = {'text':_('delete'), 'view':'document_delete', 'args':'object.id', 'famfam':'page_delete'} document_edit = {'text':_('edit'), 'view':'document_edit', 'args':'object.id', 'famfam':'page_edit'} +document_create_from_staging = {'text':_('select staging'), 'view':'document_create_from_staging', 'args':{'file_id':'object.id', 'document_type_id': 'document_type_id'}, 'famfam':'page_add'} + + register_links(Document, [document_view, document_edit, document_delete]) register_links(Document, [document_list, document_create, document_create_multiple], menu_name='sidebar') register_links(['document_list', 'document_create', 'document_create_multiple', 'upload_document_with_type', 'upload_multiple_documents_with_type'], [document_list, document_create, document_create_multiple], menu_name='sidebar') +register_links(StagingFile, [document_create_from_staging]) + register_menu([ {'text':_('documents'), 'view':'document_list', 'links':[ diff --git a/apps/documents/admin.py b/apps/documents/admin.py index 99dc063513..6ecc70a4dd 100644 --- a/apps/documents/admin.py +++ b/apps/documents/admin.py @@ -1,7 +1,8 @@ from django.contrib import admin from models import MetadataType, DocumentType, Document, \ - DocumentTypeMetadataType, DocumentMetadata, DocumentTypeFilename + DocumentTypeMetadataType, DocumentMetadata, DocumentTypeFilename, \ + DocumentFile class MetadataTypeAdmin(admin.ModelAdmin): @@ -33,9 +34,19 @@ class DocumentMetadataInline(admin.StackedInline): allow_add = True +class DocumentFileInline(admin.StackedInline): + model = DocumentFile + extra = 1 + classes = ('collapse-open',) + allow_add = True + + class DocumentAdmin(admin.ModelAdmin): - inlines = [DocumentMetadataInline,] - list_display = ('uuid', 'file_filename', 'file_extension', 'file_mimetype') + inlines = [DocumentFileInline, DocumentMetadataInline,] + list_display = ('uuid',) + + + admin.site.register(MetadataType, MetadataTypeAdmin) diff --git a/apps/documents/conf/settings.py b/apps/documents/conf/settings.py index 4aebcd897e..dac0269c8a 100644 --- a/apps/documents/conf/settings.py +++ b/apps/documents/conf/settings.py @@ -1,5 +1,6 @@ import datetime import hashlib +import uuid from django.conf import settings @@ -14,3 +15,5 @@ DELETE_LOCAL_ORIGINAL = getattr(settings, 'DOCUMENTS_DELETE_LOCAL_ORIGINAL', Fal SLUGIFY_PATH = getattr(settings, 'DOCUMENTS_SLUGIFY_PATH', False) CHECKSUM_FUNCTION = getattr(settings, 'DOCUMENTS_CHECKSUM_FUNCTION', lambda x: hashlib.sha256(x).hexdigest()) DELETE_STAGING_FILE_AFTER_UPLOAD = getattr(settings, 'DOCUMENTS_DELETE_STAGING_FILE_AFTER_UPLOAD', False) +UUID_FUNCTION = getattr(settings, 'DOCUMENTS_UUID_FUNTION', lambda:unicode(uuid.uuid4())) +STORAGE_DIRECTORY_NAME = getattr(settings, 'DOCUMENTS_STORAGE_DIRECTORY_NAME', 'documents') diff --git a/apps/documents/forms.py b/apps/documents/forms.py index 653dd47270..18eee7a999 100644 --- a/apps/documents/forms.py +++ b/apps/documents/forms.py @@ -9,12 +9,16 @@ from common.wizard import BoundFormWizard from common.utils import urlquote from common.forms import DetailForm -from models import Document, DocumentType, DocumentTypeMetadataType - +from models import Document, DocumentType, DocumentTypeMetadataType, DocumentFile from documents.conf.settings import AVAILABLE_FUNCTIONS +class DocumentFileForm(forms.ModelForm): + class Meta: + model = DocumentFile + + class DocumentForm(forms.ModelForm): def __init__(self, *args, **kwargs): super(DocumentForm, self).__init__(*args, **kwargs) @@ -104,8 +108,9 @@ class DocumentCreateWizard(BoundFormWizard): self.initial = {1:initial} if step == 1: self.urldata = [] - for metadata in form.cleaned_data: - self.urldata.append((metadata['id'],metadata['value'])) + for id, metadata in enumerate(form.cleaned_data): + self.urldata.append(('metadata%s_id' % id,metadata['id'])) + self.urldata.append(('metadata%s_value' % id,metadata['value'])) def get_template(self, step): diff --git a/apps/documents/models.py b/apps/documents/models.py index 8c20ea76e0..a22cfa8669 100644 --- a/apps/documents/models.py +++ b/apps/documents/models.py @@ -1,6 +1,5 @@ import errno import os -import uuid import mimetypes from datetime import datetime @@ -16,18 +15,21 @@ from documents.conf.settings import AVAILABLE_FUNCTIONS from documents.conf.settings import FILESERVING_PATH from documents.conf.settings import SLUGIFY_PATH from documents.conf.settings import CHECKSUM_FUNCTION +from documents.conf.settings import UUID_FUNCTION +from documents.conf.settings import STORAGE_DIRECTORY_NAME if SLUGIFY_PATH == False: #Do not slugify path or filenames and extensions slugify = lambda x:x -def get_filename_from_uuid(instance, filename, directory='documents'): - populate_file_extension_and_mimetype(instance, filename) +def get_filename_from_uuid(instance, filename, directory=STORAGE_DIRECTORY_NAME): + #populate_file_extension_and_mimetype(instance, filename) stem, extension = os.path.splitext(filename) return '%s/%s%s' % (directory, instance.uuid, extension) -def populate_file_extension_and_mimetype(instance, filename): +def populate_file_extension_and_mimetype(instance):#, filename): + filename = instance.file.name # First populate the file extension and mimetype instance.file_mimetype, encoding = mimetypes.guess_type(filename) if not instance.file_mimetype: @@ -44,33 +46,20 @@ class DocumentType(models.Model): def __unicode__(self): return self.name - -class Document(models.Model): - """ Minimum fields for a document entry. - Inherit this model to customise document metadata, see BasicDocument for an example. - """ - document_type = models.ForeignKey(DocumentType, verbose_name=_(u'document type')) +class DocumentFile(models.Model): file = models.FileField(upload_to=get_filename_from_uuid) - uuid = models.CharField(max_length=36, default=lambda:unicode(uuid.uuid4()), blank=True, editable=False) file_mimetype = models.CharField(max_length=50, default='', editable=False) file_filename = models.CharField(max_length=64, default='', editable=False) file_extension = models.CharField(max_length=10, default='', editable=False) - date_added = models.DateTimeField(verbose_name=_(u'added'), auto_now_add=True) - date_updated = models.DateTimeField(verbose_name=_(u'updated'), auto_now=True) checksum = models.TextField(blank=True, null=True, verbose_name=_(u'checksum'), editable=False) class Meta: - verbose_name = _(u'document') - verbose_name_plural = _(u'documents') - ordering = ['-date_updated', '-date_added'] - + verbose_name = _(u'document file') + verbose_name_plural = _(u'documents files') + def __unicode__(self): - return self.uuid - - @models.permalink - def get_absolute_url(self): - return ('document_view', [self.id]) - + return self.id + def update_checksum(self, save=True): self.checksum = unicode(CHECKSUM_FUNCTION(self.file.read())) if save: @@ -78,11 +67,12 @@ class Document(models.Model): def save(self, *args, **kwargs): self.update_checksum(save=False) - super(Document, self).save(*args, **kwargs) - + populate_file_extension_and_mimetype(self) + super(DocumentFile, self).save(*args, **kwargs) + def delete(self, *args, **kwargs): self.delete_fs_links() - super(Document, self).delete(*args, **kwargs) + super(DocumentFile, self).delete(*args, **kwargs) def calculate_fs_links(self): targets = [] @@ -118,7 +108,34 @@ class Document(models.Model): pass else: raise OSError(ugettext(u'Unable to delete metadata indexing symbolic link: %s') % exc) - + + +class Document(models.Model): + """ Minimum fields for a document entry. + Inherit this model to customise document metadata, see BasicDocument for an example. + """ + document_type = models.ForeignKey(DocumentType, verbose_name=_(u'document type')) + #file = models.FileField(upload_to=get_filename_from_uuid, blank=True, null=True) + uuid = models.CharField(max_length=36, default=UUID_FUNCTION(), blank=True, editable=False) + #file_mimetype = models.CharField(max_length=50, default='', editable=False) + #file_filename = models.CharField(max_length=64, default='', editable=False) + #file_extension = models.CharField(max_length=10, default='', editable=False) + date_added = models.DateTimeField(verbose_name=_(u'added'), auto_now_add=True) + date_updated = models.DateTimeField(verbose_name=_(u'updated'), auto_now=True) + #checksum = models.TextField(blank=True, null=True, verbose_name=_(u'checksum'), editable=False) + document_file = models.ForeignKey(DocumentFile, blank=True, null=True, verbose_name=_('document file')) + + class Meta: + verbose_name = _(u'document') + verbose_name_plural = _(u'documents') + ordering = ['-date_updated', '-date_added'] + + def __unicode__(self): + return self.uuid + + @models.permalink + def get_absolute_url(self): + return ('document_view', [self.id]) available_functions_string = (_(u' Available functions: %s') % ','.join(['%s()' % name for name, function in AVAILABLE_FUNCTIONS.items()])) if AVAILABLE_FUNCTIONS else '' diff --git a/apps/documents/staging.py b/apps/documents/staging.py new file mode 100644 index 0000000000..076a346414 --- /dev/null +++ b/apps/documents/staging.py @@ -0,0 +1,67 @@ +import os +import shutil + +from django.core.exceptions import ObjectDoesNotExist +from django.conf import settings +from django.core.files.storage import default_storage + +from documents.conf.settings import STAGING_DIRECTORY +from documents.conf.settings import UUID_FUNCTION +from models import Document, get_filename_from_uuid + + +def get_all_files(): + return sorted([os.path.normcase(f) for f in os.listdir(STAGING_DIRECTORY)]) + + +class StagingFile(object): + @classmethod + def get_all(cls): + staging_files = [] + for id, filename in enumerate(get_all_files()): + staging_files.append(StagingFile( + filepath=os.path.join(STAGING_DIRECTORY, filename), + id=id)) + + return staging_files + + @classmethod + def get(cls, id): + files = get_all_files() + if id <= len(files): + return StagingFile( + filepath=os.path.join(STAGING_DIRECTORY, files[id]), + id=id) + raise ObjectDoesNotExist + + def __init__(self, filepath, id): + self.filepath = filepath + self.filename = os.path.basename(filepath) + self._id = id + + def __unicode__(self): + return self.filename + + def __repr__(self): + return self.__unicode__() + + def __getattr__(self, name): + if name == 'id': + return self._id + else: + raise AttributeError, name + + def upload(self, document_type): + document = Document(document_type=document_type) + document.save(save=False) + print 'UUID', document.uuid + tmp_filepath = os.path.join(settings.MEDIA_ROOT, UUID_FUNCTION()) + #shutil.copy(self.filepath, tmp_filepath) + #document = Document(document_type=document_type, + # file=tmp_filepath) + #document.save() + #final_filepath = get_filename_from_uuid(document, filename=self.filename) + #document.save() + #print final_filepath + + diff --git a/apps/documents/urls.py b/apps/documents/urls.py index f4c1cc8613..df5470dadc 100644 --- a/apps/documents/urls.py +++ b/apps/documents/urls.py @@ -5,11 +5,14 @@ from django.views.generic.create_update import create_object, update_object urlpatterns = patterns('documents.views', url(r'^document/list/$', 'document_list', (), 'document_list'), - url(r'^document/create/single/$', 'document_create', {'multiple':False}, 'document_create'), - url(r'^document/create/multiple/$', 'document_create', {'multiple':True}, 'document_create_multiple'), + url(r'^document/create/from/local/single/$', 'document_create', {'multiple':False}, 'document_create'), + url(r'^document/create/from/local/multiple/$', 'document_create', {'multiple':True}, 'document_create_multiple'), url(r'^document/type/(?P\d+)/upload/single/$', 'upload_document_with_type', {'multiple':False}, 'upload_document_with_type'), url(r'^document/type/(?P\d+)/upload/multiple/$', 'upload_document_with_type', {'multiple':True}, 'upload_multiple_documents_with_type'), url(r'^document/(?P\d+)/$', 'document_view', (), 'document_view'), url(r'^document/(?P\d+)/delete/$', 'document_delete', (), 'document_delete'), url(r'^document/(?P\d+)/edit/$', 'document_edit', (), 'document_edit'), + + url(r'^document/type/(?P\d+)/upload/from/staging/(?P\d+)/single/$', 'document_create_from_staging', {'multiple':False}, 'document_create_from_staging'), + #url(r'^document/create/from/staging/(?P\d+)/$', '', (), 'document_create_from_staging'), ) diff --git a/apps/documents/views.py b/apps/documents/views.py index ca8ce4d37e..ff7e4ab889 100644 --- a/apps/documents/views.py +++ b/apps/documents/views.py @@ -1,6 +1,3 @@ -import datetime -import os - from django.utils.translation import ugettext as _ from django.http import HttpResponse, HttpResponseRedirect from django.shortcuts import render_to_response, get_object_or_404, redirect @@ -12,13 +9,12 @@ from django.views.generic.create_update import create_object, delete_object, upd from django.forms.formsets import formset_factory -from forms import DocumentForm_view - from models import Document, DocumentMetadata, DocumentType, MetadataType from forms import DocumentTypeSelectForm, DocumentCreateWizard, \ - MetadataForm, DocumentForm, DocumentForm_edit + MetadataForm, DocumentForm, DocumentForm_edit, DocumentForm_view, \ + DocumentFileForm -from documents.conf.settings import STAGING_DIRECTORY +from staging import StagingFile def document_list(request): return object_list( @@ -43,25 +39,39 @@ def document_create(request, multiple=True): return wizard(request) +def _save_metadata_from_request(request, document): + metadata_dict = { + 'id':{}, + 'value':{} + } + #Match out of order metadata_type ids with metadata values from request + for key, value in request.GET.items(): + if 'metadata' in key: + index, element = key[8:].split('_') + metadata_dict[element][index] = value + + #Use matched metadata now to create document metadata + for key, value in zip(metadata_dict['id'].values(), metadata_dict['value'].values()): + document_metadata = DocumentMetadata( + document=document, + metadata_type=get_object_or_404(MetadataType, pk=key), + value=value + ) + document_metadata.save() + + def upload_document_with_type(request, document_type_id, multiple=True): document_type = get_object_or_404(DocumentType, pk=document_type_id) if request.method == 'POST': - form = DocumentForm(request.POST, request.FILES, initial={'document_type':document_type}) + form = DocumentFileForm(request.POST, request.FILES)#, initial={'document_type':document_type}) if form.is_valid(): instance = form.save() if 'new_filename' in form.cleaned_data: if form.cleaned_data['new_filename']: instance.file_filename = form.cleaned_data['new_filename'].filename instance.save() - - for key, value in request.GET.items(): - document_metadata = DocumentMetadata( - document=instance, - metadata_type=get_object_or_404(MetadataType, pk=key), - value=value - ) - document_metadata.save() - + + _save_metadata_from_request(request, instance) messages.success(request, _(u'Document uploaded successfully.')) try: instance.create_fs_links() @@ -73,13 +83,14 @@ def upload_document_with_type(request, document_type_id, multiple=True): else: return HttpResponseRedirect(reverse('document_list')) else: - form = DocumentForm(initial={'document_type':document_type}) + form = DocumentFileForm()#initial={'document_type':document_type}) - filelist = sorted([os.path.normcase(f) for f in os.listdir(STAGING_DIRECTORY)]) - + filelist = StagingFile.get_all() + return render_to_response('generic_form.html', { 'form':form, 'title':_(u'upload a local document'), + 'document_type_id':document_type_id, 'subtemplates_dict':[ { 'name':'generic_list_subtemplate.html', @@ -162,3 +173,14 @@ def document_edit(request, document_id): }, context_instance=RequestContext(request)) + +def document_create_from_staging(request, file_id, document_type_id, multiple=True): + document_type = get_object_or_404(DocumentType, pk=document_type_id) + staging_file = StagingFile.get(id=int(file_id)) + staging_file.upload(document_type=document_type) + + if multiple: + return HttpResponseRedirect(request.get_full_path()) + else: + return HttpResponseRedirect(reverse('document_list')) +