matthias/mayan-edms
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Initial version of the GridFS storage driver

Browse Source
This commit is contained in:
Roberto Rosario
2011-03-04 01:08:20 -04:00
parent 3f71ee1a06
commit d0bea8ffeb
13 changed files with 218 additions and 183 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
apps/documents/__init__.py
View File

@@ -13,7 +13,6 @@ from models import Document, DocumentPage, DocumentPageTransformation
from staging import StagingFile
from common.conf import settings as common_settings
from conf.settings import ENABLE_SINGLE_DOCUMENT_UPLOAD
PERMISSION_DOCUMENT_CREATE = 'document_create'
@@ -75,6 +74,18 @@ register_links(['document_page_view', 'document_page_transformation_edit', 'docu
register_links(StagingFile, [staging_file_preview, staging_file_delete])
def document_exists(document):
try:
if document.exists():
return '<span class="famfam active famfam-tick"></span>'
else:
return '<span class="famfam active famfam-cross"></span>'
except Exception, exc:
return exc
register_model_list_columns(Document, [
{'name':_(u'thumbnail'), 'attribute':
lambda x: '<a class="fancybox" href="%s"><img src="%s" /></a>' % (reverse('document_preview', args=[x.id]),
@@ -83,7 +94,9 @@ register_model_list_columns(Document, [
{'name':_(u'metadata'), 'attribute':
lambda x: ', '.join(['%s - %s' %(metadata.metadata_type, metadata.value) for metadata in x.documentmetadata_set.all()])
},
{'name':_(u'exists'), 'attribute':
lambda x: document_exists(x)
},
])
if ENABLE_SINGLE_DOCUMENT_UPLOAD:
@@ -96,5 +109,3 @@ else:
{'text':_('documents'), 'view':'document_create_multiple', 'links':[
document_create_multiple, document_list
],'famfam':'page','position':1}])
TEMPORARY_DIRECTORY = common_settings.TEMPORARY_DIRECTORY if common_settings.TEMPORARY_DIRECTORY else tempfile.mkdtemp()

54
apps/documents/metadata.py Normal file
View File

@@ -0,0 +1,54 @@
from urllib import unquote_plus
from django.shortcuts import get_object_or_404
from django.core.exceptions import ObjectDoesNotExist
from models import DocumentMetadata, MetadataType
def decode_metadata_from_url(url_dict):
metadata_dict = {
'id':{},
'value':{}
}
metadata_list = []
#Match out of order metadata_type ids with metadata values from request
for key, value in url_dict.items():
if 'metadata' in key:
index, element = key[8:].split('_')
metadata_dict[element][index] = value
#Convert the nested dictionary into a list of id+values dictionaries
for order, id in metadata_dict['id'].items():
if order in metadata_dict['value'].keys():
metadata_list.append({'id':id, 'value':metadata_dict['value'][order]})
return metadata_list
def save_metadata_list(metadata_list, document):
for item in metadata_list:
if item['value']:
save_metadata(item, document)
else:
try:
metadata_type = MetadataType.objects.get(id=item['id'])
document_metadata = DocumentMetadata.objects.get(document=document,
metadata_type=metadata_type)
document_metadata.delete()
except ObjectDoesNotExist:
pass
def save_metadata(metadata_dict, document):
#Use matched metadata now to create document metadata
document_metadata, created = DocumentMetadata.objects.get_or_create(
document=document,
metadata_type=get_object_or_404(MetadataType, pk=metadata_dict['id']),
)
#Handle 'plus sign as space' in the url
#unquote_plus handles utf-8?!?
#http://stackoverflow.com/questions/4382875/handling-iri-in-django
document_metadata.value=unquote_plus(metadata_dict['value'])#.decode('utf-8')
document_metadata.save()

11
apps/documents/models.py
View File

@@ -19,7 +19,6 @@ from documents.conf.settings import STORAGE_BACKEND
from documents.conf.settings import AVAILABLE_TRANSFORMATIONS
from documents.conf.settings import DEFAULT_TRANSFORMATIONS
def get_filename_from_uuid(instance, filename):
filename, extension = os.path.splitext(filename)
instance.file_filename = filename
@@ -80,9 +79,9 @@ class Document(models.Model):
def update_mimetype(self, save=True):
try:
mime = magic.Magic(mime=True)
self.file_mimetype = mime.from_buffer(self.read())
self.file_mimetype = mime.from_buffer(self.open().read())
mime_encoding = magic.Magic(mime_encoding=True)
self.file_mime_encoding = mime_encoding.from_buffer(self.read())
self.file_mime_encoding = mime_encoding.from_buffer(self.open().read())
except:
self.file_mimetype = u'unknown'
self.file_mime_encoding = u'unknown'
@@ -119,8 +118,8 @@ class Document(models.Model):
def save_to_file(self, filepath, buffer_size=1024*1024):
input_descriptor = self.open()
output_descriptor = open(filepath, 'wb')
while 1:
copy_buffer = input_descriptor.read()
while True:
copy_buffer = input_descriptor.read(buffer_size)
if copy_buffer:
output_descriptor.write(copy_buffer)
else:
@@ -129,7 +128,7 @@ class Document(models.Model):
output_descriptor.close()
return filepath
def exists(self):
return self.file.storage.exists(self.file.path)

89
apps/documents/utils.py
View File

@@ -1,14 +1,9 @@
import os
import tempfile
from urllib import unquote_plus
from django.shortcuts import get_object_or_404
from django.core.exceptions import ObjectDoesNotExist
from documents import TEMPORARY_DIRECTORY
from common import TEMPORARY_DIRECTORY
from models import Document, DocumentMetadata, MetadataType
#http://stackoverflow.com/questions/123198/how-do-i-copy-a-file-in-python
def copyfile(source, dest, buffer_size=1024*1024):
@@ -22,7 +17,7 @@ def copyfile(source, dest, buffer_size=1024*1024):
if not hasattr(dest, 'write'):
dest = open(dest, 'wb')
while 1:
while True:
copy_buffer = source.read(buffer_size)
if copy_buffer:
dest.write(copy_buffer)
@@ -33,82 +28,8 @@ def copyfile(source, dest, buffer_size=1024*1024):
dest.close()
def from_descriptor_to_tempfile(input_descriptor, filename, buffer_size=1024*1024):
path = os.path.join(TEMPORARY_DIRECTORY, filename)
output_descriptor = open(path, 'wb')
while 1:
copy_buffer = input_descriptor.read(buffer_size)
if copy_buffer:
output_descriptor.write(copy_buffer)
else:
break
input_descriptor.close()
output_descriptor.close()
return path
def document_save_to_temp_dir(document, filename, buffer_size=1024*1024):
temporary_path = os.path.join(TEMPORARY_DIRECTORY, filename)
return document.save_to_file(temporary_path, buffer_size)
def from_descriptor_to_new_tempfile(input_descriptor, buffer_size=1024*1024):
output_descriptor, tmp_filename = tempfile.mkstemp()
while 1:
copy_buffer = input_descriptor.read(buffer_size)
if copy_buffer:
#output_descriptor.write(copy_buffer)
os.write(output_descriptor, copy_buffer)
else:
break
input_descriptor.close()
os.close(output_descriptor)
return tmp_filename
def decode_metadata_from_url(url_dict):
metadata_dict = {
'id':{},
'value':{}
}
metadata_list = []
#Match out of order metadata_type ids with metadata values from request
for key, value in url_dict.items():
if 'metadata' in key:
index, element = key[8:].split('_')
metadata_dict[element][index] = value
#Convert the nested dictionary into a list of id+values dictionaries
for order, id in metadata_dict['id'].items():
if order in metadata_dict['value'].keys():
metadata_list.append({'id':id, 'value':metadata_dict['value'][order]})
return metadata_list
def save_metadata_list(metadata_list, document):
for item in metadata_list:
if item['value']:
save_metadata(item, document)
else:
try:
metadata_type = MetadataType.objects.get(id=item['id'])
document_metadata = DocumentMetadata.objects.get(document=document,
metadata_type=metadata_type)
document_metadata.delete()
except ObjectDoesNotExist:
pass
def save_metadata(metadata_dict, document):
#Use matched metadata now to create document metadata
document_metadata, created = DocumentMetadata.objects.get_or_create(
document=document,
metadata_type=get_object_or_404(MetadataType, pk=metadata_dict['id']),
)
#Handle 'plus sign as space' in the url
#unquote_plus handles utf-8?!?
#http://stackoverflow.com/questions/4382875/handling-iri-in-django
document_metadata.value=unquote_plus(metadata_dict['value'])#.decode('utf-8')
document_metadata.save()

40
apps/documents/views.py
View File

@@ -16,26 +16,14 @@ from django.core.exceptions import ObjectDoesNotExist
from django.core.files.uploadedfile import SimpleUploadedFile
from common.utils import pretty_size
from permissions.api import check_permissions, Unauthorized
from filetransfers.api import serve_file
from converter.api import convert, in_image_cache, QUALITY_DEFAULT
from converter import TRANFORMATION_CHOICES
from filetransfers.api import serve_file
from filesystem_serving.api import document_create_fs_links, document_delete_fs_links
from utils import from_descriptor_to_tempfile
from models import Document, DocumentMetadata, DocumentType, MetadataType, \
DocumentPage, DocumentPageTransformation
from forms import DocumentTypeSelectForm, DocumentCreateWizard, \
MetadataForm, DocumentForm, DocumentForm_edit, DocumentForm_view, \
StagingDocumentForm, DocumentTypeMetadataType, DocumentPreviewForm, \
MetadataFormSet, DocumentPageForm, DocumentPageTransformationForm
from staging import StagingFile
from filesystem_serving.conf.settings import FILESERVING_ENABLE
from ocr.models import add_document_to_queue
from permissions.api import check_permissions, Unauthorized
from documents.conf.settings import DELETE_STAGING_FILE_AFTER_UPLOAD
from documents.conf.settings import USE_STAGING_DIRECTORY
@@ -49,16 +37,23 @@ from documents.conf.settings import AUTOMATIC_OCR
from documents.conf.settings import UNCOMPRESS_COMPRESSED_LOCAL_FILES
from documents.conf.settings import UNCOMPRESS_COMPRESSED_STAGING_FILES
from filesystem_serving.conf.settings import FILESERVING_ENABLE
from documents import PERMISSION_DOCUMENT_CREATE, \
PERMISSION_DOCUMENT_CREATE, PERMISSION_DOCUMENT_PROPERTIES_EDIT, \
PERMISSION_DOCUMENT_METADATA_EDIT, PERMISSION_DOCUMENT_VIEW, \
PERMISSION_DOCUMENT_DELETE, PERMISSION_DOCUMENT_DOWNLOAD, \
PERMISSION_DOCUMENT_TRANSFORM, PERMISSION_DOCUMENT_TOOLS
from forms import DocumentTypeSelectForm, DocumentCreateWizard, \
MetadataForm, DocumentForm, DocumentForm_edit, DocumentForm_view, \
StagingDocumentForm, DocumentTypeMetadataType, DocumentPreviewForm, \
MetadataFormSet, DocumentPageForm, DocumentPageTransformationForm
from utils import save_metadata, save_metadata_list, decode_metadata_from_url
from metadata import save_metadata, save_metadata_list, decode_metadata_from_url
from models import Document, DocumentMetadata, DocumentType, MetadataType, \
DocumentPage, DocumentPageTransformation
from staging import StagingFile
from utils import document_save_to_temp_dir
def document_list(request):
permissions = [PERMISSION_DOCUMENT_VIEW]
@@ -249,7 +244,7 @@ def upload_document_with_type(request, document_type_id, multiple=True):
return render_to_response('generic_form.html', context,
context_instance=RequestContext(request))
def document_view(request, document_id):
permissions = [PERMISSION_DOCUMENT_VIEW]
try:
@@ -503,10 +498,11 @@ def get_document_image(request, document_id, size=PREVIEW_SIZE, quality=QUALITY_
if filepath:
return serve_file(request, File(file=open(filepath, 'r')))
#Save to a temporary location
filepath = from_descriptor_to_tempfile(document.open(), document.checksum)
filepath = document_save_to_temp_dir(document, filename=document.checksum)
output_file = convert(filepath, size=size, format='jpg', quality=quality, extra_options=tranformation_string, page=page-1)
return serve_file(request, File(file=open(output_file, 'r')), content_type='image/jpeg')
except Exception, e:
#messages.error(request, e)
if size == THUMBNAIL_SIZE:
return serve_file(request, File(file=open('%simages/picture_error.png' % settings.MEDIA_ROOT, 'r')))
else: