matthias/mayan-edms
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Extracted document filesystem serving into a separate app, see docs/Changelog.txt to update database accordingly

Browse Source
This commit is contained in:
Roberto Rosario
2011-02-26 12:50:16 -04:00
parent ff77b622a9
commit 79f6897d8a
21 changed files with 328 additions and 220 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/documents/__init__.py
-1
View File
@@ -48,7 +48,6 @@ document_preview = {'text':_('preview'), 'class':'fancybox', 'view':'document_pr
document_download = {'text':_('download'), 'view':'document_download', 'args':'object.id', 'famfam':'page_save', 'permissions':{'namespace':'documents', 'permissions':[PERMISSION_DOCUMENT_DOWNLOAD]}}
document_find_duplicates = {'text':_('find duplicates'), 'view':'document_find_duplicates', 'args':'object.id', 'famfam':'page_refresh', 'permissions':{'namespace':'documents', 'permissions':[PERMISSION_DOCUMENT_VIEW]}}
document_find_all_duplicates = {'text':_('find all duplicates'), 'view':'document_find_all_duplicates', 'famfam':'page_refresh', 'permissions':{'namespace':'documents', 'permissions':[PERMISSION_DOCUMENT_VIEW]}}
document_recreate_all_links = {'text':_('recreate index links'), 'view':'document_recreate_all_links', 'famfam':'page_link', 'permissions':{'namespace':'documents', 'permissions':[PERMISSION_DOCUMENT_TOOLS]}}
document_page_transformation_create = {'text':_('create new transformation'), 'view':'document_page_transformation_create', 'args':'object.id', 'famfam':'pencil_add', 'permissions':{'namespace':'documents', 'permissions':[PERMISSION_DOCUMENT_TRANSFORM]}}
document_page_transformation_edit = {'text':_('edit'), 'view':'document_page_transformation_edit', 'args':'object.id', 'famfam':'pencil_go', 'permissions':{'namespace':'documents', 'permissions':[PERMISSION_DOCUMENT_TRANSFORM]}}
apps/documents/admin.py
+3 -10
View File
@@ -2,9 +2,11 @@ from django.contrib import admin
from models import MetadataType, DocumentType, Document, \
DocumentTypeMetadataType, DocumentMetadata, DocumentTypeFilename, \
MetadataIndex, DocumentMetadataIndex, DocumentPage, MetadataGroup, \
MetadataIndex, DocumentPage, MetadataGroup, \
MetadataGroupItem, DocumentPageTransformation
from filesystem_serving.admin import DocumentMetadataIndexInline
class MetadataTypeAdmin(admin.ModelAdmin):
list_display = ('name', 'title', 'default', 'lookup')
@@ -40,15 +42,6 @@ class DocumentMetadataInline(admin.StackedInline):
extra = 0
classes = ('collapse-open',)
allow_add = False
readonly_fields = ('metadata_type', 'value')
class DocumentMetadataIndexInline(admin.StackedInline):
model = DocumentMetadataIndex
extra = 1
classes = ('collapse-open',)
allow_add = True
readonly_fields = ('suffix', 'metadata_index', 'filename')
class DocumentPageTransformationAdmin(admin.ModelAdmin):
apps/documents/conf/settings.py
-6
View File
@@ -60,9 +60,3 @@ DEFAULT_TRANSFORMATIONS = getattr(settings, 'DOCUMENTS_DEFAULT_TRANSFORMATIONS',
#Groups
GROUP_MAX_RESULTS = getattr(settings, 'DOCUMENTS_GROUP_MAX_RESULTS', 20)
GROUP_SHOW_EMPTY = getattr(settings, 'DOCUMENTS_GROUP_SHOW_EMPTY', True)
# Serving
FILESYSTEM_FILESERVING_ENABLE = getattr(settings, 'DOCUMENTS_FILESYSTEM_FILESERVING_ENABLE', True)
FILESYSTEM_FILESERVING_PATH = getattr(settings, 'DOCUMENTS_FILESYSTEM_FILESERVING_PATH', u'/tmp/mayan/documents')
FILESYSTEM_SLUGIFY_PATHS = getattr(settings, 'DOCUMENTS_SLUGIFY_PATHS', False)
FILESYSTEM_MAX_RENAME_COUNT = getattr(settings, 'DOCUMENTS_FILESYSTEM_MAX_RENAME_COUNT', 200)
apps/documents/models.py
-132
View File
@@ -1,15 +1,11 @@
import errno
import os
import mimetypes
from datetime import datetime
import sys
from python_magic import magic
from django.conf import settings
from django.db import models
from django.template.defaultfilters import slugify
from django.utils.translation import ugettext_lazy as _
from django.utils.translation import ugettext
from django.db.models import Q
@@ -22,16 +18,9 @@ from documents.conf.settings import UUID_FUNCTION
from documents.conf.settings import PAGE_COUNT_FUNCTION
from documents.conf.settings import STORAGE_BACKEND
from documents.conf.settings import STORAGE_DIRECTORY_NAME
from documents.conf.settings import FILESYSTEM_FILESERVING_ENABLE
from documents.conf.settings import FILESYSTEM_FILESERVING_PATH
from documents.conf.settings import FILESYSTEM_SLUGIFY_PATHS
from documents.conf.settings import FILESYSTEM_MAX_RENAME_COUNT
from documents.conf.settings import AVAILABLE_TRANSFORMATIONS
from documents.conf.settings import DEFAULT_TRANSFORMATIONS
if FILESYSTEM_SLUGIFY_PATHS == False:
#Do not slugify path or filenames and extensions
slugify = lambda x:x
def get_filename_from_uuid(instance, filename, directory=STORAGE_DIRECTORY_NAME):
@@ -208,113 +197,6 @@ class Document(models.Model):
page_transformation.arguments = transformation['arguments']
page_transformation.save()
def create_fs_links(self):
if FILESYSTEM_FILESERVING_ENABLE:
if not self.exists():
raise Exception(ugettext(u'Not creating metadata indexing, document not found in document storage'))
metadata_dict = {'document':self}
metadata_dict.update(dict([(metadata.metadata_type.name, slugify(metadata.value)) for metadata in self.documentmetadata_set.all()]))
for metadata_index in self.document_type.metadataindex_set.all():
if metadata_index.enabled:
try:
fabricated_directory = eval(metadata_index.expression, metadata_dict)
target_directory = os.path.join(FILESYSTEM_FILESERVING_PATH, fabricated_directory)
try:
os.makedirs(target_directory)
except OSError, exc:
if exc.errno == errno.EEXIST:
pass
else:
raise OSError(ugettext(u'Unable to create metadata indexing directory: %s') % exc)
next_available_filename(self, metadata_index, target_directory, slugify(self.file_filename), slugify(self.file_extension))
except NameError, exc:
#raise NameError(ugettext(u'Error in metadata indexing expression: %s') % exc)
#This should be a warning not an error
pass
def delete_fs_links(self):
if FILESYSTEM_FILESERVING_ENABLE:
for document_metadata_index in self.documentmetadataindex_set.all():
try:
os.unlink(document_metadata_index.filename)
document_metadata_index.delete()
except OSError, exc:
if exc.errno == errno.ENOENT:
#No longer exits, so delete db entry anyway
document_metadata_index.delete()
else:
raise OSError(ugettext(u'Unable to delete metadata indexing symbolic link: %s') % exc)
path, filename = os.path.split(document_metadata_index.filename)
#Cleanup directory of dead stuff
#Delete siblings that are dead links
try:
for f in os.listdir(path):
filepath = os.path.join(path, f)
if os.path.islink(filepath):
#Get link's source
source = os.readlink(filepath)
if os.path.isabs(source):
if not os.path.exists(source):
#link's source is absolute and doesn't exit
os.unlink(filepath)
else:
os.unlink(os.path.join(path, filepath))
elif os.path.isdir(filepath):
#is a directory, try to delete it
try:
os.removedirs(path)
except:
pass
except OSError, exc:
pass
#Remove the directory if it is empty
try:
os.removedirs(path)
except:
pass
def next_available_filename(document, metadata_index, path, filename, extension, suffix=0):
target = filename
if suffix:
target = '_'.join([filename, unicode(suffix)])
filepath = os.path.join(path, os.extsep.join([target, extension]))
matches=DocumentMetadataIndex.objects.filter(filename=filepath)
if matches.count() == 0:
document_metadata_index = DocumentMetadataIndex(
document=document, metadata_index=metadata_index,
filename=filepath)
try:
os.symlink(document.file.path, filepath)
document_metadata_index.save()
except OSError, exc:
if exc.errno == errno.EEXIST:
#This link should not exist, try to delete it
try:
os.unlink(filepath)
#Try again with same suffix
return next_available_filename(document, metadata_index, path, filename, extension, suffix)
except Exception, exc:
raise Exception(ugettext(u'Unable to create symbolic link, filename clash: %(filepath)s; %(exc)s') % {'filepath':filepath, 'exc':exc})
else:
raise OSError(ugettext(u'Unable to create symbolic link: %(filepath)s; %(exc)s') % {'filepath':filepath, 'exc':exc})
return filepath
else:
if suffix > FILESYSTEM_MAX_RENAME_COUNT:
raise Exception(ugettext(u'Maximum rename count reached, not creating symbolic link'))
return next_available_filename(document, metadata_index, path, filename, extension, suffix+1)
available_functions_string = (_(u' Available functions: %s') % ','.join(['%s()' % name for name, function in AVAILABLE_FUNCTIONS.items()])) if AVAILABLE_FUNCTIONS else ''
@@ -368,20 +250,6 @@ class MetadataIndex(models.Model):
verbose_name_plural = _(u'metadata indexes')
class DocumentMetadataIndex(models.Model):
document = models.ForeignKey(Document, verbose_name=_(u'document'))
metadata_index = models.ForeignKey(MetadataIndex, verbose_name=_(u'metadata index'))
filename = models.CharField(max_length=255, verbose_name=_(u'filename'))
suffix = models.PositiveIntegerField(default=0, verbose_name=_(u'suffix'))
def __unicode__(self):
return unicode(self.filename)
class Meta:
verbose_name = _(u'document metadata index')
verbose_name_plural = _(u'document metadata indexes')
class DocumentMetadata(models.Model):
document = models.ForeignKey(Document, verbose_name=_(u'document'))
metadata_type = models.ForeignKey(MetadataType, verbose_name=_(u'metadata type'))
apps/documents/urls.py
-1
View File
@@ -28,7 +28,6 @@ urlpatterns = patterns('documents.views',
url(r'^document/(?P<document_id>\d+)/download/$', 'document_download', (), 'document_download'),
url(r'^document/(?P<document_id>\d+)/create/siblings/$', 'document_create_sibling', {'multiple':True if ENABLE_SINGLE_DOCUMENT_UPLOAD == False else False}, 'document_create_sibling'),
url(r'^document/(?P<document_id>\d+)/find_duplicates/$', 'document_find_duplicates', (), 'document_find_duplicates'),
url(r'^recreate_all_links/$', 'document_recreate_all_links', (), 'document_recreate_all_links'),
url(r'^duplicates/$', 'document_find_all_duplicates', (), 'document_find_all_duplicates'),
url(r'^staging_file/(?P<staging_file_id>\w+)/preview/$', 'staging_file_preview', (), 'staging_file_preview'),
apps/documents/utils.py
+1 -26
View File
@@ -111,29 +111,4 @@ def save_metadata(metadata_dict, document):
#unquote_plus handles utf-8?!?
#http://stackoverflow.com/questions/4382875/handling-iri-in-django
document_metadata.value=unquote_plus(metadata_dict['value'])#.decode('utf-8')
document_metadata.save()
def recreate_links(raise_exception=True):
errors = []
for document in Document.objects.all():
try:
document.delete_fs_links()
except Exception, e:
print document
if raise_exception:
raise Exception(e)
else:
error.append(e)
for document in Document.objects.all():
try:
document.create_fs_links()
except Exception, e:
print document
if raise_exception:
raise Exception(e)
else:
error.append(e)
return errors
document_metadata.save()
apps/documents/views.py
+11 -34
View File
@@ -20,8 +20,10 @@ from permissions.api import check_permissions, Unauthorized
from filetransfers.api import serve_file
from converter.api import convert, in_image_cache, QUALITY_DEFAULT
from converter import TRANFORMATION_CHOICES
from filesystem_serving.api import document_create_fs_links, document_delete_fs_links
from utils import from_descriptor_to_tempfile, recreate_links
from utils import from_descriptor_to_tempfile
from models import Document, DocumentMetadata, DocumentType, MetadataType, \
DocumentPage, DocumentPageTransformation
@@ -37,7 +39,6 @@ from ocr.models import add_document_to_queue
from documents.conf.settings import DELETE_STAGING_FILE_AFTER_UPLOAD
from documents.conf.settings import USE_STAGING_DIRECTORY
from documents.conf.settings import FILESYSTEM_FILESERVING_ENABLE
from documents.conf.settings import STAGING_FILES_PREVIEW_SIZE
from documents.conf.settings import PREVIEW_SIZE
from documents.conf.settings import THUMBNAIL_SIZE
@@ -48,6 +49,8 @@ from documents.conf.settings import AUTOMATIC_OCR
from documents.conf.settings import UNCOMPRESS_COMPRESSED_LOCAL_FILES
from documents.conf.settings import UNCOMPRESS_COMPRESSED_STAGING_FILES
from filesystem_serving.conf.settings import FILESERVING_ENABLE
from documents import PERMISSION_DOCUMENT_CREATE, \
PERMISSION_DOCUMENT_CREATE, PERMISSION_DOCUMENT_PROPERTIES_EDIT, \
@@ -128,7 +131,7 @@ def _handle_save_document(request, document, form=None):
save_metadata_list(decode_metadata_from_url(request.GET), document)
try:
document.create_fs_links()
document_create_fs_links(document)
except Exception, e:
messages.error(request, e)
@@ -300,7 +303,7 @@ def document_view(request, document_id):
},
]
if FILESYSTEM_FILESERVING_ENABLE:
if FILESERVING_ENABLE:
subtemplates_dict.append({
'name':'generic_list_subtemplate.html',
'title':_(u'index links'),
@@ -370,7 +373,7 @@ def document_edit(request, document_id):
form = DocumentForm_edit(request.POST, initial={'document_type':document.document_type})
if form.is_valid():
try:
document.delete_fs_links()
document_delete_fs_links(document)
except Exception, e:
messages.error(request, e)
return HttpResponseRedirect(reverse('document_list'))
@@ -386,7 +389,7 @@ def document_edit(request, document_id):
messages.success(request, _(u'Document %s edited successfully.') % document)
try:
document.create_fs_links()
document_create_fs_links(document)
messages.success(request, _(u'Document filesystem links updated successfully.'))
except Exception, e:
messages.error(request, e)
@@ -433,7 +436,7 @@ def document_edit_metadata(request, document_id):
if formset.is_valid():
save_metadata_list(formset.cleaned_data, document)
try:
document.delete_fs_links()
document_delete_fs_links(document)
except Exception, e:
messages.error(request, e)
return HttpResponseRedirect(reverse('document_list'))
@@ -441,7 +444,7 @@ def document_edit_metadata(request, document_id):
messages.success(request, _(u'Metadata for document %s edited successfully.') % document)
try:
document.create_fs_links()
document_create_fs_links(document)
messages.success(request, _(u'Document filesystem links updated successfully.'))
except Exception, e:
messages.error(request, e)
@@ -739,29 +742,3 @@ def document_find_all_duplicates(request):
raise Http404(e)
return _find_duplicate_list(request, include_source=False)
def document_recreate_all_links(request):
permissions = [PERMISSION_DOCUMENT_TOOLS]
try:
check_permissions(request.user, 'documents', permissions)
except Unauthorized, e:
raise Http404(e)
previous = request.POST.get('previous', request.GET.get('previous', request.META.get('HTTP_REFERER', None)))
next = request.POST.get('next', request.GET.get('next', request.META.get('HTTP_REFERER', None)))
if request.method != 'POST':
return render_to_response('generic_confirm.html', {
'previous':previous,
'next':next,
'message':_(u'On large databases this operation may take some time to execute.'),
}, context_instance=RequestContext(request))
else:
try:
recreate_links()
messages.success(request, _(u'Filesystem links re-creation completed successfully.'))
except Exception, e:
messages.error(request, _(u'Filesystem links re-creation error: %s') % e)
return HttpResponseRedirect(next)