matthias/mayan-edms
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Extracted document filesystem serving into a separate app, see docs/Changelog.txt to update database accordingly

Browse Source
This commit is contained in:
Roberto Rosario
2011-02-26 12:50:16 -04:00
parent ff77b622a9
commit 79f6897d8a
21 changed files with 328 additions and 220 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
apps/documents/__init__.py
View File

@@ -48,7 +48,6 @@ document_preview = {'text':_('preview'), 'class':'fancybox', 'view':'document_pr
document_download = {'text':_('download'), 'view':'document_download', 'args':'object.id', 'famfam':'page_save', 'permissions':{'namespace':'documents', 'permissions':[PERMISSION_DOCUMENT_DOWNLOAD]}} document_download = {'text':_('download'), 'view':'document_download', 'args':'object.id', 'famfam':'page_save', 'permissions':{'namespace':'documents', 'permissions':[PERMISSION_DOCUMENT_DOWNLOAD]}}
document_find_duplicates = {'text':_('find duplicates'), 'view':'document_find_duplicates', 'args':'object.id', 'famfam':'page_refresh', 'permissions':{'namespace':'documents', 'permissions':[PERMISSION_DOCUMENT_VIEW]}} document_find_duplicates = {'text':_('find duplicates'), 'view':'document_find_duplicates', 'args':'object.id', 'famfam':'page_refresh', 'permissions':{'namespace':'documents', 'permissions':[PERMISSION_DOCUMENT_VIEW]}}
document_find_all_duplicates = {'text':_('find all duplicates'), 'view':'document_find_all_duplicates', 'famfam':'page_refresh', 'permissions':{'namespace':'documents', 'permissions':[PERMISSION_DOCUMENT_VIEW]}} document_find_all_duplicates = {'text':_('find all duplicates'), 'view':'document_find_all_duplicates', 'famfam':'page_refresh', 'permissions':{'namespace':'documents', 'permissions':[PERMISSION_DOCUMENT_VIEW]}}
document_recreate_all_links = {'text':_('recreate index links'), 'view':'document_recreate_all_links', 'famfam':'page_link', 'permissions':{'namespace':'documents', 'permissions':[PERMISSION_DOCUMENT_TOOLS]}}
document_page_transformation_create = {'text':_('create new transformation'), 'view':'document_page_transformation_create', 'args':'object.id', 'famfam':'pencil_add', 'permissions':{'namespace':'documents', 'permissions':[PERMISSION_DOCUMENT_TRANSFORM]}} document_page_transformation_create = {'text':_('create new transformation'), 'view':'document_page_transformation_create', 'args':'object.id', 'famfam':'pencil_add', 'permissions':{'namespace':'documents', 'permissions':[PERMISSION_DOCUMENT_TRANSFORM]}}
document_page_transformation_edit = {'text':_('edit'), 'view':'document_page_transformation_edit', 'args':'object.id', 'famfam':'pencil_go', 'permissions':{'namespace':'documents', 'permissions':[PERMISSION_DOCUMENT_TRANSFORM]}} document_page_transformation_edit = {'text':_('edit'), 'view':'document_page_transformation_edit', 'args':'object.id', 'famfam':'pencil_go', 'permissions':{'namespace':'documents', 'permissions':[PERMISSION_DOCUMENT_TRANSFORM]}}

13
apps/documents/admin.py
View File

@@ -2,9 +2,11 @@ from django.contrib import admin
from models import MetadataType, DocumentType, Document, \ from models import MetadataType, DocumentType, Document, \
DocumentTypeMetadataType, DocumentMetadata, DocumentTypeFilename, \ DocumentTypeMetadataType, DocumentMetadata, DocumentTypeFilename, \
MetadataIndex, DocumentMetadataIndex, DocumentPage, MetadataGroup, \ MetadataIndex, DocumentPage, MetadataGroup, \
MetadataGroupItem, DocumentPageTransformation MetadataGroupItem, DocumentPageTransformation
from filesystem_serving.admin import DocumentMetadataIndexInline
class MetadataTypeAdmin(admin.ModelAdmin): class MetadataTypeAdmin(admin.ModelAdmin):
list_display = ('name', 'title', 'default', 'lookup') list_display = ('name', 'title', 'default', 'lookup')
@@ -40,15 +42,6 @@ class DocumentMetadataInline(admin.StackedInline):
extra = 0 extra = 0
classes = ('collapse-open',) classes = ('collapse-open',)
allow_add = False allow_add = False
readonly_fields = ('metadata_type', 'value')
class DocumentMetadataIndexInline(admin.StackedInline):
model = DocumentMetadataIndex
extra = 1
classes = ('collapse-open',)
allow_add = True
readonly_fields = ('suffix', 'metadata_index', 'filename')
class DocumentPageTransformationAdmin(admin.ModelAdmin): class DocumentPageTransformationAdmin(admin.ModelAdmin):

6
apps/documents/conf/settings.py
View File

@@ -60,9 +60,3 @@ DEFAULT_TRANSFORMATIONS = getattr(settings, 'DOCUMENTS_DEFAULT_TRANSFORMATIONS',
#Groups #Groups
GROUP_MAX_RESULTS = getattr(settings, 'DOCUMENTS_GROUP_MAX_RESULTS', 20) GROUP_MAX_RESULTS = getattr(settings, 'DOCUMENTS_GROUP_MAX_RESULTS', 20)
GROUP_SHOW_EMPTY = getattr(settings, 'DOCUMENTS_GROUP_SHOW_EMPTY', True) GROUP_SHOW_EMPTY = getattr(settings, 'DOCUMENTS_GROUP_SHOW_EMPTY', True)
# Serving
FILESYSTEM_FILESERVING_ENABLE = getattr(settings, 'DOCUMENTS_FILESYSTEM_FILESERVING_ENABLE', True)
FILESYSTEM_FILESERVING_PATH = getattr(settings, 'DOCUMENTS_FILESYSTEM_FILESERVING_PATH', u'/tmp/mayan/documents')
FILESYSTEM_SLUGIFY_PATHS = getattr(settings, 'DOCUMENTS_SLUGIFY_PATHS', False)
FILESYSTEM_MAX_RENAME_COUNT = getattr(settings, 'DOCUMENTS_FILESYSTEM_MAX_RENAME_COUNT', 200)

132
apps/documents/models.py
View File

@@ -1,15 +1,11 @@
import errno
import os import os
import mimetypes
from datetime import datetime from datetime import datetime
import sys import sys
from python_magic import magic from python_magic import magic
from django.conf import settings from django.conf import settings
from django.db import models from django.db import models
from django.template.defaultfilters import slugify
from django.utils.translation import ugettext_lazy as _ from django.utils.translation import ugettext_lazy as _
from django.utils.translation import ugettext
from django.db.models import Q from django.db.models import Q
@@ -22,16 +18,9 @@ from documents.conf.settings import UUID_FUNCTION
from documents.conf.settings import PAGE_COUNT_FUNCTION from documents.conf.settings import PAGE_COUNT_FUNCTION
from documents.conf.settings import STORAGE_BACKEND from documents.conf.settings import STORAGE_BACKEND
from documents.conf.settings import STORAGE_DIRECTORY_NAME from documents.conf.settings import STORAGE_DIRECTORY_NAME
from documents.conf.settings import FILESYSTEM_FILESERVING_ENABLE
from documents.conf.settings import FILESYSTEM_FILESERVING_PATH
from documents.conf.settings import FILESYSTEM_SLUGIFY_PATHS
from documents.conf.settings import FILESYSTEM_MAX_RENAME_COUNT
from documents.conf.settings import AVAILABLE_TRANSFORMATIONS from documents.conf.settings import AVAILABLE_TRANSFORMATIONS
from documents.conf.settings import DEFAULT_TRANSFORMATIONS from documents.conf.settings import DEFAULT_TRANSFORMATIONS
if FILESYSTEM_SLUGIFY_PATHS == False:
#Do not slugify path or filenames and extensions
slugify = lambda x:x
def get_filename_from_uuid(instance, filename, directory=STORAGE_DIRECTORY_NAME): def get_filename_from_uuid(instance, filename, directory=STORAGE_DIRECTORY_NAME):
@@ -208,113 +197,6 @@ class Document(models.Model):
page_transformation.arguments = transformation['arguments'] page_transformation.arguments = transformation['arguments']
page_transformation.save() page_transformation.save()
def create_fs_links(self):
if FILESYSTEM_FILESERVING_ENABLE:
if not self.exists():
raise Exception(ugettext(u'Not creating metadata indexing, document not found in document storage'))
metadata_dict = {'document':self}
metadata_dict.update(dict([(metadata.metadata_type.name, slugify(metadata.value)) for metadata in self.documentmetadata_set.all()]))
for metadata_index in self.document_type.metadataindex_set.all():
if metadata_index.enabled:
try:
fabricated_directory = eval(metadata_index.expression, metadata_dict)
target_directory = os.path.join(FILESYSTEM_FILESERVING_PATH, fabricated_directory)
try:
os.makedirs(target_directory)
except OSError, exc:
if exc.errno == errno.EEXIST:
pass
else:
raise OSError(ugettext(u'Unable to create metadata indexing directory: %s') % exc)
next_available_filename(self, metadata_index, target_directory, slugify(self.file_filename), slugify(self.file_extension))
except NameError, exc:
#raise NameError(ugettext(u'Error in metadata indexing expression: %s') % exc)
#This should be a warning not an error
pass
def delete_fs_links(self):
if FILESYSTEM_FILESERVING_ENABLE:
for document_metadata_index in self.documentmetadataindex_set.all():
try:
os.unlink(document_metadata_index.filename)
document_metadata_index.delete()
except OSError, exc:
if exc.errno == errno.ENOENT:
#No longer exits, so delete db entry anyway
document_metadata_index.delete()
else:
raise OSError(ugettext(u'Unable to delete metadata indexing symbolic link: %s') % exc)
path, filename = os.path.split(document_metadata_index.filename)
#Cleanup directory of dead stuff
#Delete siblings that are dead links
try:
for f in os.listdir(path):
filepath = os.path.join(path, f)
if os.path.islink(filepath):
#Get link's source
source = os.readlink(filepath)
if os.path.isabs(source):
if not os.path.exists(source):
#link's source is absolute and doesn't exit
os.unlink(filepath)
else:
os.unlink(os.path.join(path, filepath))
elif os.path.isdir(filepath):
#is a directory, try to delete it
try:
os.removedirs(path)
except:
pass
except OSError, exc:
pass
#Remove the directory if it is empty
try:
os.removedirs(path)
except:
pass
def next_available_filename(document, metadata_index, path, filename, extension, suffix=0):
target = filename
if suffix:
target = '_'.join([filename, unicode(suffix)])
filepath = os.path.join(path, os.extsep.join([target, extension]))
matches=DocumentMetadataIndex.objects.filter(filename=filepath)
if matches.count() == 0:
document_metadata_index = DocumentMetadataIndex(
document=document, metadata_index=metadata_index,
filename=filepath)
try:
os.symlink(document.file.path, filepath)
document_metadata_index.save()
except OSError, exc:
if exc.errno == errno.EEXIST:
#This link should not exist, try to delete it
try:
os.unlink(filepath)
#Try again with same suffix
return next_available_filename(document, metadata_index, path, filename, extension, suffix)
except Exception, exc:
raise Exception(ugettext(u'Unable to create symbolic link, filename clash: %(filepath)s; %(exc)s') % {'filepath':filepath, 'exc':exc})
else:
raise OSError(ugettext(u'Unable to create symbolic link: %(filepath)s; %(exc)s') % {'filepath':filepath, 'exc':exc})
return filepath
else:
if suffix > FILESYSTEM_MAX_RENAME_COUNT:
raise Exception(ugettext(u'Maximum rename count reached, not creating symbolic link'))
return next_available_filename(document, metadata_index, path, filename, extension, suffix+1)
available_functions_string = (_(u' Available functions: %s') % ','.join(['%s()' % name for name, function in AVAILABLE_FUNCTIONS.items()])) if AVAILABLE_FUNCTIONS else '' available_functions_string = (_(u' Available functions: %s') % ','.join(['%s()' % name for name, function in AVAILABLE_FUNCTIONS.items()])) if AVAILABLE_FUNCTIONS else ''
@@ -368,20 +250,6 @@ class MetadataIndex(models.Model):
verbose_name_plural = _(u'metadata indexes') verbose_name_plural = _(u'metadata indexes')
class DocumentMetadataIndex(models.Model):
document = models.ForeignKey(Document, verbose_name=_(u'document'))
metadata_index = models.ForeignKey(MetadataIndex, verbose_name=_(u'metadata index'))
filename = models.CharField(max_length=255, verbose_name=_(u'filename'))
suffix = models.PositiveIntegerField(default=0, verbose_name=_(u'suffix'))
def __unicode__(self):
return unicode(self.filename)
class Meta:
verbose_name = _(u'document metadata index')
verbose_name_plural = _(u'document metadata indexes')
class DocumentMetadata(models.Model): class DocumentMetadata(models.Model):
document = models.ForeignKey(Document, verbose_name=_(u'document')) document = models.ForeignKey(Document, verbose_name=_(u'document'))
metadata_type = models.ForeignKey(MetadataType, verbose_name=_(u'metadata type')) metadata_type = models.ForeignKey(MetadataType, verbose_name=_(u'metadata type'))

1
apps/documents/urls.py
View File

@@ -28,7 +28,6 @@ urlpatterns = patterns('documents.views',
url(r'^document/(?P<document_id>\d+)/download/$', 'document_download', (), 'document_download'), url(r'^document/(?P<document_id>\d+)/download/$', 'document_download', (), 'document_download'),
url(r'^document/(?P<document_id>\d+)/create/siblings/$', 'document_create_sibling', {'multiple':True if ENABLE_SINGLE_DOCUMENT_UPLOAD == False else False}, 'document_create_sibling'), url(r'^document/(?P<document_id>\d+)/create/siblings/$', 'document_create_sibling', {'multiple':True if ENABLE_SINGLE_DOCUMENT_UPLOAD == False else False}, 'document_create_sibling'),
url(r'^document/(?P<document_id>\d+)/find_duplicates/$', 'document_find_duplicates', (), 'document_find_duplicates'), url(r'^document/(?P<document_id>\d+)/find_duplicates/$', 'document_find_duplicates', (), 'document_find_duplicates'),
url(r'^recreate_all_links/$', 'document_recreate_all_links', (), 'document_recreate_all_links'),
url(r'^duplicates/$', 'document_find_all_duplicates', (), 'document_find_all_duplicates'), url(r'^duplicates/$', 'document_find_all_duplicates', (), 'document_find_all_duplicates'),
url(r'^staging_file/(?P<staging_file_id>\w+)/preview/$', 'staging_file_preview', (), 'staging_file_preview'), url(r'^staging_file/(?P<staging_file_id>\w+)/preview/$', 'staging_file_preview', (), 'staging_file_preview'),

27
apps/documents/utils.py
View File

@@ -111,29 +111,4 @@ def save_metadata(metadata_dict, document):
#unquote_plus handles utf-8?!? #unquote_plus handles utf-8?!?
#http://stackoverflow.com/questions/4382875/handling-iri-in-django #http://stackoverflow.com/questions/4382875/handling-iri-in-django
document_metadata.value=unquote_plus(metadata_dict['value'])#.decode('utf-8') document_metadata.value=unquote_plus(metadata_dict['value'])#.decode('utf-8')
document_metadata.save() document_metadata.save()
def recreate_links(raise_exception=True):
errors = []
for document in Document.objects.all():
try:
document.delete_fs_links()
except Exception, e:
print document
if raise_exception:
raise Exception(e)
else:
error.append(e)
for document in Document.objects.all():
try:
document.create_fs_links()
except Exception, e:
print document
if raise_exception:
raise Exception(e)
else:
error.append(e)
return errors

45
apps/documents/views.py
View File

@@ -20,8 +20,10 @@ from permissions.api import check_permissions, Unauthorized
from filetransfers.api import serve_file from filetransfers.api import serve_file
from converter.api import convert, in_image_cache, QUALITY_DEFAULT from converter.api import convert, in_image_cache, QUALITY_DEFAULT
from converter import TRANFORMATION_CHOICES from converter import TRANFORMATION_CHOICES
from filesystem_serving.api import document_create_fs_links, document_delete_fs_links
from utils import from_descriptor_to_tempfile, recreate_links
from utils import from_descriptor_to_tempfile
from models import Document, DocumentMetadata, DocumentType, MetadataType, \ from models import Document, DocumentMetadata, DocumentType, MetadataType, \
DocumentPage, DocumentPageTransformation DocumentPage, DocumentPageTransformation
@@ -37,7 +39,6 @@ from ocr.models import add_document_to_queue
from documents.conf.settings import DELETE_STAGING_FILE_AFTER_UPLOAD from documents.conf.settings import DELETE_STAGING_FILE_AFTER_UPLOAD
from documents.conf.settings import USE_STAGING_DIRECTORY from documents.conf.settings import USE_STAGING_DIRECTORY
from documents.conf.settings import FILESYSTEM_FILESERVING_ENABLE
from documents.conf.settings import STAGING_FILES_PREVIEW_SIZE from documents.conf.settings import STAGING_FILES_PREVIEW_SIZE
from documents.conf.settings import PREVIEW_SIZE from documents.conf.settings import PREVIEW_SIZE
from documents.conf.settings import THUMBNAIL_SIZE from documents.conf.settings import THUMBNAIL_SIZE
@@ -48,6 +49,8 @@ from documents.conf.settings import AUTOMATIC_OCR
from documents.conf.settings import UNCOMPRESS_COMPRESSED_LOCAL_FILES from documents.conf.settings import UNCOMPRESS_COMPRESSED_LOCAL_FILES
from documents.conf.settings import UNCOMPRESS_COMPRESSED_STAGING_FILES from documents.conf.settings import UNCOMPRESS_COMPRESSED_STAGING_FILES
from filesystem_serving.conf.settings import FILESERVING_ENABLE
from documents import PERMISSION_DOCUMENT_CREATE, \ from documents import PERMISSION_DOCUMENT_CREATE, \
PERMISSION_DOCUMENT_CREATE, PERMISSION_DOCUMENT_PROPERTIES_EDIT, \ PERMISSION_DOCUMENT_CREATE, PERMISSION_DOCUMENT_PROPERTIES_EDIT, \
@@ -128,7 +131,7 @@ def _handle_save_document(request, document, form=None):
save_metadata_list(decode_metadata_from_url(request.GET), document) save_metadata_list(decode_metadata_from_url(request.GET), document)
try: try:
document.create_fs_links() document_create_fs_links(document)
except Exception, e: except Exception, e:
messages.error(request, e) messages.error(request, e)
@@ -300,7 +303,7 @@ def document_view(request, document_id):
}, },
] ]
if FILESYSTEM_FILESERVING_ENABLE: if FILESERVING_ENABLE:
subtemplates_dict.append({ subtemplates_dict.append({
'name':'generic_list_subtemplate.html', 'name':'generic_list_subtemplate.html',
'title':_(u'index links'), 'title':_(u'index links'),
@@ -370,7 +373,7 @@ def document_edit(request, document_id):
form = DocumentForm_edit(request.POST, initial={'document_type':document.document_type}) form = DocumentForm_edit(request.POST, initial={'document_type':document.document_type})
if form.is_valid(): if form.is_valid():
try: try:
document.delete_fs_links() document_delete_fs_links(document)
except Exception, e: except Exception, e:
messages.error(request, e) messages.error(request, e)
return HttpResponseRedirect(reverse('document_list')) return HttpResponseRedirect(reverse('document_list'))
@@ -386,7 +389,7 @@ def document_edit(request, document_id):
messages.success(request, _(u'Document %s edited successfully.') % document) messages.success(request, _(u'Document %s edited successfully.') % document)
try: try:
document.create_fs_links() document_create_fs_links(document)
messages.success(request, _(u'Document filesystem links updated successfully.')) messages.success(request, _(u'Document filesystem links updated successfully.'))
except Exception, e: except Exception, e:
messages.error(request, e) messages.error(request, e)
@@ -433,7 +436,7 @@ def document_edit_metadata(request, document_id):
if formset.is_valid(): if formset.is_valid():
save_metadata_list(formset.cleaned_data, document) save_metadata_list(formset.cleaned_data, document)
try: try:
document.delete_fs_links() document_delete_fs_links(document)
except Exception, e: except Exception, e:
messages.error(request, e) messages.error(request, e)
return HttpResponseRedirect(reverse('document_list')) return HttpResponseRedirect(reverse('document_list'))
@@ -441,7 +444,7 @@ def document_edit_metadata(request, document_id):
messages.success(request, _(u'Metadata for document %s edited successfully.') % document) messages.success(request, _(u'Metadata for document %s edited successfully.') % document)
try: try:
document.create_fs_links() document_create_fs_links(document)
messages.success(request, _(u'Document filesystem links updated successfully.')) messages.success(request, _(u'Document filesystem links updated successfully.'))
except Exception, e: except Exception, e:
messages.error(request, e) messages.error(request, e)
@@ -739,29 +742,3 @@ def document_find_all_duplicates(request):
raise Http404(e) raise Http404(e)
return _find_duplicate_list(request, include_source=False) return _find_duplicate_list(request, include_source=False)
def document_recreate_all_links(request):
permissions = [PERMISSION_DOCUMENT_TOOLS]
try:
check_permissions(request.user, 'documents', permissions)
except Unauthorized, e:
raise Http404(e)
previous = request.POST.get('previous', request.GET.get('previous', request.META.get('HTTP_REFERER', None)))
next = request.POST.get('next', request.GET.get('next', request.META.get('HTTP_REFERER', None)))
if request.method != 'POST':
return render_to_response('generic_confirm.html', {
'previous':previous,
'next':next,
'message':_(u'On large databases this operation may take some time to execute.'),
}, context_instance=RequestContext(request))
else:
try:
recreate_links()
messages.success(request, _(u'Filesystem links re-creation completed successfully.'))
except Exception, e:
messages.error(request, _(u'Filesystem links re-creation error: %s') % e)
return HttpResponseRedirect(next)

14
apps/filesystem_serving/__init__.py Normal file
View File

@@ -0,0 +1,14 @@
from django.utils.translation import ugettext_lazy as _
from permissions.api import register_permissions
FILESYSTEM_SERVING_RECREATE_LINKS = 'recreate_links'
register_permissions('filesystem_serving', [
{'name':FILESYSTEM_SERVING_RECREATE_LINKS, 'label':_(u'Recreate filesystem links.')},
])
filesystem_serving_recreate_all_links = {'text':_('recreate index links'), 'view':'recreate_all_links', 'famfam':'page_link', 'permissions':{'namespace':'filesystem_serving', 'permissions':[FILESYSTEM_SERVING_RECREATE_LINKS]}}

10
apps/filesystem_serving/admin.py Normal file
View File

@@ -0,0 +1,10 @@
from django.contrib import admin
from models import DocumentMetadataIndex
class DocumentMetadataIndexInline(admin.StackedInline):
model = DocumentMetadataIndex
extra = 1
classes = ('collapse-open',)
allow_add = True
readonly_fields = ('suffix', 'metadata_index', 'filename')

155
apps/filesystem_serving/api.py Normal file
View File

@@ -0,0 +1,155 @@
import errno
import os
from django.template.defaultfilters import slugify
from django.utils.translation import ugettext_lazy as _
from filesystem_serving.conf.settings import FILESERVING_ENABLE
from filesystem_serving.conf.settings import FILESERVING_PATH
from filesystem_serving.conf.settings import SLUGIFY_PATHS
from filesystem_serving.conf.settings import MAX_RENAME_COUNT
from models import DocumentMetadataIndex, Document
if SLUGIFY_PATHS == False:
#Do not slugify path or filenames and extensions
slugify = lambda x:x
def document_create_fs_links(document):
if FILESERVING_ENABLE:
if not document.exists():
raise Exception(_(u'Not creating metadata indexing, document not found in document storage'))
metadata_dict = {'document':document}
metadata_dict.update(dict([(metadata.metadata_type.name, slugify(metadata.value)) for metadata in document.documentmetadata_set.all()]))
for metadata_index in document.document_type.metadataindex_set.all():
if metadata_index.enabled:
try:
fabricated_directory = eval(metadata_index.expression, metadata_dict)
target_directory = os.path.join(FILESERVING_PATH, fabricated_directory)
try:
os.makedirs(target_directory)
except OSError, exc:
if exc.errno == errno.EEXIST:
pass
else:
raise OSError(_(u'Unable to create metadata indexing directory: %s') % exc)
next_available_filename(document, metadata_index, target_directory, slugify(document.file_filename), slugify(document.file_extension))
except NameError, exc:
raise NameError(_(u'Error in metadata indexing expression: %s') % exc)
#This should be a warning not an error
#pass
except Exception, exc:
raise Exception(_(u'Unable to create metadata indexing directory: %s') % exc)
def document_delete_fs_links(document):
if FILESERVING_ENABLE:
for document_metadata_index in document.documentmetadataindex_set.all():
try:
os.unlink(document_metadata_index.filename)
document_metadata_index.delete()
except OSError, exc:
if exc.errno == errno.ENOENT:
#No longer exits, so delete db entry anyway
document_metadata_index.delete()
else:
raise OSError(_(u'Unable to delete metadata indexing symbolic link: %s') % exc)
path, filename = os.path.split(document_metadata_index.filename)
#Cleanup directory of dead stuff
#Delete siblings that are dead links
try:
for f in os.listdir(path):
filepath = os.path.join(path, f)
if os.path.islink(filepath):
#Get link's source
source = os.readlink(filepath)
if os.path.isabs(source):
if not os.path.exists(source):
#link's source is absolute and doesn't exit
os.unlink(filepath)
else:
os.unlink(os.path.join(path, filepath))
elif os.path.isdir(filepath):
#is a directory, try to delete it
try:
os.removedirs(path)
except:
pass
except OSError, exc:
pass
#Remove the directory if it is empty
try:
os.removedirs(path)
except:
pass
def next_available_filename(document, metadata_index, path, filename, extension, suffix=0):
target = filename
if suffix:
target = '_'.join([filename, unicode(suffix)])
filepath = os.path.join(path, os.extsep.join([target, extension]))
matches=DocumentMetadataIndex.objects.filter(filename=filepath)
if matches.count() == 0:
document_metadata_index = DocumentMetadataIndex(
document=document, metadata_index=metadata_index,
filename=filepath)
try:
os.symlink(document.file.path, filepath)
document_metadata_index.save()
except OSError, exc:
if exc.errno == errno.EEXIST:
#This link should not exist, try to delete it
try:
os.unlink(filepath)
#Try again with same suffix
return next_available_filename(document, metadata_index, path, filename, extension, suffix)
except Exception, exc:
raise Exception(_(u'Unable to create symbolic link, filename clash: %(filepath)s; %(exc)s') % {'filepath':filepath, 'exc':exc})
else:
raise OSError(_(u'Unable to create symbolic link: %(filepath)s; %(exc)s') % {'filepath':filepath, 'exc':exc})
return filepath
else:
if suffix > MAX_RENAME_COUNT:
raise Exception(_(u'Maximum rename count reached, not creating symbolic link'))
return next_available_filename(document, metadata_index, path, filename, extension, suffix+1)
#TODO: diferentiate between evaluation error and filesystem errors
def do_recreate_all_links(raise_exception=True):
errors = []
warnings = []
for document in Document.objects.all():
try:
document_delete_fs_links(document)
except NameError, e:
warnings.append('%s: %s' % (document, e))
except Exception, e:
if raise_exception:
raise Exception(e)
else:
errors.append('%s: %s' % (document, e))
for document in Document.objects.all():
try:
document_create_fs_links(document)
except NameError, e:
warnings.append('%s: %s' % (document, e))
except Exception, e:
if raise_exception:
raise Exception(e)
else:
errors.append('%s: %s' % (document, e))
return errors, warnings

0
apps/filesystem_serving/conf/__init__.py Normal file
View File

7
apps/filesystem_serving/conf/settings.py Normal file
View File

@@ -0,0 +1,7 @@
from django.conf import settings
# Serving
FILESERVING_ENABLE = getattr(settings, 'FILESYSTEM_FILESERVING_ENABLE', True)
FILESERVING_PATH = getattr(settings, 'FILESYSTEM_FILESERVING_PATH', u'/tmp/mayan/documents')
SLUGIFY_PATHS = getattr(settings, 'FILESYSTEM_SLUGIFY_PATHS', False)
MAX_RENAME_COUNT = getattr(settings, 'FILESYSTEM_MAX_RENAME_COUNT', 200)

18
apps/filesystem_serving/models.py Normal file
View File

@@ -0,0 +1,18 @@
from django.db import models
from django.utils.translation import ugettext_lazy as _
from documents.models import Document, MetadataIndex
class DocumentMetadataIndex(models.Model):
document = models.ForeignKey(Document, verbose_name=_(u'document'))
metadata_index = models.ForeignKey(MetadataIndex, verbose_name=_(u'metadata index'))
filename = models.CharField(max_length=255, verbose_name=_(u'filename'))
suffix = models.PositiveIntegerField(default=0, verbose_name=_(u'suffix'))
def __unicode__(self):
return unicode(self.filename)
class Meta:
verbose_name = _(u'document metadata index')
verbose_name_plural = _(u'document metadata indexes')

23
apps/filesystem_serving/tests.py Normal file
View File

@@ -0,0 +1,23 @@
"""
This file demonstrates two different styles of tests (one doctest and one
unittest). These will both pass when you run "manage.py test".
Replace these with more appropriate tests for your application.
"""
from django.test import TestCase
class SimpleTest(TestCase):
def test_basic_addition(self):
"""
Tests that 1 + 1 always equals 2.
"""
self.failUnlessEqual(1 + 1, 2)
__test__ = {"doctest": """
Another way to test that 1 + 1 is equal to 2.
>>> 1 + 1 == 2
True
"""}

5
apps/filesystem_serving/urls.py Normal file
View File

@@ -0,0 +1,5 @@
from django.conf.urls.defaults import *
urlpatterns = patterns('filesystem_serving.views',
url(r'^recreate_all_links/$', 'recreate_all_links', (), 'recreate_all_links'),
)

41
apps/filesystem_serving/views.py Normal file
View File

@@ -0,0 +1,41 @@
from django.utils.translation import ugettext_lazy as _
from django.http import HttpResponse, HttpResponseRedirect, Http404
from django.shortcuts import render_to_response
from django.template import RequestContext
from django.contrib import messages
from permissions.api import check_permissions, Unauthorized
from filesystem_serving import FILESYSTEM_SERVING_RECREATE_LINKS
from api import do_recreate_all_links
def recreate_all_links(request):
permissions = [FILESYSTEM_SERVING_RECREATE_LINKS]
try:
check_permissions(request.user, 'filesystem_serving', permissions)
except Unauthorized, e:
raise Http404(e)
previous = request.POST.get('previous', request.GET.get('previous', request.META.get('HTTP_REFERER', None)))
next = request.POST.get('next', request.GET.get('next', request.META.get('HTTP_REFERER', None)))
if request.method != 'POST':
return render_to_response('generic_confirm.html', {
'previous':previous,
'next':next,
'message':_(u'On large databases this operation may take some time to execute.'),
}, context_instance=RequestContext(request))
else:
try:
errors, warnings = do_recreate_all_links()
messages.success(request, _(u'Filesystem links re-creation completed successfully.'))
for warning in warnings:
messages.warning(request, warning)
except Exception, e:
messages.error(request, _(u'Filesystem links re-creation error: %s') % e)
return HttpResponseRedirect(next)

5
apps/main/__init__.py
View File

@@ -4,7 +4,8 @@ from common.api import register_menu
from permissions import role_list from permissions import role_list
from documents import document_find_all_duplicates, document_recreate_all_links from documents import document_find_all_duplicates
from filesystem_serving import filesystem_serving_recreate_all_links
check_settings = {'text':_(u'settings'), 'view':'check_settings', 'famfam':'cog'} check_settings = {'text':_(u'settings'), 'view':'check_settings', 'famfam':'cog'}
@@ -13,7 +14,7 @@ register_menu([
{'text':_(u'home'), 'view':'home', 'famfam':'house', 'position':0}, {'text':_(u'home'), 'view':'home', 'famfam':'house', 'position':0},
{'text':_(u'tools'), 'view':'tools_menu', 'links': [ {'text':_(u'tools'), 'view':'tools_menu', 'links': [
document_find_all_duplicates, document_recreate_all_links document_find_all_duplicates, filesystem_serving_recreate_all_links
],'famfam':'wrench', 'name':'tools','position':7}, ],'famfam':'wrench', 'name':'tools','position':7},
{'text':_(u'setup'), 'view':'check_settings', 'links': [ {'text':_(u'setup'), 'view':'check_settings', 'links': [

12
apps/main/views.py
View File

@@ -9,6 +9,7 @@ from common.conf import settings as common_settings
from documents.conf import settings as documents_settings from documents.conf import settings as documents_settings
from converter.conf import settings as converter_settings from converter.conf import settings as converter_settings
from ocr.conf import settings as ocr_settings from ocr.conf import settings as ocr_settings
from filesystem_serving.conf import settings as filesystem_serving_settings
def home(request): def home(request):
@@ -30,14 +31,17 @@ def check_settings(request):
{'name':'DOCUMENTS_PREVIEW_SIZE', 'value':documents_settings.PREVIEW_SIZE}, {'name':'DOCUMENTS_PREVIEW_SIZE', 'value':documents_settings.PREVIEW_SIZE},
{'name':'DOCUMENTS_THUMBNAIL_SIZE', 'value':documents_settings.THUMBNAIL_SIZE}, {'name':'DOCUMENTS_THUMBNAIL_SIZE', 'value':documents_settings.THUMBNAIL_SIZE},
{'name':'DOCUMENTS_DISPLAY_SIZE', 'value':documents_settings.DISPLAY_SIZE}, {'name':'DOCUMENTS_DISPLAY_SIZE', 'value':documents_settings.DISPLAY_SIZE},
{'name':'DOCUMENTS_FILESYSTEM_FILESERVING_ENABLE', 'value':documents_settings.FILESYSTEM_FILESERVING_ENABLE},
{'name':'DOCUMENTS_FILESYSTEM_FILESERVING_PATH', 'value':documents_settings.FILESYSTEM_FILESERVING_PATH, 'exists':True},
{'name':'DOCUMENTS_SLUGIFY_PATHS', 'value':documents_settings.FILESYSTEM_SLUGIFY_PATHS},
{'name':'DOCUMENTS_FILESYSTEM_MAX_RENAME_COUNT', 'value':documents_settings.FILESYSTEM_MAX_RENAME_COUNT},
{'name':'DOCUMENTS_AUTOMATIC_OCR', 'value':documents_settings.AUTOMATIC_OCR}, {'name':'DOCUMENTS_AUTOMATIC_OCR', 'value':documents_settings.AUTOMATIC_OCR},
{'name':'DOCUMENTS_ENABLE_SINGLE_DOCUMENT_UPLOAD', 'value':documents_settings.ENABLE_SINGLE_DOCUMENT_UPLOAD}, {'name':'DOCUMENTS_ENABLE_SINGLE_DOCUMENT_UPLOAD', 'value':documents_settings.ENABLE_SINGLE_DOCUMENT_UPLOAD},
{'name':'DOCUMENTS_UNCOMPRESS_COMPRESSED_LOCAL_FILES', 'value':documents_settings.UNCOMPRESS_COMPRESSED_LOCAL_FILES}, {'name':'DOCUMENTS_UNCOMPRESS_COMPRESSED_LOCAL_FILES', 'value':documents_settings.UNCOMPRESS_COMPRESSED_LOCAL_FILES},
{'name':'DOCUMENTS_UNCOMPRESS_COMPRESSED_STAGING_FILES', 'value':documents_settings.UNCOMPRESS_COMPRESSED_STAGING_FILES}, {'name':'DOCUMENTS_UNCOMPRESS_COMPRESSED_STAGING_FILES', 'value':documents_settings.UNCOMPRESS_COMPRESSED_STAGING_FILES},
#Filesystem_serving
{'name':'FILESYSTEM_FILESERVING_ENABLE', 'value':filesystem_serving_settings.FILESERVING_ENABLE},
{'name':'FILESYSTEM_FILESERVING_PATH', 'value':filesystem_serving_settings.FILESERVING_PATH, 'exists':True},
{'name':'FILESYSTEM_SLUGIFY_PATHS', 'value':filesystem_serving_settings.SLUGIFY_PATHS},
{'name':'FILESYSTEM_MAX_RENAME_COUNT', 'value':filesystem_serving_settings.MAX_RENAME_COUNT},
#Common #Common
{'name':'COMMON_TEMPORARY_DIRECTORY', 'value':common_settings.TEMPORARY_DIRECTORY, 'exists':True}, {'name':'COMMON_TEMPORARY_DIRECTORY', 'value':common_settings.TEMPORARY_DIRECTORY, 'exists':True},

23
docs/Changelog.txt
View File

@@ -20,3 +20,26 @@
* Added per document duplicate search and a tools menu option to seach all duplicated documents * Added per document duplicate search and a tools menu option to seach all duplicated documents
* Added document tool that deletes and re-creates all documents filesystem links * Added document tool that deletes and re-creates all documents filesystem links
* Increased document's and document metadata index filename field's size to 255 characters * Increased document's and document metadata index filename field's size to 255 characters
2011-Feb 26
* Added document description to the field search list
* Sort OCR queued documents according to submitted date & time
* Document filesystem serving is now a separate app
- Step to update:
1) rename the following settings:
DOCUMENTS_FILESYSTEM_FILESERVING_ENABLE
DOCUMENTS_FILESYSTEM_FILESERVING_PATH
DOCUMENTS_FILESYSTEM_SLUGIFY_PATHS
DOCUMENTS_FILESYSTEM_MAX_RENAME_COUNT
to:
FILESYSTEM_FILESERVING_ENABLE
FILESYSTEM_FILESERVING_PATH
FILESYSTEM_SLUGIFY_PATHS
FILESYSTEM_MAX_RENAME_COUNT
2) Do a ./manage.py syncdb
3) Execute 'Recreate index links' locate in the tools menu
4) Wait a few minutes
Some warnings may be returned, but these are not fatal as they might
be related to missing metadata in some documents.

9
settings.py
View File

@@ -131,6 +131,7 @@ INSTALLED_APPS = (
'sentry', 'sentry',
'sentry.client', 'sentry.client',
'sentry.client.celery', 'sentry.client.celery',
'filesystem_serving',
) )
@@ -209,10 +210,10 @@ LOGIN_EXEMPT_URLS = (
#DOCUMENTS_GROUP_SHOW_EMPTY = True #DOCUMENTS_GROUP_SHOW_EMPTY = True
# Serving # Serving
#DOCUMENTS_FILESYSTEM_FILESERVING_ENABLE = True #FILESYSTEM_FILESERVING_ENABLE = True
#DOCUMENTS_FILESYSTEM_FILESERVING_PATH = u'/tmp/mayan/documents' #FILESYSTEM_FILESERVING_PATH = u'/tmp/mayan/documents'
#DOCUMENTS_FILESYSTEM_SLUGIFY_PATHS = False #FILESYSTEM_SLUGIFY_PATHS = False
#DOCUMENTS_FILESYSTEM_MAX_RENAME_COUNT = 200 #FILESYSTEM_MAX_RENAME_COUNT = 200
# Misc # Misc
#COMMON_TEMPORARY_DIRECTORY = u'/tmp' #COMMON_TEMPORARY_DIRECTORY = u'/tmp'

1
urls.py
View File

@@ -10,6 +10,7 @@ urlpatterns = patterns('',
(r'^', include('common.urls')), (r'^', include('common.urls')),
(r'^', include('main.urls')), (r'^', include('main.urls')),
(r'^documents/', include('documents.urls')), (r'^documents/', include('documents.urls')),
(r'^filesystem_serving/', include('filesystem_serving.urls')),
(r'^search/', include('dynamic_search.urls')), (r'^search/', include('dynamic_search.urls')),
(r'^ocr/', include('ocr.urls')), (r'^ocr/', include('ocr.urls')),
(r'^permissions/', include('permissions.urls')), (r'^permissions/', include('permissions.urls')),