matthias/mayan-edms
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Started adding filesystem linking support to document indexing app

Browse Source
This commit is contained in:
Roberto Rosario
2011-05-18 19:19:37 -04:00
parent 4b0c72633d
commit b7050b4b94
5 changed files with 115 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
apps/document_indexing/admin.py
View File

@@ -2,7 +2,8 @@ from django.contrib import admin
from mptt.admin import MPTTModelAdmin
from document_indexing.models import Index, IndexInstance
from document_indexing.models import Index, IndexInstance, \
DocumentRenameCount
class IndexInstanceInline(admin.StackedInline):
@@ -23,3 +24,4 @@ class IndexInstanceAdmin(MPTTModelAdmin):
admin.site.register(Index, IndexAdmin)
admin.site.register(IndexInstance, IndexInstanceAdmin)
admin.site.register(DocumentRenameCount)

66
apps/document_indexing/api.py
View File

@@ -2,13 +2,29 @@ from django.utils.translation import ugettext_lazy as _
from django.utils.translation import ugettext
from django.core.urlresolvers import reverse
from django.utils.safestring import mark_safe
from django.template.defaultfilters import slugify
from documents.models import Document
from metadata.classes import MetadataObject
from document_indexing.models import Index, IndexInstance
from document_indexing.models import Index, IndexInstance, \
DocumentRenameCount
from document_indexing.conf.settings import AVAILABLE_INDEXING_FUNCTIONS
from document_indexing.filesystem import fs_create_index_directory
from document_indexing.conf.settings import MAX_SUFFIX_COUNT
from document_indexing.filesystem import fs_create_index_directory, \
fs_create_document_link, fs_delete_document_link
from document_indexing.conf.settings import SLUGIFY_PATHS
if SLUGIFY_PATHS == False:
# Do not slugify path or filenames and extensions
SLUGIFY_FUNCTION = lambda x: x
else:
SLUGIFY_FUNCTION = slugify
class MaxSuffixCountReached(Exception):
pass
# External functions
@@ -16,6 +32,7 @@ def update_indexes(document):
"""
Update or create all the index instances related to a document
"""
print 'update_indexes'
warnings = []
eval_dict = {}
@@ -55,12 +72,14 @@ def get_instance_link(index_instance=None, text=None, simple=False):
template = u'<a href="%(url)s">%(value)s</a>'
if index_instance:
return template % {
'url': index_instance.get_absolute_url(), 'value': text if text else index_instance
'url': index_instance.get_absolute_url(),
'value': text if text else index_instance
}
else:
# Root node
return template % {
'url': reverse('index_instance_list'), 'value': ugettext(u'root')
'url': reverse('index_instance_list'),
'value': ugettext(u'root')
}
@@ -90,11 +109,24 @@ def get_breadcrumbs(index_instance, simple=False, single_link=False):
def do_rebuild_all_indexes():
IndexInstance.objects.all().delete()
DocumentRenameCount.objects.all().delete()
for document in Document.objects.all():
update_indexes(document)
# Internal functions
def find_lowest_available_suffix(suffix_list):
print 'suffix_list', suffix_list
suffix = 0
while suffix in suffix_list:
suffix += 1
if suffix > MAX_SUFFIX_COUNT:
raise MaxSuffixCountReached(ugettext(u'Maximum suffix (%s) count reached.') % MAX_SUFFIX_COUNT)
else:
return suffix
def _evaluate_index(eval_dict, document, node, parent_index_instance=None):
"""
Evaluate an enabled index expression and update or create all the
@@ -108,11 +140,25 @@ def _evaluate_index(eval_dict, document, node, parent_index_instance=None):
index_instance, created = IndexInstance.objects.get_or_create(index=node, value=result, parent=parent_index_instance)
if created:
fs_create_index_directory(index_instance)
if node.link_documents:
if node.link_documents and document not in index_instance.documents.all():
#suffix_list = index_instance.documents.filter(file_filename=document.file_filename).filter(file_extension=document.file_extension).values_list('suffix', flat=True)
suffix_list = DocumentRenameCount.objects.filter(document__file_filename=document.file_filename).filter(document__file_extension=document.file_extension).filter(index_instance=index_instance).values_list('suffix', flat=True)
suffix = find_lowest_available_suffix(suffix_list)
print 'lower suffix: %s' % suffix
document_count = DocumentRenameCount(
index_instance=index_instance,
document=document,
suffix=suffix
)
document_count.save()
fs_create_document_link(index_instance, document, suffix)
index_instance.documents.add(document)
for children in node.get_children():
children_warnings = _evaluate_index(eval_dict, document, children, index_instance)
children_warnings = _evaluate_index(
eval_dict, document, children, index_instance
)
warnings.extend(children_warnings)
except (NameError, AttributeError), exc:
@@ -134,16 +180,22 @@ def _remove_document_from_index_instance(document, index_instance):
"""
warnings = []
try:
document_rename_count = DocumentRenameCount.objects.get(index_instance=index_instance, document=document)
fs_delete_document_link(index_instance, document, document_rename_count.suffix)
document_rename_count.delete()
index_instance.documents.remove(document)
if index_instance.documents.count() == 0 and index_instance.get_children().count() == 0:
# if there are no more documents and no children, delete
# node and check parent for the same conditions
parent = index_instance.parent
index_instance.delete()
parent_warnings = _remove_document_from_index_instance(document, parent)
parent_warnings = _remove_document_from_index_instance(
document, parent
)
warnings.extend(parent_warnings)
except Exception, exc:
print '_remove_document_from_index_instance exception: %s' % exc
warnings.append(_(u'Unable to delete document indexing node; %s') % exc)
return warnings

1
apps/document_indexing/conf/settings.py
View File

@@ -20,6 +20,7 @@ register_settings(
{'name': u'AVAILABLE_INDEXING_FUNCTIONS', 'global_name': u'DOCUMENT_INDEXING_AVAILABLE_INDEXING_FUNCTIONS', 'default': available_indexing_functions},
# Filesystem serving
{'name': u'SLUGIFY_PATHS', 'global_name': u'DOCUMENT_INDEXING_FILESYSTEM_SLUGIFY_PATHS', 'default': False},
{'name': u'MAX_SUFFIX_COUNT', 'global_name': u'DOCUMENT_INDEXING_FILESYSTEM_MAX_SUFFIX_COUNT', 'default': 1000},
{'name': u'FILESERVING_PATH', 'global_name': u'DOCUMENT_INDEXING_FILESYSTEM_FILESERVING_PATH', 'default': u'/tmp/mayan/documents', 'exists': True},
{'name': u'FILESERVING_ENABLE', 'global_name': u'DOCUMENT_INDEXING_FILESYSTEM_FILESERVING_ENABLE', 'default': True}
]

75
apps/document_indexing/filesystem.py
View File

@@ -1,24 +1,15 @@
import errno
import os
from django.template.defaultfilters import slugify
from django.utils.translation import ugettext_lazy as _
from document_indexing.models import IndexInstance
from metadata.classes import MetadataObject
from document_indexing.models import IndexInstance
from document_indexing.conf.settings import FILESERVING_ENABLE
from document_indexing.conf.settings import FILESERVING_PATH
from document_indexing.conf.settings import SLUGIFY_PATHS
#from document_indexing.conf.settings import MAX_RENAME_COUNT
if SLUGIFY_PATHS == False:
#Do not slugify path or filenames and extensions
SLUGIFY_FUNCTION = lambda x: x
else:
SLUGIFY_FUNCTION = slugify
# TODO: delete fileserving document function for use with rebuild index function
def get_instance_path(index_instance):
"""
@@ -33,29 +24,56 @@ def fs_create_index_directory(index_instance):
target_directory = os.path.join(FILESERVING_PATH, get_instance_path(index_instance))
try:
os.makedirs(target_directory)
#next_available_filename(document, metadata_index, target_directory, SLUGIFY_FUNCTION(document.file_filename), SLUGIFY_FUNCTION(document.file_extension))
except OSError, exc:
if exc.errno == errno.EEXIST:
pass
else:
raise OSError(_(u'Unable to create metadata indexing directory: %s') % exc)
raise OSError(_(u'Unable to create indexing directory; %s') % exc)
def document_delete_fs_links(document):
def fs_create_document_link(index_instance, document, suffix=0):
if FILESERVING_ENABLE:
pass
'''
for document_metadata_index in document.documentmetadataindex_set.all():
try:
os.unlink(document_metadata_index.filename)
document_metadata_index.delete()
except OSError, exc:
if exc.errno == errno.ENOENT:
#No longer exits, so delete db entry anyway
document_metadata_index.delete()
else:
raise OSError(_(u'Unable to delete metadata indexing symbolic link: %s') % exc)
name_part = document.file_filename
if suffix:
name_part = u'_'.join([name_part, unicode(suffix)])
filename = os.extsep.join([name_part, document.file_extension])
filepath = os.path.join(FILESERVING_PATH, get_instance_path(index_instance), filename)
print 'filepath', filepath
try:
os.symlink(document.file.path, filepath)
except OSError, exc:
if exc.errno == errno.EEXIST:
# This link should not exist, try to delete it
try:
os.unlink(filepath)
# Try again
os.symlink(document.file.path, filepath)
except Exception, exc:
raise Exception(_(u'Unable to create symbolic link, file exists and could not be deleted: %(filepath)s; %(exc)s') % {'filepath': filepath, 'exc': exc})
else:
raise OSError(_(u'Unable to create symbolic link: %(filepath)s; %(exc)s') % {'filepath': filepath, 'exc': exc})
def fs_delete_document_link(index_instance, document, suffix=0):
if FILESERVING_ENABLE:
name_part = document.file_filename
if suffix:
name_part = u'_'.join([name_part, unicode(suffix)])
filename = os.extsep.join([name_part, document.file_extension])
filepath = os.path.join(FILESERVING_PATH, get_instance_path(index_instance), filename)
print 'delete filepath', filepath
try:
os.unlink(filepath)
except OSError, exc:
if exc.errno != errno.ENOENT:
# Raise when any error other than doesn't exits
raise OSError(_(u'Unable to delete document symbolic link; %s') % exc)
'''
path, filename = os.path.split(document_metadata_index.filename)
#Cleanup directory of dead stuff
@@ -86,9 +104,9 @@ def document_delete_fs_links(document):
os.removedirs(path)
except:
pass
'''
'''
'''
def next_available_filename(document, metadata_index, path, filename, extension, suffix=0):
target = filename
if suffix:
@@ -149,3 +167,4 @@ def do_recreate_all_links(raise_exception=True):
for warning in create_warnings:
warnings.append('%s: %s' % (document, warning))
return errors, warnings
'''

9
apps/document_indexing/models.py
View File

@@ -46,16 +46,15 @@ class IndexInstance(MPTTModel):
verbose_name = _(u'index instance')
verbose_name_plural = _(u'indexes instances')
'''
class DocumentRenameCount(models.Model):
index = models.ForeignKey(IndexInstance, verbose_name=_(u'index instance'))
index_instance = models.ForeignKey(IndexInstance, verbose_name=_(u'index instance'))
document = models.ForeignKey(Document, verbose_name=_(u'document'))
count = models.PositiveIntegerField(blank=True, verbose_name=(u'count'))
suffix = models.PositiveIntegerField(blank=True, verbose_name=(u'suffix'))
def __unicode__(self):
return self.value
return u'%s - %s - %s' % (self.index_instance, self.document, self.suffix or u'0')
class Meta:
verbose_name = _(u'document rename count')
verbose_name_plural = _(u'documents rename count')
'''