Moved all filesystem serving app's remaining code to document indexing app
This commit is contained in:
Roberto Rosario
@@ -8,15 +8,7 @@ from metadata.classes import MetadataObject
|
||||
|
||||
from document_indexing.models import Index, IndexInstance
|
||||
from document_indexing.conf.settings import AVAILABLE_INDEXING_FUNCTIONS
|
||||
from document_indexing.conf.settings import FILESERVING_ENABLE
|
||||
from document_indexing.conf.settings import FILESERVING_PATH
|
||||
from document_indexing.conf.settings import SLUGIFY_PATHS
|
||||
|
||||
if SLUGIFY_PATHS == False:
|
||||
# Do not slugify path or filenames and extensions
|
||||
SLUGIFY_FUNCTION = lambda x: x
|
||||
else:
|
||||
SLUGIFY_FUNCTION = slugify
|
||||
from document_indexing.filesystem import fs_create_index_directory
|
||||
|
||||
|
||||
# External functions
|
||||
@@ -114,6 +106,8 @@ def _evaluate_index(eval_dict, document, node, parent_index_instance=None):
|
||||
try:
|
||||
result = eval(node.expression, eval_dict, AVAILABLE_INDEXING_FUNCTIONS)
|
||||
index_instance, created = IndexInstance.objects.get_or_create(index=node, value=result, parent=parent_index_instance)
|
||||
if created:
|
||||
fs_create_index_directory(index_instance)
|
||||
if node.link_documents:
|
||||
index_instance.documents.add(document)
|
||||
|
||||
|
||||
151
apps/document_indexing/filesystem.py
Normal file
151
apps/document_indexing/filesystem.py
Normal file
@@ -0,0 +1,151 @@
|
||||
import errno
|
||||
import os
|
||||
|
||||
from django.template.defaultfilters import slugify
|
||||
from django.utils.translation import ugettext_lazy as _
|
||||
|
||||
from document_indexing.models import IndexInstance
|
||||
|
||||
from metadata.classes import MetadataObject
|
||||
|
||||
from document_indexing.conf.settings import FILESERVING_ENABLE
|
||||
from document_indexing.conf.settings import FILESERVING_PATH
|
||||
from document_indexing.conf.settings import SLUGIFY_PATHS
|
||||
#from document_indexing.conf.settings import MAX_RENAME_COUNT
|
||||
|
||||
if SLUGIFY_PATHS == False:
|
||||
#Do not slugify path or filenames and extensions
|
||||
SLUGIFY_FUNCTION = lambda x: x
|
||||
else:
|
||||
SLUGIFY_FUNCTION = slugify
|
||||
|
||||
|
||||
def get_instance_path(index_instance):
|
||||
"""
|
||||
Return a platform formated filesytem path corresponding to an
|
||||
index instance
|
||||
"""
|
||||
return os.sep.join(index_instance.get_ancestors().values_list(u'value', flat=True))
|
||||
|
||||
|
||||
def fs_create_index_directory(index_instance):
|
||||
if FILESERVING_ENABLE:
|
||||
target_directory = os.path.join(FILESERVING_PATH, get_instance_path(index_instance))
|
||||
try:
|
||||
os.makedirs(target_directory)
|
||||
#next_available_filename(document, metadata_index, target_directory, SLUGIFY_FUNCTION(document.file_filename), SLUGIFY_FUNCTION(document.file_extension))
|
||||
except OSError, exc:
|
||||
if exc.errno == errno.EEXIST:
|
||||
pass
|
||||
else:
|
||||
raise OSError(_(u'Unable to create metadata indexing directory: %s') % exc)
|
||||
|
||||
|
||||
def document_delete_fs_links(document):
|
||||
if FILESERVING_ENABLE:
|
||||
pass
|
||||
'''
|
||||
for document_metadata_index in document.documentmetadataindex_set.all():
|
||||
try:
|
||||
os.unlink(document_metadata_index.filename)
|
||||
document_metadata_index.delete()
|
||||
except OSError, exc:
|
||||
if exc.errno == errno.ENOENT:
|
||||
#No longer exits, so delete db entry anyway
|
||||
document_metadata_index.delete()
|
||||
else:
|
||||
raise OSError(_(u'Unable to delete metadata indexing symbolic link: %s') % exc)
|
||||
|
||||
path, filename = os.path.split(document_metadata_index.filename)
|
||||
|
||||
#Cleanup directory of dead stuff
|
||||
#Delete siblings that are dead links
|
||||
try:
|
||||
for f in os.listdir(path):
|
||||
filepath = os.path.join(path, f)
|
||||
if os.path.islink(filepath):
|
||||
#Get link's source
|
||||
source = os.readlink(filepath)
|
||||
if os.path.isabs(source):
|
||||
if not os.path.exists(source):
|
||||
#link's source is absolute and doesn't exit
|
||||
os.unlink(filepath)
|
||||
else:
|
||||
os.unlink(os.path.join(path, filepath))
|
||||
elif os.path.isdir(filepath):
|
||||
#is a directory, try to delete it
|
||||
try:
|
||||
os.removedirs(path)
|
||||
except:
|
||||
pass
|
||||
except OSError, exc:
|
||||
pass
|
||||
|
||||
#Remove the directory if it is empty
|
||||
try:
|
||||
os.removedirs(path)
|
||||
except:
|
||||
pass
|
||||
'''
|
||||
|
||||
|
||||
def next_available_filename(document, metadata_index, path, filename, extension, suffix=0):
|
||||
target = filename
|
||||
if suffix:
|
||||
target = '_'.join([filename, unicode(suffix)])
|
||||
filepath = os.path.join(path, os.extsep.join([target, extension]))
|
||||
matches = DocumentMetadataIndex.objects.filter(filename=filepath)
|
||||
if matches.count() == 0:
|
||||
document_metadata_index = DocumentMetadataIndex(
|
||||
document=document, metadata_index=metadata_index,
|
||||
filename=filepath)
|
||||
try:
|
||||
os.symlink(document.file.path, filepath)
|
||||
document_metadata_index.save()
|
||||
except OSError, exc:
|
||||
if exc.errno == errno.EEXIST:
|
||||
#This link should not exist, try to delete it
|
||||
try:
|
||||
os.unlink(filepath)
|
||||
#Try again with same suffix
|
||||
return next_available_filename(document, metadata_index, path, filename, extension, suffix)
|
||||
except Exception, exc:
|
||||
raise Exception(_(u'Unable to create symbolic link, filename clash: %(filepath)s; %(exc)s') % {'filepath': filepath, 'exc': exc})
|
||||
else:
|
||||
raise OSError(_(u'Unable to create symbolic link: %(filepath)s; %(exc)s') % {'filepath': filepath, 'exc': exc})
|
||||
|
||||
return filepath
|
||||
else:
|
||||
if suffix > MAX_RENAME_COUNT:
|
||||
raise Exception(_(u'Maximum rename count reached, not creating symbolic link'))
|
||||
return next_available_filename(document, metadata_index, path, filename, extension, suffix + 1)
|
||||
|
||||
|
||||
#TODO: diferentiate between evaluation error and filesystem errors
|
||||
def do_recreate_all_links(raise_exception=True):
|
||||
errors = []
|
||||
warnings = []
|
||||
|
||||
for document in Document.objects.all():
|
||||
try:
|
||||
document_delete_fs_links(document)
|
||||
except NameError, e:
|
||||
warnings.append('%s: %s' % (document, e))
|
||||
except Exception, e:
|
||||
if raise_exception:
|
||||
raise Exception(e)
|
||||
else:
|
||||
errors.append('%s: %s' % (document, e))
|
||||
|
||||
for document in Document.objects.all():
|
||||
try:
|
||||
create_warnings = document_create_fs_links(document)
|
||||
except Exception, e:
|
||||
if raise_exception:
|
||||
raise Exception(e)
|
||||
else:
|
||||
errors.append('%s: %s' % (document, e))
|
||||
|
||||
for warning in create_warnings:
|
||||
warnings.append('%s: %s' % (document, warning))
|
||||
return errors, warnings
|
||||
@@ -46,7 +46,16 @@ class IndexInstance(MPTTModel):
|
||||
verbose_name = _(u'index instance')
|
||||
verbose_name_plural = _(u'indexes instances')
|
||||
|
||||
'''
|
||||
class DocumentRenameCount(models.Model):
|
||||
index = models.ForeignKey(IndexInstance, verbose_name=_(u'index instance'))
|
||||
document = models.ForeignKey(Document, verbose_name=_(u'document'))
|
||||
count = models.PositiveIntegerField(blank=True, verbose_name=(u'count'))
|
||||
|
||||
# TODO
|
||||
# class DocumentRenameCount
|
||||
# FK=IndexInstance
|
||||
def __unicode__(self):
|
||||
return self.value
|
||||
|
||||
class Meta:
|
||||
verbose_name = _(u'document rename count')
|
||||
verbose_name_plural = _(u'documents rename count')
|
||||
'''
|
||||
|
||||
@@ -16,7 +16,7 @@ def get_document_indexing_subtemplate(document):
|
||||
return {
|
||||
'name': 'generic_list_subtemplate.html',
|
||||
'context': {
|
||||
'title': _(u'index links'),
|
||||
'title': _(u'document indexes'),
|
||||
'object_list': object_list,
|
||||
'hide_link': True
|
||||
}
|
||||
|
||||
@@ -23,10 +23,10 @@ def index_instance_list(request, index_id=None):
|
||||
|
||||
if index_id:
|
||||
index_instance = get_object_or_404(IndexInstance, pk=index_id)
|
||||
index_instance_list = [index for index in index_instance.get_children()]
|
||||
index_instance_list = [index for index in index_instance.get_children().order_by('value')]
|
||||
breadcrumbs = get_breadcrumbs(index_instance)
|
||||
if index_instance.documents.count():
|
||||
for document in index_instance.documents.all():
|
||||
for document in index_instance.documents.all().order_by('file_filename'):
|
||||
index_instance_list.append(document)
|
||||
else:
|
||||
index_instance_list = IndexInstance.objects.filter(parent=None)
|
||||
|
||||
@@ -1,16 +1 @@
|
||||
from django.utils.translation import ugettext_lazy as _
|
||||
|
||||
|
||||
from permissions.api import register_permissions
|
||||
from main.api import register_tool
|
||||
|
||||
|
||||
FILESYSTEM_SERVING_RECREATE_LINKS = 'recreate_links'
|
||||
|
||||
register_permissions('filesystem_serving', [
|
||||
{'name': FILESYSTEM_SERVING_RECREATE_LINKS, 'label':_(u'Recreate filesystem links.')},
|
||||
])
|
||||
|
||||
filesystem_serving_recreate_all_links = {'text': _('recreate index links'), 'view': 'recreate_all_links', 'famfam': 'page_link', 'permissions': {'namespace': 'filesystem_serving', 'permissions': [FILESYSTEM_SERVING_RECREATE_LINKS]}, 'description': _(u'Deletes and creates from scratch all the file system indexing links.')}
|
||||
|
||||
register_tool(filesystem_serving_recreate_all_links, namespace='filesystem_serving', title=_(u'Filesystem'))
|
||||
|
||||
@@ -1,169 +1 @@
|
||||
import errno
|
||||
import os
|
||||
|
||||
from django.template.defaultfilters import slugify
|
||||
from django.utils.translation import ugettext_lazy as _
|
||||
|
||||
from document_indexing.models import IndexInstance
|
||||
|
||||
from metadata.classes import MetadataObject
|
||||
|
||||
from filesystem_serving.conf.settings import FILESERVING_ENABLE
|
||||
from filesystem_serving.conf.settings import FILESERVING_PATH
|
||||
from filesystem_serving.conf.settings import SLUGIFY_PATHS
|
||||
from filesystem_serving.conf.settings import MAX_RENAME_COUNT
|
||||
|
||||
#from filesystem_serving.models import DocumentMetadataIndex, Document
|
||||
|
||||
if SLUGIFY_PATHS == False:
|
||||
#Do not slugify path or filenames and extensions
|
||||
SLUGIFY_FUNCTION = lambda x: x
|
||||
else:
|
||||
SLUGIFY_FUNCTION = slugify
|
||||
|
||||
|
||||
def document_create_fs_links(document):
|
||||
warnings = []
|
||||
if FILESERVING_ENABLE:
|
||||
pass
|
||||
'''
|
||||
if not document.exists():
|
||||
raise Exception(_(u'Not creating metadata indexing, document not found in document storage'))
|
||||
eval_dict = {}
|
||||
eval_dict['document'] = document
|
||||
metadata_dict = dict([(metadata.metadata_type.name, SLUGIFY_FUNCTION(metadata.value)) for metadata in document.documentmetadata_set.all() if metadata.value])
|
||||
eval_dict['metadata'] = MetadataObject(metadata_dict)
|
||||
|
||||
for metadata_index in document.document_type.metadataindex_set.all():
|
||||
if metadata_index.enabled:
|
||||
try:
|
||||
fabricated_directory = eval(metadata_index.expression, eval_dict, AVAILABLE_INDEXING_FUNCTIONS)
|
||||
target_directory = os.path.join(FILESERVING_PATH, fabricated_directory)
|
||||
try:
|
||||
os.makedirs(target_directory)
|
||||
except OSError, exc:
|
||||
if exc.errno == errno.EEXIST:
|
||||
pass
|
||||
else:
|
||||
raise OSError(_(u'Unable to create metadata indexing directory: %s') % exc)
|
||||
|
||||
next_available_filename(document, metadata_index, target_directory, SLUGIFY_FUNCTION(document.file_filename), SLUGIFY_FUNCTION(document.file_extension))
|
||||
except NameError, exc:
|
||||
warnings.append(_(u'Error in metadata indexing expression: %s') % exc)
|
||||
#raise NameError()
|
||||
#This should be a warning not an error
|
||||
#pass
|
||||
except Exception, exc:
|
||||
warnings.append(_(u'Unable to create metadata indexing directory: %s') % exc)
|
||||
'''
|
||||
return warnings
|
||||
|
||||
|
||||
def document_delete_fs_links(document):
|
||||
if FILESERVING_ENABLE:
|
||||
pass
|
||||
'''
|
||||
for document_metadata_index in document.documentmetadataindex_set.all():
|
||||
try:
|
||||
os.unlink(document_metadata_index.filename)
|
||||
document_metadata_index.delete()
|
||||
except OSError, exc:
|
||||
if exc.errno == errno.ENOENT:
|
||||
#No longer exits, so delete db entry anyway
|
||||
document_metadata_index.delete()
|
||||
else:
|
||||
raise OSError(_(u'Unable to delete metadata indexing symbolic link: %s') % exc)
|
||||
|
||||
path, filename = os.path.split(document_metadata_index.filename)
|
||||
|
||||
#Cleanup directory of dead stuff
|
||||
#Delete siblings that are dead links
|
||||
try:
|
||||
for f in os.listdir(path):
|
||||
filepath = os.path.join(path, f)
|
||||
if os.path.islink(filepath):
|
||||
#Get link's source
|
||||
source = os.readlink(filepath)
|
||||
if os.path.isabs(source):
|
||||
if not os.path.exists(source):
|
||||
#link's source is absolute and doesn't exit
|
||||
os.unlink(filepath)
|
||||
else:
|
||||
os.unlink(os.path.join(path, filepath))
|
||||
elif os.path.isdir(filepath):
|
||||
#is a directory, try to delete it
|
||||
try:
|
||||
os.removedirs(path)
|
||||
except:
|
||||
pass
|
||||
except OSError, exc:
|
||||
pass
|
||||
|
||||
#Remove the directory if it is empty
|
||||
try:
|
||||
os.removedirs(path)
|
||||
except:
|
||||
pass
|
||||
'''
|
||||
|
||||
|
||||
def next_available_filename(document, metadata_index, path, filename, extension, suffix=0):
|
||||
target = filename
|
||||
if suffix:
|
||||
target = '_'.join([filename, unicode(suffix)])
|
||||
filepath = os.path.join(path, os.extsep.join([target, extension]))
|
||||
matches = DocumentMetadataIndex.objects.filter(filename=filepath)
|
||||
if matches.count() == 0:
|
||||
document_metadata_index = DocumentMetadataIndex(
|
||||
document=document, metadata_index=metadata_index,
|
||||
filename=filepath)
|
||||
try:
|
||||
os.symlink(document.file.path, filepath)
|
||||
document_metadata_index.save()
|
||||
except OSError, exc:
|
||||
if exc.errno == errno.EEXIST:
|
||||
#This link should not exist, try to delete it
|
||||
try:
|
||||
os.unlink(filepath)
|
||||
#Try again with same suffix
|
||||
return next_available_filename(document, metadata_index, path, filename, extension, suffix)
|
||||
except Exception, exc:
|
||||
raise Exception(_(u'Unable to create symbolic link, filename clash: %(filepath)s; %(exc)s') % {'filepath': filepath, 'exc': exc})
|
||||
else:
|
||||
raise OSError(_(u'Unable to create symbolic link: %(filepath)s; %(exc)s') % {'filepath': filepath, 'exc': exc})
|
||||
|
||||
return filepath
|
||||
else:
|
||||
if suffix > MAX_RENAME_COUNT:
|
||||
raise Exception(_(u'Maximum rename count reached, not creating symbolic link'))
|
||||
return next_available_filename(document, metadata_index, path, filename, extension, suffix + 1)
|
||||
|
||||
|
||||
#TODO: diferentiate between evaluation error and filesystem errors
|
||||
def do_recreate_all_links(raise_exception=True):
|
||||
errors = []
|
||||
warnings = []
|
||||
|
||||
for document in Document.objects.all():
|
||||
try:
|
||||
document_delete_fs_links(document)
|
||||
except NameError, e:
|
||||
warnings.append('%s: %s' % (document, e))
|
||||
except Exception, e:
|
||||
if raise_exception:
|
||||
raise Exception(e)
|
||||
else:
|
||||
errors.append('%s: %s' % (document, e))
|
||||
|
||||
for document in Document.objects.all():
|
||||
try:
|
||||
create_warnings = document_create_fs_links(document)
|
||||
except Exception, e:
|
||||
if raise_exception:
|
||||
raise Exception(e)
|
||||
else:
|
||||
errors.append('%s: %s' % (document, e))
|
||||
|
||||
for warning in create_warnings:
|
||||
warnings.append('%s: %s' % (document, warning))
|
||||
return errors, warnings
|
||||
|
||||
@@ -1,5 +1 @@
|
||||
from django.conf.urls.defaults import patterns, url
|
||||
|
||||
urlpatterns = patterns('filesystem_serving.views',
|
||||
url(r'^recreate_all_links/$', 'recreate_all_links', (), 'recreate_all_links'),
|
||||
)
|
||||
|
||||
@@ -1,35 +1 @@
|
||||
from django.utils.translation import ugettext_lazy as _
|
||||
from django.http import HttpResponseRedirect
|
||||
from django.shortcuts import render_to_response
|
||||
from django.template import RequestContext
|
||||
from django.contrib import messages
|
||||
|
||||
from permissions.api import check_permissions
|
||||
|
||||
from filesystem_serving import FILESYSTEM_SERVING_RECREATE_LINKS
|
||||
from filesystem_serving.api import do_recreate_all_links
|
||||
|
||||
|
||||
def recreate_all_links(request):
|
||||
check_permissions(request.user, 'filesystem_serving', [FILESYSTEM_SERVING_RECREATE_LINKS])
|
||||
|
||||
previous = request.POST.get('previous', request.GET.get('previous', request.META.get('HTTP_REFERER', None)))
|
||||
next = request.POST.get('next', request.GET.get('next', request.META.get('HTTP_REFERER', None)))
|
||||
|
||||
if request.method != 'POST':
|
||||
return render_to_response('generic_confirm.html', {
|
||||
'previous': previous,
|
||||
'next': next,
|
||||
'message': _(u'On large databases this operation may take some time to execute.'),
|
||||
}, context_instance=RequestContext(request))
|
||||
else:
|
||||
try:
|
||||
errors, warnings = do_recreate_all_links()
|
||||
messages.success(request, _(u'Filesystem links re-creation completed successfully.'))
|
||||
for warning in warnings:
|
||||
messages.warning(request, warning)
|
||||
|
||||
except Exception, e:
|
||||
messages.error(request, _(u'Filesystem links re-creation error: %s') % e)
|
||||
|
||||
return HttpResponseRedirect(next)
|
||||
|
||||
Reference in New Issue
Block a user