diff --git a/apps/document_indexing/api.py b/apps/document_indexing/api.py index ab23b2ad7b..1ed666f0d2 100644 --- a/apps/document_indexing/api.py +++ b/apps/document_indexing/api.py @@ -8,15 +8,7 @@ from metadata.classes import MetadataObject from document_indexing.models import Index, IndexInstance from document_indexing.conf.settings import AVAILABLE_INDEXING_FUNCTIONS -from document_indexing.conf.settings import FILESERVING_ENABLE -from document_indexing.conf.settings import FILESERVING_PATH -from document_indexing.conf.settings import SLUGIFY_PATHS - -if SLUGIFY_PATHS == False: - # Do not slugify path or filenames and extensions - SLUGIFY_FUNCTION = lambda x: x -else: - SLUGIFY_FUNCTION = slugify +from document_indexing.filesystem import fs_create_index_directory # External functions @@ -114,6 +106,8 @@ def _evaluate_index(eval_dict, document, node, parent_index_instance=None): try: result = eval(node.expression, eval_dict, AVAILABLE_INDEXING_FUNCTIONS) index_instance, created = IndexInstance.objects.get_or_create(index=node, value=result, parent=parent_index_instance) + if created: + fs_create_index_directory(index_instance) if node.link_documents: index_instance.documents.add(document) diff --git a/apps/document_indexing/filesystem.py b/apps/document_indexing/filesystem.py new file mode 100644 index 0000000000..53924640ad --- /dev/null +++ b/apps/document_indexing/filesystem.py @@ -0,0 +1,151 @@ +import errno +import os + +from django.template.defaultfilters import slugify +from django.utils.translation import ugettext_lazy as _ + +from document_indexing.models import IndexInstance + +from metadata.classes import MetadataObject + +from document_indexing.conf.settings import FILESERVING_ENABLE +from document_indexing.conf.settings import FILESERVING_PATH +from document_indexing.conf.settings import SLUGIFY_PATHS +#from document_indexing.conf.settings import MAX_RENAME_COUNT + +if SLUGIFY_PATHS == False: + #Do not slugify path or filenames and extensions + SLUGIFY_FUNCTION = lambda x: x +else: + SLUGIFY_FUNCTION = slugify + + +def get_instance_path(index_instance): + """ + Return a platform formated filesytem path corresponding to an + index instance + """ + return os.sep.join(index_instance.get_ancestors().values_list(u'value', flat=True)) + + +def fs_create_index_directory(index_instance): + if FILESERVING_ENABLE: + target_directory = os.path.join(FILESERVING_PATH, get_instance_path(index_instance)) + try: + os.makedirs(target_directory) + #next_available_filename(document, metadata_index, target_directory, SLUGIFY_FUNCTION(document.file_filename), SLUGIFY_FUNCTION(document.file_extension)) + except OSError, exc: + if exc.errno == errno.EEXIST: + pass + else: + raise OSError(_(u'Unable to create metadata indexing directory: %s') % exc) + + +def document_delete_fs_links(document): + if FILESERVING_ENABLE: + pass + ''' + for document_metadata_index in document.documentmetadataindex_set.all(): + try: + os.unlink(document_metadata_index.filename) + document_metadata_index.delete() + except OSError, exc: + if exc.errno == errno.ENOENT: + #No longer exits, so delete db entry anyway + document_metadata_index.delete() + else: + raise OSError(_(u'Unable to delete metadata indexing symbolic link: %s') % exc) + + path, filename = os.path.split(document_metadata_index.filename) + + #Cleanup directory of dead stuff + #Delete siblings that are dead links + try: + for f in os.listdir(path): + filepath = os.path.join(path, f) + if os.path.islink(filepath): + #Get link's source + source = os.readlink(filepath) + if os.path.isabs(source): + if not os.path.exists(source): + #link's source is absolute and doesn't exit + os.unlink(filepath) + else: + os.unlink(os.path.join(path, filepath)) + elif os.path.isdir(filepath): + #is a directory, try to delete it + try: + os.removedirs(path) + except: + pass + except OSError, exc: + pass + + #Remove the directory if it is empty + try: + os.removedirs(path) + except: + pass + ''' + + +def next_available_filename(document, metadata_index, path, filename, extension, suffix=0): + target = filename + if suffix: + target = '_'.join([filename, unicode(suffix)]) + filepath = os.path.join(path, os.extsep.join([target, extension])) + matches = DocumentMetadataIndex.objects.filter(filename=filepath) + if matches.count() == 0: + document_metadata_index = DocumentMetadataIndex( + document=document, metadata_index=metadata_index, + filename=filepath) + try: + os.symlink(document.file.path, filepath) + document_metadata_index.save() + except OSError, exc: + if exc.errno == errno.EEXIST: + #This link should not exist, try to delete it + try: + os.unlink(filepath) + #Try again with same suffix + return next_available_filename(document, metadata_index, path, filename, extension, suffix) + except Exception, exc: + raise Exception(_(u'Unable to create symbolic link, filename clash: %(filepath)s; %(exc)s') % {'filepath': filepath, 'exc': exc}) + else: + raise OSError(_(u'Unable to create symbolic link: %(filepath)s; %(exc)s') % {'filepath': filepath, 'exc': exc}) + + return filepath + else: + if suffix > MAX_RENAME_COUNT: + raise Exception(_(u'Maximum rename count reached, not creating symbolic link')) + return next_available_filename(document, metadata_index, path, filename, extension, suffix + 1) + + +#TODO: diferentiate between evaluation error and filesystem errors +def do_recreate_all_links(raise_exception=True): + errors = [] + warnings = [] + + for document in Document.objects.all(): + try: + document_delete_fs_links(document) + except NameError, e: + warnings.append('%s: %s' % (document, e)) + except Exception, e: + if raise_exception: + raise Exception(e) + else: + errors.append('%s: %s' % (document, e)) + + for document in Document.objects.all(): + try: + create_warnings = document_create_fs_links(document) + except Exception, e: + if raise_exception: + raise Exception(e) + else: + errors.append('%s: %s' % (document, e)) + + for warning in create_warnings: + warnings.append('%s: %s' % (document, warning)) + return errors, warnings diff --git a/apps/document_indexing/models.py b/apps/document_indexing/models.py index 527dc24459..2a07747bd8 100644 --- a/apps/document_indexing/models.py +++ b/apps/document_indexing/models.py @@ -46,7 +46,16 @@ class IndexInstance(MPTTModel): verbose_name = _(u'index instance') verbose_name_plural = _(u'indexes instances') +''' +class DocumentRenameCount(models.Model): + index = models.ForeignKey(IndexInstance, verbose_name=_(u'index instance')) + document = models.ForeignKey(Document, verbose_name=_(u'document')) + count = models.PositiveIntegerField(blank=True, verbose_name=(u'count')) -# TODO -# class DocumentRenameCount -# FK=IndexInstance + def __unicode__(self): + return self.value + + class Meta: + verbose_name = _(u'document rename count') + verbose_name_plural = _(u'documents rename count') +''' diff --git a/apps/document_indexing/utils.py b/apps/document_indexing/utils.py index 5966a5ef69..1db1caf3e5 100644 --- a/apps/document_indexing/utils.py +++ b/apps/document_indexing/utils.py @@ -16,7 +16,7 @@ def get_document_indexing_subtemplate(document): return { 'name': 'generic_list_subtemplate.html', 'context': { - 'title': _(u'index links'), + 'title': _(u'document indexes'), 'object_list': object_list, 'hide_link': True } diff --git a/apps/document_indexing/views.py b/apps/document_indexing/views.py index ee87c680cd..744c32cd0b 100644 --- a/apps/document_indexing/views.py +++ b/apps/document_indexing/views.py @@ -23,10 +23,10 @@ def index_instance_list(request, index_id=None): if index_id: index_instance = get_object_or_404(IndexInstance, pk=index_id) - index_instance_list = [index for index in index_instance.get_children()] + index_instance_list = [index for index in index_instance.get_children().order_by('value')] breadcrumbs = get_breadcrumbs(index_instance) if index_instance.documents.count(): - for document in index_instance.documents.all(): + for document in index_instance.documents.all().order_by('file_filename'): index_instance_list.append(document) else: index_instance_list = IndexInstance.objects.filter(parent=None) diff --git a/apps/filesystem_serving/__init__.py b/apps/filesystem_serving/__init__.py index 05eb6a38a5..8b13789179 100644 --- a/apps/filesystem_serving/__init__.py +++ b/apps/filesystem_serving/__init__.py @@ -1,16 +1 @@ -from django.utils.translation import ugettext_lazy as _ - -from permissions.api import register_permissions -from main.api import register_tool - - -FILESYSTEM_SERVING_RECREATE_LINKS = 'recreate_links' - -register_permissions('filesystem_serving', [ - {'name': FILESYSTEM_SERVING_RECREATE_LINKS, 'label':_(u'Recreate filesystem links.')}, -]) - -filesystem_serving_recreate_all_links = {'text': _('recreate index links'), 'view': 'recreate_all_links', 'famfam': 'page_link', 'permissions': {'namespace': 'filesystem_serving', 'permissions': [FILESYSTEM_SERVING_RECREATE_LINKS]}, 'description': _(u'Deletes and creates from scratch all the file system indexing links.')} - -register_tool(filesystem_serving_recreate_all_links, namespace='filesystem_serving', title=_(u'Filesystem')) diff --git a/apps/filesystem_serving/api.py b/apps/filesystem_serving/api.py index e9b1b0a412..8b13789179 100644 --- a/apps/filesystem_serving/api.py +++ b/apps/filesystem_serving/api.py @@ -1,169 +1 @@ -import errno -import os -from django.template.defaultfilters import slugify -from django.utils.translation import ugettext_lazy as _ - -from document_indexing.models import IndexInstance - -from metadata.classes import MetadataObject - -from filesystem_serving.conf.settings import FILESERVING_ENABLE -from filesystem_serving.conf.settings import FILESERVING_PATH -from filesystem_serving.conf.settings import SLUGIFY_PATHS -from filesystem_serving.conf.settings import MAX_RENAME_COUNT - -#from filesystem_serving.models import DocumentMetadataIndex, Document - -if SLUGIFY_PATHS == False: - #Do not slugify path or filenames and extensions - SLUGIFY_FUNCTION = lambda x: x -else: - SLUGIFY_FUNCTION = slugify - - -def document_create_fs_links(document): - warnings = [] - if FILESERVING_ENABLE: - pass - ''' - if not document.exists(): - raise Exception(_(u'Not creating metadata indexing, document not found in document storage')) - eval_dict = {} - eval_dict['document'] = document - metadata_dict = dict([(metadata.metadata_type.name, SLUGIFY_FUNCTION(metadata.value)) for metadata in document.documentmetadata_set.all() if metadata.value]) - eval_dict['metadata'] = MetadataObject(metadata_dict) - - for metadata_index in document.document_type.metadataindex_set.all(): - if metadata_index.enabled: - try: - fabricated_directory = eval(metadata_index.expression, eval_dict, AVAILABLE_INDEXING_FUNCTIONS) - target_directory = os.path.join(FILESERVING_PATH, fabricated_directory) - try: - os.makedirs(target_directory) - except OSError, exc: - if exc.errno == errno.EEXIST: - pass - else: - raise OSError(_(u'Unable to create metadata indexing directory: %s') % exc) - - next_available_filename(document, metadata_index, target_directory, SLUGIFY_FUNCTION(document.file_filename), SLUGIFY_FUNCTION(document.file_extension)) - except NameError, exc: - warnings.append(_(u'Error in metadata indexing expression: %s') % exc) - #raise NameError() - #This should be a warning not an error - #pass - except Exception, exc: - warnings.append(_(u'Unable to create metadata indexing directory: %s') % exc) - ''' - return warnings - - -def document_delete_fs_links(document): - if FILESERVING_ENABLE: - pass - ''' - for document_metadata_index in document.documentmetadataindex_set.all(): - try: - os.unlink(document_metadata_index.filename) - document_metadata_index.delete() - except OSError, exc: - if exc.errno == errno.ENOENT: - #No longer exits, so delete db entry anyway - document_metadata_index.delete() - else: - raise OSError(_(u'Unable to delete metadata indexing symbolic link: %s') % exc) - - path, filename = os.path.split(document_metadata_index.filename) - - #Cleanup directory of dead stuff - #Delete siblings that are dead links - try: - for f in os.listdir(path): - filepath = os.path.join(path, f) - if os.path.islink(filepath): - #Get link's source - source = os.readlink(filepath) - if os.path.isabs(source): - if not os.path.exists(source): - #link's source is absolute and doesn't exit - os.unlink(filepath) - else: - os.unlink(os.path.join(path, filepath)) - elif os.path.isdir(filepath): - #is a directory, try to delete it - try: - os.removedirs(path) - except: - pass - except OSError, exc: - pass - - #Remove the directory if it is empty - try: - os.removedirs(path) - except: - pass - ''' - - -def next_available_filename(document, metadata_index, path, filename, extension, suffix=0): - target = filename - if suffix: - target = '_'.join([filename, unicode(suffix)]) - filepath = os.path.join(path, os.extsep.join([target, extension])) - matches = DocumentMetadataIndex.objects.filter(filename=filepath) - if matches.count() == 0: - document_metadata_index = DocumentMetadataIndex( - document=document, metadata_index=metadata_index, - filename=filepath) - try: - os.symlink(document.file.path, filepath) - document_metadata_index.save() - except OSError, exc: - if exc.errno == errno.EEXIST: - #This link should not exist, try to delete it - try: - os.unlink(filepath) - #Try again with same suffix - return next_available_filename(document, metadata_index, path, filename, extension, suffix) - except Exception, exc: - raise Exception(_(u'Unable to create symbolic link, filename clash: %(filepath)s; %(exc)s') % {'filepath': filepath, 'exc': exc}) - else: - raise OSError(_(u'Unable to create symbolic link: %(filepath)s; %(exc)s') % {'filepath': filepath, 'exc': exc}) - - return filepath - else: - if suffix > MAX_RENAME_COUNT: - raise Exception(_(u'Maximum rename count reached, not creating symbolic link')) - return next_available_filename(document, metadata_index, path, filename, extension, suffix + 1) - - -#TODO: diferentiate between evaluation error and filesystem errors -def do_recreate_all_links(raise_exception=True): - errors = [] - warnings = [] - - for document in Document.objects.all(): - try: - document_delete_fs_links(document) - except NameError, e: - warnings.append('%s: %s' % (document, e)) - except Exception, e: - if raise_exception: - raise Exception(e) - else: - errors.append('%s: %s' % (document, e)) - - for document in Document.objects.all(): - try: - create_warnings = document_create_fs_links(document) - except Exception, e: - if raise_exception: - raise Exception(e) - else: - errors.append('%s: %s' % (document, e)) - - for warning in create_warnings: - warnings.append('%s: %s' % (document, warning)) - return errors, warnings diff --git a/apps/filesystem_serving/urls.py b/apps/filesystem_serving/urls.py index ec434df234..8b13789179 100644 --- a/apps/filesystem_serving/urls.py +++ b/apps/filesystem_serving/urls.py @@ -1,5 +1 @@ -from django.conf.urls.defaults import patterns, url -urlpatterns = patterns('filesystem_serving.views', - url(r'^recreate_all_links/$', 'recreate_all_links', (), 'recreate_all_links'), -) diff --git a/apps/filesystem_serving/views.py b/apps/filesystem_serving/views.py index 9f9c83d906..8b13789179 100644 --- a/apps/filesystem_serving/views.py +++ b/apps/filesystem_serving/views.py @@ -1,35 +1 @@ -from django.utils.translation import ugettext_lazy as _ -from django.http import HttpResponseRedirect -from django.shortcuts import render_to_response -from django.template import RequestContext -from django.contrib import messages -from permissions.api import check_permissions - -from filesystem_serving import FILESYSTEM_SERVING_RECREATE_LINKS -from filesystem_serving.api import do_recreate_all_links - - -def recreate_all_links(request): - check_permissions(request.user, 'filesystem_serving', [FILESYSTEM_SERVING_RECREATE_LINKS]) - - previous = request.POST.get('previous', request.GET.get('previous', request.META.get('HTTP_REFERER', None))) - next = request.POST.get('next', request.GET.get('next', request.META.get('HTTP_REFERER', None))) - - if request.method != 'POST': - return render_to_response('generic_confirm.html', { - 'previous': previous, - 'next': next, - 'message': _(u'On large databases this operation may take some time to execute.'), - }, context_instance=RequestContext(request)) - else: - try: - errors, warnings = do_recreate_all_links() - messages.success(request, _(u'Filesystem links re-creation completed successfully.')) - for warning in warnings: - messages.warning(request, warning) - - except Exception, e: - messages.error(request, _(u'Filesystem links re-creation error: %s') % e) - - return HttpResponseRedirect(next)