From 2a2fba9ac69145da3b13c305af90be10a6d48e0c Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 18 May 2011 14:34:55 -0400 Subject: [PATCH] Added document indexes delete support, started merge with filesystem serving app --- apps/document_indexing/api.py | 123 +++++++++++++++++------ apps/document_indexing/conf/settings.py | 4 + apps/documents/views.py | 44 ++++---- apps/filesystem_serving/conf/settings.py | 13 --- apps/metadata/views.py | 37 ++++--- settings.py | 1 - urls.py | 1 - 7 files changed, 130 insertions(+), 93 deletions(-) diff --git a/apps/document_indexing/api.py b/apps/document_indexing/api.py index ee7119231f..a762de42c3 100644 --- a/apps/document_indexing/api.py +++ b/apps/document_indexing/api.py @@ -5,57 +5,55 @@ from django.utils.safestring import mark_safe from metadata.classes import MetadataObject -#from filesystem_serving.conf.settings import MAX_RENAME_COUNT - from document_indexing.models import Index, IndexInstance from document_indexing.conf.settings import AVAILABLE_INDEXING_FUNCTIONS +from document_indexing.conf.settings import FILESERVING_ENABLE +from document_indexing.conf.settings import FILESERVING_PATH +from document_indexing.conf.settings import SLUGIFY_PATHS + +if SLUGIFY_PATHS == False: + # Do not slugify path or filenames and extensions + SLUGIFY_FUNCTION = lambda x: x +else: + SLUGIFY_FUNCTION = slugify -def evaluate_index(eval_dict, document, node, parent_index_instance=None): - warnings = [] - if node.enabled: - try: - result = eval(node.expression, eval_dict, AVAILABLE_INDEXING_FUNCTIONS) - index_instance, created = IndexInstance.objects.get_or_create(index=node, value=result, parent=parent_index_instance) - if node.link_documents: - index_instance.documents.add(document) - - for children in node.get_children(): - children_warnings = evaluate_index(eval_dict, document, children, index_instance) - warnings.extend(children_warnings) - - except NameError, exc: - warnings.append(_(u'Error in metadata indexing expression: %s') % exc) - #raise NameError() - #This should be a warning not an error - #pass - except Exception, exc: - warnings.append(_(u'Unable to create metadata indexing directory: %s') % exc) - - return warnings - - +# External functions def update_indexes(document): - print 'update_indexes' + """ + Update or create all the index instances related to a document + """ warnings = [] eval_dict = {} + document_metadata_dict = dict([(metadata.metadata_type.name, metadata.value) for metadata in document.documentmetadata_set.all() if metadata.value]) eval_dict['document'] = document - metadata_dict = dict([(metadata.metadata_type.name, metadata.value) for metadata in document.documentmetadata_set.all() if metadata.value]) - eval_dict['metadata'] = MetadataObject(metadata_dict) + eval_dict['metadata'] = MetadataObject(document_metadata_dict) for root in Index.objects.filter(parent=None): - index_warnings = evaluate_index(eval_dict, document, root) + index_warnings = _evaluate_index(eval_dict, document, root) warnings.extend(index_warnings) return warnings - - + + def delete_indexes(document): - print 'delete_indexes' + """ + Delete all the index instances related to a document + """ + warnings = [] + + for node in document.indexinstance_set.all(): + _delete_document_from_node(document, node) + + return warnings def get_instance_link(index_instance=None, text=None, simple=False): + """ + Return an HTML anchor to an index instance + """ + if simple: # Just display the instance's value or overrided text, no # HTML anchor @@ -74,6 +72,10 @@ def get_instance_link(index_instance=None, text=None, simple=False): def get_breadcrumbs(index_instance, simple=False, single_link=False): + """ + Return a joined string of HTML anchors to every index instance's + parent from the root of the tree to the index instance + """ result = [] if single_link: # Return the entire breadcrumb as a single HTML anchor @@ -90,3 +92,58 @@ def get_breadcrumbs(index_instance, simple=False, single_link=False): return mark_safe(get_instance_link(index_instance=index_instance, text=(u' / '.join(result)))) else: return mark_safe(u' / '.join(result)) + + +# Internal functions +def _evaluate_index(eval_dict, document, node, parent_index_instance=None): + """ + Evaluate an index expression and update or create all the related + index instances also recursively calling itself to evaluate all the + index's children + """ + warnings = [] + if node.enabled: + try: + result = eval(node.expression, eval_dict, AVAILABLE_INDEXING_FUNCTIONS) + index_instance, created = IndexInstance.objects.get_or_create(index=node, value=result, parent=parent_index_instance) + if node.link_documents: + index_instance.documents.add(document) + + for children in node.get_children(): + children_warnings = _evaluate_index(eval_dict, document, children, index_instance) + warnings.extend(children_warnings) + + except (NameError, AttributeError), exc: + warnings.append(_(u'Error in document indexing update expression: %(expression)s; %(exception)s') % { + 'expression': node.expression, 'exception': exc}) + #raise NameError() + #This should be a warning not an error + #pass + + except Exception, exc: + warnings.append(_(u'Error updating document index, expression: %(expression)s; %(exception)s') % { + 'expression': node.expression, 'exception': exc}) + + return warnings + + + +def _delete_document_from_node(document, node): + """ + Delete a documents reference from an index instance and call itself + recusively deleting documents and empty index instances up to the + root of the tree + """ + warnings = [] + try: + node.documents.remove(document) + if node.documents.count() == 0 and node.get_children().count() == 0: + parent = node.parent + node.delete() + parent_warnings = _delete_document_from_node(document, parent) + warnings.extend(parent_warnings) + + except Exception, exc: + warnings.append(_(u'Unable to delete document indexing node; %s') % exc) + + return warnings diff --git a/apps/document_indexing/conf/settings.py b/apps/document_indexing/conf/settings.py index 1fc19e2e51..5f8f62dff8 100644 --- a/apps/document_indexing/conf/settings.py +++ b/apps/document_indexing/conf/settings.py @@ -18,5 +18,9 @@ register_settings( settings=[ # Definition {'name': u'AVAILABLE_INDEXING_FUNCTIONS', 'global_name': u'DOCUMENT_INDEXING_AVAILABLE_INDEXING_FUNCTIONS', 'default': available_indexing_functions}, + # Filesystem serving + {'name': u'SLUGIFY_PATHS', 'global_name': u'DOCUMENT_INDEXING_FILESYSTEM_SLUGIFY_PATHS', 'default': False}, + {'name': u'FILESERVING_PATH', 'global_name': u'DOCUMENT_INDEXING_FILESYSTEM_FILESERVING_PATH', 'default': u'/tmp/mayan/documents', 'exists': True}, + {'name': u'FILESERVING_ENABLE', 'global_name': u'DOCUMENT_INDEXING_FILESYSTEM_FILESERVING_ENABLE', 'default': True} ] ) diff --git a/apps/documents/views.py b/apps/documents/views.py index 12270de4e7..45e3e2cd62 100644 --- a/apps/documents/views.py +++ b/apps/documents/views.py @@ -25,7 +25,6 @@ from converter.exceptions import UnkownConvertError, UnknownFormat from converter.api import DEFAULT_ZOOM_LEVEL, DEFAULT_ROTATION, \ DEFAULT_FILE_FORMAT, QUALITY_PRINT from document_comments.utils import get_comments_subtemplate -from filesystem_serving.api import document_create_fs_links, document_delete_fs_links from filetransfers.api import serve_file from grouping.models import DocumentGroup from grouping import document_group_link @@ -37,6 +36,7 @@ from navigation.utils import resolve_to_name from permissions.api import check_permissions from tags.utils import get_tags_subtemplate from document_indexing.utils import get_document_indexing_subtemplate +from document_indexing.api import update_indexes, delete_indexes from documents.conf.settings import DELETE_STAGING_FILE_AFTER_UPLOAD from documents.conf.settings import USE_STAGING_DIRECTORY @@ -124,13 +124,11 @@ def _handle_save_document(request, document, form=None): document.save() save_metadata_list(decode_metadata_from_url(request.GET), document, create=True) - try: - warnings = document_create_fs_links(document) - if request.user.is_staff or request.user.is_superuser: - for warning in warnings: - messages.warning(request, warning) - except Exception, e: - messages.error(request, e) + + warnings = update_indexes(document) + if request.user.is_staff or request.user.is_superuser: + for warning in warnings: + messages.warning(request, warning) def _handle_zip_file(request, uploaded_file, document_type): @@ -425,7 +423,11 @@ def document_delete(request, document_id=None, document_id_list=None): if request.method == 'POST': for document in documents: try: - document_delete_fs_links(document) + warnings = delete_indexes(document) + if request.user.is_staff or request.user.is_superuser: + for warning in warnings: + messages.warning(request, warning) + document.delete() messages.success(request, _(u'Document: %s deleted successfully.') % document) except Exception, e: @@ -469,11 +471,10 @@ def document_edit(request, document_id): if request.method == 'POST': form = DocumentForm_edit(request.POST, initial={'document_type': document.document_type}) if form.is_valid(): - try: - document_delete_fs_links(document) - except Exception, e: - messages.error(request, e) - return HttpResponseRedirect(reverse('document_list')) + warnings = delete_indexes(document) + if request.user.is_staff or request.user.is_superuser: + for warning in warnings: + messages.warning(request, warning) document.file_filename = form.cleaned_data['new_filename'] document.description = form.cleaned_data['description'] @@ -486,17 +487,10 @@ def document_edit(request, document_id): messages.success(request, _(u'Document %s edited successfully.') % document) - try: - warnings = document_create_fs_links(document) - - if request.user.is_staff or request.user.is_superuser: - for warning in warnings: - messages.warning(request, warning) - - messages.success(request, _(u'Document filesystem links updated successfully.')) - except Exception, e: - messages.error(request, e) - return HttpResponseRedirect(document.get_absolute_url()) + warnings = update_indexes(document) + if request.user.is_staff or request.user.is_superuser: + for warning in warnings: + messages.warning(request, warning) return HttpResponseRedirect(document.get_absolute_url()) else: diff --git a/apps/filesystem_serving/conf/settings.py b/apps/filesystem_serving/conf/settings.py index e014493a6d..8b13789179 100644 --- a/apps/filesystem_serving/conf/settings.py +++ b/apps/filesystem_serving/conf/settings.py @@ -1,14 +1 @@ -"""Configuration options for the filesystem_serving app""" -from smart_settings.api import register_settings - -register_settings( - namespace=u'filesystem_serving', - module=u'filesystem_serving.conf.settings', - settings=[ - {'name': u'SLUGIFY_PATHS', 'global_name': u'FILESYSTEM_SLUGIFY_PATHS', 'default': False}, - {'name': u'MAX_RENAME_COUNT', 'global_name': u'FILESYSTEM_MAX_RENAME_COUNT', 'default': 200}, - {'name': u'FILESERVING_PATH', 'global_name': u'FILESYSTEM_FILESERVING_PATH', 'default': u'/tmp/mayan/documents', 'exists': True}, - {'name': u'FILESERVING_ENABLE', 'global_name': u'FILESYSTEM_FILESERVING_ENABLE', 'default': True} - ] -) diff --git a/apps/metadata/views.py b/apps/metadata/views.py index 072c2f7e9d..c57df3b6e2 100644 --- a/apps/metadata/views.py +++ b/apps/metadata/views.py @@ -9,7 +9,6 @@ from django.utils.http import urlencode from documents.models import Document, RecentDocument from permissions.api import check_permissions -from filesystem_serving.api import document_create_fs_links, document_delete_fs_links from document_indexing.api import update_indexes, delete_indexes from metadata import PERMISSION_METADATA_DOCUMENT_EDIT, \ @@ -57,33 +56,31 @@ def metadata_edit(request, document_id=None, document_id_list=None): formset = MetadataFormSet(request.POST) if formset.is_valid(): for document in documents: - delete_indexes(document) - try: - document_delete_fs_links(document) - except Exception, e: - messages.error(request, _(u'Error deleting filesystem links for document: %(document)s; %(error)s') % { - 'document': document, 'error': e}) + + warnings = delete_indexes(document) + if request.user.is_staff or request.user.is_superuser: + for warning in warnings: + messages.warning(request, warning) + errors = [] for form in formset.forms: if form.cleaned_data['update']: try: save_metadata_list([form.cleaned_data], document) - messages.success(request, _(u'Metadata for document %s edited successfully.') % document) except Exception, e: - messages.error(request, _(u'Error editing metadata for document %(document)s; %(error)s.') % { - 'document': document, 'error': e}) - update_indexes(document) - try: - warnings = document_create_fs_links(document) + errors.append(e) - if request.user.is_staff or request.user.is_superuser: - for warning in warnings: - messages.warning(request, warning) + if errors: + for error in errors: + messages.error(request, _(u'Error editing metadata for document %(document)s; %(error)s.') % { + 'document': document, 'error': error}) + else: + messages.success(request, _(u'Metadata for document %s edited successfully.') % document) - messages.success(request, _(u'Filesystem links updated successfully for document: %s.') % document) - except Exception, e: - messages.error(request, _('Error creating filesystem links for document: %(document)s; %(error)s') % { - 'document': document, 'error': e}) + warnings = update_indexes(document) + if request.user.is_staff or request.user.is_superuser: + for warning in warnings: + messages.warning(request, warning) if len(documents) == 1: return HttpResponseRedirect(document.get_absolute_url()) diff --git a/settings.py b/settings.py index f9673a0c47..3b2bf30d48 100644 --- a/settings.py +++ b/settings.py @@ -138,7 +138,6 @@ INSTALLED_APPS = ( 'sentry', 'sentry.client', 'sentry.client.celery', - 'filesystem_serving', 'storage', 'folders', 'taggit', diff --git a/urls.py b/urls.py index b0a03cc5e9..c94d915c99 100644 --- a/urls.py +++ b/urls.py @@ -9,7 +9,6 @@ urlpatterns = patterns('', (r'^', include('main.urls')), (r'^documents/', include('documents.urls')), (r'^folders/', include('folders.urls')), - (r'^filesystem_serving/', include('filesystem_serving.urls')), (r'^search/', include('dynamic_search.urls')), (r'^ocr/', include('ocr.urls')), (r'^permissions/', include('permissions.urls')),