Added document indexes delete support, started merge with filesystem serving app

apps/document_indexing/api.py

@@ -5,57 +5,55 @@ from django.utils.safestring import mark_safe
 
 from metadata.classes import MetadataObject
 
-#from filesystem_serving.conf.settings import MAX_RENAME_COUNT
-
 from document_indexing.models import Index, IndexInstance
 from document_indexing.conf.settings import AVAILABLE_INDEXING_FUNCTIONS
+from document_indexing.conf.settings import FILESERVING_ENABLE
+from document_indexing.conf.settings import FILESERVING_PATH
+from document_indexing.conf.settings import SLUGIFY_PATHS
+
+if SLUGIFY_PATHS == False:
+    # Do not slugify path or filenames and extensions
+    SLUGIFY_FUNCTION = lambda x: x
+else:
+    SLUGIFY_FUNCTION = slugify
 
 
-def evaluate_index(eval_dict, document, node, parent_index_instance=None):
+# External functions
-    warnings = []
-    if node.enabled:
-        try:
-            result = eval(node.expression, eval_dict, AVAILABLE_INDEXING_FUNCTIONS)
-            index_instance, created = IndexInstance.objects.get_or_create(index=node, value=result, parent=parent_index_instance)
-            if node.link_documents:
-                index_instance.documents.add(document)
-                
-            for children in node.get_children():
-                children_warnings = evaluate_index(eval_dict, document, children, index_instance)
-                warnings.extend(children_warnings)
-            
-        except NameError, exc:
-            warnings.append(_(u'Error in metadata indexing expression: %s') % exc)
-            #raise NameError()
-            #This should be a warning not an error
-            #pass
-        except Exception, exc:
-            warnings.append(_(u'Unable to create metadata indexing directory: %s') % exc)
-            
-    return warnings
-
-
 def update_indexes(document):
-    print 'update_indexes'
+    """
+    Update or create all the index instances related to a document
+    """    
     warnings = []
 
     eval_dict = {}
+    document_metadata_dict = dict([(metadata.metadata_type.name, metadata.value) for metadata in document.documentmetadata_set.all() if metadata.value])
     eval_dict['document'] = document
-    metadata_dict = dict([(metadata.metadata_type.name, metadata.value) for metadata in document.documentmetadata_set.all() if metadata.value])
+    eval_dict['metadata'] = MetadataObject(document_metadata_dict)
-    eval_dict['metadata'] = MetadataObject(metadata_dict)
 
     for root in Index.objects.filter(parent=None):
-        index_warnings = evaluate_index(eval_dict, document, root)
+        index_warnings = _evaluate_index(eval_dict, document, root)
         warnings.extend(index_warnings)
 
     return warnings
 
 
 def delete_indexes(document):
-    print 'delete_indexes'
+    """
+    Delete all the index instances related to a document
+    """    
+    warnings = []
+    
+    for node in document.indexinstance_set.all():
+        _delete_document_from_node(document, node)
+            
+    return warnings
 
 
 def get_instance_link(index_instance=None, text=None, simple=False):
+    """
+    Return an HTML anchor to an index instance
+    """    
+    
     if simple:
         # Just display the instance's value or overrided text, no
         # HTML anchor
@@ -74,6 +72,10 @@ def get_instance_link(index_instance=None, text=None, simple=False):
         
 
 def get_breadcrumbs(index_instance, simple=False, single_link=False):
+    """
+    Return a joined string of HTML anchors to every index instance's
+    parent from the root of the tree to the index instance
+    """    
     result = []
     if single_link:
         # Return the entire breadcrumb as a single HTML anchor
@@ -90,3 +92,58 @@ def get_breadcrumbs(index_instance, simple=False, single_link=False):
         return mark_safe(get_instance_link(index_instance=index_instance, text=(u' / '.join(result))))
     else:
         return mark_safe(u' / '.join(result))
+
+
+# Internal functions
+def _evaluate_index(eval_dict, document, node, parent_index_instance=None):
+    """
+    Evaluate an index expression and update or create all the related
+    index instances also recursively calling itself to evaluate all the
+    index's children
+    """    
+    warnings = []
+    if node.enabled:
+        try:
+            result = eval(node.expression, eval_dict, AVAILABLE_INDEXING_FUNCTIONS)
+            index_instance, created = IndexInstance.objects.get_or_create(index=node, value=result, parent=parent_index_instance)
+            if node.link_documents:
+                index_instance.documents.add(document)
+
+            for children in node.get_children():
+                children_warnings = _evaluate_index(eval_dict, document, children, index_instance)
+                warnings.extend(children_warnings)
+
+        except (NameError, AttributeError), exc:
+            warnings.append(_(u'Error in document indexing update expression: %(expression)s; %(exception)s') % {
+                'expression': node.expression, 'exception': exc})
+            #raise NameError()
+            #This should be a warning not an error
+            #pass
+
+        except Exception, exc:
+            warnings.append(_(u'Error updating document index, expression: %(expression)s; %(exception)s') % {
+                'expression': node.expression, 'exception': exc})
+                
+    return warnings
+
+
+
+def _delete_document_from_node(document, node):
+    """
+    Delete a documents reference from an index instance and call itself
+    recusively deleting documents and empty index instances up to the 
+    root of the tree
+    """
+    warnings = []
+    try:
+        node.documents.remove(document)
+        if node.documents.count() == 0 and node.get_children().count() == 0:
+            parent = node.parent
+            node.delete()
+            parent_warnings = _delete_document_from_node(document, parent)
+            warnings.extend(parent_warnings)
+            
+    except Exception, exc:
+        warnings.append(_(u'Unable to delete document indexing node; %s') % exc)
+
+    return warnings

apps/document_indexing/conf/settings.py

@@ -18,5 +18,9 @@ register_settings(
     settings=[
         # Definition
         {'name': u'AVAILABLE_INDEXING_FUNCTIONS', 'global_name': u'DOCUMENT_INDEXING_AVAILABLE_INDEXING_FUNCTIONS', 'default': available_indexing_functions},
+        # Filesystem serving
+        {'name': u'SLUGIFY_PATHS', 'global_name': u'DOCUMENT_INDEXING_FILESYSTEM_SLUGIFY_PATHS', 'default': False},
+        {'name': u'FILESERVING_PATH', 'global_name': u'DOCUMENT_INDEXING_FILESYSTEM_FILESERVING_PATH', 'default': u'/tmp/mayan/documents', 'exists': True},
+        {'name': u'FILESERVING_ENABLE', 'global_name': u'DOCUMENT_INDEXING_FILESYSTEM_FILESERVING_ENABLE', 'default': True}
     ]
 )

apps/documents/views.py

@@ -25,7 +25,6 @@ from converter.exceptions import UnkownConvertError, UnknownFormat
 from converter.api import DEFAULT_ZOOM_LEVEL, DEFAULT_ROTATION, \
     DEFAULT_FILE_FORMAT, QUALITY_PRINT
 from document_comments.utils import get_comments_subtemplate
-from filesystem_serving.api import document_create_fs_links, document_delete_fs_links
 from filetransfers.api import serve_file
 from grouping.models import DocumentGroup
 from grouping import document_group_link
@@ -37,6 +36,7 @@ from navigation.utils import resolve_to_name
 from permissions.api import check_permissions
 from tags.utils import get_tags_subtemplate
 from document_indexing.utils import get_document_indexing_subtemplate
+from document_indexing.api import update_indexes, delete_indexes
 
 from documents.conf.settings import DELETE_STAGING_FILE_AFTER_UPLOAD
 from documents.conf.settings import USE_STAGING_DIRECTORY
@@ -124,13 +124,11 @@ def _handle_save_document(request, document, form=None):
             document.save()
 
     save_metadata_list(decode_metadata_from_url(request.GET), document, create=True)
-    try:
+
-        warnings = document_create_fs_links(document)
+    warnings = update_indexes(document)
-        if request.user.is_staff or request.user.is_superuser:
+    if request.user.is_staff or request.user.is_superuser:
-            for warning in warnings:
+        for warning in warnings:
-                messages.warning(request, warning)
+            messages.warning(request, warning)    
-    except Exception, e:
-        messages.error(request, e)
 
 
 def _handle_zip_file(request, uploaded_file, document_type):
@@ -425,7 +423,11 @@ def document_delete(request, document_id=None, document_id_list=None):
     if request.method == 'POST':
         for document in documents:
             try:
-                document_delete_fs_links(document)
+                warnings = delete_indexes(document)
+                if request.user.is_staff or request.user.is_superuser:
+                    for warning in warnings:
+                        messages.warning(request, warning)
+
                 document.delete()
                 messages.success(request, _(u'Document: %s deleted successfully.') % document)
             except Exception, e:
@@ -469,11 +471,10 @@ def document_edit(request, document_id):
     if request.method == 'POST':
         form = DocumentForm_edit(request.POST, initial={'document_type': document.document_type})
         if form.is_valid():
-            try:
+            warnings = delete_indexes(document)
-                document_delete_fs_links(document)
+            if request.user.is_staff or request.user.is_superuser:
-            except Exception, e:
+                for warning in warnings:
-                messages.error(request, e)
+                    messages.warning(request, warning)
-                return HttpResponseRedirect(reverse('document_list'))
 
             document.file_filename = form.cleaned_data['new_filename']
             document.description = form.cleaned_data['description']
@@ -486,17 +487,10 @@ def document_edit(request, document_id):
 
             messages.success(request, _(u'Document %s edited successfully.') % document)
 
-            try:
+            warnings = update_indexes(document)
-                warnings = document_create_fs_links(document)
+            if request.user.is_staff or request.user.is_superuser:
-
+                for warning in warnings:
-                if request.user.is_staff or request.user.is_superuser:
+                    messages.warning(request, warning)
-                    for warning in warnings:
-                        messages.warning(request, warning)
-
-                messages.success(request, _(u'Document filesystem links updated successfully.'))
-            except Exception, e:
-                messages.error(request, e)
-                return HttpResponseRedirect(document.get_absolute_url())
 
             return HttpResponseRedirect(document.get_absolute_url())
     else:

apps/filesystem_serving/conf/settings.py

@@ -1,14 +1 @@
-"""Configuration options for the filesystem_serving app"""
 
-from smart_settings.api import register_settings
-
-register_settings(
-    namespace=u'filesystem_serving',
-    module=u'filesystem_serving.conf.settings',
-    settings=[
-        {'name': u'SLUGIFY_PATHS', 'global_name': u'FILESYSTEM_SLUGIFY_PATHS', 'default': False},
-        {'name': u'MAX_RENAME_COUNT', 'global_name': u'FILESYSTEM_MAX_RENAME_COUNT', 'default': 200},
-        {'name': u'FILESERVING_PATH', 'global_name': u'FILESYSTEM_FILESERVING_PATH', 'default': u'/tmp/mayan/documents', 'exists': True},
-        {'name': u'FILESERVING_ENABLE', 'global_name': u'FILESYSTEM_FILESERVING_ENABLE', 'default': True}
-    ]
-)

apps/metadata/views.py

@@ -9,7 +9,6 @@ from django.utils.http import urlencode
 
 from documents.models import Document, RecentDocument
 from permissions.api import check_permissions
-from filesystem_serving.api import document_create_fs_links, document_delete_fs_links
 from document_indexing.api import update_indexes, delete_indexes
 
 from metadata import PERMISSION_METADATA_DOCUMENT_EDIT, \
@@ -57,33 +56,31 @@ def metadata_edit(request, document_id=None, document_id_list=None):
         formset = MetadataFormSet(request.POST)
         if formset.is_valid():
             for document in documents:
-                delete_indexes(document)
-                try:
-                    document_delete_fs_links(document)
-                except Exception, e:
-                    messages.error(request, _(u'Error deleting filesystem links for document: %(document)s; %(error)s') % {
-                        'document': document, 'error': e})
 
+                warnings = delete_indexes(document)
+                if request.user.is_staff or request.user.is_superuser:
+                    for warning in warnings:
+                        messages.warning(request, warning)                    
+                
+                errors = []
                 for form in formset.forms:
                     if form.cleaned_data['update']:
                         try:
                             save_metadata_list([form.cleaned_data], document)
-                            messages.success(request, _(u'Metadata for document %s edited successfully.') % document)
                         except Exception, e:
-                            messages.error(request, _(u'Error editing metadata for document %(document)s; %(error)s.') % {
+                            errors.append(e)
-                                'document': document, 'error': e})
-                update_indexes(document)
-                try:
-                    warnings = document_create_fs_links(document)
 
-                    if request.user.is_staff or request.user.is_superuser:
+                if errors:
-                        for warning in warnings:
+                    for error in errors:
-                            messages.warning(request, warning)
+                        messages.error(request, _(u'Error editing metadata for document %(document)s; %(error)s.') % {
+                        'document': document, 'error': error})
+                else:
+                    messages.success(request, _(u'Metadata for document %s edited successfully.') % document)
 
-                    messages.success(request, _(u'Filesystem links updated successfully for document: %s.') % document)
+                warnings = update_indexes(document)
-                except Exception, e:
+                if request.user.is_staff or request.user.is_superuser:
-                    messages.error(request, _('Error creating filesystem links for document: %(document)s; %(error)s') % {
+                    for warning in warnings:
-                        'document': document, 'error': e})
+                        messages.warning(request, warning)
 
             if len(documents) == 1:
                 return HttpResponseRedirect(document.get_absolute_url())

settings.py

@@ -138,7 +138,6 @@ INSTALLED_APPS = (
     'sentry',
     'sentry.client',
     'sentry.client.celery',
-    'filesystem_serving',
     'storage',
     'folders',
     'taggit',

urls.py

@@ -9,7 +9,6 @@ urlpatterns = patterns('',
     (r'^', include('main.urls')),
     (r'^documents/', include('documents.urls')),
     (r'^folders/', include('folders.urls')),
-    (r'^filesystem_serving/', include('filesystem_serving.urls')),
     (r'^search/', include('dynamic_search.urls')),
     (r'^ocr/', include('ocr.urls')),
     (r'^permissions/', include('permissions.urls')),