Started adding filesystem linking support to document indexing app

apps/document_indexing/admin.py

@@ -2,7 +2,8 @@ from django.contrib import admin
 
 from mptt.admin import MPTTModelAdmin
 
-from document_indexing.models import Index, IndexInstance
+from document_indexing.models import Index, IndexInstance, \
+    DocumentRenameCount
 
 
 class IndexInstanceInline(admin.StackedInline):
@@ -23,3 +24,4 @@ class IndexInstanceAdmin(MPTTModelAdmin):
 
 admin.site.register(Index, IndexAdmin)
 admin.site.register(IndexInstance, IndexInstanceAdmin)
+admin.site.register(DocumentRenameCount)

apps/document_indexing/api.py

@@ -2,13 +2,29 @@ from django.utils.translation import ugettext_lazy as _
 from django.utils.translation import ugettext
 from django.core.urlresolvers import reverse
 from django.utils.safestring import mark_safe
+from django.template.defaultfilters import slugify
 
 from documents.models import Document
 from metadata.classes import MetadataObject
 
-from document_indexing.models import Index, IndexInstance
+from document_indexing.models import Index, IndexInstance, \
+    DocumentRenameCount
 from document_indexing.conf.settings import AVAILABLE_INDEXING_FUNCTIONS
-from document_indexing.filesystem import fs_create_index_directory
+from document_indexing.conf.settings import MAX_SUFFIX_COUNT
+from document_indexing.filesystem import fs_create_index_directory, \
+    fs_create_document_link, fs_delete_document_link
+
+from document_indexing.conf.settings import SLUGIFY_PATHS
+
+if SLUGIFY_PATHS == False:
+    # Do not slugify path or filenames and extensions
+    SLUGIFY_FUNCTION = lambda x: x
+else:
+    SLUGIFY_FUNCTION = slugify
+
+
+class MaxSuffixCountReached(Exception):
+    pass
 
 
 # External functions
@@ -16,6 +32,7 @@ def update_indexes(document):
     """
     Update or create all the index instances related to a document
     """
+    print 'update_indexes'
     warnings = []
 
     eval_dict = {}
@@ -55,12 +72,14 @@ def get_instance_link(index_instance=None, text=None, simple=False):
         template = u'<a href="%(url)s">%(value)s</a>'
     if index_instance:
         return template % {
-            'url': index_instance.get_absolute_url(), 'value': text if text else index_instance
+            'url': index_instance.get_absolute_url(),
+            'value': text if text else index_instance
         }
     else:
         # Root node
         return template % {
-            'url': reverse('index_instance_list'), 'value': ugettext(u'root')
+            'url': reverse('index_instance_list'),
+            'value': ugettext(u'root')
         }
         
 
@@ -90,11 +109,24 @@ def get_breadcrumbs(index_instance, simple=False, single_link=False):
 
 def do_rebuild_all_indexes():
     IndexInstance.objects.all().delete()
+    DocumentRenameCount.objects.all().delete()
     for document in Document.objects.all():
         update_indexes(document)
     
     
 # Internal functions
+def find_lowest_available_suffix(suffix_list):
+    print 'suffix_list', suffix_list
+    suffix = 0
+    
+    while suffix in suffix_list:
+        suffix += 1
+        if suffix > MAX_SUFFIX_COUNT:
+            raise MaxSuffixCountReached(ugettext(u'Maximum suffix (%s) count reached.') % MAX_SUFFIX_COUNT)
+    else:
+        return suffix
+
+
 def _evaluate_index(eval_dict, document, node, parent_index_instance=None):
     """
     Evaluate an enabled index expression and update or create all the
@@ -108,11 +140,25 @@ def _evaluate_index(eval_dict, document, node, parent_index_instance=None):
             index_instance, created = IndexInstance.objects.get_or_create(index=node, value=result, parent=parent_index_instance)
             if created:
                 fs_create_index_directory(index_instance)
-            if node.link_documents:
+            if node.link_documents and document not in index_instance.documents.all():
+                #suffix_list = index_instance.documents.filter(file_filename=document.file_filename).filter(file_extension=document.file_extension).values_list('suffix', flat=True)
+                suffix_list = DocumentRenameCount.objects.filter(document__file_filename=document.file_filename).filter(document__file_extension=document.file_extension).filter(index_instance=index_instance).values_list('suffix', flat=True)
+                suffix = find_lowest_available_suffix(suffix_list)
+                print 'lower suffix: %s' % suffix
+                document_count = DocumentRenameCount(
+                    index_instance=index_instance,
+                    document=document,
+                    suffix=suffix
+                )
+                document_count.save()
+
+                fs_create_document_link(index_instance, document, suffix)
                 index_instance.documents.add(document)
 
             for children in node.get_children():
-                children_warnings = _evaluate_index(eval_dict, document, children, index_instance)
+                children_warnings = _evaluate_index(
+                    eval_dict, document, children, index_instance
+                )
                 warnings.extend(children_warnings)
 
         except (NameError, AttributeError), exc:
@@ -134,16 +180,22 @@ def _remove_document_from_index_instance(document, index_instance):
     """
     warnings = []
     try:
+        document_rename_count = DocumentRenameCount.objects.get(index_instance=index_instance, document=document)
+        fs_delete_document_link(index_instance, document, document_rename_count.suffix)
+        document_rename_count.delete()
         index_instance.documents.remove(document)
         if index_instance.documents.count() == 0 and index_instance.get_children().count() == 0:
             # if there are no more documents and no children, delete
             # node and check parent for the same conditions
             parent = index_instance.parent
             index_instance.delete()
-            parent_warnings = _remove_document_from_index_instance(document, parent)
+            parent_warnings = _remove_document_from_index_instance(
+                document, parent
+            )
             warnings.extend(parent_warnings)
             
     except Exception, exc:
+        print '_remove_document_from_index_instance exception: %s' % exc
         warnings.append(_(u'Unable to delete document indexing node; %s') % exc)
 
     return warnings

apps/document_indexing/conf/settings.py

@@ -20,6 +20,7 @@ register_settings(
         {'name': u'AVAILABLE_INDEXING_FUNCTIONS', 'global_name': u'DOCUMENT_INDEXING_AVAILABLE_INDEXING_FUNCTIONS', 'default': available_indexing_functions},
         # Filesystem serving
         {'name': u'SLUGIFY_PATHS', 'global_name': u'DOCUMENT_INDEXING_FILESYSTEM_SLUGIFY_PATHS', 'default': False},
+        {'name': u'MAX_SUFFIX_COUNT', 'global_name': u'DOCUMENT_INDEXING_FILESYSTEM_MAX_SUFFIX_COUNT', 'default': 1000},
         {'name': u'FILESERVING_PATH', 'global_name': u'DOCUMENT_INDEXING_FILESYSTEM_FILESERVING_PATH', 'default': u'/tmp/mayan/documents', 'exists': True},
         {'name': u'FILESERVING_ENABLE', 'global_name': u'DOCUMENT_INDEXING_FILESYSTEM_FILESERVING_ENABLE', 'default': True}
     ]

apps/document_indexing/filesystem.py

@@ -1,24 +1,15 @@
 import errno
 import os
 
-from django.template.defaultfilters import slugify
 from django.utils.translation import ugettext_lazy as _
 
-from document_indexing.models import IndexInstance
-
 from metadata.classes import MetadataObject
 
+from document_indexing.models import IndexInstance
 from document_indexing.conf.settings import FILESERVING_ENABLE
 from document_indexing.conf.settings import FILESERVING_PATH
-from document_indexing.conf.settings import SLUGIFY_PATHS
-#from document_indexing.conf.settings import MAX_RENAME_COUNT
-
-if SLUGIFY_PATHS == False:
-    #Do not slugify path or filenames and extensions
-    SLUGIFY_FUNCTION = lambda x: x
-else:
-    SLUGIFY_FUNCTION = slugify
 
+# TODO: delete fileserving document function for use with rebuild index function
 
 def get_instance_path(index_instance):
     """
@@ -33,29 +24,56 @@ def fs_create_index_directory(index_instance):
         target_directory = os.path.join(FILESERVING_PATH, get_instance_path(index_instance))
         try:
             os.makedirs(target_directory)
-            #next_available_filename(document, metadata_index, target_directory, SLUGIFY_FUNCTION(document.file_filename), SLUGIFY_FUNCTION(document.file_extension))
         except OSError, exc:
             if exc.errno == errno.EEXIST:
                 pass
             else:
-                raise OSError(_(u'Unable to create metadata indexing directory: %s') % exc)    
+                raise OSError(_(u'Unable to create indexing directory; %s') % exc)    
 
 
-def document_delete_fs_links(document):
+def fs_create_document_link(index_instance, document, suffix=0):
     if FILESERVING_ENABLE:
-        pass
+        name_part = document.file_filename
-        '''
+        if suffix:
-        for document_metadata_index in document.documentmetadataindex_set.all():
+            name_part = u'_'.join([name_part, unicode(suffix)])
-            try:
-                os.unlink(document_metadata_index.filename)
-                document_metadata_index.delete()
-            except OSError, exc:
-                if exc.errno == errno.ENOENT:
-                    #No longer exits, so delete db entry anyway
-                    document_metadata_index.delete()
-                else:
-                    raise OSError(_(u'Unable to delete metadata indexing symbolic link: %s') % exc)
         
+        filename = os.extsep.join([name_part, document.file_extension])
+        filepath = os.path.join(FILESERVING_PATH, get_instance_path(index_instance), filename)
+        print 'filepath', filepath
+        
+        try:
+            os.symlink(document.file.path, filepath)
+        except OSError, exc:
+            if exc.errno == errno.EEXIST:
+                # This link should not exist, try to delete it
+                try:
+                    os.unlink(filepath)
+                    # Try again
+                    os.symlink(document.file.path, filepath)
+                except Exception, exc:
+                    raise Exception(_(u'Unable to create symbolic link, file exists and could not be deleted: %(filepath)s; %(exc)s') % {'filepath': filepath, 'exc': exc})
+            else:
+                raise OSError(_(u'Unable to create symbolic link: %(filepath)s; %(exc)s') % {'filepath': filepath, 'exc': exc})
+
+
+def fs_delete_document_link(index_instance, document, suffix=0):
+    if FILESERVING_ENABLE:
+        name_part = document.file_filename
+        if suffix:
+            name_part = u'_'.join([name_part, unicode(suffix)])
+        
+        filename = os.extsep.join([name_part, document.file_extension])
+        filepath = os.path.join(FILESERVING_PATH, get_instance_path(index_instance), filename)
+        print 'delete filepath', filepath
+                
+        try:
+            os.unlink(filepath)
+        except OSError, exc:
+            if exc.errno != errno.ENOENT:
+                # Raise when any error other than doesn't exits
+                raise OSError(_(u'Unable to delete document symbolic link; %s') % exc)
+
+'''
             path, filename = os.path.split(document_metadata_index.filename)
 
             #Cleanup directory of dead stuff
@@ -86,9 +104,9 @@ def document_delete_fs_links(document):
                 os.removedirs(path)
             except:
                 pass
-        '''
+'''     
-
 
+'''
 def next_available_filename(document, metadata_index, path, filename, extension, suffix=0):
     target = filename
     if suffix:
@@ -149,3 +167,4 @@ def do_recreate_all_links(raise_exception=True):
     for warning in create_warnings:
         warnings.append('%s: %s' % (document, warning))
     return errors, warnings
+'''

apps/document_indexing/models.py

@@ -46,16 +46,15 @@ class IndexInstance(MPTTModel):
         verbose_name = _(u'index instance')
         verbose_name_plural = _(u'indexes instances')
 
-'''
+
 class DocumentRenameCount(models.Model):
-    index = models.ForeignKey(IndexInstance, verbose_name=_(u'index instance'))
+    index_instance = models.ForeignKey(IndexInstance, verbose_name=_(u'index instance'))
     document = models.ForeignKey(Document, verbose_name=_(u'document'))
-    count = models.PositiveIntegerField(blank=True, verbose_name=(u'count'))
+    suffix = models.PositiveIntegerField(blank=True, verbose_name=(u'suffix'))
 
     def __unicode__(self):
-        return self.value
+        return u'%s - %s - %s' % (self.index_instance, self.document, self.suffix or u'0')
 
     class Meta:
         verbose_name = _(u'document rename count')
         verbose_name_plural = _(u'documents rename count')
-'''