diff --git a/apps/document_indexing/admin.py b/apps/document_indexing/admin.py index 0d4dadd840..dcb7f4a5c5 100644 --- a/apps/document_indexing/admin.py +++ b/apps/document_indexing/admin.py @@ -2,7 +2,8 @@ from django.contrib import admin from mptt.admin import MPTTModelAdmin -from document_indexing.models import Index, IndexInstance +from document_indexing.models import Index, IndexInstance, \ + DocumentRenameCount class IndexInstanceInline(admin.StackedInline): @@ -23,3 +24,4 @@ class IndexInstanceAdmin(MPTTModelAdmin): admin.site.register(Index, IndexAdmin) admin.site.register(IndexInstance, IndexInstanceAdmin) +admin.site.register(DocumentRenameCount) diff --git a/apps/document_indexing/api.py b/apps/document_indexing/api.py index 1ed666f0d2..b668cdfa51 100644 --- a/apps/document_indexing/api.py +++ b/apps/document_indexing/api.py @@ -2,20 +2,37 @@ from django.utils.translation import ugettext_lazy as _ from django.utils.translation import ugettext from django.core.urlresolvers import reverse from django.utils.safestring import mark_safe +from django.template.defaultfilters import slugify from documents.models import Document from metadata.classes import MetadataObject -from document_indexing.models import Index, IndexInstance +from document_indexing.models import Index, IndexInstance, \ + DocumentRenameCount from document_indexing.conf.settings import AVAILABLE_INDEXING_FUNCTIONS -from document_indexing.filesystem import fs_create_index_directory +from document_indexing.conf.settings import MAX_SUFFIX_COUNT +from document_indexing.filesystem import fs_create_index_directory, \ + fs_create_document_link, fs_delete_document_link + +from document_indexing.conf.settings import SLUGIFY_PATHS + +if SLUGIFY_PATHS == False: + # Do not slugify path or filenames and extensions + SLUGIFY_FUNCTION = lambda x: x +else: + SLUGIFY_FUNCTION = slugify + + +class MaxSuffixCountReached(Exception): + pass # External functions def update_indexes(document): """ Update or create all the index instances related to a document - """ + """ + print 'update_indexes' warnings = [] eval_dict = {} @@ -55,12 +72,14 @@ def get_instance_link(index_instance=None, text=None, simple=False): template = u'%(value)s' if index_instance: return template % { - 'url': index_instance.get_absolute_url(), 'value': text if text else index_instance + 'url': index_instance.get_absolute_url(), + 'value': text if text else index_instance } else: # Root node return template % { - 'url': reverse('index_instance_list'), 'value': ugettext(u'root') + 'url': reverse('index_instance_list'), + 'value': ugettext(u'root') } @@ -90,11 +109,24 @@ def get_breadcrumbs(index_instance, simple=False, single_link=False): def do_rebuild_all_indexes(): IndexInstance.objects.all().delete() + DocumentRenameCount.objects.all().delete() for document in Document.objects.all(): update_indexes(document) # Internal functions +def find_lowest_available_suffix(suffix_list): + print 'suffix_list', suffix_list + suffix = 0 + + while suffix in suffix_list: + suffix += 1 + if suffix > MAX_SUFFIX_COUNT: + raise MaxSuffixCountReached(ugettext(u'Maximum suffix (%s) count reached.') % MAX_SUFFIX_COUNT) + else: + return suffix + + def _evaluate_index(eval_dict, document, node, parent_index_instance=None): """ Evaluate an enabled index expression and update or create all the @@ -108,11 +140,25 @@ def _evaluate_index(eval_dict, document, node, parent_index_instance=None): index_instance, created = IndexInstance.objects.get_or_create(index=node, value=result, parent=parent_index_instance) if created: fs_create_index_directory(index_instance) - if node.link_documents: + if node.link_documents and document not in index_instance.documents.all(): + #suffix_list = index_instance.documents.filter(file_filename=document.file_filename).filter(file_extension=document.file_extension).values_list('suffix', flat=True) + suffix_list = DocumentRenameCount.objects.filter(document__file_filename=document.file_filename).filter(document__file_extension=document.file_extension).filter(index_instance=index_instance).values_list('suffix', flat=True) + suffix = find_lowest_available_suffix(suffix_list) + print 'lower suffix: %s' % suffix + document_count = DocumentRenameCount( + index_instance=index_instance, + document=document, + suffix=suffix + ) + document_count.save() + + fs_create_document_link(index_instance, document, suffix) index_instance.documents.add(document) for children in node.get_children(): - children_warnings = _evaluate_index(eval_dict, document, children, index_instance) + children_warnings = _evaluate_index( + eval_dict, document, children, index_instance + ) warnings.extend(children_warnings) except (NameError, AttributeError), exc: @@ -134,16 +180,22 @@ def _remove_document_from_index_instance(document, index_instance): """ warnings = [] try: + document_rename_count = DocumentRenameCount.objects.get(index_instance=index_instance, document=document) + fs_delete_document_link(index_instance, document, document_rename_count.suffix) + document_rename_count.delete() index_instance.documents.remove(document) if index_instance.documents.count() == 0 and index_instance.get_children().count() == 0: # if there are no more documents and no children, delete # node and check parent for the same conditions parent = index_instance.parent index_instance.delete() - parent_warnings = _remove_document_from_index_instance(document, parent) + parent_warnings = _remove_document_from_index_instance( + document, parent + ) warnings.extend(parent_warnings) except Exception, exc: + print '_remove_document_from_index_instance exception: %s' % exc warnings.append(_(u'Unable to delete document indexing node; %s') % exc) return warnings diff --git a/apps/document_indexing/conf/settings.py b/apps/document_indexing/conf/settings.py index 5f8f62dff8..580103e7bc 100644 --- a/apps/document_indexing/conf/settings.py +++ b/apps/document_indexing/conf/settings.py @@ -20,6 +20,7 @@ register_settings( {'name': u'AVAILABLE_INDEXING_FUNCTIONS', 'global_name': u'DOCUMENT_INDEXING_AVAILABLE_INDEXING_FUNCTIONS', 'default': available_indexing_functions}, # Filesystem serving {'name': u'SLUGIFY_PATHS', 'global_name': u'DOCUMENT_INDEXING_FILESYSTEM_SLUGIFY_PATHS', 'default': False}, + {'name': u'MAX_SUFFIX_COUNT', 'global_name': u'DOCUMENT_INDEXING_FILESYSTEM_MAX_SUFFIX_COUNT', 'default': 1000}, {'name': u'FILESERVING_PATH', 'global_name': u'DOCUMENT_INDEXING_FILESYSTEM_FILESERVING_PATH', 'default': u'/tmp/mayan/documents', 'exists': True}, {'name': u'FILESERVING_ENABLE', 'global_name': u'DOCUMENT_INDEXING_FILESYSTEM_FILESERVING_ENABLE', 'default': True} ] diff --git a/apps/document_indexing/filesystem.py b/apps/document_indexing/filesystem.py index 53924640ad..7a506d9764 100644 --- a/apps/document_indexing/filesystem.py +++ b/apps/document_indexing/filesystem.py @@ -1,24 +1,15 @@ import errno import os -from django.template.defaultfilters import slugify from django.utils.translation import ugettext_lazy as _ -from document_indexing.models import IndexInstance - from metadata.classes import MetadataObject +from document_indexing.models import IndexInstance from document_indexing.conf.settings import FILESERVING_ENABLE from document_indexing.conf.settings import FILESERVING_PATH -from document_indexing.conf.settings import SLUGIFY_PATHS -#from document_indexing.conf.settings import MAX_RENAME_COUNT - -if SLUGIFY_PATHS == False: - #Do not slugify path or filenames and extensions - SLUGIFY_FUNCTION = lambda x: x -else: - SLUGIFY_FUNCTION = slugify +# TODO: delete fileserving document function for use with rebuild index function def get_instance_path(index_instance): """ @@ -33,29 +24,56 @@ def fs_create_index_directory(index_instance): target_directory = os.path.join(FILESERVING_PATH, get_instance_path(index_instance)) try: os.makedirs(target_directory) - #next_available_filename(document, metadata_index, target_directory, SLUGIFY_FUNCTION(document.file_filename), SLUGIFY_FUNCTION(document.file_extension)) except OSError, exc: if exc.errno == errno.EEXIST: pass else: - raise OSError(_(u'Unable to create metadata indexing directory: %s') % exc) + raise OSError(_(u'Unable to create indexing directory; %s') % exc) -def document_delete_fs_links(document): +def fs_create_document_link(index_instance, document, suffix=0): if FILESERVING_ENABLE: - pass - ''' - for document_metadata_index in document.documentmetadataindex_set.all(): - try: - os.unlink(document_metadata_index.filename) - document_metadata_index.delete() - except OSError, exc: - if exc.errno == errno.ENOENT: - #No longer exits, so delete db entry anyway - document_metadata_index.delete() - else: - raise OSError(_(u'Unable to delete metadata indexing symbolic link: %s') % exc) + name_part = document.file_filename + if suffix: + name_part = u'_'.join([name_part, unicode(suffix)]) + + filename = os.extsep.join([name_part, document.file_extension]) + filepath = os.path.join(FILESERVING_PATH, get_instance_path(index_instance), filename) + print 'filepath', filepath + + try: + os.symlink(document.file.path, filepath) + except OSError, exc: + if exc.errno == errno.EEXIST: + # This link should not exist, try to delete it + try: + os.unlink(filepath) + # Try again + os.symlink(document.file.path, filepath) + except Exception, exc: + raise Exception(_(u'Unable to create symbolic link, file exists and could not be deleted: %(filepath)s; %(exc)s') % {'filepath': filepath, 'exc': exc}) + else: + raise OSError(_(u'Unable to create symbolic link: %(filepath)s; %(exc)s') % {'filepath': filepath, 'exc': exc}) + +def fs_delete_document_link(index_instance, document, suffix=0): + if FILESERVING_ENABLE: + name_part = document.file_filename + if suffix: + name_part = u'_'.join([name_part, unicode(suffix)]) + + filename = os.extsep.join([name_part, document.file_extension]) + filepath = os.path.join(FILESERVING_PATH, get_instance_path(index_instance), filename) + print 'delete filepath', filepath + + try: + os.unlink(filepath) + except OSError, exc: + if exc.errno != errno.ENOENT: + # Raise when any error other than doesn't exits + raise OSError(_(u'Unable to delete document symbolic link; %s') % exc) + +''' path, filename = os.path.split(document_metadata_index.filename) #Cleanup directory of dead stuff @@ -86,9 +104,9 @@ def document_delete_fs_links(document): os.removedirs(path) except: pass - ''' - +''' +''' def next_available_filename(document, metadata_index, path, filename, extension, suffix=0): target = filename if suffix: @@ -149,3 +167,4 @@ def do_recreate_all_links(raise_exception=True): for warning in create_warnings: warnings.append('%s: %s' % (document, warning)) return errors, warnings +''' diff --git a/apps/document_indexing/models.py b/apps/document_indexing/models.py index 2a07747bd8..ca3a63ccf0 100644 --- a/apps/document_indexing/models.py +++ b/apps/document_indexing/models.py @@ -46,16 +46,15 @@ class IndexInstance(MPTTModel): verbose_name = _(u'index instance') verbose_name_plural = _(u'indexes instances') -''' + class DocumentRenameCount(models.Model): - index = models.ForeignKey(IndexInstance, verbose_name=_(u'index instance')) + index_instance = models.ForeignKey(IndexInstance, verbose_name=_(u'index instance')) document = models.ForeignKey(Document, verbose_name=_(u'document')) - count = models.PositiveIntegerField(blank=True, verbose_name=(u'count')) + suffix = models.PositiveIntegerField(blank=True, verbose_name=(u'suffix')) def __unicode__(self): - return self.value + return u'%s - %s - %s' % (self.index_instance, self.document, self.suffix or u'0') class Meta: verbose_name = _(u'document rename count') verbose_name_plural = _(u'documents rename count') -'''