From e83529ae2e8086e474155651f09f290bb26e066d Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 17 May 2011 04:27:43 -0400 Subject: [PATCH] Initial commit of the new MPTT based document indexing app --- apps/document_indexing/__init__.py | 1 + apps/document_indexing/admin.py | 30 +++++++++-------- apps/document_indexing/api.py | 53 ++++++++++++++++++++++++++++++ apps/document_indexing/models.py | 46 +++++++++++++++++++++----- apps/documents/admin.py | 5 +-- apps/documents/conf/settings.py | 7 ---- apps/filesystem_serving/admin.py | 10 ------ apps/filesystem_serving/api.py | 12 +++++-- apps/filesystem_serving/models.py | 18 ---------- apps/metadata/views.py | 4 ++- settings.py | 1 + 11 files changed, 121 insertions(+), 66 deletions(-) create mode 100644 apps/document_indexing/api.py diff --git a/apps/document_indexing/__init__.py b/apps/document_indexing/__init__.py index e69de29bb2..8b13789179 100644 --- a/apps/document_indexing/__init__.py +++ b/apps/document_indexing/__init__.py @@ -0,0 +1 @@ + diff --git a/apps/document_indexing/admin.py b/apps/document_indexing/admin.py index 13cdcdaef6..e0c71d04d9 100644 --- a/apps/document_indexing/admin.py +++ b/apps/document_indexing/admin.py @@ -1,23 +1,25 @@ from django.contrib import admin -from document_indexing.models import DocumentIndex +from mptt.admin import MPTTModelAdmin -#from filesystem_serving.admin import DocumentMetadataIndexInline +from document_indexing.models import Index, IndexInstance -#class MetadataIndexInline(admin.StackedInline): -# model = MetadataIndex -# extra = 1 -# classes = ('collapse-open',) -# allow_add = True +class IndexInstanceInline(admin.StackedInline): + model = IndexInstance + extra = 1 + classes = ('collapse-open',) + allow_add = True -class DocumentIndexAdmin(admin.ModelAdmin): - pass - #inlines = [ - # DocumentMetadataIndexInline, - # +class IndexAdmin(MPTTModelAdmin): + list_display = ('expression', 'enabled', 'link_document') + + +class IndexInstanceAdmin(MPTTModelAdmin): + model = IndexInstance + list_display = ('value', 'index', 'get_document_list_display') - -admin.site.register(DocumentIndex, DocumentIndexAdmin) +admin.site.register(Index, IndexAdmin) +admin.site.register(IndexInstance, IndexInstanceAdmin) diff --git a/apps/document_indexing/api.py b/apps/document_indexing/api.py new file mode 100644 index 0000000000..1cde0f6c4d --- /dev/null +++ b/apps/document_indexing/api.py @@ -0,0 +1,53 @@ +from django.utils.translation import ugettext_lazy as _ + +from metadata.classes import MetadataObject + +#from filesystem_serving.conf.settings import MAX_RENAME_COUNT + +from document_indexing.models import Index, IndexInstance +from document_indexing.conf.settings import AVAILABLE_INDEXING_FUNCTIONS + + +def evaluate_index(eval_dict, document, node, parent_index_instance=None): + warnings = [] + if node.enabled: + try: + result = eval(node.expression, eval_dict, AVAILABLE_INDEXING_FUNCTIONS) + index_instance, created = IndexInstance.objects.get_or_create(index=node, value=result, parent=parent_index_instance) + if node.link_document: + index_instance.documents.add(document) + + for children in node.get_children(): + children_warnings = evaluate_index(eval_dict, document, children, index_instance) + warnings.extend(children_warnings) + + except NameError, exc: + warnings.append(_(u'Error in metadata indexing expression: %s') % exc) + #raise NameError() + #This should be a warning not an error + #pass + except Exception, exc: + warnings.append(_(u'Unable to create metadata indexing directory: %s') % exc) + + return warnings + + +def update_indexes(document): + print 'update_indexes' + warnings = [] + + eval_dict = {} + eval_dict['document'] = document + metadata_dict = dict([(metadata.metadata_type.name, metadata.value) for metadata in document.documentmetadata_set.all() if metadata.value]) + eval_dict['metadata'] = MetadataObject(metadata_dict) + + for root in Index.objects.filter(parent=None): + #for node in root.get_children(): + index_warnings = evaluate_index(eval_dict, document, root) + warnings.extend(index_warnings) + + return warnings + + +def delete_indexes(document): + print 'delete_indexes' diff --git a/apps/document_indexing/models.py b/apps/document_indexing/models.py index 8865f49a5c..53a51fc527 100644 --- a/apps/document_indexing/models.py +++ b/apps/document_indexing/models.py @@ -1,20 +1,48 @@ from django.db import models from django.utils.translation import ugettext_lazy as _ +from mptt.models import MPTTModel, TreeForeignKey + +from documents.models import Document + from document_indexing.conf.settings import AVAILABLE_INDEXING_FUNCTIONS -available_indexing_functions_string = (_(u' Available functions: %s') % u','.join([u'%s()' % name for name, function in AVAILABLE_INDEXING_FUNCTIONS.items()])) if AVAILABLE_INDEXING_FUNCTIONS else u'' +available_indexing_functions_string = (_(u'Available functions: %s') % u','.join([u'%s()' % name for name, function in AVAILABLE_INDEXING_FUNCTIONS.items()])) if AVAILABLE_INDEXING_FUNCTIONS else u'' -class DocumentIndex(models.Model): - expression = models.CharField(max_length=128, - verbose_name=_(u'indexing expression'), - help_text=_(u'Enter a python string expression to be evaluated. The slash caracter "/" acts as a directory delimiter.%s') % available_indexing_functions_string) +class Index(MPTTModel): + parent = TreeForeignKey('self', null=True, blank=True, related_name='index_meta_class') + expression = models.CharField(max_length=128, verbose_name=_(u'indexing expression'), help_text=_(u'Enter a python string expression to be evaluated.')) + # % available_indexing_functions_string) enabled = models.BooleanField(default=True, verbose_name=_(u'enabled')) - + link_document = models.BooleanField(default=False, verbose_name=_(u'link document')) + def __unicode__(self): - return unicode(self.expression) + return self.expression if not self.link_document else u'%s/[document]' % self.expression class Meta: - verbose_name = _(u'metadata index') - verbose_name_plural = _(u'metadata indexes') + verbose_name = _(u'index') + verbose_name_plural = _(u'indexes') + + +class IndexInstance(MPTTModel): + parent = TreeForeignKey('self', null=True, blank=True, related_name='index_meta_instance') + index = models.ForeignKey(Index, verbose_name=_(u'index')) + value = models.CharField(max_length=128, blank=True, verbose_name=_(u'value')) + documents = models.ManyToManyField(Document, verbose_name=_(u'documents')) + + def __unicode__(self): + return self.value + + def get_document_list_display(self): + return u', '.join([d.file_filename for d in self.documents.all()]) + + + class Meta: + verbose_name = _(u'index instance') + verbose_name_plural = _(u'indexes instances') + + +# TODO +# class DocumentRenameCount +# FK=IndexInstance diff --git a/apps/documents/admin.py b/apps/documents/admin.py index 78865e6879..ff4aa33cb0 100644 --- a/apps/documents/admin.py +++ b/apps/documents/admin.py @@ -6,8 +6,6 @@ from documents.models import DocumentType, Document, \ DocumentTypeFilename, DocumentPage, \ DocumentPageTransformation, RecentDocument -from filesystem_serving.admin import DocumentMetadataIndexInline - class DocumentTypeFilenameInline(admin.StackedInline): model = DocumentTypeFilename @@ -35,8 +33,7 @@ class DocumentPageInline(admin.StackedInline): class DocumentAdmin(admin.ModelAdmin): inlines = [ - DocumentMetadataInline, DocumentMetadataIndexInline, - DocumentPageInline + DocumentMetadataInline, DocumentPageInline ] list_display = ('uuid', 'file_filename', 'file_extension') diff --git a/apps/documents/conf/settings.py b/apps/documents/conf/settings.py index 6c9cdd801e..60f229fb5e 100644 --- a/apps/documents/conf/settings.py +++ b/apps/documents/conf/settings.py @@ -5,7 +5,6 @@ import uuid from django.utils.translation import ugettext_lazy as _ -from common.utils import proper_name from storage.backends.filebasedstorage import FileBasedStorage from smart_settings.api import register_settings @@ -23,16 +22,10 @@ available_transformations = { 'rotate': {'label': _(u'Rotate [degrees]'), 'arguments': [{'name': 'degrees'}]} } -available_indexing_functions = { - 'proper_name': proper_name -} - register_settings( namespace=u'documents', module=u'documents.conf.settings', settings=[ - # Definition - {'name': u'AVAILABLE_INDEXING_FUNCTIONS', 'global_name': u'DOCUMENTS_INDEXING_AVAILABLE_FUNCTIONS', 'default': available_indexing_functions}, # Upload {'name': u'USE_STAGING_DIRECTORY', 'global_name': u'DOCUMENTS_USE_STAGING_DIRECTORY', 'default': False}, {'name': u'STAGING_DIRECTORY', 'global_name': u'DOCUMENTS_STAGING_DIRECTORY', 'default': u'/tmp/mayan/staging', 'exists': True}, diff --git a/apps/filesystem_serving/admin.py b/apps/filesystem_serving/admin.py index a8ee833907..8b13789179 100644 --- a/apps/filesystem_serving/admin.py +++ b/apps/filesystem_serving/admin.py @@ -1,11 +1 @@ -from django.contrib import admin -from filesystem_serving.models import DocumentMetadataIndex - - -class DocumentMetadataIndexInline(admin.StackedInline): - model = DocumentMetadataIndex - extra = 1 - classes = ('collapse-open',) - allow_add = True - readonly_fields = ('suffix', 'metadata_index', 'filename') diff --git a/apps/filesystem_serving/api.py b/apps/filesystem_serving/api.py index bde9185d91..e9b1b0a412 100644 --- a/apps/filesystem_serving/api.py +++ b/apps/filesystem_serving/api.py @@ -4,7 +4,8 @@ import os from django.template.defaultfilters import slugify from django.utils.translation import ugettext_lazy as _ -from documents.conf.settings import AVAILABLE_INDEXING_FUNCTIONS +from document_indexing.models import IndexInstance + from metadata.classes import MetadataObject from filesystem_serving.conf.settings import FILESERVING_ENABLE @@ -12,7 +13,7 @@ from filesystem_serving.conf.settings import FILESERVING_PATH from filesystem_serving.conf.settings import SLUGIFY_PATHS from filesystem_serving.conf.settings import MAX_RENAME_COUNT -from filesystem_serving.models import DocumentMetadataIndex, Document +#from filesystem_serving.models import DocumentMetadataIndex, Document if SLUGIFY_PATHS == False: #Do not slugify path or filenames and extensions @@ -24,6 +25,8 @@ else: def document_create_fs_links(document): warnings = [] if FILESERVING_ENABLE: + pass + ''' if not document.exists(): raise Exception(_(u'Not creating metadata indexing, document not found in document storage')) eval_dict = {} @@ -52,12 +55,14 @@ def document_create_fs_links(document): #pass except Exception, exc: warnings.append(_(u'Unable to create metadata indexing directory: %s') % exc) - + ''' return warnings def document_delete_fs_links(document): if FILESERVING_ENABLE: + pass + ''' for document_metadata_index in document.documentmetadataindex_set.all(): try: os.unlink(document_metadata_index.filename) @@ -99,6 +104,7 @@ def document_delete_fs_links(document): os.removedirs(path) except: pass + ''' def next_available_filename(document, metadata_index, path, filename, extension, suffix=0): diff --git a/apps/filesystem_serving/models.py b/apps/filesystem_serving/models.py index f513e8c299..8b13789179 100644 --- a/apps/filesystem_serving/models.py +++ b/apps/filesystem_serving/models.py @@ -1,19 +1 @@ -from django.db import models -from django.utils.translation import ugettext_lazy as _ -from documents.models import Document -from document_indexing.models import DocumentIndex - - -class DocumentMetadataIndex(models.Model): - document = models.ForeignKey(Document, verbose_name=_(u'document')) - metadata_index = models.ForeignKey(DocumentIndex, verbose_name=_(u'document index')) - filename = models.CharField(max_length=255, verbose_name=_(u'filename')) - suffix = models.PositiveIntegerField(default=0, verbose_name=_(u'suffix')) - - def __unicode__(self): - return unicode(self.filename) - - class Meta: - verbose_name = _(u'document metadata index') - verbose_name_plural = _(u'document metadata indexes') diff --git a/apps/metadata/views.py b/apps/metadata/views.py index cd99f777ee..072c2f7e9d 100644 --- a/apps/metadata/views.py +++ b/apps/metadata/views.py @@ -10,6 +10,7 @@ from django.utils.http import urlencode from documents.models import Document, RecentDocument from permissions.api import check_permissions from filesystem_serving.api import document_create_fs_links, document_delete_fs_links +from document_indexing.api import update_indexes, delete_indexes from metadata import PERMISSION_METADATA_DOCUMENT_EDIT, \ PERMISSION_METADATA_DOCUMENT_ADD, PERMISSION_METADATA_DOCUMENT_REMOVE @@ -56,6 +57,7 @@ def metadata_edit(request, document_id=None, document_id_list=None): formset = MetadataFormSet(request.POST) if formset.is_valid(): for document in documents: + delete_indexes(document) try: document_delete_fs_links(document) except Exception, e: @@ -70,7 +72,7 @@ def metadata_edit(request, document_id=None, document_id_list=None): except Exception, e: messages.error(request, _(u'Error editing metadata for document %(document)s; %(error)s.') % { 'document': document, 'error': e}) - + update_indexes(document) try: warnings = document_create_fs_links(document) diff --git a/settings.py b/settings.py index 60a37518b2..8cd9bbf6c4 100644 --- a/settings.py +++ b/settings.py @@ -148,6 +148,7 @@ INSTALLED_APPS = ( 'documents', 'grouping', 'document_indexing', + 'mptt', ) TEMPLATE_CONTEXT_PROCESSORS = (