Move index management code to the managers.py module.
This commit is contained in:
Roberto Rosario
@@ -1,91 +0,0 @@
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import logging
|
||||
|
||||
from django.utils.translation import ugettext_lazy as _
|
||||
|
||||
from .models import Index, IndexInstanceNode
|
||||
from .settings import setting_available_indexing_functions
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# External functions
|
||||
def index_document(document):
|
||||
"""
|
||||
Update or create all the index instances related to a document
|
||||
"""
|
||||
# TODO: convert this fuction into a manager method
|
||||
|
||||
warnings = []
|
||||
|
||||
for index_node in IndexInstanceNode.objects.filter(documents=document):
|
||||
index_node.documents.remove(document)
|
||||
|
||||
delete_empty_index_nodes()
|
||||
|
||||
# Only update indexes where the document type is found
|
||||
for index in Index.objects.filter(enabled=True, document_types=document.document_type):
|
||||
root_instance, created = IndexInstanceNode.objects.get_or_create(index_template_node=index.template_root, parent=None)
|
||||
for template_node in index.template_root.get_children():
|
||||
index_warnings = cascade_eval(document, template_node, root_instance)
|
||||
warnings.extend(index_warnings)
|
||||
|
||||
return warnings
|
||||
|
||||
|
||||
def cascade_eval(document, template_node, parent_index_instance=None):
|
||||
"""
|
||||
Evaluate an enabled index expression and update or create all the
|
||||
related index instances also recursively calling itself to evaluate
|
||||
all the index's children
|
||||
"""
|
||||
|
||||
warnings = []
|
||||
if template_node.enabled:
|
||||
try:
|
||||
result = eval(template_node.expression, {'document': document}, setting_available_indexing_functions.value)
|
||||
except Exception as exception:
|
||||
error_message = _('Error indexing document: %(document)s; expression: %(expression)s; %(exception)s') % {
|
||||
'document': document, 'expression': template_node.expression, 'exception': exception}
|
||||
warnings.append(error_message)
|
||||
logger.debug(error_message)
|
||||
else:
|
||||
if result:
|
||||
index_instance, created = IndexInstanceNode.objects.get_or_create(index_template_node=template_node, value=result, parent=parent_index_instance)
|
||||
|
||||
if template_node.link_documents:
|
||||
index_instance.documents.add(document)
|
||||
|
||||
for child in template_node.get_children():
|
||||
children_warnings = cascade_eval(
|
||||
document=document,
|
||||
template_node=child,
|
||||
parent_index_instance=index_instance
|
||||
)
|
||||
warnings.extend(children_warnings)
|
||||
|
||||
return warnings
|
||||
|
||||
|
||||
def delete_empty_index_nodes():
|
||||
"""
|
||||
Delete empty index instance nodes
|
||||
"""
|
||||
|
||||
for instance_node in IndexInstanceNode.objects.filter(documents__isnull=True, parent__isnull=False):
|
||||
task_delete_empty_index_nodes_recursive(instance_node)
|
||||
|
||||
|
||||
def task_delete_empty_index_nodes_recursive(instance_node):
|
||||
"""
|
||||
Calls itself recursively deleting empty index instance nodes up to root
|
||||
"""
|
||||
|
||||
if instance_node.get_children().count() == 0:
|
||||
# if there are no children, delete node and check parent for the
|
||||
# same conditions
|
||||
parent = instance_node.parent
|
||||
if parent:
|
||||
instance_node.delete()
|
||||
task_delete_empty_index_nodes_recursive(parent)
|
||||
@@ -1,6 +1,97 @@
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import logging
|
||||
|
||||
from django.db import models
|
||||
from django.utils.translation import ugettext_lazy as _
|
||||
|
||||
from documents.models import Document
|
||||
|
||||
from .settings import setting_available_indexing_functions
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class IndexManager(models.Manager):
|
||||
def get_by_natural_key(self, name):
|
||||
return self.get(name=name)
|
||||
|
||||
|
||||
class IndexInstanceNodeManager(models.Manager):
|
||||
@staticmethod
|
||||
def delete_empty_index_nodes_recursive(instance_node):
|
||||
"""
|
||||
Calls itself recursively deleting empty index instance nodes up to root
|
||||
"""
|
||||
|
||||
if instance_node.get_children().count() == 0:
|
||||
# if there are no children, delete node and check parent for the
|
||||
# same conditions
|
||||
parent = instance_node.parent
|
||||
if parent:
|
||||
instance_node.delete()
|
||||
IndexInstanceNodeManager.delete_empty_index_nodes_recursive(parent)
|
||||
|
||||
def cascade_eval(self, document, template_node, parent_index_instance=None):
|
||||
"""
|
||||
Evaluate an enabled index expression and update or create all the
|
||||
related index instances also recursively calling itself to evaluate
|
||||
all the index's children
|
||||
"""
|
||||
|
||||
if template_node.enabled:
|
||||
try:
|
||||
result = eval(template_node.expression, {'document': document}, setting_available_indexing_functions.value)
|
||||
except Exception as exception:
|
||||
error_message = _('Error indexing document: %(document)s; expression: %(expression)s; %(exception)s') % {
|
||||
'document': document, 'expression': template_node.expression, 'exception': exception}
|
||||
logger.debug(error_message)
|
||||
else:
|
||||
if result:
|
||||
index_instance, created = self.get_or_create(index_template_node=template_node, value=result, parent=parent_index_instance)
|
||||
|
||||
if template_node.link_documents:
|
||||
index_instance.documents.add(document)
|
||||
|
||||
for child in template_node.get_children():
|
||||
self.cascade_eval(
|
||||
document=document,
|
||||
template_node=child,
|
||||
parent_index_instance=index_instance
|
||||
)
|
||||
|
||||
def delete_empty_index_nodes(self):
|
||||
"""
|
||||
Delete empty index instance nodes
|
||||
"""
|
||||
|
||||
for instance_node in self.filter(documents__isnull=True, parent__isnull=False):
|
||||
IndexInstanceNodeManager.delete_empty_index_nodes_recursive(instance_node)
|
||||
|
||||
def index_document(self, document):
|
||||
"""
|
||||
Update or create all the index instances related to a document
|
||||
"""
|
||||
|
||||
from .models import Index
|
||||
|
||||
self.remove_document(document)
|
||||
|
||||
# Only update indexes where the document type is found
|
||||
for index in Index.objects.filter(enabled=True, document_types=document.document_type):
|
||||
root_instance, created = self.get_or_create(index_template_node=index.template_root, parent=None)
|
||||
for template_node in index.template_root.get_children():
|
||||
self.cascade_eval(document, template_node, root_instance)
|
||||
|
||||
def remove_document(self, document):
|
||||
for index_node in self.filter(documents=document):
|
||||
index_node.documents.remove(document)
|
||||
|
||||
self.delete_empty_index_nodes()
|
||||
|
||||
def rebuild_all_indexes():
|
||||
for instance_node in self.all():
|
||||
instance_node.delete()
|
||||
|
||||
for document in Document.objects.all():
|
||||
index_document(document)
|
||||
|
||||
@@ -10,7 +10,7 @@ from mptt.models import MPTTModel
|
||||
|
||||
from documents.models import Document, DocumentType
|
||||
|
||||
from .managers import IndexManager
|
||||
from .managers import IndexManager, IndexInstanceNodeManager
|
||||
|
||||
|
||||
@python_2_unicode_compatible
|
||||
@@ -83,6 +83,8 @@ class IndexInstanceNode(MPTTModel):
|
||||
value = models.CharField(max_length=128, blank=True, verbose_name=_('Value'))
|
||||
documents = models.ManyToManyField(Document, related_name='node_instances', verbose_name=_('Documents'))
|
||||
|
||||
objects = IndexInstanceNodeManager()
|
||||
|
||||
def __str__(self):
|
||||
return self.value
|
||||
|
||||
|
||||
@@ -6,8 +6,7 @@ from mayan.celery import app
|
||||
from documents.models import Document
|
||||
from lock_manager import Lock, LockError
|
||||
|
||||
from .api import index_document, delete_empty_index_nodes
|
||||
from .tools import do_rebuild_all_indexes
|
||||
from .models import IndexInstanceNode
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
RETRY_DELAY = 20 # TODO: convert this into a config option
|
||||
@@ -22,7 +21,7 @@ def task_delete_empty_index_nodes(self):
|
||||
raise self.retry(exc=exception, countdown=RETRY_DELAY)
|
||||
else:
|
||||
try:
|
||||
delete_empty_index_nodes()
|
||||
IndexInstanceNode.objects.delete_empty_index_nodes()
|
||||
finally:
|
||||
rebuild_lock.release()
|
||||
|
||||
@@ -47,7 +46,7 @@ def task_index_document(self, document_id):
|
||||
# Document was deleted before we could execute, abort about updating
|
||||
pass
|
||||
else:
|
||||
index_document(document)
|
||||
IndexInstanceNode.objects.index_document(document)
|
||||
finally:
|
||||
lock.release()
|
||||
finally:
|
||||
@@ -67,6 +66,6 @@ def task_do_rebuild_all_indexes(self):
|
||||
raise self.retry(exc=exception, countdown=RETRY_DELAY)
|
||||
else:
|
||||
try:
|
||||
do_rebuild_all_indexes()
|
||||
IndexInstanceNode.objects.rebuild_all_indexes()
|
||||
finally:
|
||||
lock.release()
|
||||
|
||||
@@ -1,12 +0,0 @@
|
||||
from documents.models import Document
|
||||
|
||||
from .api import index_document
|
||||
from .models import IndexInstanceNode
|
||||
|
||||
|
||||
def do_rebuild_all_indexes():
|
||||
for instance_node in IndexInstanceNode.objects.all():
|
||||
instance_node.delete()
|
||||
|
||||
for document in Document.objects.all():
|
||||
index_document(document)
|
||||
@@ -12,7 +12,6 @@ from .views import (
|
||||
SetupIndexListView,
|
||||
)
|
||||
|
||||
|
||||
urlpatterns = patterns(
|
||||
'document_indexing.views',
|
||||
url(r'^setup/index/list/$', SetupIndexListView.as_view(), name='index_setup_list'),
|
||||
|
||||
Reference in New Issue
Block a user