PEP8 cleanups, E501.

This commit is contained in:
Roberto Rosario
2015-07-23 04:05:29 -04:00
parent 0a0a92116e
commit 3b728328ad
48 changed files with 613 additions and 176 deletions

View File

@@ -13,12 +13,13 @@ from rest_api.filters import MayanObjectPermissionsFilter
from rest_api.permissions import MayanPermission
from .models import Index, IndexInstanceNode, IndexTemplateNode
from .permissions import (permission_document_indexing_create,
permission_document_indexing_delete,
permission_document_indexing_edit,
permission_document_indexing_view)
from .serializers import (IndexInstanceNodeSerializer, IndexSerializer,
IndexTemplateNodeSerializer)
from .permissions import (
permission_document_indexing_create, permission_document_indexing_delete,
permission_document_indexing_edit, permission_document_indexing_view
)
from .serializers import (
IndexInstanceNodeSerializer, IndexSerializer, IndexTemplateNodeSerializer
)
class APIIndexListView(generics.ListCreateAPIView):
@@ -69,7 +70,8 @@ class APIIndexView(generics.RetrieveUpdateDestroyAPIView):
class APIIndexNodeInstanceDocumentListView(generics.ListAPIView):
"""
Returns a list of all the documents contained by a particular index node instance.
Returns a list of all the documents contained by a particular index node
instance.
"""
filter_backends = (MayanObjectPermissionsFilter,)
@@ -80,11 +82,18 @@ class APIIndexNodeInstanceDocumentListView(generics.ListAPIView):
return DocumentSerializer
def get_queryset(self):
index_node_instance = get_object_or_404(IndexInstanceNode, pk=self.kwargs['pk'])
index_node_instance = get_object_or_404(
IndexInstanceNode, pk=self.kwargs['pk']
)
try:
Permission.check_permissions(self.request.user, [permission_document_indexing_view])
Permission.check_permissions(
self.request.user, [permission_document_indexing_view]
)
except PermissionDenied:
AccessControlList.objects.check_access(permission_document_indexing_view, self.request.user, index_node_instance.index)
AccessControlList.objects.check_access(
permission_document_indexing_view, self.request.user,
index_node_instance.index
)
return index_node_instance.documents.all()
@@ -142,8 +151,12 @@ class APIDocumentIndexListView(generics.ListAPIView):
def get_queryset(self):
document = get_object_or_404(Document, pk=self.kwargs['pk'])
try:
Permission.check_permissions(self.request.user, [permission_document_view])
Permission.check_permissions(
self.request.user, [permission_document_view]
)
except PermissionDenied:
AccessControlList.objects.check_access(permission_document_view, self.request.user, document)
AccessControlList.objects.check_access(
permission_document_view, self.request.user, document
)
return document.node_instances.all()

View File

@@ -59,15 +59,47 @@ class DocumentIndexingApp(MayanAppConfig):
}
)
menu_facet.bind_links(links=[link_document_index_list], sources=[Document])
menu_object.bind_links(links=[link_index_setup_edit, link_index_setup_view, link_index_setup_document_types, link_index_setup_delete], sources=[Index])
menu_object.bind_links(links=[link_template_node_create, link_template_node_edit, link_template_node_delete], sources=[IndexTemplateNode])
menu_facet.bind_links(
links=[link_document_index_list], sources=[Document]
)
menu_object.bind_links(
links=[
link_index_setup_edit, link_index_setup_view,
link_index_setup_document_types, link_index_setup_delete
], sources=[Index]
)
menu_object.bind_links(
links=[
link_template_node_create, link_template_node_edit,
link_template_node_delete
], sources=[IndexTemplateNode]
)
menu_main.bind_links(links=[link_index_main_menu])
menu_secondary.bind_links(links=[link_index_setup_list, link_index_setup_create], sources=[Index, 'indexing:index_setup_list', 'indexing:index_setup_create'])
menu_secondary.bind_links(
links=[link_index_setup_list, link_index_setup_create],
sources=[
Index, 'indexing:index_setup_list',
'indexing:index_setup_create'
]
)
menu_setup.bind_links(links=[link_index_setup])
menu_tools.bind_links(links=[link_rebuild_index_instances])
post_document_created.connect(document_created_index_update, dispatch_uid='document_created_index_update', sender=Document)
post_save.connect(document_metadata_index_update, dispatch_uid='document_metadata_index_update', sender=DocumentMetadata)
post_delete.connect(document_index_delete, dispatch_uid='document_index_delete', sender=Document)
post_delete.connect(document_metadata_index_post_delete, dispatch_uid='document_metadata_index_post_delete', sender=DocumentMetadata)
post_document_created.connect(
document_created_index_update,
dispatch_uid='document_created_index_update', sender=Document
)
post_save.connect(
document_metadata_index_update,
dispatch_uid='document_metadata_index_update',
sender=DocumentMetadata
)
post_delete.connect(
document_index_delete, dispatch_uid='document_index_delete',
sender=Document
)
post_delete.connect(
document_metadata_index_post_delete,
dispatch_uid='document_metadata_index_post_delete',
sender=DocumentMetadata
)

View File

@@ -16,7 +16,12 @@ class IndexTemplateNodeForm(forms.ModelForm):
super(IndexTemplateNodeForm, self).__init__(*args, **kwargs)
self.fields['index'].widget = forms.widgets.HiddenInput()
self.fields['parent'].widget = forms.widgets.HiddenInput()
self.fields['expression'].help_text = ' '.join([unicode(self.fields['expression'].help_text), ModelAttribute.help_text_for(Document, type_names=['indexing'])])
self.fields['expression'].help_text = ' '.join(
[
unicode(self.fields['expression'].help_text),
ModelAttribute.help_text_for(Document, type_names=['indexing'])
]
)
class Meta:
fields = ('parent', 'index', 'expression', 'enabled', 'link_documents')

View File

@@ -4,7 +4,9 @@ from .tasks import task_delete_empty_index_nodes, task_index_document
def document_created_index_update(sender, **kwargs):
task_index_document.apply_async(kwargs=dict(document_id=kwargs['instance'].pk))
task_index_document.apply_async(
kwargs=dict(document_id=kwargs['instance'].pk)
)
def document_index_delete(sender, **kwargs):
@@ -12,8 +14,12 @@ def document_index_delete(sender, **kwargs):
def document_metadata_index_update(sender, **kwargs):
task_index_document.apply_async(kwargs=dict(document_id=kwargs['instance'].document.pk))
task_index_document.apply_async(
kwargs=dict(document_id=kwargs['instance'].document.pk)
)
def document_metadata_index_post_delete(sender, **kwargs):
task_index_document.apply_async(kwargs=dict(document_id=kwargs['instance'].document.pk))
task_index_document.apply_async(
kwargs=dict(document_id=kwargs['instance'].document.pk)
)

View File

@@ -17,22 +17,65 @@ def is_not_root_node(context):
return not context['resolved_object'].is_root_node()
link_document_index_list = Link(permissions=[permission_document_view], text=_('Indexes'), view='indexing:document_index_list', args='object.pk')
link_index_list = Link(permissions=[permission_document_indexing_view], text=_('Index list'), view='indexing:index_list')
link_index_main_menu = Link(icon='fa fa-list-ul', text=_('Indexes'), view='indexing:index_list')
link_index_setup = Link(icon='fa fa-list-ul', permissions=[permission_document_indexing_setup], text=_('Indexes'), view='indexing:index_setup_list')
link_index_setup_list = Link(permissions=[permission_document_indexing_setup], text=_('Indexes'), view='indexing:index_setup_list')
link_index_setup_create = Link(permissions=[permission_document_indexing_create], text=_('Create index'), view='indexing:index_setup_create')
link_index_setup_edit = Link(permissions=[permission_document_indexing_edit], text=_('Edit'), view='indexing:index_setup_edit', args='resolved_object.pk')
link_index_setup_delete = Link(permissions=[permission_document_indexing_delete], tags='dangerous', text=_('Delete'), view='indexing:index_setup_delete', args='resolved_object.pk')
link_index_setup_view = Link(permissions=[permission_document_indexing_setup], text=_('Tree template'), view='indexing:index_setup_view', args='resolved_object.pk')
link_index_setup_document_types = Link(permissions=[permission_document_indexing_edit], text=_('Document types'), view='indexing:index_setup_document_types', args='resolved_object.pk')
link_document_index_list = Link(
permissions=[permission_document_view], text=_('Indexes'),
view='indexing:document_index_list', args='object.pk'
)
link_index_list = Link(
permissions=[permission_document_indexing_view], text=_('Index list'),
view='indexing:index_list'
)
link_index_main_menu = Link(
icon='fa fa-list-ul', text=_('Indexes'), view='indexing:index_list'
)
link_index_setup = Link(
icon='fa fa-list-ul', permissions=[permission_document_indexing_setup],
text=_('Indexes'), view='indexing:index_setup_list'
)
link_index_setup_list = Link(
permissions=[permission_document_indexing_setup], text=_('Indexes'),
view='indexing:index_setup_list'
)
link_index_setup_create = Link(
permissions=[permission_document_indexing_create], text=_('Create index'),
view='indexing:index_setup_create'
)
link_index_setup_edit = Link(
permissions=[permission_document_indexing_edit], text=_('Edit'),
view='indexing:index_setup_edit', args='resolved_object.pk'
)
link_index_setup_delete = Link(
permissions=[permission_document_indexing_delete], tags='dangerous',
text=_('Delete'), view='indexing:index_setup_delete',
args='resolved_object.pk'
)
link_index_setup_view = Link(
permissions=[permission_document_indexing_setup], text=_('Tree template'),
view='indexing:index_setup_view', args='resolved_object.pk'
)
link_index_setup_document_types = Link(
permissions=[permission_document_indexing_edit], text=_('Document types'),
view='indexing:index_setup_document_types', args='resolved_object.pk'
)
link_rebuild_index_instances = Link(
icon='fa fa-database',
description=_('Deletes and creates from scratch all the document indexes.'),
permissions=[permission_document_indexing_rebuild_indexes],
text=_('Rebuild indexes'), view='indexing:rebuild_index_instances'
)
link_template_node_create = Link(permissions=[permission_document_indexing_setup], text=_('New child node'), view='indexing:template_node_create', args='resolved_object.pk')
link_template_node_edit = Link(condition=is_not_root_node, permissions=[permission_document_indexing_setup], text=_('Edit'), view='indexing:template_node_edit', args='resolved_object.pk')
link_template_node_delete = Link(condition=is_not_root_node, permissions=[permission_document_indexing_setup], tags='dangerous', text=_('Delete'), view='indexing:template_node_delete', args='resolved_object.pk')
link_template_node_create = Link(
permissions=[permission_document_indexing_setup], text=_('New child node'),
view='indexing:template_node_create', args='resolved_object.pk'
)
link_template_node_edit = Link(
condition=is_not_root_node,
permissions=[permission_document_indexing_setup], text=_('Edit'),
view='indexing:template_node_edit', args='resolved_object.pk'
)
link_template_node_delete = Link(
condition=is_not_root_node, permissions=[
permission_document_indexing_setup
],
tags='dangerous', text=_('Delete'), view='indexing:template_node_delete',
args='resolved_object.pk'
)

View File

@@ -30,7 +30,9 @@ class IndexInstanceNodeManager(models.Manager):
parent = instance_node.parent
if parent:
instance_node.delete()
IndexInstanceNodeManager.delete_empty_index_nodes_recursive(parent)
IndexInstanceNodeManager.delete_empty_index_nodes_recursive(
parent
)
def cascade_eval(self, document, template_node, parent_index_instance=None):
"""
@@ -41,14 +43,26 @@ class IndexInstanceNodeManager(models.Manager):
if template_node.enabled:
try:
result = eval(template_node.expression, {'document': document}, setting_available_indexing_functions.value)
result = eval(
template_node.expression, {'document': document},
setting_available_indexing_functions.value
)
except Exception as exception:
error_message = _('Error indexing document: %(document)s; expression: %(expression)s; %(exception)s') % {
'document': document, 'expression': template_node.expression, 'exception': exception}
error_message = _(
'Error indexing document: %(document)s; expression: '
'%(expression)s; %(exception)s'
) % {
'document': document,
'expression': template_node.expression,
'exception': exception
}
logger.debug(error_message)
else:
if result:
index_instance, created = self.get_or_create(index_template_node=template_node, value=result, parent=parent_index_instance)
index_instance, created = self.get_or_create(
index_template_node=template_node, value=result,
parent=parent_index_instance
)
if template_node.link_documents:
index_instance.documents.add(document)
@@ -66,7 +80,9 @@ class IndexInstanceNodeManager(models.Manager):
"""
for instance_node in self.filter(documents__isnull=True, parent__isnull=False):
IndexInstanceNodeManager.delete_empty_index_nodes_recursive(instance_node)
IndexInstanceNodeManager.delete_empty_index_nodes_recursive(
instance_node
)
def index_document(self, document):
"""
@@ -80,7 +96,9 @@ class IndexInstanceNodeManager(models.Manager):
# Only update indexes where the document type is found
for index in Index.objects.filter(enabled=True, document_types=document.document_type):
root_instance, created = self.get_or_create(index_template_node=index.template_root, parent=None)
root_instance, created = self.get_or_create(
index_template_node=index.template_root, parent=None
)
for template_node in index.template_root.get_children():
self.cascade_eval(document, template_node, root_instance)

View File

@@ -15,9 +15,20 @@ from .managers import IndexManager, IndexInstanceNodeManager
@python_2_unicode_compatible
class Index(models.Model):
label = models.CharField(max_length=128, unique=True, verbose_name=_('Label'))
enabled = models.BooleanField(default=True, help_text=_('Causes this index to be visible and updated when document data changes.'), verbose_name=_('Enabled'))
document_types = models.ManyToManyField(DocumentType, verbose_name=_('Document types'))
label = models.CharField(
max_length=128, unique=True, verbose_name=_('Label')
)
enabled = models.BooleanField(
default=True,
help_text=_(
'Causes this index to be visible and updated when document data '
'changes.'
),
verbose_name=_('Enabled')
)
document_types = models.ManyToManyField(
DocumentType, verbose_name=_('Document types')
)
objects = IndexManager()
@@ -34,7 +45,10 @@ class Index(models.Model):
def get_absolute_url(self):
try:
return reverse('indexing:index_instance_node_view', args=[self.instance_root.pk])
return reverse(
'indexing:index_instance_node_view',
args=[self.instance_root.pk]
)
except IndexInstanceNode.DoesNotExist:
return '#'
@@ -44,7 +58,9 @@ class Index(models.Model):
IndexTemplateNode.objects.get_or_create(parent=None, index=self)
def get_document_types_names(self):
return ', '.join([unicode(document_type) for document_type in self.document_types.all()] or ['None'])
return ', '.join(
[unicode(document_type) for document_type in self.document_types.all()] or ['None']
)
def get_instance_node_count(self):
try:
@@ -59,11 +75,31 @@ class Index(models.Model):
@python_2_unicode_compatible
class IndexTemplateNode(MPTTModel):
parent = TreeForeignKey('self', null=True, blank=True)
index = models.ForeignKey(Index, verbose_name=_('Index'), related_name='node_templates')
expression = models.CharField(max_length=128, verbose_name=_('Indexing expression'), help_text=_('Enter a python string expression to be evaluated.'))
enabled = models.BooleanField(default=True, verbose_name=_('Enabled'), help_text=_('Causes this node to be visible and updated when document data changes.'))
link_documents = models.BooleanField(default=False, verbose_name=_('Link documents'), help_text=_('Check this option to have this node act as a container for documents and not as a parent for further nodes.'))
parent = TreeForeignKey('self', blank=True, null=True)
index = models.ForeignKey(
Index, related_name='node_templates', verbose_name=_('Index')
)
expression = models.CharField(
max_length=128,
help_text=_('Enter a python string expression to be evaluated.'),
verbose_name=_('Indexing expression')
)
enabled = models.BooleanField(
default=True,
help_text=_(
'Causes this node to be visible and updated when document data '
'changes.'
),
verbose_name=_('Enabled')
)
link_documents = models.BooleanField(
default=False,
help_text=_(
'Check this option to have this node act as a container for '
'documents and not as a parent for further nodes.'
),
verbose_name=_('Link documents')
)
def __str__(self):
if self.is_root_node():
@@ -79,9 +115,16 @@ class IndexTemplateNode(MPTTModel):
@python_2_unicode_compatible
class IndexInstanceNode(MPTTModel):
parent = TreeForeignKey('self', null=True, blank=True)
index_template_node = models.ForeignKey(IndexTemplateNode, related_name='node_instance', verbose_name=_('Index template node'))
value = models.CharField(max_length=128, blank=True, verbose_name=_('Value'))
documents = models.ManyToManyField(Document, related_name='node_instances', verbose_name=_('Documents'))
index_template_node = models.ForeignKey(
IndexTemplateNode, related_name='node_instance',
verbose_name=_('Index template node')
)
value = models.CharField(
blank=True, max_length=128, verbose_name=_('Value')
)
documents = models.ManyToManyField(
Document, related_name='node_instances', verbose_name=_('Documents')
)
objects = IndexInstanceNodeManager()

View File

@@ -6,9 +6,21 @@ from permissions import PermissionNamespace
namespace = PermissionNamespace('document_indexing', _('Indexing'))
permission_document_indexing_setup = namespace.add_permission(name='document_index_setup', label=_('Configure document indexes'))
permission_document_indexing_create = namespace.add_permission(name='document_index_create', label=_('Create new document indexes'))
permission_document_indexing_edit = namespace.add_permission(name='document_index_edit', label=_('Edit document indexes'))
permission_document_indexing_delete = namespace.add_permission(name='document_index_delete', label=_('Delete document indexes'))
permission_document_indexing_view = namespace.add_permission(name='document_index_view', label=_('View document indexes'))
permission_document_indexing_rebuild_indexes = namespace.add_permission(name='document_rebuild_indexes', label=_('Rebuild document indexes'))
permission_document_indexing_setup = namespace.add_permission(
name='document_index_setup', label=_('Configure document indexes')
)
permission_document_indexing_create = namespace.add_permission(
name='document_index_create', label=_('Create new document indexes')
)
permission_document_indexing_edit = namespace.add_permission(
name='document_index_edit', label=_('Edit document indexes')
)
permission_document_indexing_delete = namespace.add_permission(
name='document_index_delete', label=_('Delete document indexes')
)
permission_document_indexing_view = namespace.add_permission(
name='document_index_view', label=_('View document indexes')
)
permission_document_indexing_rebuild_indexes = namespace.add_permission(
name='document_rebuild_indexes', label=_('Rebuild document indexes')
)

View File

@@ -21,7 +21,10 @@ IndexInstanceNodeSerializer.base_fields['children'] = IndexInstanceNodeSerialize
class IndexTemplateNodeSerializer(serializers.ModelSerializer):
class Meta:
fields = ('id', 'parent', 'index', 'expression', 'enabled', 'link_documents', 'level')
fields = (
'id', 'parent', 'index', 'expression', 'enabled', 'link_documents',
'level'
)
model = IndexTemplateNode
@@ -30,5 +33,8 @@ class IndexSerializer(serializers.ModelSerializer):
instance_root = IndexInstanceNodeSerializer(read_only=True)
class Meta:
fields = ('id', 'label', 'enabled', 'document_types', 'node_templates', 'instance_root')
fields = (
'id', 'label', 'enabled', 'document_types', 'node_templates',
'instance_root'
)
model = Index

View File

@@ -8,4 +8,7 @@ available_indexing_functions = {
}
namespace = Namespace(name='document_indexing', label=_('Indexing'))
setting_available_indexing_functions = namespace.add_setting(global_name='DOCUMENT_INDEXING_AVAILABLE_INDEXING_FUNCTIONS', default=available_indexing_functions)
setting_available_indexing_functions = namespace.add_setting(
global_name='DOCUMENT_INDEXING_AVAILABLE_INDEXING_FUNCTIONS',
default=available_indexing_functions
)

View File

@@ -17,7 +17,9 @@ logger = logging.getLogger(__name__)
@app.task(bind=True, default_retry_delay=RETRY_DELAY, max_retries=None, ignore_result=True)
def task_delete_empty_index_nodes(self):
try:
rebuild_lock = Lock.acquire_lock('document_indexing_task_do_rebuild_all_indexes')
rebuild_lock = Lock.acquire_lock(
'document_indexing_task_do_rebuild_all_indexes'
)
except LockError as exception:
# A rebuild is happening, retry later
raise self.retry(exc=exception)
@@ -31,13 +33,17 @@ def task_delete_empty_index_nodes(self):
@app.task(bind=True, default_retry_delay=RETRY_DELAY, max_retries=None, ignore_result=True)
def task_index_document(self, document_id):
try:
rebuild_lock = Lock.acquire_lock('document_indexing_task_do_rebuild_all_indexes')
rebuild_lock = Lock.acquire_lock(
'document_indexing_task_do_rebuild_all_indexes'
)
except LockError as exception:
# A rebuild is happening, retry later
raise self.retry(exc=exception)
else:
try:
lock = Lock.acquire_lock('document_indexing_task_update_index_document_%d' % document_id)
lock = Lock.acquire_lock(
'document_indexing_task_update_index_document_%d' % document_id
)
except LockError as exception:
# This document is being reindexed by another task, retry later
raise self.retry(exc=exception)
@@ -45,13 +51,17 @@ def task_index_document(self, document_id):
try:
document = Document.objects.get(pk=document_id)
except Document.DoesNotExist:
# Document was deleted before we could execute, abort about updating
# Document was deleted before we could execute, abort about
# updating
pass
else:
try:
IndexInstanceNode.objects.index_document(document)
except OperationalError as exception:
logger.warning('Operational error while trying to index document: %s; %s', document, exception)
logger.warning(
'Operational error while trying to index document: %s; %s',
document, exception
)
lock.release()
raise self.retry(exc=exception)
else:
@@ -69,7 +79,9 @@ def task_do_rebuild_all_indexes(self):
raise self.retry()
try:
lock = Lock.acquire_lock('document_indexing_task_do_rebuild_all_indexes')
lock = Lock.acquire_lock(
'document_indexing_task_do_rebuild_all_indexes'
)
except LockError as exception:
# Another rebuild is happening, retry later
raise self.retry(exc=exception)

View File

@@ -12,14 +12,18 @@ from .models import Index, IndexInstanceNode, IndexTemplateNode
class IndexTestCase(TestCase):
def setUp(self):
self.document_type = DocumentType.objects.create(label=TEST_DOCUMENT_TYPE)
self.document_type = DocumentType.objects.create(
label=TEST_DOCUMENT_TYPE
)
ocr_settings = self.document_type.ocr_settings
ocr_settings.auto_ocr = False
ocr_settings.save()
with open(TEST_SMALL_DOCUMENT_PATH) as file_object:
self.document = self.document_type.new_document(file_object=File(file_object))
self.document = self.document_type.new_document(
file_object=File(file_object)
)
def tearDown(self):
for document_type in DocumentType.objects.all():
@@ -27,7 +31,9 @@ class IndexTestCase(TestCase):
def test_indexing(self):
metadata_type = MetadataType.objects.create(name='test', label='test')
DocumentTypeMetadataType.objects.create(document_type=self.document_type, metadata_type=metadata_type)
DocumentTypeMetadataType.objects.create(
document_type=self.document_type, metadata_type=metadata_type
)
# Create empty index
index = Index.objects.create(label='test')