matthias/mayan-edms
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge branch 'feature/haystack' into development

Browse Source 
Conflicts:
	requirements/production.txt
This commit is contained in:
Roberto Rosario
2012-05-22 23:42:44 -04:00
parent 9333f3c0b7 b3bcb274a0
commit 35a4a60d42
31 changed files with 638 additions and 398 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
apps/acls/managers.py
View File

@@ -137,14 +137,14 @@ class AccessEntryManager(models.Manager):
content_type = ContentType.objects.get_for_model(cls)
# Calculate actor role membership ACL query
total_queries = None
total_queries = Q()
for role in RoleMember.objects.get_roles_for_member(actor):
role_type = ContentType.objects.get_for_model(role)
if related:
query = Q(holder_type=role_type, holder_id=role.pk, permission=permission.get_stored_permission)
else:
query = Q(holder_type=role_type, holder_id=role.pk, content_type=content_type, permission=permission.get_stored_permission)
if total_queries is None:
if not total_queries:
total_queries = query
else:
total_queries = total_queries | query
@@ -161,7 +161,7 @@ class AccessEntryManager(models.Manager):
query = Q(holder_type=group_type, holder_id=group.pk, permission=permission.get_stored_permission)
else:
query = Q(holder_type=group_type, holder_id=group.pk, content_type=content_type, permission=permission.get_stored_permission)
if total_queries is None:
if not total_queries:
total_queries = query
else:
total_queries = total_queries | query

23
apps/common/templates/generic_list_subtemplate.html
View File

@@ -6,6 +6,7 @@
{% load variable_tags %}
{% load main_settings_tags %}
{% load multiselect_tags %}
{% load subtemplates_tags %}
{% get_main_setting "DISABLE_ICONS" as disable_icons %}
@@ -76,9 +77,13 @@
<th>{{ column.name|capfirst }}</th>
{% endfor %}
{% for column in object_list.0|get_model_list_columns %}
<th>{{ column.name|capfirst }}</th>
{% endfor %}
{% if object_list %}
{% get_object_list_object_name object_list.0 %}
{% for column in object|get_model_list_columns %}
<th>{{ column.name|capfirst }}</th>
{% endfor %}
{% endif %}
{% for column in extra_columns %}
<th>{{ column.name|capfirst }}</th>
@@ -89,15 +94,17 @@
{% endif %}
</tr>
{% endif %}
{% for object in object_list %}
{% get_object_list_object_name object %}
<tr class="{% cycle 'odd' 'even2' %}">
{% if multi_select or multi_select_as_buttons %}
<td>
{% if multi_select_item_properties %}
<input type="checkbox" class="checkbox" name="properties_{{ object|get_encoded_parameter:multi_select_item_properties }}" value="" />
{% else %}
<input type="checkbox" class="checkbox" name="pk_{{ object.pk }}" value="" />
{% endif %}
{% if multi_select_item_properties %}
<input type="checkbox" class="checkbox" name="properties_{{ object|get_encoded_parameter:multi_select_item_properties }}" value="" />
{% else %}
<input type="checkbox" class="checkbox" name="pk_{{ object.pk }}" value="" />
{% endif %}
</td>
{% endif %}
{% if not hide_object %}

18
apps/common/templatetags/subtemplates_tags.py
View File

@@ -1,9 +1,11 @@
import re
from django.template import Node, TemplateSyntaxError, Library, \
Variable, Context
from django.template import (Node, TemplateSyntaxError, Library,
Variable, Context)
from django.template.loader import get_template
from common.utils import return_attrib
register = Library()
@@ -49,3 +51,15 @@ def render_subtemplate(parser, token):
return RenderSubtemplateNode(template_name, template_context, var_name)
#format_string[1:-1]
@register.simple_tag(takes_context=True)
def get_object_list_object_name(context, source_object):
object_list_object_name = context.get('object_list_object_name')
if object_list_object_name:
context['object'] = return_attrib(source_object, object_list_object_name)
else:
context['object'] = source_object
return ''

8
apps/document_comments/views.py
View File

@@ -43,6 +43,7 @@ def comment_delete(request, comment_id=None, comment_id_list=None):
for comment in comments:
try:
comment.delete()
comment.content_object.mark_indexable()
messages.success(request, _(u'Comment "%s" deleted successfully.') % comment)
except Exception, e:
messages.error(request, _(u'Error deleting comment "%(comment)s": %(error)s') % {
@@ -95,7 +96,8 @@ def comment_add(request, document_id):
comment.object_pk = document.pk
comment.site = Site.objects.get_current()
comment.save()
document.mark_indexable()
messages.success(request, _(u'Comment added successfully.'))
return HttpResponseRedirect(next)
else:
@@ -110,9 +112,9 @@ def comment_add(request, document_id):
def comments_for_document(request, document_id):
'''
"""
Show a list of all the comments related to the passed object
'''
"""
document = get_object_or_404(Document, pk=document_id)
try:

42
apps/documents/models.py
View File

@@ -13,13 +13,14 @@ try:
except ImportError:
from StringIO import StringIO
from unidecode import unidecode
from django.db import models
from django.utils.translation import ugettext_lazy as _
from django.utils.translation import ugettext
from django.contrib.auth.models import User
from django.core.exceptions import ValidationError
from dynamic_search.api import register
from converter.api import get_page_count
from converter.api import get_available_transformations_choices
from converter.api import convert
@@ -102,6 +103,7 @@ class Document(models.Model):
self.uuid = UUID_FUNCTION()
self.date_added = datetime.datetime.now()
super(Document, self).save(*args, **kwargs)
self.mark_indexable()
def get_cached_image_name(self, page, version):
document_version = DocumentVersion.objects.get(pk=version)
@@ -275,8 +277,22 @@ class Document(models.Model):
version.filename = value
return version.save()
@property
def content(self):
return self.latest_version.content
filename = property(_get_filename, _set_filename)
@property
def cleaned_filename(self):
return unidecode(self.extension_split[0])
@property
def extension_split(self):
filename, extension = os.path.splitext(self.filename)
return filename, extension[1:]
class DocumentVersion(models.Model):
"""
@@ -524,6 +540,15 @@ class DocumentVersion(models.Model):
self.filename = u''.join([new_name, extension])
self.save()
@property
def content(self):
content = []
for page in self.document.pages.all():
if page.content:
content.append(page.content)
return u''.join(content)
class DocumentTypeFilename(models.Model):
"""
@@ -657,18 +682,3 @@ class RecentDocument(models.Model):
ordering = ('-datetime_accessed',)
verbose_name = _(u'recent document')
verbose_name_plural = _(u'recent documents')
# Register the fields that will be searchable
register('document', Document, _(u'document'), [
{'name': u'document_type__name', 'title': _(u'Document type')},
{'name': u'documentversion__mimetype', 'title': _(u'MIME type')},
{'name': u'documentversion__filename', 'title': _(u'Filename')},
{'name': u'documentmetadata__value', 'title': _(u'Metadata value')},
{'name': u'documentversion__documentpage__content', 'title': _(u'Content')},
{'name': u'description', 'title': _(u'Description')},
{'name': u'tags__name', 'title': _(u'Tags')},
{'name': u'comments__comment', 'title': _(u'Comments')},
]
)
#register(Document, _(u'document'), ['document_type__name', 'file_mimetype', 'documentmetadata__value', 'documentpage__content', 'description', {'field_name':'file_filename', 'comparison':'iexact'}])

61
apps/dynamic_search/__init__.py
View File

@@ -1,14 +1,63 @@
from __future__ import absolute_import
import logging
from django.utils.translation import ugettext_lazy as _
from django.dispatch import receiver
from django.core.management import call_command
from navigation.api import register_sidebar_template, register_links
from documents.models import Document
from scheduler.runtime import scheduler
from signaler.signals import post_update_index, pre_update_index
from scheduler.api import register_interval_job
from lock_manager import Lock, LockError
from .models import IndexableObject
from .conf.settings import INDEX_UPDATE_INTERVAL
logger = logging.getLogger(__name__)
search = {'text': _(u'search'), 'view': 'search', 'famfam': 'zoom'}
search_advanced = {'text': _(u'advanced search'), 'view': 'search_advanced', 'famfam': 'zoom_in'}
search_again = {'text': _(u'search again'), 'view': 'search_again', 'famfam': 'arrow_undo'}
register_sidebar_template(['search', 'search_advanced'], 'search_help.html')
register_sidebar_template(['search'], 'search_help.html')
register_links(['search', 'search_advanced', 'results'], [search, search_advanced], menu_name='form_header')
register_links(['results'], [search_again], menu_name='sidebar')
register_links(['search'], [search], menu_name='form_header')
register_sidebar_template(['search', 'search_advanced', 'results'], 'recent_searches.html')
register_sidebar_template(['search'], 'recent_searches.html')
Document.add_to_class('mark_indexable', lambda obj: IndexableObject.objects.mark_indexable(obj))
@receiver(pre_update_index, dispatch_uid='scheduler_shutdown_pre_update_index')
def scheduler_shutdown_pre_update_index(sender, mayan_runtime, **kwargs):
logger.debug('Scheduler shut down on pre update index signal')
logger.debug('Runtime variable: %s' % mayan_runtime)
# Only shutdown the scheduler if the command is called from the command
# line
if not mayan_runtime:
scheduler.shutdown()
def search_index_update():
lock_id = u'search_index_update'
try:
logger.debug('trying to acquire lock: %s' % lock_id)
lock = Lock.acquire_lock(lock_id)
logger.debug('acquired lock: %s' % lock_id)
logger.debug('Executing haystack\'s index update command')
call_command('update_index', '--mayan_runtime')
lock.release()
except LockError:
logger.debug('unable to obtain lock')
pass
except Exception, instance:
logger.debug('Unhandled exception: %s' % instance)
lock.release()
pass
register_interval_job('search_index_update', _(u'Update the search index with the most recent modified documents.'), search_index_update, seconds=INDEX_UPDATE_INTERVAL)

5
apps/dynamic_search/admin.py
View File

@@ -1,6 +1,8 @@
from __future__ import absolute_import
from django.contrib import admin
from dynamic_search.models import RecentSearch
from .models import RecentSearch, IndexableObject
class RecentSearchAdmin(admin.ModelAdmin):
@@ -9,3 +11,4 @@ class RecentSearchAdmin(admin.ModelAdmin):
readonly_fields = ('user', 'query', 'datetime_created', 'hits')
admin.site.register(RecentSearch, RecentSearchAdmin)
admin.site.register(IndexableObject)

138
apps/dynamic_search/api.py
View File

@@ -1,138 +0,0 @@
# original code from:
# http://www.julienphalip.com/blog/2008/08/16/adding-search-django-site-snap/
import re
import types
import datetime
from django.db.models import Q
from dynamic_search.conf.settings import LIMIT
registered_search_dict = {}
def register(model_name, model, title, fields):
registered_search_dict.setdefault(model_name, {'model': model, 'fields': [], 'title': title})
registered_search_dict[model_name]['fields'].extend(fields)
def normalize_query(query_string,
findterms=re.compile(r'"([^"]+)"|(\S+)').findall,
normspace=re.compile(r'\s{2,}').sub):
"""
Splits the query string in invidual keywords, getting rid of unecessary spaces
and grouping quoted words together.
Example:
>>> normalize_query(' some random words "with quotes " and spaces')
['some', 'random', 'words', 'with quotes', 'and', 'spaces']
"""
return [normspace(' ', (t[0] or t[1]).strip()) for t in findterms(query_string)]
def get_query(terms, search_fields):
"""
Returns a query, that is a combination of Q objects. That combination
aims to search keywords within a model by testing the given search fields.
"""
queries = []
for term in terms:
or_query = None
for field in search_fields:
if isinstance(field, types.StringTypes):
comparison = u'icontains'
field_name = field
elif isinstance(field, types.DictType):
comparison = field.get('comparison', u'icontains')
field_name = field.get('field_name', u'')
if field_name:
q = Q(**{'%s__%s' % (field_name, comparison): term})
if or_query is None:
or_query = q
else:
or_query = or_query | q
queries.append(or_query)
return queries
def perform_search(query_string, field_list=None):
model_list = {}
flat_list = []
result_count = 0
shown_result_count = 0
elapsed_time = 0
start_time = datetime.datetime.now()
search_dict = {}
if query_string:
simple_query_string = query_string.get('q', u'').strip()
if simple_query_string:
for model, values in registered_search_dict.items():
search_dict.setdefault(values['model'], {'query_entries': [], 'title': values['title']})
field_names = [field['name'] for field in values['fields']]
# One entry, single set of terms for all fields names
search_dict[values['model']]['query_entries'].append(
{
'field_name': field_names,
'terms': normalize_query(simple_query_string)
}
)
else:
for key, value in query_string.items():
try:
model, field_name = key.split('__', 1)
model_entry = registered_search_dict.get(model, {})
if model_entry:
for model_field in model_entry.get('fields', [{}]):
if model_field.get('name') == field_name:
search_dict.setdefault(model_entry['model'], {'query_entries': [], 'title': model_entry['title']})
search_dict[model_entry['model']]['query_entries'].append(
{
'field_name': [field_name],
'terms': normalize_query(value.strip())
}
)
except ValueError:
pass
for model, data in search_dict.items():
title = data['title']
queries = []
for query_entry in data['query_entries']:
queries.extend(get_query(query_entry['terms'], query_entry['field_name']))
model_result_ids = None
for query in queries:
single_result_ids = set(model.objects.filter(query).values_list('pk', flat=True))
#Convert queryset to python set and perform the
#AND operation on the program and not as a query
if model_result_ids == None:
model_result_ids = single_result_ids
else:
model_result_ids &= single_result_ids
if model_result_ids == None:
model_result_ids = []
result_count += len(model_result_ids)
results = model.objects.in_bulk(list(model_result_ids)[: LIMIT]).values()
shown_result_count += len(results)
if results:
model_list[title] = results
for result in results:
if result not in flat_list:
flat_list.append(result)
elapsed_time = unicode(datetime.datetime.now() - start_time).split(':')[2]
return {
'model_list': model_list,
'flat_list': flat_list,
'shown_result_count': shown_result_count,
'result_count': result_count,
'elapsed_time': elapsed_time
}

24
apps/dynamic_search/conf/settings.py
View File

@@ -6,22 +6,6 @@ from smart_settings.api import Setting, SettingNamespace
namespace = SettingNamespace('dynamic_search', _(u'Searching'), module='dynamic_search.conf.settings')
Setting(
namespace=namespace,
name='SHOW_OBJECT_TYPE',
global_name='SEARCH_SHOW_OBJECT_TYPE',
default=True,
hidden=True
)
Setting(
namespace=namespace,
name='LIMIT',
global_name='SEARCH_LIMIT',
default=100,
description=_(u'Maximum amount search hits to fetch and display.')
)
Setting(
namespace=namespace,
name='RECENT_COUNT',
@@ -29,3 +13,11 @@ Setting(
default=5,
description=_(u'Maximum number of search queries to remember per user.')
)
Setting(
namespace=namespace,
name='INDEX_UPDATE_INTERVAL',
global_name='SEARCH_INDEX_UPDATE_INTERVAL',
default=1800,
description=_(u'Interval in second on which to trigger the search index update.')
)

25
apps/dynamic_search/forms.py
View File

@@ -1,25 +0,0 @@
from django import forms
from django.utils.translation import ugettext_lazy as _
from dynamic_search.api import registered_search_dict
class SearchForm(forms.Form):
q = forms.CharField(max_length=128, label=_(u'Search terms'))
source = forms.CharField(
max_length=32,
required=False,
widget=forms.widgets.HiddenInput()
)
class AdvancedSearchForm(forms.Form):
def __init__(self, *args, **kwargs):
super(AdvancedSearchForm, self).__init__(*args, **kwargs)
for model_name, values in registered_search_dict.items():
for field in values['fields']:
self.fields['%s__%s' % (model_name, field['name'])] = forms.CharField(
label=field['title'],
required=False
)

61
apps/dynamic_search/managers.py
View File

@@ -1,32 +1,47 @@
import urlparse
from __future__ import absolute_import
from django.db import models
from django.utils.http import urlencode
from urlparse import urlparse, parse_qs
from urllib import unquote_plus
from django.utils.simplejson import dumps
from django.db.models import Manager
from django.contrib.auth.models import AnonymousUser
from dynamic_search.conf.settings import RECENT_COUNT
from django.contrib.contenttypes.models import ContentType
from django.utils.encoding import smart_str
from .conf.settings import RECENT_COUNT
class RecentSearchManager(models.Manager):
def add_query_for_user(self, user, query, hits):
parsed_query = urlparse.parse_qs(query)
for key, value in parsed_query.items():
parsed_query[key] = ' '.join(value)
class RecentSearchManager(Manager):
def add_query_for_user(self, search_view):
query_dict = parse_qs(unquote_plus(smart_str(urlparse(search_view.request.get_full_path()).query)))
if 'q=' in query:
# Is a simple query
if not parsed_query.get('q'):
# Don't store empty simple searches
return
else:
# Cleanup query string and only store the q parameter
parsed_query = {'q': parsed_query['q']}
if parsed_query and not isinstance(user, AnonymousUser):
if query_dict and not isinstance(search_view.request.user, AnonymousUser):
# If the URL query has at least one variable with a value
new_recent, created = self.model.objects.get_or_create(user=user, query=urlencode(parsed_query), defaults={'hits': hits})
new_recent.hits = hits
new_recent, created = self.model.objects.get_or_create(user=search_view.request.user, query=dumps(query_dict), defaults={'hits': 0})
new_recent.hits = search_view.results.count()
new_recent.save()
to_delete = self.model.objects.filter(user=user)[RECENT_COUNT:]
to_delete = self.model.objects.filter(user=search_view.request.user)[RECENT_COUNT:]
for recent_to_delete in to_delete:
recent_to_delete.delete()
def get_for_user(self, user):
return [entry for entry in self.model.objects.filter(user=user) if entry.get_query()]
class IndexableObjectManager(Manager):
def get_indexables(self, datetime=None):
if datetime:
return self.model.objects.filter(datetime__gte=datetime)
else:
return self.model.objects.all()
def get_indexables_pk_list(self, datetime=None):
return self.get_indexables(datetime).values_list('object_id', flat=True)
def mark_indexable(self, obj):
content_type = ContentType.objects.get_for_model(obj)
self.model.objects.get_or_create(content_type=content_type, object_id=obj.pk)
def clear_all(self):
self.model.objects.all().delete()

74
apps/dynamic_search/migrations/0001_initial.py Normal file
View File

@@ -0,0 +1,74 @@
# -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
class Migration(SchemaMigration):
def forwards(self, orm):
# Adding model 'RecentSearch'
db.create_table('dynamic_search_recentsearch', (
('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
('user', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'])),
('query', self.gf('django.db.models.fields.TextField')()),
('datetime_created', self.gf('django.db.models.fields.DateTimeField')()),
('hits', self.gf('django.db.models.fields.IntegerField')()),
))
db.send_create_signal('dynamic_search', ['RecentSearch'])
def backwards(self, orm):
# Deleting model 'RecentSearch'
db.delete_table('dynamic_search_recentsearch')
models = {
'auth.group': {
'Meta': {'object_name': 'Group'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
},
'auth.permission': {
'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
},
'auth.user': {
'Meta': {'object_name': 'User'},
'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
},
'contenttypes.contenttype': {
'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
},
'dynamic_search.recentsearch': {
'Meta': {'ordering': "('-datetime_created',)", 'object_name': 'RecentSearch'},
'datetime_created': ('django.db.models.fields.DateTimeField', [], {}),
'hits': ('django.db.models.fields.IntegerField', [], {}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'query': ('django.db.models.fields.TextField', [], {}),
'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
}
}
complete_apps = ['dynamic_search']

80
apps/dynamic_search/migrations/0002_auto__add_indexableobject.py Normal file
View File

@@ -0,0 +1,80 @@
# -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
class Migration(SchemaMigration):
def forwards(self, orm):
# Adding model 'IndexableObject'
db.create_table('dynamic_search_indexableobject', (
('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
('datetime', self.gf('django.db.models.fields.DateTimeField')()),
('content_type', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['contenttypes.ContentType'])),
('object_id', self.gf('django.db.models.fields.PositiveIntegerField')()),
))
db.send_create_signal('dynamic_search', ['IndexableObject'])
def backwards(self, orm):
# Deleting model 'IndexableObject'
db.delete_table('dynamic_search_indexableobject')
models = {
'auth.group': {
'Meta': {'object_name': 'Group'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
},
'auth.permission': {
'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
},
'auth.user': {
'Meta': {'object_name': 'User'},
'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
},
'contenttypes.contenttype': {
'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
},
'dynamic_search.indexableobject': {
'Meta': {'object_name': 'IndexableObject'},
'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
'datetime': ('django.db.models.fields.DateTimeField', [], {}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'object_id': ('django.db.models.fields.PositiveIntegerField', [], {})
},
'dynamic_search.recentsearch': {
'Meta': {'ordering': "('-datetime_created',)", 'object_name': 'RecentSearch'},
'datetime_created': ('django.db.models.fields.DateTimeField', [], {}),
'hits': ('django.db.models.fields.IntegerField', [], {}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'query': ('django.db.models.fields.TextField', [], {}),
'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
}
}
complete_apps = ['dynamic_search']

0
apps/dynamic_search/migrations/__init__.py Normal file
View File

82
apps/dynamic_search/models.py
View File

@@ -1,22 +1,23 @@
import urlparse
import urllib
from __future__ import absolute_import
from datetime import datetime
import datetime
from django.db import models
from django.utils.translation import ugettext as _
from django.contrib.auth.models import User
from django.core.urlresolvers import reverse
from django.utils.encoding import smart_unicode, smart_str
from django.utils.translation import ugettext_lazy as _
from django.contrib.contenttypes.models import ContentType
from django.contrib.contenttypes import generic
from django.utils.simplejson import loads
from django.utils.http import urlencode
from dynamic_search.managers import RecentSearchManager
from dynamic_search.api import registered_search_dict
from .managers import RecentSearchManager, IndexableObjectManager
class RecentSearch(models.Model):
'''
"""
Keeps a list of the n most recent search keywords for a given user
'''
"""
user = models.ForeignKey(User, verbose_name=_(u'user'), editable=False)
query = models.TextField(verbose_name=_(u'query'), editable=False)
datetime_created = models.DateTimeField(verbose_name=_(u'datetime created'), editable=False)
@@ -25,38 +26,55 @@ class RecentSearch(models.Model):
objects = RecentSearchManager()
def __unicode__(self):
query_dict = urlparse.parse_qs(urllib.unquote_plus(smart_str(self.query)))
if 'q' in query_dict:
# Is a simple search
display_string = smart_unicode(' '.join(query_dict['q']))
else:
# Advanced search
advanced_string = []
for key, value in query_dict.items():
# Get model name
model, field_name = key.split('__', 1)
model_entry = registered_search_dict.get(model, {})
if model_entry:
# Find the field name title
for model_field in model_entry.get('fields', [{}]):
if model_field.get('name') == field_name:
advanced_string.append(u'%s: %s' % (model_field.get('title', model_field['name']), smart_unicode(' '.join(value))))
return self.form_string()
display_string = u', '.join(advanced_string)
return u'%s (%s)' % (display_string, self.hits)
def form_string(self):
if self.is_advanced():
return u'%s (%s)' % (self.get_query(), self.hits)
else:
return u'%s (%s)' % (u' '.join(self.get_query().get('q')), self.hits)
def save(self, *args, **kwargs):
self.datetime_created = datetime.now()
self.datetime_created = datetime.datetime.now()
super(RecentSearch, self).save(*args, **kwargs)
def url(self):
view = 'results' if self.is_advanced() else 'search'
return '%s?%s' % (reverse(view), self.query)
def get_query(self):
try:
return loads(self.query)
except ValueError:
return {}
def is_advanced(self):
return 'q' not in urlparse.parse_qs(self.query)
return 'q' not in self.get_query()
def get_absolute_url(self):
return '?'.join([reverse('search'), urlencode(self.get_query(), doseq=True)])
class Meta:
ordering = ('-datetime_created',)
verbose_name = _(u'recent search')
verbose_name_plural = _(u'recent searches')
class IndexableObject(models.Model):
"""
Store a list of object links that have been modified and are
meant to be indexed in the next search index update
"""
datetime = models.DateTimeField(verbose_name=_(u'date time'))
content_type = models.ForeignKey(ContentType)
object_id = models.PositiveIntegerField()
content_object = generic.GenericForeignKey('content_type', 'object_id')
objects = IndexableObjectManager()
def __unicode__(self):
return unicode(self.content_object)
def save(self, *args, **kwargs):
self.datetime = datetime.datetime.now()
super(IndexableObject, self).save(*args, **kwargs)
class Meta:
verbose_name = _(u'indexable object')
verbose_name_plural = _(u'indexable objects')

27
apps/dynamic_search/search_indexes.py Normal file
View File

@@ -0,0 +1,27 @@
from __future__ import absolute_import
import logging
import copy
from haystack import indexes
from documents.models import Document
from .models import IndexableObject
logger = logging.getLogger(__name__)
class DocumentIndex(indexes.SearchIndex, indexes.Indexable):
text = indexes.CharField(document=True, use_template=True)
def get_model(self):
return Document
def build_queryset(self, start_date=None, end_date=None):
indexable_query_set = IndexableObject.objects.get_indexables()
logger.debug('indexable_query_set: %s' % indexable_query_set)
object_list = copy.copy(self.get_model().objects.filter(pk__in=indexable_query_set.values_list('object_id', flat=True)))
logger.debug('object_list: %s' % object_list)
indexable_query_set.delete()
return object_list

22
apps/dynamic_search/templates/search/indexes/documents/document_text.txt Normal file
View File

@@ -0,0 +1,22 @@
{{ object.content }}
{% for document_metadata in object.documentmetadata_set.all %}
{{ document_metadata.value }}
{% endfor %}
{{ object.cleaned_filename }}
{{ object.filename }}
{{ object.extension_split.1 }}
{{ object.document_type }}
{{ object.file_mimetype }}
{{ object.description|default:' ' }}
{% for comment in object.comments.all %}
{{ comment.comment }}
{% endfor %}
{% for tag in object.tags.all %}
{{ tag }}
{% endfor %}
{{ object.uuid }}

25
apps/dynamic_search/templates/search/search.html Normal file
View File

@@ -0,0 +1,25 @@
{% extends 'base.html' %}
{% load i18n %}
{% load search_tags %}
{% block title %} :: {% trans "Search results" %}{% endblock %}
{% block content %}
{% if form %}
{% include "search_results_subtemplate.html" %}
{% endif %}
{% if query %}
{% with list_title as title %}
{% include "generic_list_subtemplate.html" %}
{% endwith %}
{% endif %}
{% if not form and not query %}
{% include "generic_list_subtemplate.html" %}
{% endif %}
{% endblock %}
{% block footer %}
{% if query %}
{% blocktrans %}Elapsed time: {{ elapsed_time }} seconds{% endblocktrans %}
{% endif %}
{% endblock %}

1
apps/dynamic_search/templates/search_results.html
View File

@@ -12,6 +12,7 @@
{% if not form and not query_string %}
{% include "generic_list_subtemplate.html" %}
{% endif %}
{% endblock %}
{% block footer %}

21
apps/dynamic_search/templatetags/search_tags.py
View File

@@ -1,10 +1,13 @@
from __future__ import absolute_import
from django.core.urlresolvers import reverse
from django.template import Library
from django.utils.translation import ugettext as _
from dynamic_search.forms import SearchForm
from dynamic_search.models import RecentSearch
from dynamic_search.conf.settings import RECENT_COUNT
from haystack.forms import SearchForm
from ..models import RecentSearch
from ..conf.settings import RECENT_COUNT
register = Library()
@@ -20,12 +23,12 @@ def search_form(context):
'submit_label': _(u'Search'),
'submit_icon_famfam': 'zoom',
})
return context
return ''
@register.inclusion_tag('generic_subtemplate.html', takes_context=True)
def recent_searches_template(context):
recent_searches = RecentSearch.objects.filter(user=context['user'])
recent_searches = RecentSearch.objects.get_for_user(user=context['user'])
context.update({
'request': context['request'],
'STATIC_URL': context['STATIC_URL'],
@@ -33,10 +36,10 @@ def recent_searches_template(context):
'title': _(u'recent searches (maximum of %d)') % RECENT_COUNT,
'paragraphs': [
u'<a href="%(url)s"><span class="famfam active famfam-%(icon)s"></span>%(text)s</a>' % {
'text': rs,
'url': rs.url(),
'icon': 'zoom_in' if rs.is_advanced() else 'zoom',
} for rs in recent_searches
'text': recent_search,
'url': recent_search.get_absolute_url(),
'icon': 'zoom_in' if recent_search.is_advanced() else 'zoom',
} for recent_search in recent_searches
]
})
return context

9
apps/dynamic_search/urls.py
View File

@@ -1,8 +1,9 @@
from django.conf.urls.defaults import patterns, url
from haystack.forms import SearchForm
from .views import CustomSearchView
urlpatterns = patterns('dynamic_search.views',
url(r'^$', 'search', (), 'search'),
url(r'^advanced/$', 'search', {'advanced': True}, 'search_advanced'),
url(r'^again/$', 'search_again', (), 'search_again'),
url(r'^results/$', 'results', (), 'results'),
url(r'^$', CustomSearchView(form_class=SearchForm), (), 'search'),
)

139
apps/dynamic_search/views.py
View File

@@ -1,107 +1,54 @@
import urlparse
from __future__ import absolute_import
import datetime
from django.shortcuts import render_to_response
from django.template import RequestContext
from django.utils.translation import ugettext_lazy as _
from django.contrib import messages
from django.conf import settings
from django.http import HttpResponseRedirect
from django.core.urlresolvers import reverse
from django.utils.http import urlencode
from django.core.exceptions import PermissionDenied
from dynamic_search.models import RecentSearch
from dynamic_search.api import perform_search
from dynamic_search.forms import SearchForm, AdvancedSearchForm
from dynamic_search.conf.settings import SHOW_OBJECT_TYPE
from dynamic_search.conf.settings import LIMIT
from haystack.views import SearchView
from documents.permissions import PERMISSION_DOCUMENT_VIEW
from permissions import Permission
from acls.models import AccessEntry
from .models import RecentSearch
def results(request, extra_context=None):
context = {}
class CustomSearchView(SearchView):
def __call__(self, *args, **kwargs):
self.start_time = datetime.datetime.now()
context.update({
'query_string': request.GET,
#'hide_header': True,
'hide_links': True,
'multi_select_as_buttons': True,
'search_results_limit': LIMIT,
})
return super(CustomSearchView, self).__call__(*args, **kwargs)
try:
response = perform_search(request.GET)
if response['shown_result_count'] != response['result_count']:
title = _(u'results, (showing only %(shown_result_count)s out of %(result_count)s)') % {
'shown_result_count': response['shown_result_count'],
'result_count': response['result_count']}
else:
title = _(u'results')
def create_response(self):
"""
Generates the actual HttpResponse to send back to the user.
"""
object_list = self.results.values_list('object', flat=True)
try:
Permission.objects.check_permissions(self.request.user, [PERMISSION_DOCUMENT_VIEW])
except PermissionDenied:
if self.query:
object_list = AccessEntry.objects.filter_objects_by_access(PERMISSION_DOCUMENT_VIEW, self.request.user, object_list)
if extra_context:
context.update(extra_context)
query = urlencode(dict(request.GET.items()))
context = {
'query': self.query,
'form': self.form,
'object_list': object_list,
'suggestion': None,
'submit_label': _(u'Search'),
'submit_icon_famfam': 'zoom',
'form_title': _(u'Search'),
'form_hide_required_text': True,
'list_title': _(u'results for: %s') % self.query,
'hide_links': True,
'multi_select_as_buttons': True,
'elapsed_time': unicode(datetime.datetime.now() - self.start_time).split(':')[2],
}
if query:
RecentSearch.objects.add_query_for_user(request.user, query, response['result_count'])
RecentSearch.objects.add_query_for_user(self)
context.update({
'found_entries': response['model_list'],
'object_list': response['flat_list'],
'title': title,
'time_delta': response['elapsed_time'],
})
except Exception, e:
if settings.DEBUG:
raise
elif request.user.is_staff or request.user.is_superuser:
messages.error(request, _(u'Search error: %s') % e)
if SHOW_OBJECT_TYPE:
context.update({'extra_columns':
[{'name': _(u'type'), 'attribute': lambda x: x._meta.verbose_name[0].upper() + x._meta.verbose_name[1:]}]})
return render_to_response('search_results.html', context,
context_instance=RequestContext(request))
def search(request, advanced=False):
if advanced:
form = AdvancedSearchForm(data=request.GET)
return render_to_response('generic_form.html',
{
'form': form,
'title': _(u'advanced search'),
'form_action': reverse('results'),
'submit_method': 'GET',
'search_results_limit': LIMIT,
'submit_label': _(u'Search'),
'submit_icon_famfam': 'zoom',
},
context_instance=RequestContext(request)
)
else:
if request.GET.get('source') != 'sidebar':
# Don't include a form a top of the results if the search
# was originated from the sidebar search form
extra_context = {
'submit_label': _(u'Search'),
'submit_icon_famfam': 'zoom',
'form_title': _(u'Search'),
'form_hide_required_text': True,
}
if ('q' in request.GET) and request.GET['q'].strip():
query_string = request.GET['q']
form = SearchForm(initial={'q': query_string})
extra_context.update({'form': form})
return results(request, extra_context=extra_context)
else:
form = SearchForm()
extra_context.update({'form': form})
return results(request, extra_context=extra_context)
else:
# Already has a form with data, go to results
return results(request)
def search_again(request):
query = urlparse.urlparse(request.META.get('HTTP_REFERER', u'/')).query
return HttpResponseRedirect('%s?%s' % (reverse('search_advanced'), query))
context.update(self.extra_context())
return render_to_response(self.template, context, context_instance=self.context_class(self.request))

4
apps/metadata/views.py
View File

@@ -92,6 +92,7 @@ def metadata_edit(request, document_id=None, document_id_list=None):
messages.error(request, _(u'Error editing metadata for document %(document)s; %(error)s.') % {
'document': document, 'error': error})
else:
document.mark_indexable()
messages.success(request, _(u'Metadata for document %s edited successfully.') % document)
return HttpResponseRedirect(next)
@@ -149,6 +150,8 @@ def metadata_add(request, document_id=None, document_id_list=None):
else:
messages.warning(request, _(u'Metadata type: %(metadata_type)s already present in document %(document)s.') % {
'metadata_type': metadata_type, 'document': document})
document.mark_indexable()
if len(documents) == 1:
return HttpResponseRedirect(u'%s?%s' % (
@@ -236,6 +239,7 @@ def metadata_remove(request, document_id=None, document_id_list=None):
try:
document_metadata = DocumentMetadata.objects.get(document=document, metadata_type=metadata_type)
document_metadata.delete()
document.mark_indexable()
messages.success(request, _(u'Successfully remove metadata type: %(metadata_type)s from document: %(document)s.') % {
'metadata_type': metadata_type, 'document': document})
except:

32
apps/signaler/management/commands/update_index.py Normal file
View File

@@ -0,0 +1,32 @@
import logging
from optparse import make_option
from haystack.management.commands import update_index
from signaler.signals import post_update_index, pre_update_index
logger = logging.getLogger(__name__)
MAYAN_RUNTIME = 'mayan_runtime'
MAYAN_RUNTIME_OPT = '--%s' % MAYAN_RUNTIME
class Command(update_index.Command):
"""
Wrapper for the haystack's update_index command
"""
option_list = update_index.Command.option_list + (
make_option(None, MAYAN_RUNTIME_OPT, action='store_true', dest=MAYAN_RUNTIME, default=False),
)
def handle(self, *args, **kwargs):
mayan_runtime = kwargs.pop(MAYAN_RUNTIME)
args = list(args)
if MAYAN_RUNTIME_OPT in args:
# Being called from another app
mayan_runtime = True
args.remove(MAYAN_RUNTIME_OPT)
logger.debug('mayan_runtime: %s' % mayan_runtime)
pre_update_index.send(sender=self, mayan_runtime=mayan_runtime)
super(Command, self).handle(*args, **kwargs)
post_update_index.send(sender=self, mayan_runtime=mayan_runtime)

2
apps/signaler/signals.py
View File

@@ -1,3 +1,5 @@
from django.dispatch import Signal
pre_collectstatic = Signal()
pre_update_index = Signal(providing_args=['mayan_runtime'])
post_update_index = Signal(providing_args=['mayan_runtime'])

7
apps/tags/views.py
View File

@@ -78,7 +78,7 @@ def tag_attach(request, document_id):
return HttpResponseRedirect(next)
document.tags.add(tag)
document.mark_indexable()
messages.success(request, _(u'Tag "%s" attached successfully.') % tag)
return HttpResponseRedirect(next)
else:
@@ -141,6 +141,8 @@ def tag_delete(request, tag_id=None, tag_id_list=None):
if request.method == 'POST':
for tag in tags:
try:
for document in Document.objects.filter(tags__in=[tag]):
document.mark_indexable()
tag.delete()
messages.success(request, _(u'Tag "%s" deleted successfully.') % tag)
except Exception, e:
@@ -188,6 +190,8 @@ def tag_edit(request, tag_id):
if form.is_valid():
tag.name = form.cleaned_data['name']
tag.save()
for document in Document.objects.filter(tags__in=[tag]):
document.mark_indexable()
tag_properties = tag.tagproperties_set.get()
tag_properties.color = form.cleaned_data['color']
tag_properties.save()
@@ -264,6 +268,7 @@ def tag_remove(request, document_id, tag_id=None, tag_id_list=None):
for tag in tags:
try:
document.tags.remove(tag)
document.mark_indexable()
messages.success(request, _(u'Tag "%s" removed successfully.') % tag)
except Exception, e:
messages.error(request, _(u'Error deleting tag "%(tag)s": %(error)s') % {

31
docs/releases/0.13.rst
View File

@@ -13,11 +13,13 @@ Overview
* Send document or document links via E-mail
* Document previews and page previews navigation improvements
* Ability to delete detached signatures
* Specialized search via Haystack
What's new in Mayan EDMS v0.13
==============================
E-mail document source
~~~~~~~~~~~~~~~~~~~~~~
POP3 and IMAP
@@ -27,6 +29,14 @@ Send document or document links via E-mail
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Haystack
~~~~~~~~
Haystack is setup to use Whoosh by default but user can choose to use
any of the other search backends supported by Haystack. The interval
between search index updates is controlled with the new :setting:`SEARCH_INDEX_UPDATE_INTERVAL`
configuration option. This addition closes issues #20 and #12.
Upgrading from a previous version
=================================
@@ -42,20 +52,31 @@ Afterwards migrate existing database schema with::
$ ./manage.py migrate sources 0001 --fake
$ ./manage.py migrate sources
$ ./manage.py migrate dynamic_search 0001 --fake
$ ./manage.py migrate dynamic_search
The upgrade procedure is now complete.
Issue the following command to index existing documents in the new full text search database::
$ ./manage.py rebuild_index --noinput
Depending on the existing size of you current document base this make take from a few minutes up to an hour.
Once the full text initial indexing ends, the upgrade procedure is complete.
Backward incompatible changes
=============================
None so far
None
Bugs fixed
==========
* Metadata unicode fixes, while creating new documents.
* Update indexes when renaming document
* Metadata unicode fixes while creating new documents.
* Fixed document index link filename when renaming document the source document.
* Issues #20 and #12 with the addition of a dedicated search solution and
not rely on SQL based searches.
Stuff removed
=============
None so far
Removal of the internal SEARCH_SHOW_OBJECT_TYPE option
Removal of the SEARCH_LIMIT configuration option

8
docs/topics/settings.rst
View File

@@ -461,13 +461,13 @@ Allow non authenticated users, access to all views.
Search
======
.. setting:: SEARCH_LIMIT
.. setting:: SEARCH_INDEX_UPDATE_INTERVAL
**SEARCH_LIMIT**
**SEARCH_INDEX_UPDATE_INTERVAL**
Default: ``100``
Default: ``1800``
Maximum amount search hits to fetch and display.
Interval in second on which to trigger the search index update.
.. setting:: SEARCH_RECENT_COUNT

8
requirements/development.txt
View File

@@ -1,4 +1,4 @@
Werkzeug==0.6.2
django-extensions==0.7.1
django-rosetta==0.6.2
transifex-client==0.6.1
Werkzeug==0.8.3
django-extensions==0.8
django-rosetta==0.6.6
transifex-client==0.7.2

45
requirements/production.txt
View File

@@ -1,19 +1,60 @@
# Base
Django==1.3.1
django-pagination==1.0.7
wsgiref==0.1.2
# Tags
django-taggit==0.9.3
# Indexes
-e git://github.com/django-mptt/django-mptt.git@0af02a95877041b2fd6d458bd95413dc1666c321#egg=django-mptt
# Mimetypes
-e git://github.com/ahupp/python-magic.git@a75cf0a4a7790eb106155c947af9612f15693b6e#egg=python-magic
slate==0.3
ghostscript==0.4.1
pdfminer==20110227
# Scheduler
APScheduler==2.0.2
# Python converter backend
Pillow==1.7.4
ghostscript==0.4.1
# Asset compression
cssmin==0.1.4
django-compressor==1.1.1
# Sendfile
-e git://github.com/rosarior/django-sendfile.git#egg=django-sendfile
# API
djangorestframework==0.2.3
# Migrations
South==0.7.5
# Keys and signing
https://github.com/rosarior/python-gnupg/zipball/0.2.8
python-hkp==0.1.3
# Feedback
requests==0.10.1
# Search
-e git+https://github.com/toastdriven/django-haystack.git@c6fd81d4163eb816476a853d416b68757e0c7ca4#egg=django_haystack-dev
Whoosh==2.3.2
Unidecode==0.04.9

8
settings.py
View File

@@ -131,6 +131,7 @@ INSTALLED_APPS = (
# 3rd party
# South
'south',
'haystack',
# Others
'filetransfers',
'taggit',
@@ -198,6 +199,13 @@ COMPRESS_CSS_FILTERS = ['compressor.filters.css_default.CssAbsoluteFilter', 'com
COMPRESS_ENABLED=False
SENDFILE_BACKEND = 'sendfile.backends.simple'
#--------- Haystack -----------
HAYSTACK_CONNECTIONS = {
'default': {
'ENGINE': 'haystack.backends.whoosh_backend.WhooshEngine',
'PATH': os.path.join(os.path.dirname(__file__), 'whoosh_index'),
},
}
#--------- Web theme ---------------
WEB_THEME_ENABLE_SCROLL_JS = False
#--------- Django -------------------