diff --git a/apps/acls/managers.py b/apps/acls/managers.py index 6ba807a897..529643c7ec 100644 --- a/apps/acls/managers.py +++ b/apps/acls/managers.py @@ -137,14 +137,14 @@ class AccessEntryManager(models.Manager): content_type = ContentType.objects.get_for_model(cls) # Calculate actor role membership ACL query - total_queries = None + total_queries = Q() for role in RoleMember.objects.get_roles_for_member(actor): role_type = ContentType.objects.get_for_model(role) if related: query = Q(holder_type=role_type, holder_id=role.pk, permission=permission.get_stored_permission) else: query = Q(holder_type=role_type, holder_id=role.pk, content_type=content_type, permission=permission.get_stored_permission) - if total_queries is None: + if not total_queries: total_queries = query else: total_queries = total_queries | query @@ -161,7 +161,7 @@ class AccessEntryManager(models.Manager): query = Q(holder_type=group_type, holder_id=group.pk, permission=permission.get_stored_permission) else: query = Q(holder_type=group_type, holder_id=group.pk, content_type=content_type, permission=permission.get_stored_permission) - if total_queries is None: + if not total_queries: total_queries = query else: total_queries = total_queries | query diff --git a/apps/common/templates/generic_list_subtemplate.html b/apps/common/templates/generic_list_subtemplate.html index d1494312bf..1e0868a227 100644 --- a/apps/common/templates/generic_list_subtemplate.html +++ b/apps/common/templates/generic_list_subtemplate.html @@ -6,6 +6,7 @@ {% load variable_tags %} {% load main_settings_tags %} {% load multiselect_tags %} +{% load subtemplates_tags %} {% get_main_setting "DISABLE_ICONS" as disable_icons %} @@ -76,9 +77,13 @@ {{ column.name|capfirst }} {% endfor %} - {% for column in object_list.0|get_model_list_columns %} - {{ column.name|capfirst }} - {% endfor %} + {% if object_list %} + {% get_object_list_object_name object_list.0 %} + + {% for column in object|get_model_list_columns %} + {{ column.name|capfirst }} + {% endfor %} + {% endif %} {% for column in extra_columns %} {{ column.name|capfirst }} @@ -89,15 +94,17 @@ {% endif %} {% endif %} + {% for object in object_list %} + {% get_object_list_object_name object %} {% if multi_select or multi_select_as_buttons %} - {% if multi_select_item_properties %} - - {% else %} - - {% endif %} + {% if multi_select_item_properties %} + + {% else %} + + {% endif %} {% endif %} {% if not hide_object %} diff --git a/apps/common/templatetags/subtemplates_tags.py b/apps/common/templatetags/subtemplates_tags.py index d759df4bbb..228c5ce8af 100644 --- a/apps/common/templatetags/subtemplates_tags.py +++ b/apps/common/templatetags/subtemplates_tags.py @@ -1,9 +1,11 @@ import re -from django.template import Node, TemplateSyntaxError, Library, \ - Variable, Context +from django.template import (Node, TemplateSyntaxError, Library, + Variable, Context) from django.template.loader import get_template +from common.utils import return_attrib + register = Library() @@ -49,3 +51,15 @@ def render_subtemplate(parser, token): return RenderSubtemplateNode(template_name, template_context, var_name) #format_string[1:-1] + + +@register.simple_tag(takes_context=True) +def get_object_list_object_name(context, source_object): + object_list_object_name = context.get('object_list_object_name') + if object_list_object_name: + context['object'] = return_attrib(source_object, object_list_object_name) + else: + context['object'] = source_object + + return '' + diff --git a/apps/document_comments/views.py b/apps/document_comments/views.py index 4a87435fa3..005ee109cd 100644 --- a/apps/document_comments/views.py +++ b/apps/document_comments/views.py @@ -43,6 +43,7 @@ def comment_delete(request, comment_id=None, comment_id_list=None): for comment in comments: try: comment.delete() + comment.content_object.mark_indexable() messages.success(request, _(u'Comment "%s" deleted successfully.') % comment) except Exception, e: messages.error(request, _(u'Error deleting comment "%(comment)s": %(error)s') % { @@ -95,7 +96,8 @@ def comment_add(request, document_id): comment.object_pk = document.pk comment.site = Site.objects.get_current() comment.save() - + document.mark_indexable() + messages.success(request, _(u'Comment added successfully.')) return HttpResponseRedirect(next) else: @@ -110,9 +112,9 @@ def comment_add(request, document_id): def comments_for_document(request, document_id): - ''' + """ Show a list of all the comments related to the passed object - ''' + """ document = get_object_or_404(Document, pk=document_id) try: diff --git a/apps/documents/models.py b/apps/documents/models.py index e3ad7be96a..01e8d6c809 100644 --- a/apps/documents/models.py +++ b/apps/documents/models.py @@ -13,13 +13,14 @@ try: except ImportError: from StringIO import StringIO +from unidecode import unidecode + from django.db import models from django.utils.translation import ugettext_lazy as _ from django.utils.translation import ugettext from django.contrib.auth.models import User from django.core.exceptions import ValidationError -from dynamic_search.api import register from converter.api import get_page_count from converter.api import get_available_transformations_choices from converter.api import convert @@ -102,6 +103,7 @@ class Document(models.Model): self.uuid = UUID_FUNCTION() self.date_added = datetime.datetime.now() super(Document, self).save(*args, **kwargs) + self.mark_indexable() def get_cached_image_name(self, page, version): document_version = DocumentVersion.objects.get(pk=version) @@ -275,8 +277,22 @@ class Document(models.Model): version.filename = value return version.save() + @property + def content(self): + return self.latest_version.content + filename = property(_get_filename, _set_filename) + @property + def cleaned_filename(self): + return unidecode(self.extension_split[0]) + + @property + def extension_split(self): + filename, extension = os.path.splitext(self.filename) + return filename, extension[1:] + + class DocumentVersion(models.Model): """ @@ -524,6 +540,15 @@ class DocumentVersion(models.Model): self.filename = u''.join([new_name, extension]) self.save() + @property + def content(self): + content = [] + for page in self.document.pages.all(): + if page.content: + content.append(page.content) + + return u''.join(content) + class DocumentTypeFilename(models.Model): """ @@ -657,18 +682,3 @@ class RecentDocument(models.Model): ordering = ('-datetime_accessed',) verbose_name = _(u'recent document') verbose_name_plural = _(u'recent documents') - - -# Register the fields that will be searchable -register('document', Document, _(u'document'), [ - {'name': u'document_type__name', 'title': _(u'Document type')}, - {'name': u'documentversion__mimetype', 'title': _(u'MIME type')}, - {'name': u'documentversion__filename', 'title': _(u'Filename')}, - {'name': u'documentmetadata__value', 'title': _(u'Metadata value')}, - {'name': u'documentversion__documentpage__content', 'title': _(u'Content')}, - {'name': u'description', 'title': _(u'Description')}, - {'name': u'tags__name', 'title': _(u'Tags')}, - {'name': u'comments__comment', 'title': _(u'Comments')}, - ] -) -#register(Document, _(u'document'), ['document_type__name', 'file_mimetype', 'documentmetadata__value', 'documentpage__content', 'description', {'field_name':'file_filename', 'comparison':'iexact'}]) diff --git a/apps/dynamic_search/__init__.py b/apps/dynamic_search/__init__.py index 2d3d73c5f4..cb35778be2 100644 --- a/apps/dynamic_search/__init__.py +++ b/apps/dynamic_search/__init__.py @@ -1,14 +1,63 @@ +from __future__ import absolute_import + +import logging + from django.utils.translation import ugettext_lazy as _ +from django.dispatch import receiver + +from django.core.management import call_command from navigation.api import register_sidebar_template, register_links +from documents.models import Document +from scheduler.runtime import scheduler +from signaler.signals import post_update_index, pre_update_index +from scheduler.api import register_interval_job +from lock_manager import Lock, LockError + +from .models import IndexableObject +from .conf.settings import INDEX_UPDATE_INTERVAL + +logger = logging.getLogger(__name__) search = {'text': _(u'search'), 'view': 'search', 'famfam': 'zoom'} -search_advanced = {'text': _(u'advanced search'), 'view': 'search_advanced', 'famfam': 'zoom_in'} -search_again = {'text': _(u'search again'), 'view': 'search_again', 'famfam': 'arrow_undo'} -register_sidebar_template(['search', 'search_advanced'], 'search_help.html') +register_sidebar_template(['search'], 'search_help.html') -register_links(['search', 'search_advanced', 'results'], [search, search_advanced], menu_name='form_header') -register_links(['results'], [search_again], menu_name='sidebar') +register_links(['search'], [search], menu_name='form_header') -register_sidebar_template(['search', 'search_advanced', 'results'], 'recent_searches.html') +register_sidebar_template(['search'], 'recent_searches.html') + +Document.add_to_class('mark_indexable', lambda obj: IndexableObject.objects.mark_indexable(obj)) + + +@receiver(pre_update_index, dispatch_uid='scheduler_shutdown_pre_update_index') +def scheduler_shutdown_pre_update_index(sender, mayan_runtime, **kwargs): + logger.debug('Scheduler shut down on pre update index signal') + logger.debug('Runtime variable: %s' % mayan_runtime) + # Only shutdown the scheduler if the command is called from the command + # line + if not mayan_runtime: + scheduler.shutdown() + + +def search_index_update(): + lock_id = u'search_index_update' + try: + logger.debug('trying to acquire lock: %s' % lock_id) + lock = Lock.acquire_lock(lock_id) + logger.debug('acquired lock: %s' % lock_id) + + logger.debug('Executing haystack\'s index update command') + call_command('update_index', '--mayan_runtime') + + lock.release() + except LockError: + logger.debug('unable to obtain lock') + pass + except Exception, instance: + logger.debug('Unhandled exception: %s' % instance) + lock.release() + pass + + +register_interval_job('search_index_update', _(u'Update the search index with the most recent modified documents.'), search_index_update, seconds=INDEX_UPDATE_INTERVAL) diff --git a/apps/dynamic_search/admin.py b/apps/dynamic_search/admin.py index 3592e29bc6..504f7c2160 100644 --- a/apps/dynamic_search/admin.py +++ b/apps/dynamic_search/admin.py @@ -1,6 +1,8 @@ +from __future__ import absolute_import + from django.contrib import admin -from dynamic_search.models import RecentSearch +from .models import RecentSearch, IndexableObject class RecentSearchAdmin(admin.ModelAdmin): @@ -9,3 +11,4 @@ class RecentSearchAdmin(admin.ModelAdmin): readonly_fields = ('user', 'query', 'datetime_created', 'hits') admin.site.register(RecentSearch, RecentSearchAdmin) +admin.site.register(IndexableObject) diff --git a/apps/dynamic_search/api.py b/apps/dynamic_search/api.py deleted file mode 100644 index fdf098ba8c..0000000000 --- a/apps/dynamic_search/api.py +++ /dev/null @@ -1,138 +0,0 @@ -# original code from: -# http://www.julienphalip.com/blog/2008/08/16/adding-search-django-site-snap/ - -import re -import types -import datetime - -from django.db.models import Q - -from dynamic_search.conf.settings import LIMIT - -registered_search_dict = {} - - -def register(model_name, model, title, fields): - registered_search_dict.setdefault(model_name, {'model': model, 'fields': [], 'title': title}) - registered_search_dict[model_name]['fields'].extend(fields) - - -def normalize_query(query_string, - findterms=re.compile(r'"([^"]+)"|(\S+)').findall, - normspace=re.compile(r'\s{2,}').sub): - """ - Splits the query string in invidual keywords, getting rid of unecessary spaces - and grouping quoted words together. - Example: - >>> normalize_query(' some random words "with quotes " and spaces') - ['some', 'random', 'words', 'with quotes', 'and', 'spaces'] - """ - return [normspace(' ', (t[0] or t[1]).strip()) for t in findterms(query_string)] - - -def get_query(terms, search_fields): - """ - Returns a query, that is a combination of Q objects. That combination - aims to search keywords within a model by testing the given search fields. - """ - queries = [] - for term in terms: - or_query = None - for field in search_fields: - if isinstance(field, types.StringTypes): - comparison = u'icontains' - field_name = field - elif isinstance(field, types.DictType): - comparison = field.get('comparison', u'icontains') - field_name = field.get('field_name', u'') - - if field_name: - q = Q(**{'%s__%s' % (field_name, comparison): term}) - if or_query is None: - or_query = q - else: - or_query = or_query | q - - queries.append(or_query) - return queries - - -def perform_search(query_string, field_list=None): - model_list = {} - flat_list = [] - result_count = 0 - shown_result_count = 0 - elapsed_time = 0 - start_time = datetime.datetime.now() - - search_dict = {} - - if query_string: - simple_query_string = query_string.get('q', u'').strip() - if simple_query_string: - for model, values in registered_search_dict.items(): - search_dict.setdefault(values['model'], {'query_entries': [], 'title': values['title']}) - field_names = [field['name'] for field in values['fields']] - # One entry, single set of terms for all fields names - search_dict[values['model']]['query_entries'].append( - { - 'field_name': field_names, - 'terms': normalize_query(simple_query_string) - } - ) - else: - for key, value in query_string.items(): - try: - model, field_name = key.split('__', 1) - model_entry = registered_search_dict.get(model, {}) - if model_entry: - for model_field in model_entry.get('fields', [{}]): - if model_field.get('name') == field_name: - search_dict.setdefault(model_entry['model'], {'query_entries': [], 'title': model_entry['title']}) - search_dict[model_entry['model']]['query_entries'].append( - { - 'field_name': [field_name], - 'terms': normalize_query(value.strip()) - } - ) - except ValueError: - pass - - for model, data in search_dict.items(): - title = data['title'] - queries = [] - - for query_entry in data['query_entries']: - queries.extend(get_query(query_entry['terms'], query_entry['field_name'])) - - model_result_ids = None - for query in queries: - single_result_ids = set(model.objects.filter(query).values_list('pk', flat=True)) - #Convert queryset to python set and perform the - #AND operation on the program and not as a query - if model_result_ids == None: - model_result_ids = single_result_ids - else: - model_result_ids &= single_result_ids - - if model_result_ids == None: - model_result_ids = [] - - result_count += len(model_result_ids) - results = model.objects.in_bulk(list(model_result_ids)[: LIMIT]).values() - shown_result_count += len(results) - if results: - model_list[title] = results - for result in results: - if result not in flat_list: - flat_list.append(result) - - elapsed_time = unicode(datetime.datetime.now() - start_time).split(':')[2] - - return { - 'model_list': model_list, - 'flat_list': flat_list, - 'shown_result_count': shown_result_count, - 'result_count': result_count, - 'elapsed_time': elapsed_time - } diff --git a/apps/dynamic_search/conf/settings.py b/apps/dynamic_search/conf/settings.py index 78ff433087..1c7c07559f 100644 --- a/apps/dynamic_search/conf/settings.py +++ b/apps/dynamic_search/conf/settings.py @@ -6,22 +6,6 @@ from smart_settings.api import Setting, SettingNamespace namespace = SettingNamespace('dynamic_search', _(u'Searching'), module='dynamic_search.conf.settings') -Setting( - namespace=namespace, - name='SHOW_OBJECT_TYPE', - global_name='SEARCH_SHOW_OBJECT_TYPE', - default=True, - hidden=True -) - -Setting( - namespace=namespace, - name='LIMIT', - global_name='SEARCH_LIMIT', - default=100, - description=_(u'Maximum amount search hits to fetch and display.') -) - Setting( namespace=namespace, name='RECENT_COUNT', @@ -29,3 +13,11 @@ Setting( default=5, description=_(u'Maximum number of search queries to remember per user.') ) + +Setting( + namespace=namespace, + name='INDEX_UPDATE_INTERVAL', + global_name='SEARCH_INDEX_UPDATE_INTERVAL', + default=1800, + description=_(u'Interval in second on which to trigger the search index update.') +) diff --git a/apps/dynamic_search/forms.py b/apps/dynamic_search/forms.py deleted file mode 100644 index 82fc2f6c9d..0000000000 --- a/apps/dynamic_search/forms.py +++ /dev/null @@ -1,25 +0,0 @@ -from django import forms -from django.utils.translation import ugettext_lazy as _ - -from dynamic_search.api import registered_search_dict - - -class SearchForm(forms.Form): - q = forms.CharField(max_length=128, label=_(u'Search terms')) - source = forms.CharField( - max_length=32, - required=False, - widget=forms.widgets.HiddenInput() - ) - - -class AdvancedSearchForm(forms.Form): - def __init__(self, *args, **kwargs): - super(AdvancedSearchForm, self).__init__(*args, **kwargs) - - for model_name, values in registered_search_dict.items(): - for field in values['fields']: - self.fields['%s__%s' % (model_name, field['name'])] = forms.CharField( - label=field['title'], - required=False - ) diff --git a/apps/dynamic_search/managers.py b/apps/dynamic_search/managers.py index e825bc7350..57a4ad30ab 100644 --- a/apps/dynamic_search/managers.py +++ b/apps/dynamic_search/managers.py @@ -1,32 +1,47 @@ -import urlparse +from __future__ import absolute_import -from django.db import models -from django.utils.http import urlencode +from urlparse import urlparse, parse_qs +from urllib import unquote_plus + +from django.utils.simplejson import dumps +from django.db.models import Manager from django.contrib.auth.models import AnonymousUser - -from dynamic_search.conf.settings import RECENT_COUNT +from django.contrib.contenttypes.models import ContentType +from django.utils.encoding import smart_str + +from .conf.settings import RECENT_COUNT -class RecentSearchManager(models.Manager): - def add_query_for_user(self, user, query, hits): - parsed_query = urlparse.parse_qs(query) - for key, value in parsed_query.items(): - parsed_query[key] = ' '.join(value) +class RecentSearchManager(Manager): + def add_query_for_user(self, search_view): + query_dict = parse_qs(unquote_plus(smart_str(urlparse(search_view.request.get_full_path()).query))) - if 'q=' in query: - # Is a simple query - if not parsed_query.get('q'): - # Don't store empty simple searches - return - else: - # Cleanup query string and only store the q parameter - parsed_query = {'q': parsed_query['q']} - - if parsed_query and not isinstance(user, AnonymousUser): + if query_dict and not isinstance(search_view.request.user, AnonymousUser): # If the URL query has at least one variable with a value - new_recent, created = self.model.objects.get_or_create(user=user, query=urlencode(parsed_query), defaults={'hits': hits}) - new_recent.hits = hits + new_recent, created = self.model.objects.get_or_create(user=search_view.request.user, query=dumps(query_dict), defaults={'hits': 0}) + new_recent.hits = search_view.results.count() new_recent.save() - to_delete = self.model.objects.filter(user=user)[RECENT_COUNT:] + to_delete = self.model.objects.filter(user=search_view.request.user)[RECENT_COUNT:] for recent_to_delete in to_delete: recent_to_delete.delete() + + def get_for_user(self, user): + return [entry for entry in self.model.objects.filter(user=user) if entry.get_query()] + + +class IndexableObjectManager(Manager): + def get_indexables(self, datetime=None): + if datetime: + return self.model.objects.filter(datetime__gte=datetime) + else: + return self.model.objects.all() + + def get_indexables_pk_list(self, datetime=None): + return self.get_indexables(datetime).values_list('object_id', flat=True) + + def mark_indexable(self, obj): + content_type = ContentType.objects.get_for_model(obj) + self.model.objects.get_or_create(content_type=content_type, object_id=obj.pk) + + def clear_all(self): + self.model.objects.all().delete() diff --git a/apps/dynamic_search/migrations/0001_initial.py b/apps/dynamic_search/migrations/0001_initial.py new file mode 100644 index 0000000000..5f42fc047a --- /dev/null +++ b/apps/dynamic_search/migrations/0001_initial.py @@ -0,0 +1,74 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding model 'RecentSearch' + db.create_table('dynamic_search_recentsearch', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('user', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'])), + ('query', self.gf('django.db.models.fields.TextField')()), + ('datetime_created', self.gf('django.db.models.fields.DateTimeField')()), + ('hits', self.gf('django.db.models.fields.IntegerField')()), + )) + db.send_create_signal('dynamic_search', ['RecentSearch']) + + + def backwards(self, orm): + # Deleting model 'RecentSearch' + db.delete_table('dynamic_search_recentsearch') + + + models = { + 'auth.group': { + 'Meta': {'object_name': 'Group'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + 'auth.permission': { + 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + 'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'dynamic_search.recentsearch': { + 'Meta': {'ordering': "('-datetime_created',)", 'object_name': 'RecentSearch'}, + 'datetime_created': ('django.db.models.fields.DateTimeField', [], {}), + 'hits': ('django.db.models.fields.IntegerField', [], {}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'query': ('django.db.models.fields.TextField', [], {}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}) + } + } + + complete_apps = ['dynamic_search'] \ No newline at end of file diff --git a/apps/dynamic_search/migrations/0002_auto__add_indexableobject.py b/apps/dynamic_search/migrations/0002_auto__add_indexableobject.py new file mode 100644 index 0000000000..eeac9ecc5e --- /dev/null +++ b/apps/dynamic_search/migrations/0002_auto__add_indexableobject.py @@ -0,0 +1,80 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding model 'IndexableObject' + db.create_table('dynamic_search_indexableobject', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('datetime', self.gf('django.db.models.fields.DateTimeField')()), + ('content_type', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['contenttypes.ContentType'])), + ('object_id', self.gf('django.db.models.fields.PositiveIntegerField')()), + )) + db.send_create_signal('dynamic_search', ['IndexableObject']) + + + def backwards(self, orm): + # Deleting model 'IndexableObject' + db.delete_table('dynamic_search_indexableobject') + + + models = { + 'auth.group': { + 'Meta': {'object_name': 'Group'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + 'auth.permission': { + 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + 'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'dynamic_search.indexableobject': { + 'Meta': {'object_name': 'IndexableObject'}, + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'datetime': ('django.db.models.fields.DateTimeField', [], {}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'object_id': ('django.db.models.fields.PositiveIntegerField', [], {}) + }, + 'dynamic_search.recentsearch': { + 'Meta': {'ordering': "('-datetime_created',)", 'object_name': 'RecentSearch'}, + 'datetime_created': ('django.db.models.fields.DateTimeField', [], {}), + 'hits': ('django.db.models.fields.IntegerField', [], {}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'query': ('django.db.models.fields.TextField', [], {}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}) + } + } + + complete_apps = ['dynamic_search'] \ No newline at end of file diff --git a/apps/dynamic_search/migrations/__init__.py b/apps/dynamic_search/migrations/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/apps/dynamic_search/models.py b/apps/dynamic_search/models.py index 9e7cc552bc..6cae642507 100644 --- a/apps/dynamic_search/models.py +++ b/apps/dynamic_search/models.py @@ -1,22 +1,23 @@ -import urlparse -import urllib +from __future__ import absolute_import -from datetime import datetime +import datetime from django.db import models -from django.utils.translation import ugettext as _ from django.contrib.auth.models import User from django.core.urlresolvers import reverse -from django.utils.encoding import smart_unicode, smart_str +from django.utils.translation import ugettext_lazy as _ +from django.contrib.contenttypes.models import ContentType +from django.contrib.contenttypes import generic +from django.utils.simplejson import loads +from django.utils.http import urlencode -from dynamic_search.managers import RecentSearchManager -from dynamic_search.api import registered_search_dict +from .managers import RecentSearchManager, IndexableObjectManager class RecentSearch(models.Model): - ''' + """ Keeps a list of the n most recent search keywords for a given user - ''' + """ user = models.ForeignKey(User, verbose_name=_(u'user'), editable=False) query = models.TextField(verbose_name=_(u'query'), editable=False) datetime_created = models.DateTimeField(verbose_name=_(u'datetime created'), editable=False) @@ -25,38 +26,55 @@ class RecentSearch(models.Model): objects = RecentSearchManager() def __unicode__(self): - query_dict = urlparse.parse_qs(urllib.unquote_plus(smart_str(self.query))) - if 'q' in query_dict: - # Is a simple search - display_string = smart_unicode(' '.join(query_dict['q'])) - else: - # Advanced search - advanced_string = [] - for key, value in query_dict.items(): - # Get model name - model, field_name = key.split('__', 1) - model_entry = registered_search_dict.get(model, {}) - if model_entry: - # Find the field name title - for model_field in model_entry.get('fields', [{}]): - if model_field.get('name') == field_name: - advanced_string.append(u'%s: %s' % (model_field.get('title', model_field['name']), smart_unicode(' '.join(value)))) + return self.form_string() - display_string = u', '.join(advanced_string) - return u'%s (%s)' % (display_string, self.hits) + def form_string(self): + if self.is_advanced(): + return u'%s (%s)' % (self.get_query(), self.hits) + else: + return u'%s (%s)' % (u' '.join(self.get_query().get('q')), self.hits) def save(self, *args, **kwargs): - self.datetime_created = datetime.now() + self.datetime_created = datetime.datetime.now() super(RecentSearch, self).save(*args, **kwargs) - def url(self): - view = 'results' if self.is_advanced() else 'search' - return '%s?%s' % (reverse(view), self.query) + def get_query(self): + try: + return loads(self.query) + except ValueError: + return {} def is_advanced(self): - return 'q' not in urlparse.parse_qs(self.query) + return 'q' not in self.get_query() + + def get_absolute_url(self): + return '?'.join([reverse('search'), urlencode(self.get_query(), doseq=True)]) class Meta: ordering = ('-datetime_created',) verbose_name = _(u'recent search') verbose_name_plural = _(u'recent searches') + + +class IndexableObject(models.Model): + """ + Store a list of object links that have been modified and are + meant to be indexed in the next search index update + """ + datetime = models.DateTimeField(verbose_name=_(u'date time')) + content_type = models.ForeignKey(ContentType) + object_id = models.PositiveIntegerField() + content_object = generic.GenericForeignKey('content_type', 'object_id') + + objects = IndexableObjectManager() + + def __unicode__(self): + return unicode(self.content_object) + + def save(self, *args, **kwargs): + self.datetime = datetime.datetime.now() + super(IndexableObject, self).save(*args, **kwargs) + + class Meta: + verbose_name = _(u'indexable object') + verbose_name_plural = _(u'indexable objects') diff --git a/apps/dynamic_search/search_indexes.py b/apps/dynamic_search/search_indexes.py new file mode 100644 index 0000000000..b053ff7066 --- /dev/null +++ b/apps/dynamic_search/search_indexes.py @@ -0,0 +1,27 @@ +from __future__ import absolute_import + +import logging +import copy + +from haystack import indexes + +from documents.models import Document + +from .models import IndexableObject + +logger = logging.getLogger(__name__) + + +class DocumentIndex(indexes.SearchIndex, indexes.Indexable): + text = indexes.CharField(document=True, use_template=True) + + def get_model(self): + return Document + + def build_queryset(self, start_date=None, end_date=None): + indexable_query_set = IndexableObject.objects.get_indexables() + logger.debug('indexable_query_set: %s' % indexable_query_set) + object_list = copy.copy(self.get_model().objects.filter(pk__in=indexable_query_set.values_list('object_id', flat=True))) + logger.debug('object_list: %s' % object_list) + indexable_query_set.delete() + return object_list diff --git a/apps/dynamic_search/templates/search/indexes/documents/document_text.txt b/apps/dynamic_search/templates/search/indexes/documents/document_text.txt new file mode 100644 index 0000000000..77dfc80213 --- /dev/null +++ b/apps/dynamic_search/templates/search/indexes/documents/document_text.txt @@ -0,0 +1,22 @@ +{{ object.content }} + +{% for document_metadata in object.documentmetadata_set.all %} + {{ document_metadata.value }} +{% endfor %} + +{{ object.cleaned_filename }} +{{ object.filename }} +{{ object.extension_split.1 }} +{{ object.document_type }} +{{ object.file_mimetype }} +{{ object.description|default:' ' }} + +{% for comment in object.comments.all %} + {{ comment.comment }} +{% endfor %} + +{% for tag in object.tags.all %} + {{ tag }} +{% endfor %} + +{{ object.uuid }} diff --git a/apps/dynamic_search/templates/search/search.html b/apps/dynamic_search/templates/search/search.html new file mode 100644 index 0000000000..1e113f5a0d --- /dev/null +++ b/apps/dynamic_search/templates/search/search.html @@ -0,0 +1,25 @@ +{% extends 'base.html' %} +{% load i18n %} +{% load search_tags %} +{% block title %} :: {% trans "Search results" %}{% endblock %} + +{% block content %} + {% if form %} + {% include "search_results_subtemplate.html" %} + {% endif %} + {% if query %} + {% with list_title as title %} + {% include "generic_list_subtemplate.html" %} + {% endwith %} + {% endif %} + {% if not form and not query %} + {% include "generic_list_subtemplate.html" %} + {% endif %} +{% endblock %} + +{% block footer %} + {% if query %} + {% blocktrans %}Elapsed time: {{ elapsed_time }} seconds{% endblocktrans %} + {% endif %} +{% endblock %} + diff --git a/apps/dynamic_search/templates/search_results.html b/apps/dynamic_search/templates/search_results.html index 79ded014eb..f364ab0c45 100644 --- a/apps/dynamic_search/templates/search_results.html +++ b/apps/dynamic_search/templates/search_results.html @@ -12,6 +12,7 @@ {% if not form and not query_string %} {% include "generic_list_subtemplate.html" %} {% endif %} + {% endblock %} {% block footer %} diff --git a/apps/dynamic_search/templatetags/search_tags.py b/apps/dynamic_search/templatetags/search_tags.py index aa20e22149..e48852685f 100644 --- a/apps/dynamic_search/templatetags/search_tags.py +++ b/apps/dynamic_search/templatetags/search_tags.py @@ -1,10 +1,13 @@ +from __future__ import absolute_import + from django.core.urlresolvers import reverse from django.template import Library from django.utils.translation import ugettext as _ -from dynamic_search.forms import SearchForm -from dynamic_search.models import RecentSearch -from dynamic_search.conf.settings import RECENT_COUNT +from haystack.forms import SearchForm + +from ..models import RecentSearch +from ..conf.settings import RECENT_COUNT register = Library() @@ -20,12 +23,12 @@ def search_form(context): 'submit_label': _(u'Search'), 'submit_icon_famfam': 'zoom', }) - return context + return '' @register.inclusion_tag('generic_subtemplate.html', takes_context=True) def recent_searches_template(context): - recent_searches = RecentSearch.objects.filter(user=context['user']) + recent_searches = RecentSearch.objects.get_for_user(user=context['user']) context.update({ 'request': context['request'], 'STATIC_URL': context['STATIC_URL'], @@ -33,10 +36,10 @@ def recent_searches_template(context): 'title': _(u'recent searches (maximum of %d)') % RECENT_COUNT, 'paragraphs': [ u'%(text)s' % { - 'text': rs, - 'url': rs.url(), - 'icon': 'zoom_in' if rs.is_advanced() else 'zoom', - } for rs in recent_searches + 'text': recent_search, + 'url': recent_search.get_absolute_url(), + 'icon': 'zoom_in' if recent_search.is_advanced() else 'zoom', + } for recent_search in recent_searches ] }) return context diff --git a/apps/dynamic_search/urls.py b/apps/dynamic_search/urls.py index 8069a50342..aa1cd26796 100644 --- a/apps/dynamic_search/urls.py +++ b/apps/dynamic_search/urls.py @@ -1,8 +1,9 @@ from django.conf.urls.defaults import patterns, url +from haystack.forms import SearchForm + +from .views import CustomSearchView + urlpatterns = patterns('dynamic_search.views', - url(r'^$', 'search', (), 'search'), - url(r'^advanced/$', 'search', {'advanced': True}, 'search_advanced'), - url(r'^again/$', 'search_again', (), 'search_again'), - url(r'^results/$', 'results', (), 'results'), + url(r'^$', CustomSearchView(form_class=SearchForm), (), 'search'), ) diff --git a/apps/dynamic_search/views.py b/apps/dynamic_search/views.py index 0fff396b85..de9636a779 100644 --- a/apps/dynamic_search/views.py +++ b/apps/dynamic_search/views.py @@ -1,107 +1,54 @@ -import urlparse +from __future__ import absolute_import + +import datetime from django.shortcuts import render_to_response -from django.template import RequestContext from django.utils.translation import ugettext_lazy as _ -from django.contrib import messages -from django.conf import settings -from django.http import HttpResponseRedirect -from django.core.urlresolvers import reverse -from django.utils.http import urlencode +from django.core.exceptions import PermissionDenied -from dynamic_search.models import RecentSearch -from dynamic_search.api import perform_search -from dynamic_search.forms import SearchForm, AdvancedSearchForm -from dynamic_search.conf.settings import SHOW_OBJECT_TYPE -from dynamic_search.conf.settings import LIMIT +from haystack.views import SearchView + +from documents.permissions import PERMISSION_DOCUMENT_VIEW +from permissions import Permission +from acls.models import AccessEntry + +from .models import RecentSearch -def results(request, extra_context=None): - context = {} +class CustomSearchView(SearchView): + def __call__(self, *args, **kwargs): + self.start_time = datetime.datetime.now() - context.update({ - 'query_string': request.GET, - #'hide_header': True, - 'hide_links': True, - 'multi_select_as_buttons': True, - 'search_results_limit': LIMIT, - }) + return super(CustomSearchView, self).__call__(*args, **kwargs) - try: - response = perform_search(request.GET) - if response['shown_result_count'] != response['result_count']: - title = _(u'results, (showing only %(shown_result_count)s out of %(result_count)s)') % { - 'shown_result_count': response['shown_result_count'], - 'result_count': response['result_count']} - else: - title = _(u'results') + def create_response(self): + """ + Generates the actual HttpResponse to send back to the user. + """ + object_list = self.results.values_list('object', flat=True) + + try: + Permission.objects.check_permissions(self.request.user, [PERMISSION_DOCUMENT_VIEW]) + except PermissionDenied: + if self.query: + object_list = AccessEntry.objects.filter_objects_by_access(PERMISSION_DOCUMENT_VIEW, self.request.user, object_list) - if extra_context: - context.update(extra_context) - query = urlencode(dict(request.GET.items())) + context = { + 'query': self.query, + 'form': self.form, + 'object_list': object_list, + 'suggestion': None, + 'submit_label': _(u'Search'), + 'submit_icon_famfam': 'zoom', + 'form_title': _(u'Search'), + 'form_hide_required_text': True, + 'list_title': _(u'results for: %s') % self.query, + 'hide_links': True, + 'multi_select_as_buttons': True, + 'elapsed_time': unicode(datetime.datetime.now() - self.start_time).split(':')[2], + } - if query: - RecentSearch.objects.add_query_for_user(request.user, query, response['result_count']) + RecentSearch.objects.add_query_for_user(self) - context.update({ - 'found_entries': response['model_list'], - 'object_list': response['flat_list'], - 'title': title, - 'time_delta': response['elapsed_time'], - }) - except Exception, e: - if settings.DEBUG: - raise - elif request.user.is_staff or request.user.is_superuser: - messages.error(request, _(u'Search error: %s') % e) - - if SHOW_OBJECT_TYPE: - context.update({'extra_columns': - [{'name': _(u'type'), 'attribute': lambda x: x._meta.verbose_name[0].upper() + x._meta.verbose_name[1:]}]}) - - return render_to_response('search_results.html', context, - context_instance=RequestContext(request)) - - -def search(request, advanced=False): - if advanced: - form = AdvancedSearchForm(data=request.GET) - return render_to_response('generic_form.html', - { - 'form': form, - 'title': _(u'advanced search'), - 'form_action': reverse('results'), - 'submit_method': 'GET', - 'search_results_limit': LIMIT, - 'submit_label': _(u'Search'), - 'submit_icon_famfam': 'zoom', - }, - context_instance=RequestContext(request) - ) - else: - if request.GET.get('source') != 'sidebar': - # Don't include a form a top of the results if the search - # was originated from the sidebar search form - extra_context = { - 'submit_label': _(u'Search'), - 'submit_icon_famfam': 'zoom', - 'form_title': _(u'Search'), - 'form_hide_required_text': True, - } - if ('q' in request.GET) and request.GET['q'].strip(): - query_string = request.GET['q'] - form = SearchForm(initial={'q': query_string}) - extra_context.update({'form': form}) - return results(request, extra_context=extra_context) - else: - form = SearchForm() - extra_context.update({'form': form}) - return results(request, extra_context=extra_context) - else: - # Already has a form with data, go to results - return results(request) - - -def search_again(request): - query = urlparse.urlparse(request.META.get('HTTP_REFERER', u'/')).query - return HttpResponseRedirect('%s?%s' % (reverse('search_advanced'), query)) + context.update(self.extra_context()) + return render_to_response(self.template, context, context_instance=self.context_class(self.request)) diff --git a/apps/metadata/views.py b/apps/metadata/views.py index b380b81025..b5b834a081 100644 --- a/apps/metadata/views.py +++ b/apps/metadata/views.py @@ -92,6 +92,7 @@ def metadata_edit(request, document_id=None, document_id_list=None): messages.error(request, _(u'Error editing metadata for document %(document)s; %(error)s.') % { 'document': document, 'error': error}) else: + document.mark_indexable() messages.success(request, _(u'Metadata for document %s edited successfully.') % document) return HttpResponseRedirect(next) @@ -149,6 +150,8 @@ def metadata_add(request, document_id=None, document_id_list=None): else: messages.warning(request, _(u'Metadata type: %(metadata_type)s already present in document %(document)s.') % { 'metadata_type': metadata_type, 'document': document}) + + document.mark_indexable() if len(documents) == 1: return HttpResponseRedirect(u'%s?%s' % ( @@ -236,6 +239,7 @@ def metadata_remove(request, document_id=None, document_id_list=None): try: document_metadata = DocumentMetadata.objects.get(document=document, metadata_type=metadata_type) document_metadata.delete() + document.mark_indexable() messages.success(request, _(u'Successfully remove metadata type: %(metadata_type)s from document: %(document)s.') % { 'metadata_type': metadata_type, 'document': document}) except: diff --git a/apps/signaler/management/commands/update_index.py b/apps/signaler/management/commands/update_index.py new file mode 100644 index 0000000000..8f468c1086 --- /dev/null +++ b/apps/signaler/management/commands/update_index.py @@ -0,0 +1,32 @@ +import logging +from optparse import make_option + +from haystack.management.commands import update_index + +from signaler.signals import post_update_index, pre_update_index + +logger = logging.getLogger(__name__) + +MAYAN_RUNTIME = 'mayan_runtime' +MAYAN_RUNTIME_OPT = '--%s' % MAYAN_RUNTIME + + +class Command(update_index.Command): + """ + Wrapper for the haystack's update_index command + """ + option_list = update_index.Command.option_list + ( + make_option(None, MAYAN_RUNTIME_OPT, action='store_true', dest=MAYAN_RUNTIME, default=False), + ) + + def handle(self, *args, **kwargs): + mayan_runtime = kwargs.pop(MAYAN_RUNTIME) + args = list(args) + if MAYAN_RUNTIME_OPT in args: + # Being called from another app + mayan_runtime = True + args.remove(MAYAN_RUNTIME_OPT) + logger.debug('mayan_runtime: %s' % mayan_runtime) + pre_update_index.send(sender=self, mayan_runtime=mayan_runtime) + super(Command, self).handle(*args, **kwargs) + post_update_index.send(sender=self, mayan_runtime=mayan_runtime) diff --git a/apps/signaler/signals.py b/apps/signaler/signals.py index c102b00357..ae5c121049 100644 --- a/apps/signaler/signals.py +++ b/apps/signaler/signals.py @@ -1,3 +1,5 @@ from django.dispatch import Signal pre_collectstatic = Signal() +pre_update_index = Signal(providing_args=['mayan_runtime']) +post_update_index = Signal(providing_args=['mayan_runtime']) diff --git a/apps/tags/views.py b/apps/tags/views.py index a153506e02..1cf79abdf8 100644 --- a/apps/tags/views.py +++ b/apps/tags/views.py @@ -78,7 +78,7 @@ def tag_attach(request, document_id): return HttpResponseRedirect(next) document.tags.add(tag) - + document.mark_indexable() messages.success(request, _(u'Tag "%s" attached successfully.') % tag) return HttpResponseRedirect(next) else: @@ -141,6 +141,8 @@ def tag_delete(request, tag_id=None, tag_id_list=None): if request.method == 'POST': for tag in tags: try: + for document in Document.objects.filter(tags__in=[tag]): + document.mark_indexable() tag.delete() messages.success(request, _(u'Tag "%s" deleted successfully.') % tag) except Exception, e: @@ -188,6 +190,8 @@ def tag_edit(request, tag_id): if form.is_valid(): tag.name = form.cleaned_data['name'] tag.save() + for document in Document.objects.filter(tags__in=[tag]): + document.mark_indexable() tag_properties = tag.tagproperties_set.get() tag_properties.color = form.cleaned_data['color'] tag_properties.save() @@ -264,6 +268,7 @@ def tag_remove(request, document_id, tag_id=None, tag_id_list=None): for tag in tags: try: document.tags.remove(tag) + document.mark_indexable() messages.success(request, _(u'Tag "%s" removed successfully.') % tag) except Exception, e: messages.error(request, _(u'Error deleting tag "%(tag)s": %(error)s') % { diff --git a/docs/releases/0.13.rst b/docs/releases/0.13.rst index 9e11dcd124..194da2ecc1 100644 --- a/docs/releases/0.13.rst +++ b/docs/releases/0.13.rst @@ -13,11 +13,13 @@ Overview * Send document or document links via E-mail * Document previews and page previews navigation improvements * Ability to delete detached signatures +* Specialized search via Haystack What's new in Mayan EDMS v0.13 ============================== + E-mail document source ~~~~~~~~~~~~~~~~~~~~~~ POP3 and IMAP @@ -27,6 +29,14 @@ Send document or document links via E-mail ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Haystack +~~~~~~~~ +Haystack is setup to use Whoosh by default but user can choose to use +any of the other search backends supported by Haystack. The interval +between search index updates is controlled with the new :setting:`SEARCH_INDEX_UPDATE_INTERVAL` +configuration option. This addition closes issues #20 and #12. + + Upgrading from a previous version ================================= @@ -42,20 +52,31 @@ Afterwards migrate existing database schema with:: $ ./manage.py migrate sources 0001 --fake $ ./manage.py migrate sources + $ ./manage.py migrate dynamic_search 0001 --fake + $ ./manage.py migrate dynamic_search -The upgrade procedure is now complete. +Issue the following command to index existing documents in the new full text search database:: + + $ ./manage.py rebuild_index --noinput + +Depending on the existing size of you current document base this make take from a few minutes up to an hour. +Once the full text initial indexing ends, the upgrade procedure is complete. Backward incompatible changes ============================= -None so far +None + Bugs fixed ========== -* Metadata unicode fixes, while creating new documents. -* Update indexes when renaming document +* Metadata unicode fixes while creating new documents. +* Fixed document index link filename when renaming document the source document. +* Issues #20 and #12 with the addition of a dedicated search solution and + not rely on SQL based searches. Stuff removed ============= -None so far +Removal of the internal SEARCH_SHOW_OBJECT_TYPE option +Removal of the SEARCH_LIMIT configuration option diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index ae395a435e..b3af96d500 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -461,13 +461,13 @@ Allow non authenticated users, access to all views. Search ====== -.. setting:: SEARCH_LIMIT +.. setting:: SEARCH_INDEX_UPDATE_INTERVAL -**SEARCH_LIMIT** +**SEARCH_INDEX_UPDATE_INTERVAL** -Default: ``100`` +Default: ``1800`` -Maximum amount search hits to fetch and display. +Interval in second on which to trigger the search index update. .. setting:: SEARCH_RECENT_COUNT diff --git a/requirements/development.txt b/requirements/development.txt index 427ff2cf00..fafe9eeaa8 100644 --- a/requirements/development.txt +++ b/requirements/development.txt @@ -1,4 +1,4 @@ -Werkzeug==0.6.2 -django-extensions==0.7.1 -django-rosetta==0.6.2 -transifex-client==0.6.1 +Werkzeug==0.8.3 +django-extensions==0.8 +django-rosetta==0.6.6 +transifex-client==0.7.2 diff --git a/requirements/production.txt b/requirements/production.txt index 6666ede069..0e4c94e327 100644 --- a/requirements/production.txt +++ b/requirements/production.txt @@ -1,19 +1,60 @@ +# Base + Django==1.3.1 django-pagination==1.0.7 -wsgiref==0.1.2 + +# Tags + django-taggit==0.9.3 + +# Indexes + -e git://github.com/django-mptt/django-mptt.git@0af02a95877041b2fd6d458bd95413dc1666c321#egg=django-mptt + +# Mimetypes + -e git://github.com/ahupp/python-magic.git@a75cf0a4a7790eb106155c947af9612f15693b6e#egg=python-magic + slate==0.3 -ghostscript==0.4.1 pdfminer==20110227 + +# Scheduler + APScheduler==2.0.2 + +# Python converter backend + Pillow==1.7.4 +ghostscript==0.4.1 + +# Asset compression + cssmin==0.1.4 django-compressor==1.1.1 + +# Sendfile + -e git://github.com/rosarior/django-sendfile.git#egg=django-sendfile + +# API + djangorestframework==0.2.3 + +# Migrations + South==0.7.5 + +# Keys and signing + https://github.com/rosarior/python-gnupg/zipball/0.2.8 python-hkp==0.1.3 + +# Feedback + requests==0.10.1 + +# Search + +-e git+https://github.com/toastdriven/django-haystack.git@c6fd81d4163eb816476a853d416b68757e0c7ca4#egg=django_haystack-dev +Whoosh==2.3.2 +Unidecode==0.04.9 diff --git a/settings.py b/settings.py index c6894408c2..b1f4ba0929 100644 --- a/settings.py +++ b/settings.py @@ -131,6 +131,7 @@ INSTALLED_APPS = ( # 3rd party # South 'south', + 'haystack', # Others 'filetransfers', 'taggit', @@ -198,6 +199,13 @@ COMPRESS_CSS_FILTERS = ['compressor.filters.css_default.CssAbsoluteFilter', 'com COMPRESS_ENABLED=False SENDFILE_BACKEND = 'sendfile.backends.simple' +#--------- Haystack ----------- +HAYSTACK_CONNECTIONS = { + 'default': { + 'ENGINE': 'haystack.backends.whoosh_backend.WhooshEngine', + 'PATH': os.path.join(os.path.dirname(__file__), 'whoosh_index'), + }, +} #--------- Web theme --------------- WEB_THEME_ENABLE_SCROLL_JS = False #--------- Django -------------------