From 021c75c665ac77ed46dedc97fa7cd51f5dc01890 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Thu, 5 Apr 2012 23:43:10 -0400 Subject: [PATCH 01/46] Add a content producing method to the document version and document class --- apps/documents/models.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/apps/documents/models.py b/apps/documents/models.py index e3ad7be96a..d03be1df90 100644 --- a/apps/documents/models.py +++ b/apps/documents/models.py @@ -275,6 +275,10 @@ class Document(models.Model): version.filename = value return version.save() + @property + def content(self): + return self.latest_version.content + filename = property(_get_filename, _set_filename) @@ -524,6 +528,15 @@ class DocumentVersion(models.Model): self.filename = u''.join([new_name, extension]) self.save() + @property + def content(self): + content = [] + for page in self.document.pages.all(): + if page.content: + content.append(page.content) + + return u''.join(content) + class DocumentTypeFilename(models.Model): """ From 95cf290e2c537c488409f3730b1ed620ecfa45a0 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Thu, 5 Apr 2012 23:44:20 -0400 Subject: [PATCH 02/46] Add haystack to the requirements --- requirements/production.txt | 45 +++++++++++++++++++++++++++++++++++-- settings.py | 8 +++++++ urls.py | 1 + 3 files changed, 52 insertions(+), 2 deletions(-) diff --git a/requirements/production.txt b/requirements/production.txt index 41ab10ed77..14dc4aeb91 100644 --- a/requirements/production.txt +++ b/requirements/production.txt @@ -1,19 +1,60 @@ +# Base + Django==1.3.1 django-pagination==1.0.7 -wsgiref==0.1.2 + +# Tags + django-taggit==0.9.3 + +# Indexes + -e git://github.com/django-mptt/django-mptt.git@0af02a95877041b2fd6d458bd95413dc1666c321#egg=django-mptt + +# Mimetypes + -e git://github.com/ahupp/python-magic.git@a75cf0a4a7790eb106155c947af9612f15693b6e#egg=python-magic + slate==0.3 -ghostscript==0.4.1 pdfminer==20110227 + +# Scheduler + APScheduler==2.0.2 + +# Python converter backend + Pillow==1.7.4 +ghostscript==0.4.1 + +# Asset compression + cssmin==0.1.4 django-compressor==1.1.1 + +# Sendfile + -e git://github.com/rosarior/django-sendfile.git#egg=django-sendfile + +# API + djangorestframework==0.2.3 + +# Migrations + South==0.7.3 + +# Keys and signing + python-gnupg==0.2.8 python-hkp==0.1.3 + +# Feedback + requests==0.10.1 + +# Search + +-e git+https://github.com/toastdriven/django-haystack.git@c6fd81d4163eb816476a853d416b68757e0c7ca4#egg=django_haystack-dev +Whoosh==2.3.2 +Unidecode==0.04.9 diff --git a/settings.py b/settings.py index c6894408c2..206f22c1c4 100644 --- a/settings.py +++ b/settings.py @@ -131,6 +131,7 @@ INSTALLED_APPS = ( # 3rd party # South 'south', + 'haystack', # Others 'filetransfers', 'taggit', @@ -198,6 +199,13 @@ COMPRESS_CSS_FILTERS = ['compressor.filters.css_default.CssAbsoluteFilter', 'com COMPRESS_ENABLED=False SENDFILE_BACKEND = 'sendfile.backends.simple' +#--------- Haystack ----------- +HAYSTACK_CONNECTIONS = { + 'default': { + 'ENGINE': 'haystack.backends.whoosh_backend.WhooshEngine', + 'PATH': os.path.join(os.path.dirname(__file__), 'whoosh_index'), + }, +} #--------- Web theme --------------- WEB_THEME_ENABLE_SCROLL_JS = False #--------- Django ------------------- diff --git a/urls.py b/urls.py index ee1f199449..80d8cdef7d 100644 --- a/urls.py +++ b/urls.py @@ -9,6 +9,7 @@ urlpatterns = patterns('', (r'^', include('main.urls')), (r'^documents/', include('documents.urls')), (r'^folders/', include('folders.urls')), + #(r'^search/', include('haystack.urls')), (r'^search/', include('dynamic_search.urls')), (r'^ocr/', include('ocr.urls')), (r'^permissions/', include('permissions.urls')), From 5dbceb5584a376639a4a2a97f730258d8d799eb8 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Thu, 5 Apr 2012 23:45:50 -0400 Subject: [PATCH 03/46] Initial changes to support haystack from the dynamic search app --- .../templates/search/search.html | 38 ++++++++++++++++++ .../templates/search_results.html | 2 + apps/dynamic_search/urls.py | 4 +- apps/dynamic_search/views.py | 40 +++++++++++++++++++ 4 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 apps/dynamic_search/templates/search/search.html diff --git a/apps/dynamic_search/templates/search/search.html b/apps/dynamic_search/templates/search/search.html new file mode 100644 index 0000000000..13cb21427e --- /dev/null +++ b/apps/dynamic_search/templates/search/search.html @@ -0,0 +1,38 @@ +{% extends 'base.html' %} +{% load i18n %} +{% load search_tags %} +{% block title %} :: {% trans "Search results" %}{% endblock %} + +{% block content %} + {% if form %} + {% include "search_results_subtemplate.html" %} + {% endif %} + {% if query %} + {% with list_title as title %} + {% include "generic_list_subtemplate.html" %} + {% endwith %} + {% endif %} + {% if not form and not query %} + {% include "generic_list_subtemplate.html" %} + {% endif %} + +{% comment %} + +{% for result in page.object_list %} + {% with result|search_include as fragment %} + {% include fragment %} + {% endwith %} +{% empty %} +

No results found.

+{% endfor %} + +{% endcomment %} + +{% endblock %} + +{% block footer %} + {% if query %} + {% blocktrans %}Elapsed time: {{ elapsed_time }} seconds{% endblocktrans %} + {% endif %} +{% endblock %} + diff --git a/apps/dynamic_search/templates/search_results.html b/apps/dynamic_search/templates/search_results.html index 79ded014eb..04c0dd1f8e 100644 --- a/apps/dynamic_search/templates/search_results.html +++ b/apps/dynamic_search/templates/search_results.html @@ -3,6 +3,7 @@ {% block title %} :: {% trans "Search results" %}{% endblock %} {% block content %} + {% if form %} {% include "search_results_subtemplate.html" %} {% endif %} @@ -12,6 +13,7 @@ {% if not form and not query_string %} {% include "generic_list_subtemplate.html" %} {% endif %} + {% endblock %} {% block footer %} diff --git a/apps/dynamic_search/urls.py b/apps/dynamic_search/urls.py index 8069a50342..ed2932609e 100644 --- a/apps/dynamic_search/urls.py +++ b/apps/dynamic_search/urls.py @@ -1,7 +1,9 @@ from django.conf.urls.defaults import patterns, url +from .views import CustomSearchView + urlpatterns = patterns('dynamic_search.views', - url(r'^$', 'search', (), 'search'), + url(r'^$', CustomSearchView(), (), 'search'), url(r'^advanced/$', 'search', {'advanced': True}, 'search_advanced'), url(r'^again/$', 'search_again', (), 'search_again'), url(r'^results/$', 'results', (), 'results'), diff --git a/apps/dynamic_search/views.py b/apps/dynamic_search/views.py index 0fff396b85..67820c7251 100644 --- a/apps/dynamic_search/views.py +++ b/apps/dynamic_search/views.py @@ -1,3 +1,4 @@ +import datetime import urlparse from django.shortcuts import render_to_response @@ -9,6 +10,8 @@ from django.http import HttpResponseRedirect from django.core.urlresolvers import reverse from django.utils.http import urlencode +from haystack.views import SearchView + from dynamic_search.models import RecentSearch from dynamic_search.api import perform_search from dynamic_search.forms import SearchForm, AdvancedSearchForm @@ -16,6 +19,43 @@ from dynamic_search.conf.settings import SHOW_OBJECT_TYPE from dynamic_search.conf.settings import LIMIT +class CustomSearchView(SearchView): + def __call__(self, *args, **kwargs): + self.start_time = datetime.datetime.now() + + return super(CustomSearchView, self).__call__(*args, **kwargs) + + def create_response(self): + """ + Generates the actual HttpResponse to send back to the user. + """ + #(paginator, page) = self.build_page() + + context = { + 'query': self.query, + 'form': self.form, + 'object_list': [result.object for result in self.results], + #'page': page, + #'paginator': paginator, + 'suggestion': None, + 'submit_label': _(u'Search'), + 'submit_icon_famfam': 'zoom', + 'form_title': _(u'Search'), + 'form_hide_required_text': True, + 'list_title': _(u'results for: %s') % self.query, + 'hide_links': True, + 'multi_select_as_buttons': True, + 'elapsed_time': unicode(datetime.datetime.now() - self.start_time).split(':')[2] + + } + + #if self.results and hasattr(self.results, 'query') and self.results.query.backend.include_spelling: + # context['suggestion'] = self.form.get_suggestion() + + context.update(self.extra_context()) + return render_to_response(self.template, context, context_instance=self.context_class(self.request)) + + def results(request, extra_context=None): context = {} From a5b88d3683f983f3fa6feb3f0dbb8dd59934ab25 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Thu, 5 Apr 2012 23:46:28 -0400 Subject: [PATCH 04/46] Add document search result templates --- apps/documents/search_indexes.py | 80 +++++++++++++++++++ .../indexes/documents/document_text.txt | 3 + 2 files changed, 83 insertions(+) create mode 100644 apps/documents/search_indexes.py create mode 100644 apps/documents/templates/search/indexes/documents/document_text.txt diff --git a/apps/documents/search_indexes.py b/apps/documents/search_indexes.py new file mode 100644 index 0000000000..4d51335446 --- /dev/null +++ b/apps/documents/search_indexes.py @@ -0,0 +1,80 @@ +from __future__ import absolute_import + +import datetime + +from unidecode import unidecode +from haystack import indexes + +from .models import Document + +''' + uuid = models.CharField(max_length=48, blank=True, editable=False) + document_type = models.ForeignKey(DocumentType, verbose_name=_(u'document type'), null=True, blank=True) + description = models.TextField(blank=True, null=True, verbose_name=_(u'description')) + date_added = models.DateTimeField(verbose_name=_(u'added'), db_index=True, editable=False) + document = models.ForeignKey(Document, verbose_name=_(u'document'), editable=False) + major = models.PositiveIntegerField(verbose_name=_(u'mayor'), default=1, editable=False) + minor = models.PositiveIntegerField(verbose_name=_(u'minor'), default=0, editable=False) + micro = models.PositiveIntegerField(verbose_name=_(u'micro'), default=0, editable=False) + release_level = models.PositiveIntegerField(choices=RELEASE_LEVEL_CHOICES, default=RELEASE_LEVEL_FINAL, verbose_name=_(u'release level'), editable=False) + serial = models.PositiveIntegerField(verbose_name=_(u'serial'), default=0, editable=False) + timestamp = models.DateTimeField(verbose_name=_(u'timestamp'), editable=False) + comment = models.TextField(blank=True, verbose_name=_(u'comment')) + + # File related fields + file = models.FileField(upload_to=get_filename_from_uuid, storage=STORAGE_BACKEND(), verbose_name=_(u'file')) + mimetype = models.CharField(max_length=64, default='', editable=False) + encoding = models.CharField(max_length=64, default='', editable=False) + filename = models.CharField(max_length=255, default=u'', editable=False, db_index=True) + checksum = models.TextField(blank=True, null=True, verbose_name=_(u'checksum'), editable=False) + +class DocumentPage(models.Model): + """ + Model that describes a document version page including it's content + """ + document_version = models.ForeignKey(DocumentVersion, verbose_name=_(u'document version')) + content = models.TextField(blank=True, null=True, verbose_name=_(u'content')) + page_label = models.CharField(max_length=32, blank=True, null=True, verbose_name=_(u'page label')) + page_number = models.PositiveIntegerField(default=1, editable=False, verbose_name=_(u'page number'), db_index=True) + + +register('document', Document, _(u'document'), [ + {'name': u'document_type__name', 'title': _(u'Document type')}, + {'name': u'documentversion__mimetype', 'title': _(u'MIME type')}, + {'name': u'documentversion__filename', 'title': _(u'Filename')}, + {'name': u'documentmetadata__value', 'title': _(u'Metadata value')}, + {'name': u'documentversion__documentpage__content', 'title': _(u'Content')}, + {'name': u'description', 'title': _(u'Description')}, + {'name': u'tags__name', 'title': _(u'Tags')}, + {'name': u'comments__comment', 'title': _(u'Comments')}, + ] +) +''' + +class DocumentIndex(indexes.SearchIndex, indexes.Indexable): + # Content + text = indexes.CharField(document=True, use_template=True) + # filename + filename = indexes.CharField(model_attr='filename', boost=1.125) + cleaned_filename = indexes.CharField(model_attr='filename', boost=1.125) + # description + #description = indexes.CharField(null=True, model_attr='description') + # tags + #tags = indexes.CharField(null=True, model_attr='tags') + + + #date_added = indexes.DateTimeField(model_attr='date_added') + + def get_model(self): + return Document + + def index_queryset(self): + """Used when the entire index for model is updated.""" + #return self.get_model().objects.filter(date_added__lte=datetime.datetime.now()) + return self.get_model().objects.filter(pk__lte=300) + + def prepare_cleaned_filename(self, obj): + #print 'CLEAN' + return unidecode(obj.filename) + #print "1,2: %s - %s" % (obj.filename, after) + #return after diff --git a/apps/documents/templates/search/indexes/documents/document_text.txt b/apps/documents/templates/search/indexes/documents/document_text.txt new file mode 100644 index 0000000000..4c142984a8 --- /dev/null +++ b/apps/documents/templates/search/indexes/documents/document_text.txt @@ -0,0 +1,3 @@ +{{ object.filename }} +{{ object.description }} +{{ object.latest_version.content }} From c5bce945c4881b62468e654b6be6ccb1d1b5f797 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Fri, 6 Apr 2012 01:06:35 -0400 Subject: [PATCH 05/46] Don't install python-gnupg from upstream but from my github fork Conflicts: requirements/production.txt --- requirements/production.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/production.txt b/requirements/production.txt index 14dc4aeb91..95d4d6df37 100644 --- a/requirements/production.txt +++ b/requirements/production.txt @@ -46,7 +46,7 @@ South==0.7.3 # Keys and signing -python-gnupg==0.2.8 +https://github.com/rosarior/python-gnupg/zipball/0.2.8 python-hkp==0.1.3 # Feedback From 0050fe4eef6a812e3b59dafdd2b04a5ac9837ed2 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Mon, 9 Apr 2012 07:41:03 -0400 Subject: [PATCH 06/46] Haystack updates --- apps/documents/search_indexes.py | 80 ------------------- .../indexes/documents/document_text.txt | 3 - apps/dynamic_search/search_indexes.py | 45 +++++++++++ .../indexes/documents/document_text.txt | 11 +++ 4 files changed, 56 insertions(+), 83 deletions(-) delete mode 100644 apps/documents/search_indexes.py delete mode 100644 apps/documents/templates/search/indexes/documents/document_text.txt create mode 100644 apps/dynamic_search/search_indexes.py create mode 100644 apps/dynamic_search/templates/search/indexes/documents/document_text.txt diff --git a/apps/documents/search_indexes.py b/apps/documents/search_indexes.py deleted file mode 100644 index 4d51335446..0000000000 --- a/apps/documents/search_indexes.py +++ /dev/null @@ -1,80 +0,0 @@ -from __future__ import absolute_import - -import datetime - -from unidecode import unidecode -from haystack import indexes - -from .models import Document - -''' - uuid = models.CharField(max_length=48, blank=True, editable=False) - document_type = models.ForeignKey(DocumentType, verbose_name=_(u'document type'), null=True, blank=True) - description = models.TextField(blank=True, null=True, verbose_name=_(u'description')) - date_added = models.DateTimeField(verbose_name=_(u'added'), db_index=True, editable=False) - document = models.ForeignKey(Document, verbose_name=_(u'document'), editable=False) - major = models.PositiveIntegerField(verbose_name=_(u'mayor'), default=1, editable=False) - minor = models.PositiveIntegerField(verbose_name=_(u'minor'), default=0, editable=False) - micro = models.PositiveIntegerField(verbose_name=_(u'micro'), default=0, editable=False) - release_level = models.PositiveIntegerField(choices=RELEASE_LEVEL_CHOICES, default=RELEASE_LEVEL_FINAL, verbose_name=_(u'release level'), editable=False) - serial = models.PositiveIntegerField(verbose_name=_(u'serial'), default=0, editable=False) - timestamp = models.DateTimeField(verbose_name=_(u'timestamp'), editable=False) - comment = models.TextField(blank=True, verbose_name=_(u'comment')) - - # File related fields - file = models.FileField(upload_to=get_filename_from_uuid, storage=STORAGE_BACKEND(), verbose_name=_(u'file')) - mimetype = models.CharField(max_length=64, default='', editable=False) - encoding = models.CharField(max_length=64, default='', editable=False) - filename = models.CharField(max_length=255, default=u'', editable=False, db_index=True) - checksum = models.TextField(blank=True, null=True, verbose_name=_(u'checksum'), editable=False) - -class DocumentPage(models.Model): - """ - Model that describes a document version page including it's content - """ - document_version = models.ForeignKey(DocumentVersion, verbose_name=_(u'document version')) - content = models.TextField(blank=True, null=True, verbose_name=_(u'content')) - page_label = models.CharField(max_length=32, blank=True, null=True, verbose_name=_(u'page label')) - page_number = models.PositiveIntegerField(default=1, editable=False, verbose_name=_(u'page number'), db_index=True) - - -register('document', Document, _(u'document'), [ - {'name': u'document_type__name', 'title': _(u'Document type')}, - {'name': u'documentversion__mimetype', 'title': _(u'MIME type')}, - {'name': u'documentversion__filename', 'title': _(u'Filename')}, - {'name': u'documentmetadata__value', 'title': _(u'Metadata value')}, - {'name': u'documentversion__documentpage__content', 'title': _(u'Content')}, - {'name': u'description', 'title': _(u'Description')}, - {'name': u'tags__name', 'title': _(u'Tags')}, - {'name': u'comments__comment', 'title': _(u'Comments')}, - ] -) -''' - -class DocumentIndex(indexes.SearchIndex, indexes.Indexable): - # Content - text = indexes.CharField(document=True, use_template=True) - # filename - filename = indexes.CharField(model_attr='filename', boost=1.125) - cleaned_filename = indexes.CharField(model_attr='filename', boost=1.125) - # description - #description = indexes.CharField(null=True, model_attr='description') - # tags - #tags = indexes.CharField(null=True, model_attr='tags') - - - #date_added = indexes.DateTimeField(model_attr='date_added') - - def get_model(self): - return Document - - def index_queryset(self): - """Used when the entire index for model is updated.""" - #return self.get_model().objects.filter(date_added__lte=datetime.datetime.now()) - return self.get_model().objects.filter(pk__lte=300) - - def prepare_cleaned_filename(self, obj): - #print 'CLEAN' - return unidecode(obj.filename) - #print "1,2: %s - %s" % (obj.filename, after) - #return after diff --git a/apps/documents/templates/search/indexes/documents/document_text.txt b/apps/documents/templates/search/indexes/documents/document_text.txt deleted file mode 100644 index 4c142984a8..0000000000 --- a/apps/documents/templates/search/indexes/documents/document_text.txt +++ /dev/null @@ -1,3 +0,0 @@ -{{ object.filename }} -{{ object.description }} -{{ object.latest_version.content }} diff --git a/apps/dynamic_search/search_indexes.py b/apps/dynamic_search/search_indexes.py new file mode 100644 index 0000000000..c1a0e5e80a --- /dev/null +++ b/apps/dynamic_search/search_indexes.py @@ -0,0 +1,45 @@ +from __future__ import absolute_import + +import datetime + +from unidecode import unidecode +from haystack import indexes + +from documents.models import Document + +''' + date_added = models.DateTimeField(verbose_name=_(u'added'), db_index=True, editable=False) + document = models.ForeignKey(Document, verbose_name=_(u'document'), editable=False) + major = models.PositiveIntegerField(verbose_name=_(u'mayor'), default=1, editable=False) + minor = models.PositiveIntegerField(verbose_name=_(u'minor'), default=0, editable=False) + micro = models.PositiveIntegerField(verbose_name=_(u'micro'), default=0, editable=False) + release_level = models.PositiveIntegerField(choices=RELEASE_LEVEL_CHOICES, default=RELEASE_LEVEL_FINAL, verbose_name=_(u'release level'), editable=False) + serial = models.PositiveIntegerField(verbose_name=_(u'serial'), default=0, editable=False) + timestamp = models.DateTimeField(verbose_name=_(u'timestamp'), editable=False) + comment = models.TextField(blank=True, verbose_name=_(u'comment')) + + checksum = models.TextField(blank=True, null=True, verbose_name=_(u'checksum'), editable=False) + + page_label = models.CharField(max_length=32, blank=True, null=True, verbose_name=_(u'page label')) + page_number = models.PositiveIntegerField(default=1, editable=False, verbose_name=_(u'page number'), db_index=True) +''' + +class DocumentIndex(indexes.SearchIndex, indexes.Indexable): + text = indexes.CharField(document=True, use_template=True) + #fractional_filename = indexes.CharField(model_attr='filename')#, boost=1.125) + cleaned_filename = indexes.CharField(model_attr='filename')#, boost=1.125) + + def get_model(self): + return Document + + #def index_queryset(self): + # """Used when the entire index for model is updated.""" + # #return self.get_model().objects.filter(date_added__lte=datetime.datetime.now()) + # return self.get_model().objects.filter(pk__lte=3000) + + #def prepare_cleaned_filename(self, obj): + # #print 'CLEAN' + # return 'qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq' + # return unidecode(obj.filename) + # #print "1,2: %s - %s" % (obj.filename, after) + # #return after diff --git a/apps/dynamic_search/templates/search/indexes/documents/document_text.txt b/apps/dynamic_search/templates/search/indexes/documents/document_text.txt new file mode 100644 index 0000000000..3226d8f509 --- /dev/null +++ b/apps/dynamic_search/templates/search/indexes/documents/document_text.txt @@ -0,0 +1,11 @@ +{{ object.content }} +{{ object.metadata_values_string }} +{{ object.cleaned_filename }} +{{ object.filename }} +{{ object.extension_split.1 }} +{{ object.document_type }} +{{ object.file_mimetype }} +{{ object.description|default:' ' }} +{{ object.flat_comments }} +{{ object.flat_tags }} +{{ object.uuid }} From ea49395c23eb659e04618cb80c112ff1773844a8 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 11 Apr 2012 01:56:32 -0400 Subject: [PATCH 07/46] Add flat_comments helper method to Document class --- apps/document_comments/__init__.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/apps/document_comments/__init__.py b/apps/document_comments/__init__.py index 2356f36885..53bc133a94 100644 --- a/apps/document_comments/__init__.py +++ b/apps/document_comments/__init__.py @@ -40,6 +40,11 @@ register_links(['comments_for_document', 'comment_add', 'comment_delete', 'comme register_links(Comment, [comment_delete]) register_links(Document, [comments_for_document], menu_name='form_header') + +def flat_comments(document): + return u' '.join(document.comments.values_list('comment', flat=True)) + + Document.add_to_class( 'comments', generic.GenericRelation( @@ -49,6 +54,8 @@ Document.add_to_class( ) ) +Document.add_to_class('flat_comments', flat_comments) + class_permissions(Document, [ PERMISSION_COMMENT_CREATE, PERMISSION_COMMENT_DELETE, From e3505d05ec1496238c45b58137a7f9ab0f694f06 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 11 Apr 2012 01:57:08 -0400 Subject: [PATCH 08/46] Add extension_split and cleaned_filename to the Document class --- apps/documents/models.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/apps/documents/models.py b/apps/documents/models.py index d03be1df90..ed8aa39186 100644 --- a/apps/documents/models.py +++ b/apps/documents/models.py @@ -13,6 +13,8 @@ try: except ImportError: from StringIO import StringIO +from unidecode import unidecode + from django.db import models from django.utils.translation import ugettext_lazy as _ from django.utils.translation import ugettext @@ -281,6 +283,16 @@ class Document(models.Model): filename = property(_get_filename, _set_filename) + @property + def cleaned_filename(self): + return unidecode(self.extension_split()[0]) + + @property + def extension_split(self): + filename, extension = os.path.splitext(self.filename) + return filename, extension[1:] + + class DocumentVersion(models.Model): """ From b6fcf977e58bc0e96a3320e63c5ec7140501243a Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 11 Apr 2012 01:59:12 -0400 Subject: [PATCH 09/46] Add flat_tags search helper method to Document class --- apps/tags/__init__.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/apps/tags/__init__.py b/apps/tags/__init__.py index 2bfcb2f508..5263c3b240 100644 --- a/apps/tags/__init__.py +++ b/apps/tags/__init__.py @@ -66,4 +66,11 @@ class_permissions(Tag, [ PERMISSION_TAG_VIEW, ]) + +def flat_tags(document): + return u' '.join(document.tags.values_list('name', flat=True)) + + Document.add_to_class('tags', TaggableManager()) +Document.add_to_class('flat_tags', flat_tags) + From a7b7195370d159bee0d205ca0b20f75d96812311 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 11 Apr 2012 01:59:46 -0400 Subject: [PATCH 10/46] Add metadata search helper methods --- apps/metadata/models.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/apps/metadata/models.py b/apps/metadata/models.py index 5e13f65c48..93aeb921df 100644 --- a/apps/metadata/models.py +++ b/apps/metadata/models.py @@ -98,3 +98,18 @@ class DocumentTypeDefaults(models.Model): class Meta: verbose_name = _(u'document type defaults') verbose_name_plural = _(u'document types defaults') + + +def document_metadata_values(document): + return DocumentMetadata.objects.filter(document=document).values_list('value', flat=True) + + +def document_metadata_values_string(document): + return u' '.join(document_metadata_values(document)) + + +def document_metadata_dict_list(document): + return [{name: value} for name, value in DocumentMetadata.objects.filter(document=document).values_list('metadata_type__name', 'value')] + + +Document.add_to_class('metadata_values_string', document_metadata_values_string) From c8129b1f5fd56cdf1e01747fd771257598ff8e1b Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 11 Apr 2012 03:11:08 -0400 Subject: [PATCH 11/46] Add get_object_list_object_name tag to override the object to process from an object_list --- .../templates/generic_list_subtemplate.html | 17 +++++++++++------ apps/common/templatetags/subtemplates_tags.py | 18 ++++++++++++++++-- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/apps/common/templates/generic_list_subtemplate.html b/apps/common/templates/generic_list_subtemplate.html index d1494312bf..15732c8998 100644 --- a/apps/common/templates/generic_list_subtemplate.html +++ b/apps/common/templates/generic_list_subtemplate.html @@ -6,6 +6,7 @@ {% load variable_tags %} {% load main_settings_tags %} {% load multiselect_tags %} +{% load subtemplates_tags %} {% get_main_setting "DISABLE_ICONS" as disable_icons %} @@ -76,7 +77,9 @@ {{ column.name|capfirst }} {% endfor %} - {% for column in object_list.0|get_model_list_columns %} + {% get_object_list_object_name object_list.0 %} + + {% for column in object|get_model_list_columns %} {{ column.name|capfirst }} {% endfor %} @@ -89,15 +92,17 @@ {% endif %} {% endif %} + {% for object in object_list %} + {% get_object_list_object_name object %} {% if multi_select or multi_select_as_buttons %} - {% if multi_select_item_properties %} - - {% else %} - - {% endif %} + {% if multi_select_item_properties %} + + {% else %} + + {% endif %} {% endif %} {% if not hide_object %} diff --git a/apps/common/templatetags/subtemplates_tags.py b/apps/common/templatetags/subtemplates_tags.py index d759df4bbb..228c5ce8af 100644 --- a/apps/common/templatetags/subtemplates_tags.py +++ b/apps/common/templatetags/subtemplates_tags.py @@ -1,9 +1,11 @@ import re -from django.template import Node, TemplateSyntaxError, Library, \ - Variable, Context +from django.template import (Node, TemplateSyntaxError, Library, + Variable, Context) from django.template.loader import get_template +from common.utils import return_attrib + register = Library() @@ -49,3 +51,15 @@ def render_subtemplate(parser, token): return RenderSubtemplateNode(template_name, template_context, var_name) #format_string[1:-1] + + +@register.simple_tag(takes_context=True) +def get_object_list_object_name(context, source_object): + object_list_object_name = context.get('object_list_object_name') + if object_list_object_name: + context['object'] = return_attrib(source_object, object_list_object_name) + else: + context['object'] = source_object + + return '' + From 34ebf6a263f73323e0bf41bbc06b704b2adf9412 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 11 Apr 2012 03:11:45 -0400 Subject: [PATCH 12/46] Use a simple for the basic search --- apps/dynamic_search/urls.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/dynamic_search/urls.py b/apps/dynamic_search/urls.py index ed2932609e..98660d6742 100644 --- a/apps/dynamic_search/urls.py +++ b/apps/dynamic_search/urls.py @@ -1,9 +1,11 @@ from django.conf.urls.defaults import patterns, url +from haystack.forms import SearchForm + from .views import CustomSearchView urlpatterns = patterns('dynamic_search.views', - url(r'^$', CustomSearchView(), (), 'search'), + url(r'^$', CustomSearchView(form_class=SearchForm), (), 'search'), url(r'^advanced/$', 'search', {'advanced': True}, 'search_advanced'), url(r'^again/$', 'search_again', (), 'search_again'), url(r'^results/$', 'results', (), 'results'), From 0d4031630525bb091966f2fb7397b508b371be6c Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 11 Apr 2012 03:32:50 -0400 Subject: [PATCH 13/46] Cleanup search view --- apps/dynamic_search/views.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/apps/dynamic_search/views.py b/apps/dynamic_search/views.py index 67820c7251..ddd7fe4501 100644 --- a/apps/dynamic_search/views.py +++ b/apps/dynamic_search/views.py @@ -12,6 +12,8 @@ from django.utils.http import urlencode from haystack.views import SearchView +from common.utils import encapsulate + from dynamic_search.models import RecentSearch from dynamic_search.api import perform_search from dynamic_search.forms import SearchForm, AdvancedSearchForm @@ -24,19 +26,16 @@ class CustomSearchView(SearchView): self.start_time = datetime.datetime.now() return super(CustomSearchView, self).__call__(*args, **kwargs) - + def create_response(self): """ Generates the actual HttpResponse to send back to the user. """ - #(paginator, page) = self.build_page() - context = { 'query': self.query, 'form': self.form, - 'object_list': [result.object for result in self.results], - #'page': page, - #'paginator': paginator, + 'results': self.results, + 'object_list': self.results, 'suggestion': None, 'submit_label': _(u'Search'), 'submit_icon_famfam': 'zoom', @@ -45,13 +44,10 @@ class CustomSearchView(SearchView): 'list_title': _(u'results for: %s') % self.query, 'hide_links': True, 'multi_select_as_buttons': True, - 'elapsed_time': unicode(datetime.datetime.now() - self.start_time).split(':')[2] - + 'elapsed_time': unicode(datetime.datetime.now() - self.start_time).split(':')[2], + 'object_list_object_name': 'object', } - #if self.results and hasattr(self.results, 'query') and self.results.query.backend.include_spelling: - # context['suggestion'] = self.form.get_suggestion() - context.update(self.extra_context()) return render_to_response(self.template, context, context_instance=self.context_class(self.request)) From 9bcf7574bfa16467826bdfdb22b98bd7f1b533fa Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 11 Apr 2012 03:33:23 -0400 Subject: [PATCH 14/46] Remove reduntant code from the object_list object default column detection code --- apps/common/templates/generic_list_subtemplate.html | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/apps/common/templates/generic_list_subtemplate.html b/apps/common/templates/generic_list_subtemplate.html index 15732c8998..1e0868a227 100644 --- a/apps/common/templates/generic_list_subtemplate.html +++ b/apps/common/templates/generic_list_subtemplate.html @@ -77,11 +77,13 @@ {{ column.name|capfirst }} {% endfor %} - {% get_object_list_object_name object_list.0 %} + {% if object_list %} + {% get_object_list_object_name object_list.0 %} - {% for column in object|get_model_list_columns %} - {{ column.name|capfirst }} - {% endfor %} + {% for column in object|get_model_list_columns %} + {{ column.name|capfirst }} + {% endfor %} + {% endif %} {% for column in extra_columns %} {{ column.name|capfirst }} From 71a3e16a01baabe9f9893e77dec77b9aebd52938 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 11 Apr 2012 03:33:56 -0400 Subject: [PATCH 15/46] Remove test template code --- apps/dynamic_search/templates/search/search.html | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/apps/dynamic_search/templates/search/search.html b/apps/dynamic_search/templates/search/search.html index 13cb21427e..1e113f5a0d 100644 --- a/apps/dynamic_search/templates/search/search.html +++ b/apps/dynamic_search/templates/search/search.html @@ -15,19 +15,6 @@ {% if not form and not query %} {% include "generic_list_subtemplate.html" %} {% endif %} - -{% comment %} - -{% for result in page.object_list %} - {% with result|search_include as fragment %} - {% include fragment %} - {% endwith %} -{% empty %} -

No results found.

-{% endfor %} - -{% endcomment %} - {% endblock %} {% block footer %} From 6a7ca922f0d5e7b16bb6e50fe9abbf1b6d55a833 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 11 Apr 2012 16:46:28 -0400 Subject: [PATCH 16/46] Add pre and post signals to haystack's update index command --- apps/signaler/management/commands/update_index.py | 14 ++++++++++++++ apps/signaler/signals.py | 2 ++ 2 files changed, 16 insertions(+) create mode 100644 apps/signaler/management/commands/update_index.py diff --git a/apps/signaler/management/commands/update_index.py b/apps/signaler/management/commands/update_index.py new file mode 100644 index 0000000000..bfc87a0fe0 --- /dev/null +++ b/apps/signaler/management/commands/update_index.py @@ -0,0 +1,14 @@ +from haystack.management.commands import update_index + +from signaler.signals import post_update_index, pre_update_index + + +class Command(update_index.Command): + """ + Wrapper for the haystack's update_index command + """ + + def handle(self, *args, **kwargs): + pre_update_index.send(sender=self) + super(Command, self).handle(*args, **kwargs) + post_update_index.send(sender=self) diff --git a/apps/signaler/signals.py b/apps/signaler/signals.py index c102b00357..66a764ab57 100644 --- a/apps/signaler/signals.py +++ b/apps/signaler/signals.py @@ -1,3 +1,5 @@ from django.dispatch import Signal pre_collectstatic = Signal() +pre_update_index = Signal() +post_update_index = Signal() From 5adffd4e75281e0f964d059b54e7958c53579168 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 11 Apr 2012 21:43:30 -0400 Subject: [PATCH 17/46] Implement a dirty flag to index newly modified objects --- apps/dynamic_search/__init__.py | 30 ++++++++++++++ apps/dynamic_search/admin.py | 3 +- apps/dynamic_search/models.py | 60 +++++++++++++++++++++++---- apps/dynamic_search/search_indexes.py | 46 ++++++++++---------- 4 files changed, 108 insertions(+), 31 deletions(-) diff --git a/apps/dynamic_search/__init__.py b/apps/dynamic_search/__init__.py index 2d3d73c5f4..a1c7ca32bd 100644 --- a/apps/dynamic_search/__init__.py +++ b/apps/dynamic_search/__init__.py @@ -1,6 +1,18 @@ +from __future__ import absolute_import + +import logging + from django.utils.translation import ugettext_lazy as _ +from django.dispatch import receiver from navigation.api import register_sidebar_template, register_links +from documents.models import Document +from scheduler.runtime import scheduler +from signaler.signals import post_update_index, pre_update_index + +from .models import IndexableObject + +logger = logging.getLogger(__name__) search = {'text': _(u'search'), 'view': 'search', 'famfam': 'zoom'} search_advanced = {'text': _(u'advanced search'), 'view': 'search_advanced', 'famfam': 'zoom_in'} @@ -12,3 +24,21 @@ register_links(['search', 'search_advanced', 'results'], [search, search_advance register_links(['results'], [search_again], menu_name='sidebar') register_sidebar_template(['search', 'search_advanced', 'results'], 'recent_searches.html') + + +def mark_dirty(obj): + IndexableObject.objects.mark_dirty(content_object=obj) + +Document.add_to_class('mark_dirty', lambda obj: IndexableObject.objects.mark_dirty(obj)) + + +@receiver(post_update_index, dispatch_uid='clear_dirty_indexables') +def clear_dirty_indexables(sender, **kwargs): + logger.debug('Clearing all indexable flags post update index signal') + IndexableObject.objects.clear_all() + + +@receiver(pre_update_index, dispatch_uid='scheduler_shutdown_pre_update_index') +def scheduler_shutdown_pre_update_index(sender, **kwargs): + logger.debug('Scheduler shut down on pre update index signal') + scheduler.shutdown() diff --git a/apps/dynamic_search/admin.py b/apps/dynamic_search/admin.py index 3592e29bc6..cd7892c299 100644 --- a/apps/dynamic_search/admin.py +++ b/apps/dynamic_search/admin.py @@ -1,6 +1,6 @@ from django.contrib import admin -from dynamic_search.models import RecentSearch +from dynamic_search.models import RecentSearch, IndexableObject class RecentSearchAdmin(admin.ModelAdmin): @@ -9,3 +9,4 @@ class RecentSearchAdmin(admin.ModelAdmin): readonly_fields = ('user', 'query', 'datetime_created', 'hits') admin.site.register(RecentSearch, RecentSearchAdmin) +admin.site.register(IndexableObject) diff --git a/apps/dynamic_search/models.py b/apps/dynamic_search/models.py index 9e7cc552bc..3a69e3533f 100644 --- a/apps/dynamic_search/models.py +++ b/apps/dynamic_search/models.py @@ -1,22 +1,26 @@ +from __future__ import absolute_import + import urlparse import urllib - -from datetime import datetime +import datetime from django.db import models from django.utils.translation import ugettext as _ from django.contrib.auth.models import User from django.core.urlresolvers import reverse from django.utils.encoding import smart_unicode, smart_str +from django.contrib.contenttypes.models import ContentType +from django.contrib.contenttypes import generic +from django.utils.translation import ugettext_lazy as _ -from dynamic_search.managers import RecentSearchManager -from dynamic_search.api import registered_search_dict +from .managers import RecentSearchManager +from .api import registered_search_dict class RecentSearch(models.Model): - ''' + """ Keeps a list of the n most recent search keywords for a given user - ''' + """ user = models.ForeignKey(User, verbose_name=_(u'user'), editable=False) query = models.TextField(verbose_name=_(u'query'), editable=False) datetime_created = models.DateTimeField(verbose_name=_(u'datetime created'), editable=False) @@ -46,7 +50,7 @@ class RecentSearch(models.Model): return u'%s (%s)' % (display_string, self.hits) def save(self, *args, **kwargs): - self.datetime_created = datetime.now() + self.datetime_created = datetime.datetime.now() super(RecentSearch, self).save(*args, **kwargs) def url(self): @@ -60,3 +64,45 @@ class RecentSearch(models.Model): ordering = ('-datetime_created',) verbose_name = _(u'recent search') verbose_name_plural = _(u'recent searches') + + +class IndexableObjectManager(models.Manager): + def get_dirty(self, datetime=None): + if datetime: + return self.model.objects.filter(datetime__gte=datetime) + else: + return self.model.objects.all() + + def get_dirty_pk_list(self, datetime=None): + return self.get_dirty(datetime).values_list('object_id', flat=True) + + def mark_dirty(self, obj): + content_type = ContentType.objects.get_for_model(obj) + self.model.objects.get_or_create(content_type=content_type, object_id=obj.pk) + + def clear_all(self): + self.model.objects.all().delete() + + +class IndexableObject(models.Model): + """ + Store a list of object links that have been modified and are + meant to be indexed in the next search index update + """ + datetime = models.DateTimeField(verbose_name=_(u'date time')) + content_type = models.ForeignKey(ContentType, blank=True, null=True) + object_id = models.PositiveIntegerField(blank=True, null=True) + content_object = generic.GenericForeignKey('content_type', 'object_id') + + objects = IndexableObjectManager() + + def __unicode__(self): + return unicode(self.content_object) + + def save(self, *args, **kwargs): + self.datetime = datetime.datetime.now() + super(IndexableObject, self).save(*args, **kwargs) + + class Meta: + verbose_name = _(u'indexable object') + verbose_name_plural = _(u'indexable objects') diff --git a/apps/dynamic_search/search_indexes.py b/apps/dynamic_search/search_indexes.py index c1a0e5e80a..016c85864b 100644 --- a/apps/dynamic_search/search_indexes.py +++ b/apps/dynamic_search/search_indexes.py @@ -7,39 +7,39 @@ from haystack import indexes from documents.models import Document +from .models import IndexableObject + ''' - date_added = models.DateTimeField(verbose_name=_(u'added'), db_index=True, editable=False) - document = models.ForeignKey(Document, verbose_name=_(u'document'), editable=False) - major = models.PositiveIntegerField(verbose_name=_(u'mayor'), default=1, editable=False) - minor = models.PositiveIntegerField(verbose_name=_(u'minor'), default=0, editable=False) - micro = models.PositiveIntegerField(verbose_name=_(u'micro'), default=0, editable=False) - release_level = models.PositiveIntegerField(choices=RELEASE_LEVEL_CHOICES, default=RELEASE_LEVEL_FINAL, verbose_name=_(u'release level'), editable=False) - serial = models.PositiveIntegerField(verbose_name=_(u'serial'), default=0, editable=False) - timestamp = models.DateTimeField(verbose_name=_(u'timestamp'), editable=False) comment = models.TextField(blank=True, verbose_name=_(u'comment')) - checksum = models.TextField(blank=True, null=True, verbose_name=_(u'checksum'), editable=False) - page_label = models.CharField(max_length=32, blank=True, null=True, verbose_name=_(u'page label')) page_number = models.PositiveIntegerField(default=1, editable=False, verbose_name=_(u'page number'), db_index=True) + +# Register the fields that will be searchable +register('document', Document, _(u'document'), [ + {'name': u'document_type__name', 'title': _(u'Document type')}, + {'name': u'documentversion__mimetype', 'title': _(u'MIME type')}, + {'name': u'documentversion__filename', 'title': _(u'Filename')}, + {'name': u'documentmetadata__value', 'title': _(u'Metadata value')}, + {'name': u'documentversion__documentpage__content', 'title': _(u'Content')}, + {'name': u'description', 'title': _(u'Description')}, + {'name': u'tags__name', 'title': _(u'Tags')}, + {'name': u'comments__comment', 'title': _(u'Comments')}, + ] +) +#register(Document, _(u'document'), ['document_type__name', 'file_mimetype', 'documentmetadata__value', 'documentpage__content', 'description', {'field_name':'file_filename', 'comparison':'iexact'}]) + ''' class DocumentIndex(indexes.SearchIndex, indexes.Indexable): text = indexes.CharField(document=True, use_template=True) - #fractional_filename = indexes.CharField(model_attr='filename')#, boost=1.125) - cleaned_filename = indexes.CharField(model_attr='filename')#, boost=1.125) def get_model(self): return Document - #def index_queryset(self): - # """Used when the entire index for model is updated.""" - # #return self.get_model().objects.filter(date_added__lte=datetime.datetime.now()) - # return self.get_model().objects.filter(pk__lte=3000) - - #def prepare_cleaned_filename(self, obj): - # #print 'CLEAN' - # return 'qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq' - # return unidecode(obj.filename) - # #print "1,2: %s - %s" % (obj.filename, after) - # #return after + def build_queryset(self, start_date=None, end_date=None): + print "DIRTY", IndexableObject.objects.get_dirty_pk_list() + + #return self.get_model().objects.filter(date_added__lte=datetime.datetime.now()) + return self.get_model().objects.filter(pk__in=IndexableObject.objects.get_dirty_pk_list()) + #return self.get_model().objects.all() From 007b8c10ab7edc88ef6abfadda8c591f1e23cc12 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 11 Apr 2012 21:43:59 -0400 Subject: [PATCH 18/46] Make documents dirty when edited --- apps/documents/models.py | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/apps/documents/models.py b/apps/documents/models.py index ed8aa39186..5c1c69e732 100644 --- a/apps/documents/models.py +++ b/apps/documents/models.py @@ -21,7 +21,6 @@ from django.utils.translation import ugettext from django.contrib.auth.models import User from django.core.exceptions import ValidationError -from dynamic_search.api import register from converter.api import get_page_count from converter.api import get_available_transformations_choices from converter.api import convert @@ -104,6 +103,7 @@ class Document(models.Model): self.uuid = UUID_FUNCTION() self.date_added = datetime.datetime.now() super(Document, self).save(*args, **kwargs) + self.mark_dirty() def get_cached_image_name(self, page, version): document_version = DocumentVersion.objects.get(pk=version) @@ -682,18 +682,3 @@ class RecentDocument(models.Model): ordering = ('-datetime_accessed',) verbose_name = _(u'recent document') verbose_name_plural = _(u'recent documents') - - -# Register the fields that will be searchable -register('document', Document, _(u'document'), [ - {'name': u'document_type__name', 'title': _(u'Document type')}, - {'name': u'documentversion__mimetype', 'title': _(u'MIME type')}, - {'name': u'documentversion__filename', 'title': _(u'Filename')}, - {'name': u'documentmetadata__value', 'title': _(u'Metadata value')}, - {'name': u'documentversion__documentpage__content', 'title': _(u'Content')}, - {'name': u'description', 'title': _(u'Description')}, - {'name': u'tags__name', 'title': _(u'Tags')}, - {'name': u'comments__comment', 'title': _(u'Comments')}, - ] -) -#register(Document, _(u'document'), ['document_type__name', 'file_mimetype', 'documentmetadata__value', 'documentpage__content', 'description', {'field_name':'file_filename', 'comparison':'iexact'}]) From 97ae34a06d6b951af305eff2903fdcb2425d2bbe Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 11 Apr 2012 21:45:09 -0400 Subject: [PATCH 19/46] Initial changes to support advanced search via haystack --- apps/dynamic_search/forms.py | 36 +++++++++++++++++++++++++----------- apps/dynamic_search/urls.py | 3 ++- 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/apps/dynamic_search/forms.py b/apps/dynamic_search/forms.py index 82fc2f6c9d..272fca1687 100644 --- a/apps/dynamic_search/forms.py +++ b/apps/dynamic_search/forms.py @@ -1,19 +1,14 @@ +from __future__ import absolute_import + from django import forms from django.utils.translation import ugettext_lazy as _ -from dynamic_search.api import registered_search_dict +from haystack.forms import SearchForm + +from .api import registered_search_dict -class SearchForm(forms.Form): - q = forms.CharField(max_length=128, label=_(u'Search terms')) - source = forms.CharField( - max_length=32, - required=False, - widget=forms.widgets.HiddenInput() - ) - - -class AdvancedSearchForm(forms.Form): +class AdvancedSearchForm(SearchForm): def __init__(self, *args, **kwargs): super(AdvancedSearchForm, self).__init__(*args, **kwargs) @@ -23,3 +18,22 @@ class AdvancedSearchForm(forms.Form): label=field['title'], required=False ) + + def search(self): + if not self.is_valid(): + return self.no_query_found() + + #if not self.cleaned_data.get('q'): + # return self.no_query_found() + for field in self.fields: + print 'field', field + #sqs = self.searchqueryset.auto_query(self.cleaned_data['q']) + + if self.load_all: + sqs = sqs.load_all() + + return sqs + + def search(self): + sqs = super(ModelSearchForm, self).search() + return sqs.models(*self.get_models()) diff --git a/apps/dynamic_search/urls.py b/apps/dynamic_search/urls.py index 98660d6742..5c226e4838 100644 --- a/apps/dynamic_search/urls.py +++ b/apps/dynamic_search/urls.py @@ -3,10 +3,11 @@ from django.conf.urls.defaults import patterns, url from haystack.forms import SearchForm from .views import CustomSearchView +from .forms import AdvancedSearchForm urlpatterns = patterns('dynamic_search.views', url(r'^$', CustomSearchView(form_class=SearchForm), (), 'search'), - url(r'^advanced/$', 'search', {'advanced': True}, 'search_advanced'), + url(r'^advanced/$', CustomSearchView(form_class=AdvancedSearchForm), (), 'search_advanced'), url(r'^again/$', 'search_again', (), 'search_again'), url(r'^results/$', 'results', (), 'results'), ) From 520d6cc3af168cba5de1efcbc22768c548c5491f Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Thu, 12 Apr 2012 02:04:50 -0400 Subject: [PATCH 20/46] Improve IndexableObject class methods --- apps/dynamic_search/__init__.py | 27 ++++---- apps/dynamic_search/admin.py | 4 +- apps/dynamic_search/forms.py | 8 ++- apps/dynamic_search/managers.py | 67 +++++++++++++------ apps/dynamic_search/models.py | 62 +++++------------ apps/dynamic_search/search_indexes.py | 22 ++---- .../templates/search_results.html | 1 - .../templatetags/search_tags.py | 4 +- apps/dynamic_search/urls.py | 7 +- apps/dynamic_search/views.py | 10 ++- 10 files changed, 107 insertions(+), 105 deletions(-) diff --git a/apps/dynamic_search/__init__.py b/apps/dynamic_search/__init__.py index a1c7ca32bd..53d427e0a0 100644 --- a/apps/dynamic_search/__init__.py +++ b/apps/dynamic_search/__init__.py @@ -15,25 +15,24 @@ from .models import IndexableObject logger = logging.getLogger(__name__) search = {'text': _(u'search'), 'view': 'search', 'famfam': 'zoom'} -search_advanced = {'text': _(u'advanced search'), 'view': 'search_advanced', 'famfam': 'zoom_in'} -search_again = {'text': _(u'search again'), 'view': 'search_again', 'famfam': 'arrow_undo'} +#search_advanced = {'text': _(u'advanced search'), 'view': 'search_advanced', 'famfam': 'zoom_in'} +#search_again = {'text': _(u'search again'), 'view': 'search_again', 'famfam': 'arrow_undo'} -register_sidebar_template(['search', 'search_advanced'], 'search_help.html') +#register_sidebar_template(['search', 'search_advanced'], 'search_help.html') +register_sidebar_template(['search'], 'search_help.html') -register_links(['search', 'search_advanced', 'results'], [search, search_advanced], menu_name='form_header') -register_links(['results'], [search_again], menu_name='sidebar') +#register_links(['search', 'search_advanced', 'results'], [search, search_advanced], menu_name='form_header') +register_links(['search'], [search], menu_name='form_header') +#register_links(['results'], [search_again], menu_name='sidebar') -register_sidebar_template(['search', 'search_advanced', 'results'], 'recent_searches.html') +#register_sidebar_template(['search', 'search_advanced', 'results'], 'recent_searches.html') +register_sidebar_template(['search'], 'recent_searches.html') + +Document.add_to_class('mark_indexable', lambda obj: IndexableObject.objects.mark_indexable(obj)) -def mark_dirty(obj): - IndexableObject.objects.mark_dirty(content_object=obj) - -Document.add_to_class('mark_dirty', lambda obj: IndexableObject.objects.mark_dirty(obj)) - - -@receiver(post_update_index, dispatch_uid='clear_dirty_indexables') -def clear_dirty_indexables(sender, **kwargs): +@receiver(post_update_index, dispatch_uid='clear_pending_indexables') +def clear_pending_indexables(sender, **kwargs): logger.debug('Clearing all indexable flags post update index signal') IndexableObject.objects.clear_all() diff --git a/apps/dynamic_search/admin.py b/apps/dynamic_search/admin.py index cd7892c299..504f7c2160 100644 --- a/apps/dynamic_search/admin.py +++ b/apps/dynamic_search/admin.py @@ -1,6 +1,8 @@ +from __future__ import absolute_import + from django.contrib import admin -from dynamic_search.models import RecentSearch, IndexableObject +from .models import RecentSearch, IndexableObject class RecentSearchAdmin(admin.ModelAdmin): diff --git a/apps/dynamic_search/forms.py b/apps/dynamic_search/forms.py index 272fca1687..56ca16e348 100644 --- a/apps/dynamic_search/forms.py +++ b/apps/dynamic_search/forms.py @@ -3,11 +3,16 @@ from __future__ import absolute_import from django import forms from django.utils.translation import ugettext_lazy as _ -from haystack.forms import SearchForm +from haystack.forms import SearchForm as BasicSearchForm from .api import registered_search_dict +class SearchForm(BasicSearchForm): + pass + + +""" class AdvancedSearchForm(SearchForm): def __init__(self, *args, **kwargs): super(AdvancedSearchForm, self).__init__(*args, **kwargs) @@ -37,3 +42,4 @@ class AdvancedSearchForm(SearchForm): def search(self): sqs = super(ModelSearchForm, self).search() return sqs.models(*self.get_models()) +""" diff --git a/apps/dynamic_search/managers.py b/apps/dynamic_search/managers.py index e825bc7350..db0c99d111 100644 --- a/apps/dynamic_search/managers.py +++ b/apps/dynamic_search/managers.py @@ -1,32 +1,59 @@ -import urlparse +from urlparse import urlparse, parse_qs +from urllib import unquote_plus -from django.db import models +from django.utils.simplejson import dumps, loads, JSONEncoder +from django.db.models import Manager from django.utils.http import urlencode from django.contrib.auth.models import AnonymousUser - +from django.contrib.contenttypes.models import ContentType +from django.contrib.contenttypes import generic +from django.utils.encoding import smart_unicode, smart_str + from dynamic_search.conf.settings import RECENT_COUNT -class RecentSearchManager(models.Manager): - def add_query_for_user(self, user, query, hits): - parsed_query = urlparse.parse_qs(query) - for key, value in parsed_query.items(): - parsed_query[key] = ' '.join(value) +class RecentSearchManager(Manager): + def add_query_for_user(self, search_view):#user, url, hits): + query_dict = parse_qs(unquote_plus(smart_str(urlparse(search_view.request.get_full_path()).query))) + print 'query_dict', query_dict + print 'serial', dumps(query_dict) + + #parsed_query = urlparse.parse_qs(query) + #for key, value in parsed_query.items(): + # parsed_query[key] = ' '.join(value)# - if 'q=' in query: - # Is a simple query - if not parsed_query.get('q'): - # Don't store empty simple searches - return - else: - # Cleanup query string and only store the q parameter - parsed_query = {'q': parsed_query['q']} + #if 'q=' in query: + # # Is a simple query + # if not parsed_query.get('q'): + # # Don't store empty simple searches + # return + # else: + # # Cleanup query string and only store the q parameter + # parsed_query = {'q': parsed_query['q']} - if parsed_query and not isinstance(user, AnonymousUser): + if query_dict and not isinstance(search_view.request.user, AnonymousUser): # If the URL query has at least one variable with a value - new_recent, created = self.model.objects.get_or_create(user=user, query=urlencode(parsed_query), defaults={'hits': hits}) - new_recent.hits = hits + new_recent, created = self.model.objects.get_or_create(user=search_view.request.user, query=dumps(query_dict), defaults={'hits': 0}) + new_recent.hits = search_view.results.count() new_recent.save() - to_delete = self.model.objects.filter(user=user)[RECENT_COUNT:] + to_delete = self.model.objects.filter(user=search_view.request.user)[RECENT_COUNT:] for recent_to_delete in to_delete: recent_to_delete.delete() + + +class IndexableObjectManager(Manager): + def get_indexables(self, datetime=None): + if datetime: + return self.model.objects.filter(datetime__gte=datetime) + else: + return self.model.objects.all() + + def get_indexables_pk_list(self, datetime=None): + return self.get_indexables(datetime).values_list('object_id', flat=True) + + def mark_indexable(self, obj): + content_type = ContentType.objects.get_for_model(obj) + self.model.objects.get_or_create(content_type=content_type, object_id=obj.pk) + + def clear_all(self): + self.model.objects.all().delete() diff --git a/apps/dynamic_search/models.py b/apps/dynamic_search/models.py index 3a69e3533f..f0e29408fb 100644 --- a/apps/dynamic_search/models.py +++ b/apps/dynamic_search/models.py @@ -9,11 +9,12 @@ from django.utils.translation import ugettext as _ from django.contrib.auth.models import User from django.core.urlresolvers import reverse from django.utils.encoding import smart_unicode, smart_str +from django.utils.translation import ugettext_lazy as _ from django.contrib.contenttypes.models import ContentType from django.contrib.contenttypes import generic -from django.utils.translation import ugettext_lazy as _ +from django.utils.simplejson import loads -from .managers import RecentSearchManager +from .managers import RecentSearchManager, IndexableObjectManager from .api import registered_search_dict @@ -29,36 +30,27 @@ class RecentSearch(models.Model): objects = RecentSearchManager() def __unicode__(self): - query_dict = urlparse.parse_qs(urllib.unquote_plus(smart_str(self.query))) - if 'q' in query_dict: - # Is a simple search - display_string = smart_unicode(' '.join(query_dict['q'])) + if self.is_advanced(): + return u'%s (%s)' % (self.get_query(), self.hits) else: - # Advanced search - advanced_string = [] - for key, value in query_dict.items(): - # Get model name - model, field_name = key.split('__', 1) - model_entry = registered_search_dict.get(model, {}) - if model_entry: - # Find the field name title - for model_field in model_entry.get('fields', [{}]): - if model_field.get('name') == field_name: - advanced_string.append(u'%s: %s' % (model_field.get('title', model_field['name']), smart_unicode(' '.join(value)))) - - display_string = u', '.join(advanced_string) - return u'%s (%s)' % (display_string, self.hits) + return u'%s (%s)' % (self.get_query().get('q'), self.hits) def save(self, *args, **kwargs): self.datetime_created = datetime.datetime.now() super(RecentSearch, self).save(*args, **kwargs) - def url(self): - view = 'results' if self.is_advanced() else 'search' - return '%s?%s' % (reverse(view), self.query) + #def readable_query(self): + # return self. + + #def url(self): + # view = 'results' if self.is_advanced() else 'search' + # return '%s?%s' % (reverse(view), self.query) + + def get_query(self): + return loads(self.query) def is_advanced(self): - return 'q' not in urlparse.parse_qs(self.query) + return 'q' not in self.get_query() class Meta: ordering = ('-datetime_created',) @@ -66,32 +58,14 @@ class RecentSearch(models.Model): verbose_name_plural = _(u'recent searches') -class IndexableObjectManager(models.Manager): - def get_dirty(self, datetime=None): - if datetime: - return self.model.objects.filter(datetime__gte=datetime) - else: - return self.model.objects.all() - - def get_dirty_pk_list(self, datetime=None): - return self.get_dirty(datetime).values_list('object_id', flat=True) - - def mark_dirty(self, obj): - content_type = ContentType.objects.get_for_model(obj) - self.model.objects.get_or_create(content_type=content_type, object_id=obj.pk) - - def clear_all(self): - self.model.objects.all().delete() - - class IndexableObject(models.Model): """ Store a list of object links that have been modified and are meant to be indexed in the next search index update """ datetime = models.DateTimeField(verbose_name=_(u'date time')) - content_type = models.ForeignKey(ContentType, blank=True, null=True) - object_id = models.PositiveIntegerField(blank=True, null=True) + content_type = models.ForeignKey(ContentType) + object_id = models.PositiveIntegerField() content_object = generic.GenericForeignKey('content_type', 'object_id') objects = IndexableObjectManager() diff --git a/apps/dynamic_search/search_indexes.py b/apps/dynamic_search/search_indexes.py index 016c85864b..7fc4294d6f 100644 --- a/apps/dynamic_search/search_indexes.py +++ b/apps/dynamic_search/search_indexes.py @@ -1,6 +1,7 @@ from __future__ import absolute_import import datetime +import logging from unidecode import unidecode from haystack import indexes @@ -9,25 +10,16 @@ from documents.models import Document from .models import IndexableObject +logger = logging.getLogger(__name__) + ''' comment = models.TextField(blank=True, verbose_name=_(u'comment')) checksum = models.TextField(blank=True, null=True, verbose_name=_(u'checksum'), editable=False) page_label = models.CharField(max_length=32, blank=True, null=True, verbose_name=_(u'page label')) page_number = models.PositiveIntegerField(default=1, editable=False, verbose_name=_(u'page number'), db_index=True) -# Register the fields that will be searchable -register('document', Document, _(u'document'), [ - {'name': u'document_type__name', 'title': _(u'Document type')}, - {'name': u'documentversion__mimetype', 'title': _(u'MIME type')}, - {'name': u'documentversion__filename', 'title': _(u'Filename')}, - {'name': u'documentmetadata__value', 'title': _(u'Metadata value')}, {'name': u'documentversion__documentpage__content', 'title': _(u'Content')}, {'name': u'description', 'title': _(u'Description')}, - {'name': u'tags__name', 'title': _(u'Tags')}, - {'name': u'comments__comment', 'title': _(u'Comments')}, - ] -) -#register(Document, _(u'document'), ['document_type__name', 'file_mimetype', 'documentmetadata__value', 'documentpage__content', 'description', {'field_name':'file_filename', 'comparison':'iexact'}]) ''' @@ -38,8 +30,6 @@ class DocumentIndex(indexes.SearchIndex, indexes.Indexable): return Document def build_queryset(self, start_date=None, end_date=None): - print "DIRTY", IndexableObject.objects.get_dirty_pk_list() - - #return self.get_model().objects.filter(date_added__lte=datetime.datetime.now()) - return self.get_model().objects.filter(pk__in=IndexableObject.objects.get_dirty_pk_list()) - #return self.get_model().objects.all() + indexable_list = IndexableObject.objects.get_indexables_pk_list() + logger.debug('indexable list: %s' % indexable_list) + return self.get_model().objects.filter(pk__in=indexable_list) diff --git a/apps/dynamic_search/templates/search_results.html b/apps/dynamic_search/templates/search_results.html index 04c0dd1f8e..f364ab0c45 100644 --- a/apps/dynamic_search/templates/search_results.html +++ b/apps/dynamic_search/templates/search_results.html @@ -3,7 +3,6 @@ {% block title %} :: {% trans "Search results" %}{% endblock %} {% block content %} - {% if form %} {% include "search_results_subtemplate.html" %} {% endif %} diff --git a/apps/dynamic_search/templatetags/search_tags.py b/apps/dynamic_search/templatetags/search_tags.py index aa20e22149..555c7a7c87 100644 --- a/apps/dynamic_search/templatetags/search_tags.py +++ b/apps/dynamic_search/templatetags/search_tags.py @@ -20,7 +20,7 @@ def search_form(context): 'submit_label': _(u'Search'), 'submit_icon_famfam': 'zoom', }) - return context + return '' @register.inclusion_tag('generic_subtemplate.html', takes_context=True) @@ -34,7 +34,7 @@ def recent_searches_template(context): 'paragraphs': [ u'%(text)s' % { 'text': rs, - 'url': rs.url(), + 'url': reverse('search'), 'icon': 'zoom_in' if rs.is_advanced() else 'zoom', } for rs in recent_searches ] diff --git a/apps/dynamic_search/urls.py b/apps/dynamic_search/urls.py index 5c226e4838..3c7ef4e752 100644 --- a/apps/dynamic_search/urls.py +++ b/apps/dynamic_search/urls.py @@ -3,11 +3,10 @@ from django.conf.urls.defaults import patterns, url from haystack.forms import SearchForm from .views import CustomSearchView -from .forms import AdvancedSearchForm urlpatterns = patterns('dynamic_search.views', url(r'^$', CustomSearchView(form_class=SearchForm), (), 'search'), - url(r'^advanced/$', CustomSearchView(form_class=AdvancedSearchForm), (), 'search_advanced'), - url(r'^again/$', 'search_again', (), 'search_again'), - url(r'^results/$', 'results', (), 'results'), + #url(r'^advanced/$', CustomSearchView(form_class=AdvancedSearchForm), (), 'search_advanced'), + #url(r'^again/$', 'search_again', (), 'search_again'), + #url(r'^results/$', 'results', (), 'results'), ) diff --git a/apps/dynamic_search/views.py b/apps/dynamic_search/views.py index ddd7fe4501..a5c08a6080 100644 --- a/apps/dynamic_search/views.py +++ b/apps/dynamic_search/views.py @@ -16,8 +16,11 @@ from common.utils import encapsulate from dynamic_search.models import RecentSearch from dynamic_search.api import perform_search -from dynamic_search.forms import SearchForm, AdvancedSearchForm + +#TODO: DEPRECATED from dynamic_search.conf.settings import SHOW_OBJECT_TYPE + +#TODO: DEPRECATED? from dynamic_search.conf.settings import LIMIT @@ -47,11 +50,13 @@ class CustomSearchView(SearchView): 'elapsed_time': unicode(datetime.datetime.now() - self.start_time).split(':')[2], 'object_list_object_name': 'object', } + + RecentSearch.objects.add_query_for_user(self) context.update(self.extra_context()) return render_to_response(self.template, context, context_instance=self.context_class(self.request)) - +""" def results(request, extra_context=None): context = {} @@ -141,3 +146,4 @@ def search(request, advanced=False): def search_again(request): query = urlparse.urlparse(request.META.get('HTTP_REFERER', u'/')).query return HttpResponseRedirect('%s?%s' % (reverse('search_advanced'), query)) +""" From da48d2c28a89b937fc2b0181f2471e1e8f7cd907 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Thu, 12 Apr 2012 02:05:18 -0400 Subject: [PATCH 21/46] Remove old code line --- urls.py | 1 - 1 file changed, 1 deletion(-) diff --git a/urls.py b/urls.py index 80d8cdef7d..ee1f199449 100644 --- a/urls.py +++ b/urls.py @@ -9,7 +9,6 @@ urlpatterns = patterns('', (r'^', include('main.urls')), (r'^documents/', include('documents.urls')), (r'^folders/', include('folders.urls')), - #(r'^search/', include('haystack.urls')), (r'^search/', include('dynamic_search.urls')), (r'^ocr/', include('ocr.urls')), (r'^permissions/', include('permissions.urls')), From a697b7415700b906f5baa4b8a57f7eec65f02bf5 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Thu, 12 Apr 2012 02:05:49 -0400 Subject: [PATCH 22/46] Update apps to flag edited documents as dirty --- apps/document_comments/views.py | 8 +++++--- apps/documents/models.py | 2 +- apps/metadata/views.py | 4 ++++ apps/tags/views.py | 7 ++++++- 4 files changed, 16 insertions(+), 5 deletions(-) diff --git a/apps/document_comments/views.py b/apps/document_comments/views.py index 4a87435fa3..005ee109cd 100644 --- a/apps/document_comments/views.py +++ b/apps/document_comments/views.py @@ -43,6 +43,7 @@ def comment_delete(request, comment_id=None, comment_id_list=None): for comment in comments: try: comment.delete() + comment.content_object.mark_indexable() messages.success(request, _(u'Comment "%s" deleted successfully.') % comment) except Exception, e: messages.error(request, _(u'Error deleting comment "%(comment)s": %(error)s') % { @@ -95,7 +96,8 @@ def comment_add(request, document_id): comment.object_pk = document.pk comment.site = Site.objects.get_current() comment.save() - + document.mark_indexable() + messages.success(request, _(u'Comment added successfully.')) return HttpResponseRedirect(next) else: @@ -110,9 +112,9 @@ def comment_add(request, document_id): def comments_for_document(request, document_id): - ''' + """ Show a list of all the comments related to the passed object - ''' + """ document = get_object_or_404(Document, pk=document_id) try: diff --git a/apps/documents/models.py b/apps/documents/models.py index 5c1c69e732..784c97d2f8 100644 --- a/apps/documents/models.py +++ b/apps/documents/models.py @@ -103,7 +103,7 @@ class Document(models.Model): self.uuid = UUID_FUNCTION() self.date_added = datetime.datetime.now() super(Document, self).save(*args, **kwargs) - self.mark_dirty() + self.mark_indexable() def get_cached_image_name(self, page, version): document_version = DocumentVersion.objects.get(pk=version) diff --git a/apps/metadata/views.py b/apps/metadata/views.py index b380b81025..b5b834a081 100644 --- a/apps/metadata/views.py +++ b/apps/metadata/views.py @@ -92,6 +92,7 @@ def metadata_edit(request, document_id=None, document_id_list=None): messages.error(request, _(u'Error editing metadata for document %(document)s; %(error)s.') % { 'document': document, 'error': error}) else: + document.mark_indexable() messages.success(request, _(u'Metadata for document %s edited successfully.') % document) return HttpResponseRedirect(next) @@ -149,6 +150,8 @@ def metadata_add(request, document_id=None, document_id_list=None): else: messages.warning(request, _(u'Metadata type: %(metadata_type)s already present in document %(document)s.') % { 'metadata_type': metadata_type, 'document': document}) + + document.mark_indexable() if len(documents) == 1: return HttpResponseRedirect(u'%s?%s' % ( @@ -236,6 +239,7 @@ def metadata_remove(request, document_id=None, document_id_list=None): try: document_metadata = DocumentMetadata.objects.get(document=document, metadata_type=metadata_type) document_metadata.delete() + document.mark_indexable() messages.success(request, _(u'Successfully remove metadata type: %(metadata_type)s from document: %(document)s.') % { 'metadata_type': metadata_type, 'document': document}) except: diff --git a/apps/tags/views.py b/apps/tags/views.py index a153506e02..1cf79abdf8 100644 --- a/apps/tags/views.py +++ b/apps/tags/views.py @@ -78,7 +78,7 @@ def tag_attach(request, document_id): return HttpResponseRedirect(next) document.tags.add(tag) - + document.mark_indexable() messages.success(request, _(u'Tag "%s" attached successfully.') % tag) return HttpResponseRedirect(next) else: @@ -141,6 +141,8 @@ def tag_delete(request, tag_id=None, tag_id_list=None): if request.method == 'POST': for tag in tags: try: + for document in Document.objects.filter(tags__in=[tag]): + document.mark_indexable() tag.delete() messages.success(request, _(u'Tag "%s" deleted successfully.') % tag) except Exception, e: @@ -188,6 +190,8 @@ def tag_edit(request, tag_id): if form.is_valid(): tag.name = form.cleaned_data['name'] tag.save() + for document in Document.objects.filter(tags__in=[tag]): + document.mark_indexable() tag_properties = tag.tagproperties_set.get() tag_properties.color = form.cleaned_data['color'] tag_properties.save() @@ -264,6 +268,7 @@ def tag_remove(request, document_id, tag_id=None, tag_id_list=None): for tag in tags: try: document.tags.remove(tag) + document.mark_indexable() messages.success(request, _(u'Tag "%s" removed successfully.') % tag) except Exception, e: messages.error(request, _(u'Error deleting tag "%(tag)s": %(error)s') % { From 738e304545c3bf93fe2bcf0f8d6a14e9c51983d3 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Thu, 12 Apr 2012 02:06:28 -0400 Subject: [PATCH 23/46] Small 0.13 release notes update --- docs/releases/0.13.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/releases/0.13.rst b/docs/releases/0.13.rst index 9e11dcd124..20bb0e0591 100644 --- a/docs/releases/0.13.rst +++ b/docs/releases/0.13.rst @@ -43,6 +43,8 @@ Afterwards migrate existing database schema with:: $ ./manage.py migrate sources 0001 --fake $ ./manage.py migrate sources +#Haystack - build index + The upgrade procedure is now complete. From 8e829612af15c9ef793f2fb2eeb5a1c6496a92b9 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Thu, 12 Apr 2012 12:22:58 -0400 Subject: [PATCH 24/46] Refactor and simply recent search functionality --- apps/dynamic_search/managers.py | 18 ++------ apps/dynamic_search/models.py | 22 ++++++---- .../templatetags/search_tags.py | 41 +++++++++++-------- 3 files changed, 39 insertions(+), 42 deletions(-) diff --git a/apps/dynamic_search/managers.py b/apps/dynamic_search/managers.py index db0c99d111..0806712b45 100644 --- a/apps/dynamic_search/managers.py +++ b/apps/dynamic_search/managers.py @@ -15,21 +15,6 @@ from dynamic_search.conf.settings import RECENT_COUNT class RecentSearchManager(Manager): def add_query_for_user(self, search_view):#user, url, hits): query_dict = parse_qs(unquote_plus(smart_str(urlparse(search_view.request.get_full_path()).query))) - print 'query_dict', query_dict - print 'serial', dumps(query_dict) - - #parsed_query = urlparse.parse_qs(query) - #for key, value in parsed_query.items(): - # parsed_query[key] = ' '.join(value)# - - #if 'q=' in query: - # # Is a simple query - # if not parsed_query.get('q'): - # # Don't store empty simple searches - # return - # else: - # # Cleanup query string and only store the q parameter - # parsed_query = {'q': parsed_query['q']} if query_dict and not isinstance(search_view.request.user, AnonymousUser): # If the URL query has at least one variable with a value @@ -40,6 +25,9 @@ class RecentSearchManager(Manager): for recent_to_delete in to_delete: recent_to_delete.delete() + def get_for_user(self, user): + return [entry for entry in self.model.objects.filter(user=user) if entry.get_query()] + class IndexableObjectManager(Manager): def get_indexables(self, datetime=None): diff --git a/apps/dynamic_search/models.py b/apps/dynamic_search/models.py index f0e29408fb..bcb1eb0fa9 100644 --- a/apps/dynamic_search/models.py +++ b/apps/dynamic_search/models.py @@ -13,6 +13,7 @@ from django.utils.translation import ugettext_lazy as _ from django.contrib.contenttypes.models import ContentType from django.contrib.contenttypes import generic from django.utils.simplejson import loads +from django.utils.http import urlencode from .managers import RecentSearchManager, IndexableObjectManager from .api import registered_search_dict @@ -30,28 +31,31 @@ class RecentSearch(models.Model): objects = RecentSearchManager() def __unicode__(self): + return self.form_string() + + def form_string(self): + query = self.get_query() if self.is_advanced(): return u'%s (%s)' % (self.get_query(), self.hits) else: - return u'%s (%s)' % (self.get_query().get('q'), self.hits) + return u'%s (%s)' % (u' '.join(self.get_query().get('q')), self.hits) def save(self, *args, **kwargs): self.datetime_created = datetime.datetime.now() super(RecentSearch, self).save(*args, **kwargs) - #def readable_query(self): - # return self. - - #def url(self): - # view = 'results' if self.is_advanced() else 'search' - # return '%s?%s' % (reverse(view), self.query) - def get_query(self): - return loads(self.query) + try: + return loads(self.query) + except ValueError: + return {} def is_advanced(self): return 'q' not in self.get_query() + def get_absolute_url(self): + return '?'.join([reverse('search'), urlencode(self.get_query(), doseq=True)]) + class Meta: ordering = ('-datetime_created',) verbose_name = _(u'recent search') diff --git a/apps/dynamic_search/templatetags/search_tags.py b/apps/dynamic_search/templatetags/search_tags.py index 555c7a7c87..9874169d26 100644 --- a/apps/dynamic_search/templatetags/search_tags.py +++ b/apps/dynamic_search/templatetags/search_tags.py @@ -1,10 +1,12 @@ +from __future__ import absolute_import + from django.core.urlresolvers import reverse from django.template import Library from django.utils.translation import ugettext as _ -from dynamic_search.forms import SearchForm -from dynamic_search.models import RecentSearch -from dynamic_search.conf.settings import RECENT_COUNT +from ..forms import SearchForm +from ..models import RecentSearch +from ..conf.settings import RECENT_COUNT register = Library() @@ -25,18 +27,21 @@ def search_form(context): @register.inclusion_tag('generic_subtemplate.html', takes_context=True) def recent_searches_template(context): - recent_searches = RecentSearch.objects.filter(user=context['user']) - context.update({ - 'request': context['request'], - 'STATIC_URL': context['STATIC_URL'], - 'side_bar': True, - 'title': _(u'recent searches (maximum of %d)') % RECENT_COUNT, - 'paragraphs': [ - u'%(text)s' % { - 'text': rs, - 'url': reverse('search'), - 'icon': 'zoom_in' if rs.is_advanced() else 'zoom', - } for rs in recent_searches - ] - }) - return context + try: + recent_searches = RecentSearch.objects.get_for_user(user=context['user']) + context.update({ + 'request': context['request'], + 'STATIC_URL': context['STATIC_URL'], + 'side_bar': True, + 'title': _(u'recent searches (maximum of %d)') % RECENT_COUNT, + 'paragraphs': [ + u'%(text)s' % { + 'text': recent_search, + 'url': recent_search.get_absolute_url(), + 'icon': 'zoom_in' if recent_search.is_advanced() else 'zoom', + } for recent_search in recent_searches + ] + }) + return context + except Exception, e: + print 'EEEEEEEEEEEE', e From 2ee59df866dd5c00e256348e87a8d64fa2aec71f Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Thu, 12 Apr 2012 12:25:59 -0400 Subject: [PATCH 25/46] Removal of the SEARCH_SHOW_OBJECT_TYPE and SEARCH_LIMIT configuration options --- apps/dynamic_search/conf/settings.py | 16 ---------------- apps/dynamic_search/views.py | 8 +------- docs/releases/0.13.rst | 3 ++- 3 files changed, 3 insertions(+), 24 deletions(-) diff --git a/apps/dynamic_search/conf/settings.py b/apps/dynamic_search/conf/settings.py index 78ff433087..ac9c80a476 100644 --- a/apps/dynamic_search/conf/settings.py +++ b/apps/dynamic_search/conf/settings.py @@ -6,22 +6,6 @@ from smart_settings.api import Setting, SettingNamespace namespace = SettingNamespace('dynamic_search', _(u'Searching'), module='dynamic_search.conf.settings') -Setting( - namespace=namespace, - name='SHOW_OBJECT_TYPE', - global_name='SEARCH_SHOW_OBJECT_TYPE', - default=True, - hidden=True -) - -Setting( - namespace=namespace, - name='LIMIT', - global_name='SEARCH_LIMIT', - default=100, - description=_(u'Maximum amount search hits to fetch and display.') -) - Setting( namespace=namespace, name='RECENT_COUNT', diff --git a/apps/dynamic_search/views.py b/apps/dynamic_search/views.py index a5c08a6080..65e2c4c83e 100644 --- a/apps/dynamic_search/views.py +++ b/apps/dynamic_search/views.py @@ -4,11 +4,11 @@ import urlparse from django.shortcuts import render_to_response from django.template import RequestContext from django.utils.translation import ugettext_lazy as _ +from django.utils.http import urlencode from django.contrib import messages from django.conf import settings from django.http import HttpResponseRedirect from django.core.urlresolvers import reverse -from django.utils.http import urlencode from haystack.views import SearchView @@ -17,12 +17,6 @@ from common.utils import encapsulate from dynamic_search.models import RecentSearch from dynamic_search.api import perform_search -#TODO: DEPRECATED -from dynamic_search.conf.settings import SHOW_OBJECT_TYPE - -#TODO: DEPRECATED? -from dynamic_search.conf.settings import LIMIT - class CustomSearchView(SearchView): def __call__(self, *args, **kwargs): diff --git a/docs/releases/0.13.rst b/docs/releases/0.13.rst index 20bb0e0591..bcea977354 100644 --- a/docs/releases/0.13.rst +++ b/docs/releases/0.13.rst @@ -60,4 +60,5 @@ Bugs fixed Stuff removed ============= -None so far +Removal of the internal SEARCH_SHOW_OBJECT_TYPE option +Removal of the SEARCH_LIMIT configuration option From e89959b71be0510650b903af94aaed1cb2062e1f Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Thu, 12 Apr 2012 12:59:06 -0400 Subject: [PATCH 26/46] Remove remaining code for the old search method and the advanced search support, misc cleanups too --- apps/dynamic_search/__init__.py | 6 - apps/dynamic_search/api.py | 138 ------------------ apps/dynamic_search/forms.py | 45 ------ apps/dynamic_search/managers.py | 16 +- apps/dynamic_search/models.py | 14 +- apps/dynamic_search/search_indexes.py | 12 -- .../templatetags/search_tags.py | 36 +++-- apps/dynamic_search/urls.py | 3 - apps/dynamic_search/views.py | 108 +------------- 9 files changed, 33 insertions(+), 345 deletions(-) delete mode 100644 apps/dynamic_search/api.py delete mode 100644 apps/dynamic_search/forms.py diff --git a/apps/dynamic_search/__init__.py b/apps/dynamic_search/__init__.py index 53d427e0a0..c46817f759 100644 --- a/apps/dynamic_search/__init__.py +++ b/apps/dynamic_search/__init__.py @@ -15,17 +15,11 @@ from .models import IndexableObject logger = logging.getLogger(__name__) search = {'text': _(u'search'), 'view': 'search', 'famfam': 'zoom'} -#search_advanced = {'text': _(u'advanced search'), 'view': 'search_advanced', 'famfam': 'zoom_in'} -#search_again = {'text': _(u'search again'), 'view': 'search_again', 'famfam': 'arrow_undo'} -#register_sidebar_template(['search', 'search_advanced'], 'search_help.html') register_sidebar_template(['search'], 'search_help.html') -#register_links(['search', 'search_advanced', 'results'], [search, search_advanced], menu_name='form_header') register_links(['search'], [search], menu_name='form_header') -#register_links(['results'], [search_again], menu_name='sidebar') -#register_sidebar_template(['search', 'search_advanced', 'results'], 'recent_searches.html') register_sidebar_template(['search'], 'recent_searches.html') Document.add_to_class('mark_indexable', lambda obj: IndexableObject.objects.mark_indexable(obj)) diff --git a/apps/dynamic_search/api.py b/apps/dynamic_search/api.py deleted file mode 100644 index fdf098ba8c..0000000000 --- a/apps/dynamic_search/api.py +++ /dev/null @@ -1,138 +0,0 @@ -# original code from: -# http://www.julienphalip.com/blog/2008/08/16/adding-search-django-site-snap/ - -import re -import types -import datetime - -from django.db.models import Q - -from dynamic_search.conf.settings import LIMIT - -registered_search_dict = {} - - -def register(model_name, model, title, fields): - registered_search_dict.setdefault(model_name, {'model': model, 'fields': [], 'title': title}) - registered_search_dict[model_name]['fields'].extend(fields) - - -def normalize_query(query_string, - findterms=re.compile(r'"([^"]+)"|(\S+)').findall, - normspace=re.compile(r'\s{2,}').sub): - """ - Splits the query string in invidual keywords, getting rid of unecessary spaces - and grouping quoted words together. - Example: - >>> normalize_query(' some random words "with quotes " and spaces') - ['some', 'random', 'words', 'with quotes', 'and', 'spaces'] - """ - return [normspace(' ', (t[0] or t[1]).strip()) for t in findterms(query_string)] - - -def get_query(terms, search_fields): - """ - Returns a query, that is a combination of Q objects. That combination - aims to search keywords within a model by testing the given search fields. - """ - queries = [] - for term in terms: - or_query = None - for field in search_fields: - if isinstance(field, types.StringTypes): - comparison = u'icontains' - field_name = field - elif isinstance(field, types.DictType): - comparison = field.get('comparison', u'icontains') - field_name = field.get('field_name', u'') - - if field_name: - q = Q(**{'%s__%s' % (field_name, comparison): term}) - if or_query is None: - or_query = q - else: - or_query = or_query | q - - queries.append(or_query) - return queries - - -def perform_search(query_string, field_list=None): - model_list = {} - flat_list = [] - result_count = 0 - shown_result_count = 0 - elapsed_time = 0 - start_time = datetime.datetime.now() - - search_dict = {} - - if query_string: - simple_query_string = query_string.get('q', u'').strip() - if simple_query_string: - for model, values in registered_search_dict.items(): - search_dict.setdefault(values['model'], {'query_entries': [], 'title': values['title']}) - field_names = [field['name'] for field in values['fields']] - # One entry, single set of terms for all fields names - search_dict[values['model']]['query_entries'].append( - { - 'field_name': field_names, - 'terms': normalize_query(simple_query_string) - } - ) - else: - for key, value in query_string.items(): - try: - model, field_name = key.split('__', 1) - model_entry = registered_search_dict.get(model, {}) - if model_entry: - for model_field in model_entry.get('fields', [{}]): - if model_field.get('name') == field_name: - search_dict.setdefault(model_entry['model'], {'query_entries': [], 'title': model_entry['title']}) - search_dict[model_entry['model']]['query_entries'].append( - { - 'field_name': [field_name], - 'terms': normalize_query(value.strip()) - } - ) - except ValueError: - pass - - for model, data in search_dict.items(): - title = data['title'] - queries = [] - - for query_entry in data['query_entries']: - queries.extend(get_query(query_entry['terms'], query_entry['field_name'])) - - model_result_ids = None - for query in queries: - single_result_ids = set(model.objects.filter(query).values_list('pk', flat=True)) - #Convert queryset to python set and perform the - #AND operation on the program and not as a query - if model_result_ids == None: - model_result_ids = single_result_ids - else: - model_result_ids &= single_result_ids - - if model_result_ids == None: - model_result_ids = [] - - result_count += len(model_result_ids) - results = model.objects.in_bulk(list(model_result_ids)[: LIMIT]).values() - shown_result_count += len(results) - if results: - model_list[title] = results - for result in results: - if result not in flat_list: - flat_list.append(result) - - elapsed_time = unicode(datetime.datetime.now() - start_time).split(':')[2] - - return { - 'model_list': model_list, - 'flat_list': flat_list, - 'shown_result_count': shown_result_count, - 'result_count': result_count, - 'elapsed_time': elapsed_time - } diff --git a/apps/dynamic_search/forms.py b/apps/dynamic_search/forms.py deleted file mode 100644 index 56ca16e348..0000000000 --- a/apps/dynamic_search/forms.py +++ /dev/null @@ -1,45 +0,0 @@ -from __future__ import absolute_import - -from django import forms -from django.utils.translation import ugettext_lazy as _ - -from haystack.forms import SearchForm as BasicSearchForm - -from .api import registered_search_dict - - -class SearchForm(BasicSearchForm): - pass - - -""" -class AdvancedSearchForm(SearchForm): - def __init__(self, *args, **kwargs): - super(AdvancedSearchForm, self).__init__(*args, **kwargs) - - for model_name, values in registered_search_dict.items(): - for field in values['fields']: - self.fields['%s__%s' % (model_name, field['name'])] = forms.CharField( - label=field['title'], - required=False - ) - - def search(self): - if not self.is_valid(): - return self.no_query_found() - - #if not self.cleaned_data.get('q'): - # return self.no_query_found() - for field in self.fields: - print 'field', field - #sqs = self.searchqueryset.auto_query(self.cleaned_data['q']) - - if self.load_all: - sqs = sqs.load_all() - - return sqs - - def search(self): - sqs = super(ModelSearchForm, self).search() - return sqs.models(*self.get_models()) -""" diff --git a/apps/dynamic_search/managers.py b/apps/dynamic_search/managers.py index 0806712b45..57a4ad30ab 100644 --- a/apps/dynamic_search/managers.py +++ b/apps/dynamic_search/managers.py @@ -1,19 +1,19 @@ +from __future__ import absolute_import + from urlparse import urlparse, parse_qs from urllib import unquote_plus -from django.utils.simplejson import dumps, loads, JSONEncoder +from django.utils.simplejson import dumps from django.db.models import Manager -from django.utils.http import urlencode from django.contrib.auth.models import AnonymousUser from django.contrib.contenttypes.models import ContentType -from django.contrib.contenttypes import generic -from django.utils.encoding import smart_unicode, smart_str +from django.utils.encoding import smart_str -from dynamic_search.conf.settings import RECENT_COUNT +from .conf.settings import RECENT_COUNT class RecentSearchManager(Manager): - def add_query_for_user(self, search_view):#user, url, hits): + def add_query_for_user(self, search_view): query_dict = parse_qs(unquote_plus(smart_str(urlparse(search_view.request.get_full_path()).query))) if query_dict and not isinstance(search_view.request.user, AnonymousUser): @@ -35,13 +35,13 @@ class IndexableObjectManager(Manager): return self.model.objects.filter(datetime__gte=datetime) else: return self.model.objects.all() - + def get_indexables_pk_list(self, datetime=None): return self.get_indexables(datetime).values_list('object_id', flat=True) def mark_indexable(self, obj): content_type = ContentType.objects.get_for_model(obj) self.model.objects.get_or_create(content_type=content_type, object_id=obj.pk) - + def clear_all(self): self.model.objects.all().delete() diff --git a/apps/dynamic_search/models.py b/apps/dynamic_search/models.py index bcb1eb0fa9..6cae642507 100644 --- a/apps/dynamic_search/models.py +++ b/apps/dynamic_search/models.py @@ -1,14 +1,10 @@ from __future__ import absolute_import -import urlparse -import urllib import datetime from django.db import models -from django.utils.translation import ugettext as _ from django.contrib.auth.models import User from django.core.urlresolvers import reverse -from django.utils.encoding import smart_unicode, smart_str from django.utils.translation import ugettext_lazy as _ from django.contrib.contenttypes.models import ContentType from django.contrib.contenttypes import generic @@ -16,7 +12,6 @@ from django.utils.simplejson import loads from django.utils.http import urlencode from .managers import RecentSearchManager, IndexableObjectManager -from .api import registered_search_dict class RecentSearch(models.Model): @@ -34,7 +29,6 @@ class RecentSearch(models.Model): return self.form_string() def form_string(self): - query = self.get_query() if self.is_advanced(): return u'%s (%s)' % (self.get_query(), self.hits) else: @@ -64,16 +58,16 @@ class RecentSearch(models.Model): class IndexableObject(models.Model): """ - Store a list of object links that have been modified and are - meant to be indexed in the next search index update + Store a list of object links that have been modified and are + meant to be indexed in the next search index update """ datetime = models.DateTimeField(verbose_name=_(u'date time')) content_type = models.ForeignKey(ContentType) object_id = models.PositiveIntegerField() content_object = generic.GenericForeignKey('content_type', 'object_id') - + objects = IndexableObjectManager() - + def __unicode__(self): return unicode(self.content_object) diff --git a/apps/dynamic_search/search_indexes.py b/apps/dynamic_search/search_indexes.py index 7fc4294d6f..e2bfa0c69c 100644 --- a/apps/dynamic_search/search_indexes.py +++ b/apps/dynamic_search/search_indexes.py @@ -1,9 +1,7 @@ from __future__ import absolute_import -import datetime import logging -from unidecode import unidecode from haystack import indexes from documents.models import Document @@ -12,16 +10,6 @@ from .models import IndexableObject logger = logging.getLogger(__name__) -''' - comment = models.TextField(blank=True, verbose_name=_(u'comment')) - checksum = models.TextField(blank=True, null=True, verbose_name=_(u'checksum'), editable=False) - page_label = models.CharField(max_length=32, blank=True, null=True, verbose_name=_(u'page label')) - page_number = models.PositiveIntegerField(default=1, editable=False, verbose_name=_(u'page number'), db_index=True) - - {'name': u'documentversion__documentpage__content', 'title': _(u'Content')}, - {'name': u'description', 'title': _(u'Description')}, - -''' class DocumentIndex(indexes.SearchIndex, indexes.Indexable): text = indexes.CharField(document=True, use_template=True) diff --git a/apps/dynamic_search/templatetags/search_tags.py b/apps/dynamic_search/templatetags/search_tags.py index 9874169d26..e48852685f 100644 --- a/apps/dynamic_search/templatetags/search_tags.py +++ b/apps/dynamic_search/templatetags/search_tags.py @@ -4,7 +4,8 @@ from django.core.urlresolvers import reverse from django.template import Library from django.utils.translation import ugettext as _ -from ..forms import SearchForm +from haystack.forms import SearchForm + from ..models import RecentSearch from ..conf.settings import RECENT_COUNT @@ -27,21 +28,18 @@ def search_form(context): @register.inclusion_tag('generic_subtemplate.html', takes_context=True) def recent_searches_template(context): - try: - recent_searches = RecentSearch.objects.get_for_user(user=context['user']) - context.update({ - 'request': context['request'], - 'STATIC_URL': context['STATIC_URL'], - 'side_bar': True, - 'title': _(u'recent searches (maximum of %d)') % RECENT_COUNT, - 'paragraphs': [ - u'%(text)s' % { - 'text': recent_search, - 'url': recent_search.get_absolute_url(), - 'icon': 'zoom_in' if recent_search.is_advanced() else 'zoom', - } for recent_search in recent_searches - ] - }) - return context - except Exception, e: - print 'EEEEEEEEEEEE', e + recent_searches = RecentSearch.objects.get_for_user(user=context['user']) + context.update({ + 'request': context['request'], + 'STATIC_URL': context['STATIC_URL'], + 'side_bar': True, + 'title': _(u'recent searches (maximum of %d)') % RECENT_COUNT, + 'paragraphs': [ + u'%(text)s' % { + 'text': recent_search, + 'url': recent_search.get_absolute_url(), + 'icon': 'zoom_in' if recent_search.is_advanced() else 'zoom', + } for recent_search in recent_searches + ] + }) + return context diff --git a/apps/dynamic_search/urls.py b/apps/dynamic_search/urls.py index 3c7ef4e752..aa1cd26796 100644 --- a/apps/dynamic_search/urls.py +++ b/apps/dynamic_search/urls.py @@ -6,7 +6,4 @@ from .views import CustomSearchView urlpatterns = patterns('dynamic_search.views', url(r'^$', CustomSearchView(form_class=SearchForm), (), 'search'), - #url(r'^advanced/$', CustomSearchView(form_class=AdvancedSearchForm), (), 'search_advanced'), - #url(r'^again/$', 'search_again', (), 'search_again'), - #url(r'^results/$', 'results', (), 'results'), ) diff --git a/apps/dynamic_search/views.py b/apps/dynamic_search/views.py index 65e2c4c83e..cac54e2a7c 100644 --- a/apps/dynamic_search/views.py +++ b/apps/dynamic_search/views.py @@ -1,21 +1,13 @@ +from __future__ import absolute_import + import datetime -import urlparse from django.shortcuts import render_to_response -from django.template import RequestContext from django.utils.translation import ugettext_lazy as _ -from django.utils.http import urlencode -from django.contrib import messages -from django.conf import settings -from django.http import HttpResponseRedirect -from django.core.urlresolvers import reverse from haystack.views import SearchView -from common.utils import encapsulate - -from dynamic_search.models import RecentSearch -from dynamic_search.api import perform_search +from .models import RecentSearch class CustomSearchView(SearchView): @@ -44,100 +36,8 @@ class CustomSearchView(SearchView): 'elapsed_time': unicode(datetime.datetime.now() - self.start_time).split(':')[2], 'object_list_object_name': 'object', } - + RecentSearch.objects.add_query_for_user(self) context.update(self.extra_context()) return render_to_response(self.template, context, context_instance=self.context_class(self.request)) - -""" -def results(request, extra_context=None): - context = {} - - context.update({ - 'query_string': request.GET, - #'hide_header': True, - 'hide_links': True, - 'multi_select_as_buttons': True, - 'search_results_limit': LIMIT, - }) - - try: - response = perform_search(request.GET) - if response['shown_result_count'] != response['result_count']: - title = _(u'results, (showing only %(shown_result_count)s out of %(result_count)s)') % { - 'shown_result_count': response['shown_result_count'], - 'result_count': response['result_count']} - else: - title = _(u'results') - - if extra_context: - context.update(extra_context) - query = urlencode(dict(request.GET.items())) - - if query: - RecentSearch.objects.add_query_for_user(request.user, query, response['result_count']) - - context.update({ - 'found_entries': response['model_list'], - 'object_list': response['flat_list'], - 'title': title, - 'time_delta': response['elapsed_time'], - }) - except Exception, e: - if settings.DEBUG: - raise - elif request.user.is_staff or request.user.is_superuser: - messages.error(request, _(u'Search error: %s') % e) - - if SHOW_OBJECT_TYPE: - context.update({'extra_columns': - [{'name': _(u'type'), 'attribute': lambda x: x._meta.verbose_name[0].upper() + x._meta.verbose_name[1:]}]}) - - return render_to_response('search_results.html', context, - context_instance=RequestContext(request)) - - -def search(request, advanced=False): - if advanced: - form = AdvancedSearchForm(data=request.GET) - return render_to_response('generic_form.html', - { - 'form': form, - 'title': _(u'advanced search'), - 'form_action': reverse('results'), - 'submit_method': 'GET', - 'search_results_limit': LIMIT, - 'submit_label': _(u'Search'), - 'submit_icon_famfam': 'zoom', - }, - context_instance=RequestContext(request) - ) - else: - if request.GET.get('source') != 'sidebar': - # Don't include a form a top of the results if the search - # was originated from the sidebar search form - extra_context = { - 'submit_label': _(u'Search'), - 'submit_icon_famfam': 'zoom', - 'form_title': _(u'Search'), - 'form_hide_required_text': True, - } - if ('q' in request.GET) and request.GET['q'].strip(): - query_string = request.GET['q'] - form = SearchForm(initial={'q': query_string}) - extra_context.update({'form': form}) - return results(request, extra_context=extra_context) - else: - form = SearchForm() - extra_context.update({'form': form}) - return results(request, extra_context=extra_context) - else: - # Already has a form with data, go to results - return results(request) - - -def search_again(request): - query = urlparse.urlparse(request.META.get('HTTP_REFERER', u'/')).query - return HttpResponseRedirect('%s?%s' % (reverse('search_advanced'), query)) -""" From ed517b24f36e5cd5a7f449d69481179d0ca5d868 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Fri, 13 Apr 2012 01:32:54 -0400 Subject: [PATCH 27/46] Initial commit of the index update scheduling support --- apps/dynamic_search/__init__.py | 29 ++++++++++++++++++++++++++++ apps/dynamic_search/conf/settings.py | 8 ++++++++ docs/releases/0.13.rst | 9 +++++++-- 3 files changed, 44 insertions(+), 2 deletions(-) diff --git a/apps/dynamic_search/__init__.py b/apps/dynamic_search/__init__.py index c46817f759..88ee1b9489 100644 --- a/apps/dynamic_search/__init__.py +++ b/apps/dynamic_search/__init__.py @@ -5,12 +5,17 @@ import logging from django.utils.translation import ugettext_lazy as _ from django.dispatch import receiver +from haystack.management.commands.update_index import Command + from navigation.api import register_sidebar_template, register_links from documents.models import Document from scheduler.runtime import scheduler from signaler.signals import post_update_index, pre_update_index +from scheduler.api import register_interval_job +from lock_manager import Lock, LockError from .models import IndexableObject +from .conf.settings import INDEX_UPDATE_INTERVAL logger = logging.getLogger(__name__) @@ -35,3 +40,27 @@ def clear_pending_indexables(sender, **kwargs): def scheduler_shutdown_pre_update_index(sender, **kwargs): logger.debug('Scheduler shut down on pre update index signal') scheduler.shutdown() + + +def search_index_update(): + lock_id = u'search_index_update' + try: + logger.debug('trying to acquire lock: %s' % lock_id) + lock = Lock.acquire_lock(lock_id) + logger.debug('acquired lock: %s' % lock_id) + + logger.debug('Executing haystack\'s index update command') + command = Command() + command.handle() + + lock.release() + except LockError: + logger.debug('unable to obtain lock') + pass + + +register_interval_job('search_index_update', _(u'Update the search index with the most recent modified documents.'), search_index_update, seconds=INDEX_UPDATE_INTERVAL) + + + + diff --git a/apps/dynamic_search/conf/settings.py b/apps/dynamic_search/conf/settings.py index ac9c80a476..1c7c07559f 100644 --- a/apps/dynamic_search/conf/settings.py +++ b/apps/dynamic_search/conf/settings.py @@ -13,3 +13,11 @@ Setting( default=5, description=_(u'Maximum number of search queries to remember per user.') ) + +Setting( + namespace=namespace, + name='INDEX_UPDATE_INTERVAL', + global_name='SEARCH_INDEX_UPDATE_INTERVAL', + default=1800, + description=_(u'Interval in second on which to trigger the search index update.') +) diff --git a/docs/releases/0.13.rst b/docs/releases/0.13.rst index bcea977354..172bbe41b5 100644 --- a/docs/releases/0.13.rst +++ b/docs/releases/0.13.rst @@ -26,6 +26,8 @@ POP3 and IMAP Send document or document links via E-mail ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Haystack +~~~~~~~~ Upgrading from a previous version ================================= @@ -43,9 +45,12 @@ Afterwards migrate existing database schema with:: $ ./manage.py migrate sources 0001 --fake $ ./manage.py migrate sources -#Haystack - build index +Issue the following command to index existing documents in the new full text search database:: -The upgrade procedure is now complete. + $ ./manage.py rebuild_index --noinput + +Depending on the existing size of you current document base this make take from a few minutes up to an hour. +Once the full text initial indexing ends, the upgrade procedure is complete. Backward incompatible changes From 83fd8ab5cffb4b6039ce04de18c4c94f8041d328 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Sun, 15 Apr 2012 22:47:14 -0400 Subject: [PATCH 28/46] Fix ACL calculation when user doesn't belong to any group or role --- apps/acls/managers.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/acls/managers.py b/apps/acls/managers.py index 6ba807a897..529643c7ec 100644 --- a/apps/acls/managers.py +++ b/apps/acls/managers.py @@ -137,14 +137,14 @@ class AccessEntryManager(models.Manager): content_type = ContentType.objects.get_for_model(cls) # Calculate actor role membership ACL query - total_queries = None + total_queries = Q() for role in RoleMember.objects.get_roles_for_member(actor): role_type = ContentType.objects.get_for_model(role) if related: query = Q(holder_type=role_type, holder_id=role.pk, permission=permission.get_stored_permission) else: query = Q(holder_type=role_type, holder_id=role.pk, content_type=content_type, permission=permission.get_stored_permission) - if total_queries is None: + if not total_queries: total_queries = query else: total_queries = total_queries | query @@ -161,7 +161,7 @@ class AccessEntryManager(models.Manager): query = Q(holder_type=group_type, holder_id=group.pk, permission=permission.get_stored_permission) else: query = Q(holder_type=group_type, holder_id=group.pk, content_type=content_type, permission=permission.get_stored_permission) - if total_queries is None: + if not total_queries: total_queries = query else: total_queries = total_queries | query From 01b6ff168ef0c987c8e6839d8bd7cbc7ad2be15e Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Sun, 15 Apr 2012 22:47:49 -0400 Subject: [PATCH 29/46] Apply permission and ACL checks to the search results --- apps/dynamic_search/views.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/apps/dynamic_search/views.py b/apps/dynamic_search/views.py index cac54e2a7c..de9636a779 100644 --- a/apps/dynamic_search/views.py +++ b/apps/dynamic_search/views.py @@ -4,9 +4,14 @@ import datetime from django.shortcuts import render_to_response from django.utils.translation import ugettext_lazy as _ +from django.core.exceptions import PermissionDenied from haystack.views import SearchView +from documents.permissions import PERMISSION_DOCUMENT_VIEW +from permissions import Permission +from acls.models import AccessEntry + from .models import RecentSearch @@ -20,11 +25,18 @@ class CustomSearchView(SearchView): """ Generates the actual HttpResponse to send back to the user. """ + object_list = self.results.values_list('object', flat=True) + + try: + Permission.objects.check_permissions(self.request.user, [PERMISSION_DOCUMENT_VIEW]) + except PermissionDenied: + if self.query: + object_list = AccessEntry.objects.filter_objects_by_access(PERMISSION_DOCUMENT_VIEW, self.request.user, object_list) + context = { 'query': self.query, 'form': self.form, - 'results': self.results, - 'object_list': self.results, + 'object_list': object_list, 'suggestion': None, 'submit_label': _(u'Search'), 'submit_icon_famfam': 'zoom', @@ -34,7 +46,6 @@ class CustomSearchView(SearchView): 'hide_links': True, 'multi_select_as_buttons': True, 'elapsed_time': unicode(datetime.datetime.now() - self.start_time).split(':')[2], - 'object_list_object_name': 'object', } RecentSearch.objects.add_query_for_user(self) From 4d8460ad8aa5e0669588adcaa5d679d60b40cde9 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Mon, 21 May 2012 10:28:46 -0400 Subject: [PATCH 30/46] Remove document metadata value helper, update document index template accordingly --- .../search/indexes/documents/document_text.txt | 6 +++++- apps/metadata/models.py | 15 --------------- 2 files changed, 5 insertions(+), 16 deletions(-) diff --git a/apps/dynamic_search/templates/search/indexes/documents/document_text.txt b/apps/dynamic_search/templates/search/indexes/documents/document_text.txt index 3226d8f509..a3a280a7ce 100644 --- a/apps/dynamic_search/templates/search/indexes/documents/document_text.txt +++ b/apps/dynamic_search/templates/search/indexes/documents/document_text.txt @@ -1,5 +1,9 @@ {{ object.content }} -{{ object.metadata_values_string }} + +{% for document_metadata in object.documentmetadata_set.all %} + {{ document_metadata.value }} +{% endfor %} + {{ object.cleaned_filename }} {{ object.filename }} {{ object.extension_split.1 }} diff --git a/apps/metadata/models.py b/apps/metadata/models.py index 93aeb921df..5e13f65c48 100644 --- a/apps/metadata/models.py +++ b/apps/metadata/models.py @@ -98,18 +98,3 @@ class DocumentTypeDefaults(models.Model): class Meta: verbose_name = _(u'document type defaults') verbose_name_plural = _(u'document types defaults') - - -def document_metadata_values(document): - return DocumentMetadata.objects.filter(document=document).values_list('value', flat=True) - - -def document_metadata_values_string(document): - return u' '.join(document_metadata_values(document)) - - -def document_metadata_dict_list(document): - return [{name: value} for name, value in DocumentMetadata.objects.filter(document=document).values_list('metadata_type__name', 'value')] - - -Document.add_to_class('metadata_values_string', document_metadata_values_string) From 98c387fbd094e560b68165b083fbc471758a98e8 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Mon, 21 May 2012 10:34:19 -0400 Subject: [PATCH 31/46] Remove document flat_tags helper, update document index template accordingly --- .../templates/search/indexes/documents/document_text.txt | 6 +++++- apps/tags/__init__.py | 7 ------- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/apps/dynamic_search/templates/search/indexes/documents/document_text.txt b/apps/dynamic_search/templates/search/indexes/documents/document_text.txt index a3a280a7ce..12ce19fa6f 100644 --- a/apps/dynamic_search/templates/search/indexes/documents/document_text.txt +++ b/apps/dynamic_search/templates/search/indexes/documents/document_text.txt @@ -11,5 +11,9 @@ {{ object.file_mimetype }} {{ object.description|default:' ' }} {{ object.flat_comments }} -{{ object.flat_tags }} + +{% for tag in object.tags.all %} + {{ tag }} +{% endfor %} + {{ object.uuid }} diff --git a/apps/tags/__init__.py b/apps/tags/__init__.py index 5263c3b240..2bfcb2f508 100644 --- a/apps/tags/__init__.py +++ b/apps/tags/__init__.py @@ -66,11 +66,4 @@ class_permissions(Tag, [ PERMISSION_TAG_VIEW, ]) - -def flat_tags(document): - return u' '.join(document.tags.values_list('name', flat=True)) - - Document.add_to_class('tags', TaggableManager()) -Document.add_to_class('flat_tags', flat_tags) - From ccc26f9566ba457f0bd3b32d39c96dbeab114d39 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Mon, 21 May 2012 10:42:17 -0400 Subject: [PATCH 32/46] Remove flat comment document helper, update document indexing template accordingly --- apps/document_comments/__init__.py | 7 ------- .../templates/search/indexes/documents/document_text.txt | 5 ++++- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/apps/document_comments/__init__.py b/apps/document_comments/__init__.py index 53bc133a94..2356f36885 100644 --- a/apps/document_comments/__init__.py +++ b/apps/document_comments/__init__.py @@ -40,11 +40,6 @@ register_links(['comments_for_document', 'comment_add', 'comment_delete', 'comme register_links(Comment, [comment_delete]) register_links(Document, [comments_for_document], menu_name='form_header') - -def flat_comments(document): - return u' '.join(document.comments.values_list('comment', flat=True)) - - Document.add_to_class( 'comments', generic.GenericRelation( @@ -54,8 +49,6 @@ Document.add_to_class( ) ) -Document.add_to_class('flat_comments', flat_comments) - class_permissions(Document, [ PERMISSION_COMMENT_CREATE, PERMISSION_COMMENT_DELETE, diff --git a/apps/dynamic_search/templates/search/indexes/documents/document_text.txt b/apps/dynamic_search/templates/search/indexes/documents/document_text.txt index 12ce19fa6f..77dfc80213 100644 --- a/apps/dynamic_search/templates/search/indexes/documents/document_text.txt +++ b/apps/dynamic_search/templates/search/indexes/documents/document_text.txt @@ -10,7 +10,10 @@ {{ object.document_type }} {{ object.file_mimetype }} {{ object.description|default:' ' }} -{{ object.flat_comments }} + +{% for comment in object.comments.all %} + {{ comment.comment }} +{% endfor %} {% for tag in object.tags.all %} {{ tag }} From 1c2a89e3e80ae9d6c2097c8d99a7343d6aff3597 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Mon, 21 May 2012 10:47:45 -0400 Subject: [PATCH 33/46] Fix cleaned_filename document property --- apps/documents/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/documents/models.py b/apps/documents/models.py index 784c97d2f8..01e8d6c809 100644 --- a/apps/documents/models.py +++ b/apps/documents/models.py @@ -285,7 +285,7 @@ class Document(models.Model): @property def cleaned_filename(self): - return unidecode(self.extension_split()[0]) + return unidecode(self.extension_split[0]) @property def extension_split(self): From ce0082ee3fff70706e4eaeb8d13bb261c4f39bef Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Mon, 21 May 2012 15:51:56 -0400 Subject: [PATCH 34/46] Add the mayan_runtime option to the intercepted update_index command --- apps/signaler/management/commands/update_index.py | 10 ++++++++-- apps/signaler/signals.py | 4 ++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/apps/signaler/management/commands/update_index.py b/apps/signaler/management/commands/update_index.py index bfc87a0fe0..e810de5b82 100644 --- a/apps/signaler/management/commands/update_index.py +++ b/apps/signaler/management/commands/update_index.py @@ -1,3 +1,5 @@ +from optparse import make_option + from haystack.management.commands import update_index from signaler.signals import post_update_index, pre_update_index @@ -7,8 +9,12 @@ class Command(update_index.Command): """ Wrapper for the haystack's update_index command """ + option_list = update_index.Command.option_list + ( + make_option('--mayan_runtime', action='store_true', dest='mayan_runtime', default=False), + ) def handle(self, *args, **kwargs): - pre_update_index.send(sender=self) + mayan_runtime = kwargs.pop('mayan_runtime') + pre_update_index.send(sender=self, mayan_runtime=mayan_runtime) super(Command, self).handle(*args, **kwargs) - post_update_index.send(sender=self) + post_update_index.send(sender=self, mayan_runtime=mayan_runtime) diff --git a/apps/signaler/signals.py b/apps/signaler/signals.py index 66a764ab57..ae5c121049 100644 --- a/apps/signaler/signals.py +++ b/apps/signaler/signals.py @@ -1,5 +1,5 @@ from django.dispatch import Signal pre_collectstatic = Signal() -pre_update_index = Signal() -post_update_index = Signal() +pre_update_index = Signal(providing_args=['mayan_runtime']) +post_update_index = Signal(providing_args=['mayan_runtime']) From 675e1d3df7a9b8d2296b799624e43ff6da198575 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Mon, 21 May 2012 15:53:06 -0400 Subject: [PATCH 35/46] Call Haystack's update_index using Django's call_command and use the new mayan_runtime option --- apps/dynamic_search/__init__.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/apps/dynamic_search/__init__.py b/apps/dynamic_search/__init__.py index 88ee1b9489..8edc3e5f34 100644 --- a/apps/dynamic_search/__init__.py +++ b/apps/dynamic_search/__init__.py @@ -5,7 +5,7 @@ import logging from django.utils.translation import ugettext_lazy as _ from django.dispatch import receiver -from haystack.management.commands.update_index import Command +from django.core.management import call_command from navigation.api import register_sidebar_template, register_links from documents.models import Document @@ -37,9 +37,13 @@ def clear_pending_indexables(sender, **kwargs): @receiver(pre_update_index, dispatch_uid='scheduler_shutdown_pre_update_index') -def scheduler_shutdown_pre_update_index(sender, **kwargs): +def scheduler_shutdown_pre_update_index(sender, mayan_runtime, **kwargs): logger.debug('Scheduler shut down on pre update index signal') - scheduler.shutdown() + logger.debug('Runtime variable: %s' % mayan_runtime) + # Only shutdown the scheduler if the command is called from the command + # line + if not mayan_runtime: + scheduler.shutdown() def search_index_update(): @@ -50,15 +54,18 @@ def search_index_update(): logger.debug('acquired lock: %s' % lock_id) logger.debug('Executing haystack\'s index update command') - command = Command() - command.handle() + call_command('update_index --mayan_runtime') lock.release() except LockError: logger.debug('unable to obtain lock') pass - - + except Exception, instance: + logger.debug('Unhandled exception: %s' % instance) + lock.release() + pass + + register_interval_job('search_index_update', _(u'Update the search index with the most recent modified documents.'), search_index_update, seconds=INDEX_UPDATE_INTERVAL) From 80e236723b0157bab6b93438eb827ebfbc61852e Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Mon, 21 May 2012 23:08:31 -0400 Subject: [PATCH 36/46] Add support to clear the indexable list returned --- apps/dynamic_search/managers.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/apps/dynamic_search/managers.py b/apps/dynamic_search/managers.py index 57a4ad30ab..53ebe7860f 100644 --- a/apps/dynamic_search/managers.py +++ b/apps/dynamic_search/managers.py @@ -2,6 +2,7 @@ from __future__ import absolute_import from urlparse import urlparse, parse_qs from urllib import unquote_plus +import copy from django.utils.simplejson import dumps from django.db.models import Manager @@ -36,8 +37,12 @@ class IndexableObjectManager(Manager): else: return self.model.objects.all() - def get_indexables_pk_list(self, datetime=None): - return self.get_indexables(datetime).values_list('object_id', flat=True) + def get_indexables_pk_list(self, datetime=None, clear_indexable=False): + indexable_query_set = self.get_indexables(datetime) + indexable_value_list = copy.copy(indexable_query_set.values_list('object_id', flat=True)) + if clear_indexable: + indexable_query_set.delete() + return indexable_value_list def mark_indexable(self, obj): content_type = ContentType.objects.get_for_model(obj) From 5b349cb0105c3d5971a96776315e0e18fe0a159a Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Mon, 21 May 2012 23:09:10 -0400 Subject: [PATCH 37/46] Do not clear the indexable list upon post update index signal --- apps/dynamic_search/__init__.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/apps/dynamic_search/__init__.py b/apps/dynamic_search/__init__.py index 8edc3e5f34..894cf9e1a3 100644 --- a/apps/dynamic_search/__init__.py +++ b/apps/dynamic_search/__init__.py @@ -30,12 +30,6 @@ register_sidebar_template(['search'], 'recent_searches.html') Document.add_to_class('mark_indexable', lambda obj: IndexableObject.objects.mark_indexable(obj)) -@receiver(post_update_index, dispatch_uid='clear_pending_indexables') -def clear_pending_indexables(sender, **kwargs): - logger.debug('Clearing all indexable flags post update index signal') - IndexableObject.objects.clear_all() - - @receiver(pre_update_index, dispatch_uid='scheduler_shutdown_pre_update_index') def scheduler_shutdown_pre_update_index(sender, mayan_runtime, **kwargs): logger.debug('Scheduler shut down on pre update index signal') @@ -67,7 +61,3 @@ def search_index_update(): register_interval_job('search_index_update', _(u'Update the search index with the most recent modified documents.'), search_index_update, seconds=INDEX_UPDATE_INTERVAL) - - - - From fac274c9c2861b3a56ab2b7adac41243ea653620 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 22 May 2012 12:53:59 -0400 Subject: [PATCH 38/46] Update the required version of South to latest (0.7.5) Conflicts: requirements/production.txt --- requirements/production.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/production.txt b/requirements/production.txt index 95d4d6df37..0e4c94e327 100644 --- a/requirements/production.txt +++ b/requirements/production.txt @@ -42,7 +42,7 @@ djangorestframework==0.2.3 # Migrations -South==0.7.3 +South==0.7.5 # Keys and signing From 09eb1b20debb04e05885616787ffa6495a3e3047 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 22 May 2012 15:20:01 -0400 Subject: [PATCH 39/46] Add migration for the dynamic_search app --- .../dynamic_search/migrations/0001_initial.py | 74 +++++++++++++++++ .../0002_auto__add_indexableobject.py | 80 +++++++++++++++++++ apps/dynamic_search/migrations/__init__.py | 0 3 files changed, 154 insertions(+) create mode 100644 apps/dynamic_search/migrations/0001_initial.py create mode 100644 apps/dynamic_search/migrations/0002_auto__add_indexableobject.py create mode 100644 apps/dynamic_search/migrations/__init__.py diff --git a/apps/dynamic_search/migrations/0001_initial.py b/apps/dynamic_search/migrations/0001_initial.py new file mode 100644 index 0000000000..5f42fc047a --- /dev/null +++ b/apps/dynamic_search/migrations/0001_initial.py @@ -0,0 +1,74 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding model 'RecentSearch' + db.create_table('dynamic_search_recentsearch', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('user', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'])), + ('query', self.gf('django.db.models.fields.TextField')()), + ('datetime_created', self.gf('django.db.models.fields.DateTimeField')()), + ('hits', self.gf('django.db.models.fields.IntegerField')()), + )) + db.send_create_signal('dynamic_search', ['RecentSearch']) + + + def backwards(self, orm): + # Deleting model 'RecentSearch' + db.delete_table('dynamic_search_recentsearch') + + + models = { + 'auth.group': { + 'Meta': {'object_name': 'Group'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + 'auth.permission': { + 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + 'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'dynamic_search.recentsearch': { + 'Meta': {'ordering': "('-datetime_created',)", 'object_name': 'RecentSearch'}, + 'datetime_created': ('django.db.models.fields.DateTimeField', [], {}), + 'hits': ('django.db.models.fields.IntegerField', [], {}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'query': ('django.db.models.fields.TextField', [], {}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}) + } + } + + complete_apps = ['dynamic_search'] \ No newline at end of file diff --git a/apps/dynamic_search/migrations/0002_auto__add_indexableobject.py b/apps/dynamic_search/migrations/0002_auto__add_indexableobject.py new file mode 100644 index 0000000000..eeac9ecc5e --- /dev/null +++ b/apps/dynamic_search/migrations/0002_auto__add_indexableobject.py @@ -0,0 +1,80 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding model 'IndexableObject' + db.create_table('dynamic_search_indexableobject', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('datetime', self.gf('django.db.models.fields.DateTimeField')()), + ('content_type', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['contenttypes.ContentType'])), + ('object_id', self.gf('django.db.models.fields.PositiveIntegerField')()), + )) + db.send_create_signal('dynamic_search', ['IndexableObject']) + + + def backwards(self, orm): + # Deleting model 'IndexableObject' + db.delete_table('dynamic_search_indexableobject') + + + models = { + 'auth.group': { + 'Meta': {'object_name': 'Group'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + 'auth.permission': { + 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + 'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'dynamic_search.indexableobject': { + 'Meta': {'object_name': 'IndexableObject'}, + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'datetime': ('django.db.models.fields.DateTimeField', [], {}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'object_id': ('django.db.models.fields.PositiveIntegerField', [], {}) + }, + 'dynamic_search.recentsearch': { + 'Meta': {'ordering': "('-datetime_created',)", 'object_name': 'RecentSearch'}, + 'datetime_created': ('django.db.models.fields.DateTimeField', [], {}), + 'hits': ('django.db.models.fields.IntegerField', [], {}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'query': ('django.db.models.fields.TextField', [], {}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}) + } + } + + complete_apps = ['dynamic_search'] \ No newline at end of file diff --git a/apps/dynamic_search/migrations/__init__.py b/apps/dynamic_search/migrations/__init__.py new file mode 100644 index 0000000000..e69de29bb2 From a4da16d8a189572e96d046cc05a18250444b1956 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 22 May 2012 15:20:15 -0400 Subject: [PATCH 40/46] Update upgrade steps to include dynamic_search app migration --- docs/releases/0.13.rst | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/docs/releases/0.13.rst b/docs/releases/0.13.rst index 172bbe41b5..ff9ce4fac8 100644 --- a/docs/releases/0.13.rst +++ b/docs/releases/0.13.rst @@ -13,11 +13,16 @@ Overview * Send document or document links via E-mail * Document previews and page previews navigation improvements * Ability to delete detached signatures +* Specialized search via Haystack + + * Haystack is setup to use Whoosh by default but user can choose to use + any of the other search backends supported by Haystack. What's new in Mayan EDMS v0.13 ============================== + E-mail document source ~~~~~~~~~~~~~~~~~~~~~~ POP3 and IMAP @@ -26,9 +31,11 @@ POP3 and IMAP Send document or document links via E-mail ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Haystack ~~~~~~~~ + Upgrading from a previous version ================================= @@ -44,6 +51,8 @@ Afterwards migrate existing database schema with:: $ ./manage.py migrate sources 0001 --fake $ ./manage.py migrate sources + $ ./manage.py migrate dynamic_search 0001 --fake + $ ./manage.py migrate dynamic_search Issue the following command to index existing documents in the new full text search database:: @@ -55,12 +64,13 @@ Once the full text initial indexing ends, the upgrade procedure is complete. Backward incompatible changes ============================= -None so far +None + Bugs fixed ========== -* Metadata unicode fixes, while creating new documents. -* Update indexes when renaming document +* Metadata unicode fixes while creating new documents. +* Fixed document index link filename when renaming document the source document. Stuff removed From 3d5fa93be09743e1f6ebe254cf467c153b2b01bd Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 22 May 2012 15:20:43 -0400 Subject: [PATCH 41/46] Update the requirement file development tool versions --- requirements/development.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements/development.txt b/requirements/development.txt index 427ff2cf00..fafe9eeaa8 100644 --- a/requirements/development.txt +++ b/requirements/development.txt @@ -1,4 +1,4 @@ -Werkzeug==0.6.2 -django-extensions==0.7.1 -django-rosetta==0.6.2 -transifex-client==0.6.1 +Werkzeug==0.8.3 +django-extensions==0.8 +django-rosetta==0.6.6 +transifex-client==0.7.2 From c771177666f671d2039204dfd15e896759001795 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 22 May 2012 23:24:03 -0400 Subject: [PATCH 42/46] Fix tab indentation --- settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings.py b/settings.py index 206f22c1c4..b1f4ba0929 100644 --- a/settings.py +++ b/settings.py @@ -131,7 +131,7 @@ INSTALLED_APPS = ( # 3rd party # South 'south', - 'haystack', + 'haystack', # Others 'filetransfers', 'taggit', From 7e3e3af503e970e9d51ffb36f0b26de9a8d1a1cf Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 22 May 2012 23:24:17 -0400 Subject: [PATCH 43/46] Improve handling of the inserted --mayan_runtime update_index flags --- .../signaler/management/commands/update_index.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/apps/signaler/management/commands/update_index.py b/apps/signaler/management/commands/update_index.py index e810de5b82..8f468c1086 100644 --- a/apps/signaler/management/commands/update_index.py +++ b/apps/signaler/management/commands/update_index.py @@ -1,20 +1,32 @@ +import logging from optparse import make_option from haystack.management.commands import update_index from signaler.signals import post_update_index, pre_update_index +logger = logging.getLogger(__name__) + +MAYAN_RUNTIME = 'mayan_runtime' +MAYAN_RUNTIME_OPT = '--%s' % MAYAN_RUNTIME + class Command(update_index.Command): """ Wrapper for the haystack's update_index command """ option_list = update_index.Command.option_list + ( - make_option('--mayan_runtime', action='store_true', dest='mayan_runtime', default=False), + make_option(None, MAYAN_RUNTIME_OPT, action='store_true', dest=MAYAN_RUNTIME, default=False), ) def handle(self, *args, **kwargs): - mayan_runtime = kwargs.pop('mayan_runtime') + mayan_runtime = kwargs.pop(MAYAN_RUNTIME) + args = list(args) + if MAYAN_RUNTIME_OPT in args: + # Being called from another app + mayan_runtime = True + args.remove(MAYAN_RUNTIME_OPT) + logger.debug('mayan_runtime: %s' % mayan_runtime) pre_update_index.send(sender=self, mayan_runtime=mayan_runtime) super(Command, self).handle(*args, **kwargs) post_update_index.send(sender=self, mayan_runtime=mayan_runtime) From 1792599411c4a0f6e64d5494386b18856cb492f3 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 22 May 2012 23:25:06 -0400 Subject: [PATCH 44/46] Move removal of items marked for indexing to the search_indexes.py file --- apps/dynamic_search/managers.py | 9 ++------- apps/dynamic_search/search_indexes.py | 10 +++++++--- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/apps/dynamic_search/managers.py b/apps/dynamic_search/managers.py index 53ebe7860f..57a4ad30ab 100644 --- a/apps/dynamic_search/managers.py +++ b/apps/dynamic_search/managers.py @@ -2,7 +2,6 @@ from __future__ import absolute_import from urlparse import urlparse, parse_qs from urllib import unquote_plus -import copy from django.utils.simplejson import dumps from django.db.models import Manager @@ -37,12 +36,8 @@ class IndexableObjectManager(Manager): else: return self.model.objects.all() - def get_indexables_pk_list(self, datetime=None, clear_indexable=False): - indexable_query_set = self.get_indexables(datetime) - indexable_value_list = copy.copy(indexable_query_set.values_list('object_id', flat=True)) - if clear_indexable: - indexable_query_set.delete() - return indexable_value_list + def get_indexables_pk_list(self, datetime=None): + return self.get_indexables(datetime).values_list('object_id', flat=True) def mark_indexable(self, obj): content_type = ContentType.objects.get_for_model(obj) diff --git a/apps/dynamic_search/search_indexes.py b/apps/dynamic_search/search_indexes.py index e2bfa0c69c..b053ff7066 100644 --- a/apps/dynamic_search/search_indexes.py +++ b/apps/dynamic_search/search_indexes.py @@ -1,6 +1,7 @@ from __future__ import absolute_import import logging +import copy from haystack import indexes @@ -18,6 +19,9 @@ class DocumentIndex(indexes.SearchIndex, indexes.Indexable): return Document def build_queryset(self, start_date=None, end_date=None): - indexable_list = IndexableObject.objects.get_indexables_pk_list() - logger.debug('indexable list: %s' % indexable_list) - return self.get_model().objects.filter(pk__in=indexable_list) + indexable_query_set = IndexableObject.objects.get_indexables() + logger.debug('indexable_query_set: %s' % indexable_query_set) + object_list = copy.copy(self.get_model().objects.filter(pk__in=indexable_query_set.values_list('object_id', flat=True))) + logger.debug('object_list: %s' % object_list) + indexable_query_set.delete() + return object_list From 895a5c7e76a57bc00d03fdaddad949ab890c52e3 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 22 May 2012 23:25:55 -0400 Subject: [PATCH 45/46] Call the update_index in the correct manner --- apps/dynamic_search/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/dynamic_search/__init__.py b/apps/dynamic_search/__init__.py index 894cf9e1a3..cb35778be2 100644 --- a/apps/dynamic_search/__init__.py +++ b/apps/dynamic_search/__init__.py @@ -48,7 +48,7 @@ def search_index_update(): logger.debug('acquired lock: %s' % lock_id) logger.debug('Executing haystack\'s index update command') - call_command('update_index --mayan_runtime') + call_command('update_index', '--mayan_runtime') lock.release() except LockError: From b3bcb274a0e4a0de54b4434fff452a33a5b96ddb Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 22 May 2012 23:33:35 -0400 Subject: [PATCH 46/46] Add SEARCH_INDEX_UPDATE_INTERVAL to the documentation --- docs/releases/0.13.rst | 9 ++++++--- docs/topics/settings.rst | 8 ++++---- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/docs/releases/0.13.rst b/docs/releases/0.13.rst index ff9ce4fac8..194da2ecc1 100644 --- a/docs/releases/0.13.rst +++ b/docs/releases/0.13.rst @@ -15,9 +15,6 @@ Overview * Ability to delete detached signatures * Specialized search via Haystack - * Haystack is setup to use Whoosh by default but user can choose to use - any of the other search backends supported by Haystack. - What's new in Mayan EDMS v0.13 ============================== @@ -34,6 +31,10 @@ Send document or document links via E-mail Haystack ~~~~~~~~ +Haystack is setup to use Whoosh by default but user can choose to use +any of the other search backends supported by Haystack. The interval +between search index updates is controlled with the new :setting:`SEARCH_INDEX_UPDATE_INTERVAL` +configuration option. This addition closes issues #20 and #12. Upgrading from a previous version @@ -71,6 +72,8 @@ Bugs fixed ========== * Metadata unicode fixes while creating new documents. * Fixed document index link filename when renaming document the source document. +* Issues #20 and #12 with the addition of a dedicated search solution and + not rely on SQL based searches. Stuff removed diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index ae395a435e..b3af96d500 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -461,13 +461,13 @@ Allow non authenticated users, access to all views. Search ====== -.. setting:: SEARCH_LIMIT +.. setting:: SEARCH_INDEX_UPDATE_INTERVAL -**SEARCH_LIMIT** +**SEARCH_INDEX_UPDATE_INTERVAL** -Default: ``100`` +Default: ``1800`` -Maximum amount search hits to fetch and display. +Interval in second on which to trigger the search index update. .. setting:: SEARCH_RECENT_COUNT