Initial commit to support per page search.

mayan/apps/appearance/templates/appearance/home.html

@@ -49,15 +49,17 @@
             </div>
         </div>
     </div>
+
+{% comment %}
     <div class="row">
         <div class="col-xs-12">
             <div class="well center-block">
-                <form action="{% url 'search:results' %}" method="get" role="search">
+                <form action="{% url 'search:results' search_model='documents.Document' %}" method="get" role="search">
                     <div class="input-group">
                         <input class="form-control" name="q" placeholder="{% trans 'Space separated terms' %}" type="text" value="{{ search_terms|default:'' }}">
                         <span class="input-group-btn">
                             <button class="btn btn-default" type="submit">{% trans 'Search' %}</button>
-                            <a class="btn btn-primary" href="{% url 'search:search_advanced' %}">{% trans 'Advanced' %}</a>
+                            <a class="btn btn-primary" href="{% url 'search:search_advanced' search_model='documents.Document' %}">{% trans 'Advanced' %}</a>
                         </span>
                     </div>
                 </form>
@@ -67,4 +69,48 @@
             </div>
         </div>
     </div>
+{% endcomment %}
+    <div class="row">
+        <div class="col-xs-12">
+            <div class="panel panel-default center-block">
+                <div class="panel-heading">{% trans 'Search documents' %}</div>
+                <div class="panel-body">
+                    <form action="{% url 'search:results' search_model='documents.Document' %}" method="get" role="search">
+                        <div class="input-group">
+                            <input class="form-control" name="q" placeholder="{% trans 'Space separated terms' %}" type="text" value="{{ search_terms|default:'' }}">
+                            <span class="input-group-btn">
+                                <button class="btn btn-default" type="submit">{% trans 'Search' %}</button>
+                                <a class="btn btn-primary" href="{% url 'search:search_advanced' search_model='documents.Document' %}">{% trans 'Advanced' %}</a>
+                            </span>
+                        </div>
+                    </form>
+                    {% if search_terms %}
+                        {% include 'appearance/generic_list_subtemplate.html' %}
+                    {% endif %}
+                </div>
+            </div>
+        </div>
+    </div>
+
+    <div class="row">
+        <div class="col-xs-12">
+            <div class="panel panel-default center-block">
+                <div class="panel-heading">{% trans 'Search pages' %}</div>
+                <div class="panel-body">
+                    <form action="{% url 'search:results' search_model='documents.DocumentPageResult' %}" method="get" role="search">
+                        <div class="input-group">
+                            <input class="form-control" name="q" placeholder="{% trans 'Space separated terms' %}" type="text" value="{{ search_terms|default:'' }}">
+                            <span class="input-group-btn">
+                                <button class="btn btn-default" type="submit">{% trans 'Search' %}</button>
+                                <a class="btn btn-primary" href="{% url 'search:search_advanced' search_model='documents.DocumentPageResult' %}">{% trans 'Advanced' %}</a>
+                            </span>
+                        </div>
+                    </form>
+                    {% if search_terms %}
+                        {% include 'appearance/generic_list_subtemplate.html' %}
+                    {% endif %}
+                </div>
+            </div>
+        </div>
+    </div>
 {% endblock %}

mayan/apps/documents/apps.py

@@ -68,13 +68,14 @@ from .permissions import (
     permission_document_trash, permission_document_version_revert,
     permission_document_view
 )
+from .search import document_search, document_page_search
 from .settings import setting_thumbnail_size
 from .statistics import (
     new_documents_per_month, new_document_pages_per_month,
     new_document_versions_per_month, total_document_per_month,
     total_document_page_per_month, total_document_version_per_month
 )
-from .widgets import document_thumbnail
+from .widgets import document_html_widget, document_thumbnail
 
 
 class DocumentsApp(MayanAppConfig):
@@ -90,6 +91,7 @@ class DocumentsApp(MayanAppConfig):
         DeletedDocument = self.get_model('DeletedDocument')
         Document = self.get_model('Document')
         DocumentPage = self.get_model('DocumentPage')
+        DocumentPageResult = self.get_model('DocumentPageResult')
         DocumentType = self.get_model('DocumentType')
         DocumentTypeFilename = self.get_model('DocumentTypeFilename')
         DocumentVersion = self.get_model('DocumentVersion')
@@ -159,6 +161,36 @@ class DocumentsApp(MayanAppConfig):
             source=Document, label=_('Type'), attribute='document_type'
         )
 
+        SourceColumn(
+            source=DocumentPage, label=_('Thumbnail'),
+            func=lambda context: document_html_widget(
+                document_page=context['object'],
+                click_view='documents:document_display',
+                click_view_arguments=(context['object'].document.pk,),
+                gallery_name='documents:document_page_list',
+                preview_click_view='documents:document_page_view',
+                size=setting_thumbnail_size.value,
+                title=unicode(context['object']),
+            )
+        )
+
+        SourceColumn(
+            source=DocumentPageResult, label=_('Thumbnail'),
+            func=lambda context: document_html_widget(
+                document_page=context['object'],
+                click_view='documents:document_display',
+                click_view_arguments=(context['object'].document.pk,),
+                gallery_name='documents:document_page_list',
+                preview_click_view='documents:document_page_view',
+                size=setting_thumbnail_size.value,
+                title=unicode(context['object']),
+            )
+        )
+        SourceColumn(
+            source=DocumentPageResult, label=_('Type'),
+            attribute='document_version.document.document_type'
+        )
+
         SourceColumn(
             source=DocumentType, label=_('Documents'),
             func=lambda context: context['object'].get_document_count(

mayan/apps/documents/models.py

@@ -774,6 +774,14 @@ class DocumentPage(models.Model):
         return '{}-{}'.format(self.document_version.uuid, self.pk)
 
 
+class DocumentPageResult(DocumentPage):
+    class Meta:
+        ordering = ('document_version__document', 'page_number')
+        proxy = True
+        verbose_name = _('Document page')
+        verbose_name_plural = _('Document pages')
+
+
 class NewVersionBlock(models.Model):
     document = models.ForeignKey(Document, verbose_name=_('Document'))
 

mayan/apps/documents/search.py

@@ -7,7 +7,8 @@ from dynamic_search.classes import SearchModel
 from .permissions import permission_document_view
 
 document_search = SearchModel(
-    'documents', 'Document', permission=permission_document_view,
+    app_label='documents', model_name='Document',
+    permission=permission_document_view,
     serializer_string='documents.serializers.DocumentSerializer'
 )
 
@@ -19,3 +20,24 @@ document_search.add_model_field(
 )
 document_search.add_model_field(field='label', label=_('Label'))
 document_search.add_model_field(field='description', label=_('Description'))
+
+document_page_search = SearchModel(
+    app_label='documents', model_name='DocumentPageResult',
+    permission=permission_document_view,
+    serializer_string='documents.serializers.DocumentPageSerializer'
+)
+
+document_page_search.add_model_field(
+    field='document_version__document__document_type__label',
+    label=_('Document type')
+)
+document_page_search.add_model_field(
+    field='document_version__document__versions__mimetype',
+    label=_('MIME type')
+)
+document_page_search.add_model_field(
+    field='document_version__document__label', label=_('Label')
+)
+document_page_search.add_model_field(
+    field='document_version__document__description', label=_('Description')
+)

mayan/apps/documents/widgets.py

@@ -83,7 +83,7 @@ class DocumentPagesCarouselWidget(forms.widgets.Widget):
 
 def document_thumbnail(document, **kwargs):
     return document_html_widget(
-        document.latest_version.pages.first(),
+        document_page=document.latest_version.pages.first(),
         click_view='documents:document_display', **kwargs
     )
 
@@ -94,7 +94,7 @@ def document_link(document):
     )
 
 
-def document_html_widget(document_page, click_view=None, click_view_arguments=None, zoom=DEFAULT_ZOOM_LEVEL, rotation=DEFAULT_ROTATION, gallery_name=None, fancybox_class='fancybox', image_class='lazy-load', title=None, size=setting_thumbnail_size.value, nolazyload=False, post_load_class=None, disable_title_link=False):
+def document_html_widget(document_page, click_view=None, click_view_arguments=None, zoom=DEFAULT_ZOOM_LEVEL, rotation=DEFAULT_ROTATION, gallery_name=None, fancybox_class='fancybox', image_class='lazy-load', title=None, size=setting_thumbnail_size.value, nolazyload=False, post_load_class=None, disable_title_link=False, preview_click_view=None):
     result = []
 
     alt_text = _('Document page image')
@@ -110,6 +110,7 @@ def document_html_widget(document_page, click_view=None, click_view_arguments=No
         'zoom': zoom,
         'rotation': rotation,
         'size': size,
+        'page': document_page.page_number
     }
 
     if gallery_name:
@@ -132,7 +133,13 @@ def document_html_widget(document_page, click_view=None, click_view_arguments=No
 
     if title:
         if not disable_title_link:
+            if not preview_click_view:
                 preview_click_link = document.get_absolute_url()
+            else:
+                preview_click_link = reverse(
+                    preview_click_view, args=(document_page.pk,)
+                )
+
             title_template = 'data-caption="<a class=\'a-caption\' href=\'{url}\'>{title}</a>"'.format(
                 title=strip_tags(title), url=preview_click_link or '#'
             )

mayan/apps/dynamic_search/classes.py

@@ -33,23 +33,55 @@ class SearchModel(object):
         self.app_label = app_label
         self.model_name = model_name
         self.search_fields = []
-        self.model = None  # Lazy
+        self._model = None  # Lazy
-        self.label = label
+        self._label = label
         self.serializer_string = serializer_string
         self.permission = permission
         self.__class__.registry[self.get_full_name()] = self
 
-    def get_full_name(self):
+    @property
-        return '%s.%s' % (self.app_label, self.model_name)
+    def model(self):
+        if not self._model:
+            self._model = apps.get_model(self.app_label, self.model_name)
+        return self._model
+
+    @property
+    def label(self):
+        if not self._label:
+            self._label = self.model._meta.verbose_name
+        return self._label
+
+    def add_model_field(self, *args, **kwargs):
+        """
+        Add a search field that directly belongs to the parent SearchModel
+        """
+        search_field = SearchField(self, *args, **kwargs)
+        self.search_fields.append(search_field)
+
+    def assemble_query(self, terms, search_fields):
+        """
+        Returns a query, that is a combination of Q objects. That combination
+        aims to search keywords within a model by testing the given search
+        fields.
+        """
+        queries = []
+        for term in terms:
+            or_query = None
+            for field in search_fields:
+                q = Q(**{'%s__%s' % (field, 'icontains'): term})
+                if or_query is None:
+                    or_query = q
+                else:
+                    or_query = or_query | q
+
+            queries.append(or_query)
+        return queries
 
     def get_all_search_fields(self):
         return self.search_fields
 
-    def get_search_field(self, full_name):
+    def get_full_name(self):
-        try:
+        return '%s.%s' % (self.app_label, self.model_name)
-            return self.search_fields[full_name]
-        except KeyError:
-            raise KeyError('No search field named: %s' % full_name)
 
     def get_fields_simple_list(self):
         """
@@ -61,12 +93,11 @@ class SearchModel(object):
 
         return result
 
-    def add_model_field(self, *args, **kwargs):
+    def get_search_field(self, full_name):
-        """
+        try:
-        Add a search field that directly belongs to the parent SearchModel
+            return self.search_fields[full_name]
-        """
+        except KeyError:
-        search_field = SearchField(self, *args, **kwargs)
+            raise KeyError('No search field named: %s' % full_name)
-        self.search_fields.append(search_field)
 
     def normalize_query(self, query_string,
                         findterms=re.compile(r'"([^"]+)"|(\S+)').findall,
@@ -88,11 +119,6 @@ class SearchModel(object):
         result_set = set()
         search_dict = {}
 
-        if not self.model:
-            self.model = apps.get_model(self.app_label, self.model_name)
-            if not self.label:
-                self.label = self.model._meta.verbose_name
-
         if 'q' in query_string:
             # Simple search
             for search_field in self.get_all_search_fields():
@@ -110,7 +136,6 @@ class SearchModel(object):
                     }
                 )
         else:
-
             for search_field in self.get_all_search_fields():
                 if search_field.field in query_string and query_string[search_field.field]:
                     search_dict.setdefault(search_field.get_model(), {
@@ -183,6 +208,8 @@ class SearchModel(object):
             datetime.datetime.now() - start_time
         ).split(':')[2]
 
+        logger.debug('elapsed_time: %s', elapsed_time)
+
         queryset = self.model.objects.filter(
             pk__in=list(result_set)[:setting_limit.value]
         )
@@ -201,25 +228,6 @@ class SearchModel(object):
 
         return queryset, result_set, elapsed_time
 
-    def assemble_query(self, terms, search_fields):
-        """
-        Returns a query, that is a combination of Q objects. That combination
-        aims to search keywords within a model by testing the given search
-        fields.
-        """
-        queries = []
-        for term in terms:
-            or_query = None
-            for field in search_fields:
-                q = Q(**{'%s__%s' % (field, 'icontains'): term})
-                if or_query is None:
-                    or_query = q
-                else:
-                    or_query = or_query | q
-
-            queries.append(or_query)
-        return queries
-
 
 # SearchField classes
 class SearchField(object):

mayan/apps/dynamic_search/forms.py

@@ -5,6 +5,14 @@ from django.utils.translation import ugettext_lazy as _
 
 
 class AdvancedSearchForm(forms.Form):
+    _match_all = forms.BooleanField(
+        label=_('Match all'), help_text=_(
+            'When checked, only results that match all fields will be '
+            'returned. When unchecked results that match at least one field '
+            'will be returned.'
+        ), required=False
+    )
+
     def __init__(self, *args, **kwargs):
         self.search_model = kwargs.pop('search_model')
         super(AdvancedSearchForm, self).__init__(*args, **kwargs)

mayan/apps/dynamic_search/links.py

@@ -4,8 +4,14 @@ from django.utils.translation import ugettext_lazy as _
 
 from navigation import Link
 
-link_search = Link(text=_('Search'), view='search:search')
+link_search = Link(
-link_search_advanced = Link(
+    text=_('Search'), view='search:search', args='search_model.get_full_name'
-    text=_('Advanced search'), view='search:search_advanced'
+)
+link_search_advanced = Link(
+    text=_('Advanced search'), view='search:search_advanced',
+    args='search_model.get_full_name'
+)
+link_search_again = Link(
+    text=_('Search again'), view='search:search_again',
+    args='search_model.get_full_name', keep_query=True
 )
-link_search_again = Link(text=_('Search again'), view='search:search_again')

mayan/apps/dynamic_search/settings.py

@@ -6,9 +6,6 @@ from smart_settings import Namespace
 
 
 namespace = Namespace(name='dynamic_search', label=_('Search'))
-setting_show_object_type = namespace.add_setting(
-    global_name='SEARCH_SHOW_OBJECT_TYPE', default=False
-)
 setting_limit = namespace.add_setting(
     global_name='SEARCH_LIMIT', default=100,
     help_text=_('Maximum amount search hits to fetch and display.')

mayan/apps/dynamic_search/urls.py

@@ -5,14 +5,25 @@ from django.conf.urls import patterns, url
 from .api_views import (
     APIRecentSearchListView, APIRecentSearchView, APISearchView
 )
-from .views import AdvancedSearchView, ResultsView, SearchView
+from .views import (
+    AdvancedSearchView, ResultsView, SearchAgainView, SearchView
+)
 
 urlpatterns = patterns(
     'dynamic_search.views',
-    url(r'^$', SearchView.as_view(), name='search'),
+    url(r'^(?P<search_model>[\.\w]+)/$', SearchView.as_view(), name='search'),
-    url(r'^advanced/$', AdvancedSearchView.as_view(), name='search_advanced'),
+    url(
-    url(r'^again/$', 'search_again', name='search_again'),
+        r'^advanced/(?P<search_model>[\.\w]+)/$', AdvancedSearchView.as_view(),
-    url(r'^results/$', ResultsView.as_view(), name='results'),
+        name='search_advanced'
+    ),
+    url(
+        r'^again/(?P<search_model>[\.\w]+)/$', SearchAgainView.as_view(),
+        name='search_again'
+    ),
+    url(
+        r'^results/(?P<search_model>[\.\w]+)/$', ResultsView.as_view(),
+        name='results'
+    ),
 )
 
 api_urls = patterns(

mayan/apps/dynamic_search/views.py

@@ -7,12 +7,13 @@ from django.conf import settings
 from django.core.urlresolvers import reverse
 from django.http import HttpResponseRedirect
 from django.utils.translation import ugettext_lazy as _
+from django.views.generic.base import RedirectView
 
 from common.generics import SimpleView, SingleObjectListView
 
 from .classes import SearchModel
 from .forms import SearchForm, AdvancedSearchForm
-from .settings import setting_limit, setting_show_object_type
+from .settings import setting_limit
 
 logger = logging.getLogger(__name__)
 
@@ -21,40 +22,52 @@ class ResultsView(SingleObjectListView):
     def get_extra_context(self):
         context = {
             'hide_links': True,
+            'search_model': self.search_model,
             'search_results_limit': setting_limit.value,
-            'title': _('Search results'),
+            'title': _('Search results for: %s') % self.search_model.label,
         }
 
-        if setting_show_object_type.value:
-            context.update({
-                'extra_columns': (
-                    {
-                        'name': _('Type'),
-                        'attribute': lambda x: x._meta.verbose_name[0].upper() + x._meta.verbose_name[1:]
-                    },
-                )
-            })
-
         return context
 
     def get_queryset(self):
-        document_search = SearchModel.get('documents.Document')
+        self.search_model = self.get_search_model()
 
         if self.request.GET:
             # Only do search if there is user input, otherwise just render
             # the template with the extra_context
 
-            queryset, ids, timedelta = document_search.search(
+            if self.request.GET.get('_match_all', 'off') == 'on':
-                self.request.GET, self.request.user
+                global_and_search=True
+            else:
+                global_and_search=False
+
+            queryset, ids, timedelta = self.search_model.search(
+                query_string=self.request.GET, user=self.request.user,
+                global_and_search=global_and_search
             )
 
             return queryset
 
+    def get_search_model(self):
+        return SearchModel.get(self.kwargs['search_model'])
+
 
 class SearchView(SimpleView):
     template_name = 'appearance/generic_form.html'
     title = _('Search')
 
+    def get_extra_context(self):
+        self.search_model = self.get_search_model()
+        return {
+            'form': self.get_form(),
+            'form_action': reverse('search:results', args=(self.search_model.get_full_name(),)),
+            'search_model': self.search_model,
+            'submit_icon': 'fa fa-search',
+            'submit_label': _('Search'),
+            'submit_method': 'GET',
+            'title': _('Search for: %s') % self.search_model.label,
+        }
+
     def get_form(self):
         if ('q' in self.request.GET) and self.request.GET['q'].strip():
             query_string = self.request.GET['q']
@@ -62,32 +75,19 @@ class SearchView(SimpleView):
         else:
             return SearchForm()
 
-    def get_extra_context(self):
+    def get_search_model(self):
-        return {
+        return SearchModel.get(self.kwargs['search_model'])
-            'form': self.get_form(),
-            'form_action': reverse('search:results'),
-            'submit_icon': 'fa fa-search',
-            'submit_label': _('Search'),
-            'submit_method': 'GET',
-            'title': self.title,
-        }
 
 
 class AdvancedSearchView(SearchView):
     title = _('Advanced search')
 
     def get_form(self):
-        document_search = SearchModel.get('documents.Document')
-
         return AdvancedSearchForm(
-            data=self.request.GET, search_model=document_search
+            data=self.request.GET, search_model=self.get_search_model()
         )
 
 
-def search_again(request):
+class SearchAgainView(RedirectView):
-    query = urlparse.urlparse(
+    pattern_name = 'search:search_advanced'
-        request.META.get('HTTP_REFERER', reverse(settings.LOGIN_REDIRECT_URL))
+    query_string = True
-    ).query
-    return HttpResponseRedirect(
-        '{}?{}'.format(reverse('search:search_advanced'), query)
-    )

mayan/apps/metadata/apps.py

@@ -15,7 +15,7 @@ from common import (
 )
 from common.classes import ModelAttribute, Filter
 from common.widgets import two_state_template
-from documents.search import document_search
+from documents.search import document_page_search, document_search
 from documents.signals import post_document_type_change
 from documents.permissions import permission_document_view
 from mayan.celery import app
@@ -57,6 +57,9 @@ class MetadataApp(MayanAppConfig):
         Document = apps.get_model(
             app_label='documents', model_name='Document'
         )
+        DocumentPageResult = apps.get_model(
+            app_label='documents', model_name='DocumentPageResult'
+        )
 
         DocumentType = apps.get_model(
             app_label='documents', model_name='DocumentType'
@@ -143,6 +146,13 @@ class MetadataApp(MayanAppConfig):
             func=lambda context: get_metadata_string(context['object'])
         )
 
+        SourceColumn(
+            source=DocumentPageResult, label=_('Metadata'),
+            func=lambda context: get_metadata_string(
+                context['object'].document
+            )
+        )
+
         SourceColumn(
             source=DocumentMetadata, label=_('Value'),
             attribute='value'
@@ -176,6 +186,15 @@ class MetadataApp(MayanAppConfig):
             field='metadata__value', label=_('Metadata value')
         )
 
+        document_page_search.add_model_field(
+            field='document_version__document__metadata__metadata_type__name',
+            label=_('Metadata type')
+        )
+        document_page_search.add_model_field(
+            field='document_version__document__metadata__value',
+            label=_('Metadata value')
+        )
+
         menu_facet.bind_links(links=(link_metadata_view,), sources=(Document,))
         menu_multi_item.bind_links(
             links=(

mayan/apps/ocr/apps.py

@@ -15,7 +15,7 @@ from common import (
     menu_tools
 )
 from common.settings import settings_db_sync_task_delay
-from documents.search import document_search
+from documents.search import document_search, document_page_search
 from documents.signals import post_version_upload
 from documents.widgets import document_link
 from mayan.celery import app
@@ -115,6 +115,10 @@ class OCRApp(MayanAppConfig):
             field='versions__pages__ocr_content__content', label=_('OCR')
         )
 
+        document_page_search.add_model_field(
+            field='ocr_content__content', label=_('OCR')
+        )
+
         menu_facet.bind_links(
             links=(link_document_content,), sources=(Document,)
         )

mayan/apps/tags/apps.py

@@ -10,7 +10,7 @@ from common import (
     MayanAppConfig, menu_facet, menu_secondary, menu_object, menu_main,
     menu_multi_item, menu_sidebar
 )
-from documents.search import document_search
+from documents.search import document_page_search, document_search
 from navigation import SourceColumn
 from rest_api.classes import APIEndPoint
 
@@ -39,6 +39,10 @@ class TagsApp(MayanAppConfig):
             app_label='documents', model_name='Document'
         )
 
+        DocumentPageResult = apps.get_model(
+            app_label='documents', model_name='DocumentPageResult'
+        )
+
         DocumentTag = self.get_model('DocumentTag')
         Tag = self.get_model('Tag')
 
@@ -76,6 +80,14 @@ class TagsApp(MayanAppConfig):
             )
         )
 
+        SourceColumn(
+            source=DocumentPageResult, label=_('Tags'),
+            func=lambda context: widget_document_tags(
+                document=context['object'].document,
+                user=context['request'].user
+            )
+        )
+
         SourceColumn(
             source=Tag, label=_('Preview'),
             func=lambda context: widget_single_tag(context['object'])
@@ -87,6 +99,9 @@ class TagsApp(MayanAppConfig):
             )
         )
 
+        document_page_search.add_model_field(
+            field='document_version__document__tags__label', label=_('Tags')
+        )
         document_search.add_model_field(field='tags__label', label=_('Tags'))
 
         menu_facet.bind_links(