From 030d0e8806a99df4a8642f50d234315f13921588 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 9 Dec 2014 03:26:25 -0400 Subject: [PATCH 1/3] Merge new document API view with the document list API view (as a POST action). Move search result filtering to clases module, this way it filters results from the API too. --- mayan/apps/documents/api_views.py | 15 +++++++-------- mayan/apps/documents/tests.py | 4 ++-- mayan/apps/documents/urls.py | 4 +--- mayan/apps/dynamic_search/classes.py | 23 +++++++++++++++++------ mayan/apps/dynamic_search/views.py | 8 +------- 5 files changed, 28 insertions(+), 26 deletions(-) diff --git a/mayan/apps/documents/api_views.py b/mayan/apps/documents/api_views.py index 51b2cb5033..859754d246 100644 --- a/mayan/apps/documents/api_views.py +++ b/mayan/apps/documents/api_views.py @@ -41,19 +41,18 @@ class APIDocumentListView(generics.ListAPIView): Returns a list of all the documents. """ - serializer_class = DocumentSerializer queryset = Document.objects.all() permission_classes = (MayanPermission,) filter_backends = (MayanObjectPermissionsFilter,) - mayan_object_permissions = {'GET': [PERMISSION_DOCUMENT_VIEW]} + mayan_object_permissions = {'GET': [PERMISSION_DOCUMENT_VIEW], + 'POST': [PERMISSION_DOCUMENT_CREATE]} - -class APINewDocumentView(generics.GenericAPIView): - serializer_class = NewDocumentSerializer - - permission_classes = (MayanPermission,) - mayan_view_permissions = {'POST': [PERMISSION_DOCUMENT_CREATE]} + def get_serializer_class(self): + if self.request.method == 'GET': + return DocumentSerializer + elif self.request.method == 'POST': + return NewDocumentSerializer def post(self, request, *args, **kwargs): """Create a new document.""" diff --git a/mayan/apps/documents/tests.py b/mayan/apps/documents/tests.py index 50eae7c41a..c99a9573ec 100644 --- a/mayan/apps/documents/tests.py +++ b/mayan/apps/documents/tests.py @@ -146,7 +146,7 @@ class DocumentAPICreateDocumentTestCase(TestCase): # Create a blank document with no token in the header with open(TEST_SMALL_DOCUMENT_PATH) as file_descriptor: - response = document_client.post(reverse('newdocument-view'), {'document_type': self.document_type.pk, 'file': file_descriptor}) + response = document_client.post(reverse('document-list'), {'document_type': self.document_type.pk, 'file': file_descriptor}) # Make sure toke authentication is working, should fail self.assertEqual(response.status_code, status.HTTP_401_UNAUTHORIZED) @@ -155,7 +155,7 @@ class DocumentAPICreateDocumentTestCase(TestCase): # Create a blank document with open(TEST_SMALL_DOCUMENT_PATH) as file_descriptor: - document_response = document_client.post(reverse('newdocument-view'), {'document_type': self.document_type.pk, 'file': file_descriptor}) + document_response = document_client.post(reverse('document-list'), {'document_type': self.document_type.pk, 'file': file_descriptor}) self.assertEqual(document_response.status_code, status.HTTP_202_ACCEPTED) diff --git a/mayan/apps/documents/urls.py b/mayan/apps/documents/urls.py index b6de002001..1c43bd5012 100644 --- a/mayan/apps/documents/urls.py +++ b/mayan/apps/documents/urls.py @@ -5,8 +5,7 @@ from django.conf.urls import patterns, url from .api_views import (APIDocumentView, APIDocumentImageView, APIDocumentListView, APIDocumentPageView, APIDocumentTypeDocumentListView, APIDocumentTypeListView, APIDocumentTypeView, - APIDocumentVersionCreateView, APIDocumentVersionView, - APINewDocumentView) + APIDocumentVersionCreateView, APIDocumentVersionView) from .settings import PRINT_SIZE, DISPLAY_SIZE from .views import DocumentListView @@ -74,7 +73,6 @@ urlpatterns = patterns('documents.views', api_urls = patterns('', url(r'^documents/$', APIDocumentListView.as_view(), name='document-list'), - url(r'^documents/new/$', APINewDocumentView.as_view(), name='newdocument-view'), url(r'^documents/(?P[0-9]+)/$', APIDocumentView.as_view(), name='document-detail'), url(r'^document_version/(?P[0-9]+)/$', APIDocumentVersionView.as_view(), name='documentversion-detail'), url(r'^document_page/(?P[0-9]+)/$', APIDocumentPageView.as_view(), name='documentpage-detail'), diff --git a/mayan/apps/dynamic_search/classes.py b/mayan/apps/dynamic_search/classes.py index c0b5df93ee..fc8768eea5 100644 --- a/mayan/apps/dynamic_search/classes.py +++ b/mayan/apps/dynamic_search/classes.py @@ -4,10 +4,13 @@ import datetime import logging import re +from django.core.exceptions import PermissionDenied from django.db.models import Q from django.db.models.loading import get_model +from acls.models import AccessEntry from common.utils import load_backend +from permissions.models import Permission from .settings import LIMIT @@ -84,7 +87,7 @@ class SearchModel(object): """ return [normspace(' ', (t[0] or t[1]).strip()) for t in findterms(query_string)] - def simple_search(self, query_string): + def simple_search(self, query_string, user): search_dict = {} for search_field in self.get_all_search_fields(): @@ -102,9 +105,9 @@ class SearchModel(object): logger.debug('search_dict: %s', search_dict) - return self.execute_search(search_dict, global_and_search=False) + return self.execute_search(search_dict, user=user, global_and_search=False) - def advanced_search(self, dictionary): + def advanced_search(self, dictionary, user): search_dict = {} for key, value in dictionary.items(): @@ -130,9 +133,9 @@ class SearchModel(object): logger.debug('search_dict: %s', search_dict) - return self.execute_search(search_dict, global_and_search=True) + return self.execute_search(search_dict, user=user, global_and_search=True) - def execute_search(self, search_dict, global_and_search=False): + def execute_search(self, search_dict, user, global_and_search=False): elapsed_time = 0 start_time = datetime.datetime.now() result_set = set() @@ -183,7 +186,15 @@ class SearchModel(object): elapsed_time = unicode(datetime.datetime.now() - start_time).split(':')[2] - return self.model.objects.in_bulk(list(result_set)[: LIMIT]).values(), result_set, elapsed_time + queryset = self.model.objects.in_bulk(list(result_set)[: LIMIT]).values() + + if self.permission: + try: + Permission.objects.check_permissions(user, [self.permission]) + except PermissionDenied: + queryset = AccessEntry.objects.filter_objects_by_access(self.permission, user, queryset) + + return queryset, result_set, elapsed_time def assemble_query(self, terms, search_fields): """ diff --git a/mayan/apps/dynamic_search/views.py b/mayan/apps/dynamic_search/views.py index bcf6d39c89..d884fe552e 100644 --- a/mayan/apps/dynamic_search/views.py +++ b/mayan/apps/dynamic_search/views.py @@ -44,12 +44,6 @@ def results(request, extra_context=None): logger.debug('advanced search') queryset, ids, timedelta = document_search.advanced_search(request.GET) - if document_search.permission: - try: - Permission.objects.check_permissions(request.user, [document_search.permission]) - except PermissionDenied: - queryset = AccessEntry.objects.filter_objects_by_access(document_search.permission, request.user, queryset) - # Update the context with the search results context.update({ 'object_list': queryset, @@ -57,7 +51,7 @@ def results(request, extra_context=None): 'title': _(u'Results'), }) - RecentSearch.objects.add_query_for_user(request.user, request.GET, len(ids)) + RecentSearch.objects.add_query_for_user(request.user, document_search, request.GET, len(ids)) if extra_context: context.update(extra_context) From 1a8ab7956fe867bf7e0e50428f4420fde69613f1 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Thu, 11 Dec 2014 19:18:03 -0400 Subject: [PATCH 2/3] Update django rest framework version --- requirements/common.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/common.txt b/requirements/common.txt index ea8e23279a..59d6e86ef0 100644 --- a/requirements/common.txt +++ b/requirements/common.txt @@ -13,7 +13,7 @@ django-rest-swagger==0.2.0 django-sendfile==0.3.6 django-solo==1.1.0 django-suit==0.2.12 -djangorestframework==2.4.3 +djangorestframework==2.4.4 GitPython==0.3.2.RC1 From d0aaa256cd017e14017bb36917d6d27ccae1cb8b Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Thu, 11 Dec 2014 19:52:24 -0400 Subject: [PATCH 3/3] Don't search blindly for all the fields passed in the advanced search, use the defined fields, avoid hard coded rejection of &page arguments in the query string. The SearchModel class is now responsible for storing the search in the recent entries instead of the view. --- mayan/apps/dynamic_search/api_views.py | 10 +-- mayan/apps/dynamic_search/classes.py | 92 +++++++------------------- mayan/apps/dynamic_search/managers.py | 9 +-- mayan/apps/dynamic_search/models.py | 8 +-- mayan/apps/dynamic_search/views.py | 12 +--- 5 files changed, 36 insertions(+), 95 deletions(-) diff --git a/mayan/apps/dynamic_search/api_views.py b/mayan/apps/dynamic_search/api_views.py index 39df548a5d..7d2f082ea5 100644 --- a/mayan/apps/dynamic_search/api_views.py +++ b/mayan/apps/dynamic_search/api_views.py @@ -49,16 +49,8 @@ class APISearchView(generics.ListAPIView): self.mayan_object_permissions = {'GET': [document_search.permission]} try: - if 'q' in self.request.GET: - # Simple query - query_string = self.request.GET.get('q', u'').strip() - queryset, ids, timedelta = document_search.simple_search(query_string) - else: - # Advanced search - queryset, ids, timedelta = document_search.advanced_search(self.request.GET) + queryset, ids, timedelta = document_search.search(self.request.GET, self.request.user) except Exception as exception: raise ParseError(unicode(exception)) - RecentSearch.objects.add_query_for_user(self.request.user, self.request.GET, len(ids)) - return queryset diff --git a/mayan/apps/dynamic_search/classes.py b/mayan/apps/dynamic_search/classes.py index fc8768eea5..0d5b7aa9d7 100644 --- a/mayan/apps/dynamic_search/classes.py +++ b/mayan/apps/dynamic_search/classes.py @@ -12,6 +12,7 @@ from acls.models import AccessEntry from common.utils import load_backend from permissions.models import Permission +from .models import RecentSearch from .settings import LIMIT logger = logging.getLogger(__name__) @@ -67,14 +68,6 @@ class SearchModel(object): search_field = SearchField(self, *args, **kwargs) self.search_fields[search_field.get_full_name()] = search_field - def add_related_field(self, *args, **kwargs): - """ - Add a search field that will search content in a related field in - a separate model - """ - search_field = RelatedSearchField(self, *args, **kwargs) - self.search_fields[search_field.get_full_name()] = search_field - def normalize_query(self, query_string, findterms=re.compile(r'"([^"]+)"|(\S+)').findall, normspace=re.compile(r'\s{2,}').sub): @@ -87,38 +80,15 @@ class SearchModel(object): """ return [normspace(' ', (t[0] or t[1]).strip()) for t in findterms(query_string)] - def simple_search(self, query_string, user): + def search(self, query_string, user, global_and_search=False): + elapsed_time = 0 + start_time = datetime.datetime.now() + result_set = set() search_dict = {} - for search_field in self.get_all_search_fields(): - search_dict.setdefault(search_field.get_model(), { - 'searches': [], - 'label': search_field.label, - 'return_value': search_field.return_value - }) - search_dict[search_field.get_model()]['searches'].append( - { - 'field_name': [search_field.field], - 'terms': self.normalize_query(query_string) - } - ) - - logger.debug('search_dict: %s', search_dict) - - return self.execute_search(search_dict, user=user, global_and_search=False) - - def advanced_search(self, dictionary, user): - search_dict = {} - - for key, value in dictionary.items(): - logger.debug('key: %s', key) - logger.debug('value: %s', value) - - if key == 'page': - continue - if value: - search_field = self.get_search_field(key) - logger.debug('search_field: %s', search_field) + if 'q' in query_string: + # Simple search + for search_field in self.get_all_search_fields(): search_dict.setdefault(search_field.get_model(), { 'searches': [], 'label': search_field.label, @@ -127,18 +97,24 @@ class SearchModel(object): search_dict[search_field.get_model()]['searches'].append( { 'field_name': [search_field.field], - 'terms': self.normalize_query(value) + 'terms': self.normalize_query(query_string.get('q', u'').strip()) } ) + else: - logger.debug('search_dict: %s', search_dict) - - return self.execute_search(search_dict, user=user, global_and_search=True) - - def execute_search(self, search_dict, user, global_and_search=False): - elapsed_time = 0 - start_time = datetime.datetime.now() - result_set = set() + for search_field in self.get_all_search_fields(): + if search_field.field in query_string and query_string[search_field.field]: + search_dict.setdefault(search_field.get_model(), { + 'searches': [], + 'label': search_field.label, + 'return_value': search_field.return_value + }) + search_dict[search_field.get_model()]['searches'].append( + { + 'field_name': [search_field.field], + 'terms': self.normalize_query(query_string[search_field.field]) + } + ) for model, data in search_dict.items(): logger.debug('model: %s', model) @@ -194,6 +170,8 @@ class SearchModel(object): except PermissionDenied: queryset = AccessEntry.objects.filter_objects_by_access(self.permission, user, queryset) + RecentSearch.objects.add_query_for_user(user, query_string, len(result_set)) + return queryset, result_set, elapsed_time def assemble_query(self, terms, search_fields): @@ -231,23 +209,3 @@ class SearchField(object): def get_model(self): return self.search_model.model - - -class RelatedSearchField(object): - """ - Search for terms in fields that are related to the parent SearchModel - """ - def __init__(self, search_model, app_label, model_name, field, return_value, label): - self.search_model = search_model - self.app_label = app_label - self.model_name = model_name - self.field = field - self.return_value = return_value - self.model = get_model(app_label, model_name) - self.label = label - - def get_full_name(self): - return '%s.%s.%s' % (self.app_label, self.model_name, self.field) - - def get_model(self): - return self.model diff --git a/mayan/apps/dynamic_search/managers.py b/mayan/apps/dynamic_search/managers.py index aa4d3a545e..f295b64e03 100644 --- a/mayan/apps/dynamic_search/managers.py +++ b/mayan/apps/dynamic_search/managers.py @@ -10,15 +10,16 @@ from .settings import RECENT_COUNT class RecentSearchManager(models.Manager): - def add_query_for_user(self, user, query, hits): - parsed_query = urlparse.parse_qs(urlencode(dict(query.items()))) + def add_query_for_user(self, user, query_string, hits): + parsed_query = urlparse.parse_qs(urlencode(dict(query_string.items()))) for key, value in parsed_query.items(): parsed_query[key] = ' '.join(value) - if 'q=' in query: + if 'q' in query_string: # Is a simple query - if not parsed_query.get('q'): + + if not query_string['q']: # Don't store empty simple searches return else: diff --git a/mayan/apps/dynamic_search/models.py b/mayan/apps/dynamic_search/models.py index 0e44a4bfa4..f58641e962 100644 --- a/mayan/apps/dynamic_search/models.py +++ b/mayan/apps/dynamic_search/models.py @@ -9,7 +9,6 @@ from django.db import models from django.utils.encoding import smart_str, smart_unicode from django.utils.translation import ugettext as _ -from .classes import SearchModel from .managers import RecentSearchManager @@ -30,6 +29,8 @@ class RecentSearch(models.Model): objects = RecentSearchManager() def __unicode__(self): + # TODO: Fix this hack, store the search model name in the recent search entry + from .classes import SearchModel document_search = SearchModel.get('documents.Document') query_dict = urlparse.parse_qs(urllib.unquote_plus(smart_str(self.query))) @@ -38,9 +39,8 @@ class RecentSearch(models.Model): # Advanced search advanced_string = [] for key, value in query_dict.items(): - if key != 'page': - search_field = document_search.get_search_field(key) - advanced_string.append(u'%s: %s' % (search_field.label, smart_unicode(' '.join(value)))) + search_field = document_search.get_search_field(key) + advanced_string.append(u'%s: %s' % (search_field.label, smart_unicode(' '.join(value)))) display_string = u', '.join(advanced_string) else: diff --git a/mayan/apps/dynamic_search/views.py b/mayan/apps/dynamic_search/views.py index d884fe552e..95e2665774 100644 --- a/mayan/apps/dynamic_search/views.py +++ b/mayan/apps/dynamic_search/views.py @@ -34,15 +34,7 @@ def results(request, extra_context=None): # Only do search if there is user input, otherwise just render # the template with the extra_context - if 'q' in request.GET: - # Simple query - logger.debug('simple search') - query_string = request.GET.get('q', u'').strip() - queryset, ids, timedelta = document_search.simple_search(query_string) - else: - # Advanced search - logger.debug('advanced search') - queryset, ids, timedelta = document_search.advanced_search(request.GET) + queryset, ids, timedelta = document_search.search(request.GET, request.user) # Update the context with the search results context.update({ @@ -51,8 +43,6 @@ def results(request, extra_context=None): 'title': _(u'Results'), }) - RecentSearch.objects.add_query_for_user(request.user, document_search, request.GET, len(ids)) - if extra_context: context.update(extra_context)