diff --git a/README.md b/README.md index ca908b8835..70691cac63 100755 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ Features * User defined document checksum algorithm * Previews for a great deal of image formats, including PDF * Document OCR and searching - +* Group documents by metadata automatically Requirements --- diff --git a/apps/common/templates/generic_list_subtemplate.html b/apps/common/templates/generic_list_subtemplate.html index 2e8d626918..824e597731 100755 --- a/apps/common/templates/generic_list_subtemplate.html +++ b/apps/common/templates/generic_list_subtemplate.html @@ -6,7 +6,7 @@ {% if side_bar %}

- {{ title }} + {{ title|capfirst }}

diff --git a/apps/documents/conf/settings.py b/apps/documents/conf/settings.py index 78ae6d52df..f35d48dc10 100755 --- a/apps/documents/conf/settings.py +++ b/apps/documents/conf/settings.py @@ -39,6 +39,10 @@ PREVIEW_SIZE = getattr(settings, 'DOCUMENTS_PREVIEW_SIZE', '640x480') THUMBNAIL_SIZE = getattr(settings, 'DOCUMENTS_THUMBNAIL_SIZE', '50x50') DISPLAY_SIZE = getattr(settings, 'DOCUMENTS_DISPLAY_SIZE', '1024x768') +#Groups +GROUP_MAX_RESULTS = getattr(settings, 'DOCUMENTS_GROUP_MAX_RESULTS', 20) +GROUP_SHOW_EMPTY = getattr(settings, 'DOCUMENTS_GROUP_SHOW_EMPTY', True) + # Serving FILESYSTEM_FILESERVING_ENABLE = getattr(settings, 'DOCUMENTS_FILESYSTEM_FILESERVING_ENABLE', True) FILESYSTEM_FILESERVING_PATH = getattr(settings, 'DOCUMENTS_FILESYSTEM_FILESERVING_PATH', u'/tmp/mayan/documents') diff --git a/apps/documents/models.py b/apps/documents/models.py index 9336ef5526..790d50fd8d 100755 --- a/apps/documents/models.py +++ b/apps/documents/models.py @@ -10,7 +10,8 @@ from django.db import models from django.template.defaultfilters import slugify from django.utils.translation import ugettext_lazy as _ from django.utils.translation import ugettext - +from django.db.models import Q + from dynamic_search.api import register from documents.conf.settings import AVAILABLE_FUNCTIONS @@ -110,6 +111,36 @@ class Document(models.Model): #topics/db/queries.html#topics-db-queries-delete self.delete_fs_links() super(Document, self).delete(*args, **kwargs) + + def get_metadata_groups(self): + errors = [] + metadata_groups = {} + if MetadataGroup.objects.all().count(): + metadata_dict = {} + for document_metadata in self.documentmetadata_set.all(): + metadata_dict['metadata_%s' % document_metadata.metadata_type.name] = document_metadata.value + + for group in MetadataGroup.objects.filter((Q(document_type=self.document_type) | Q(document_type=None)) & Q(enabled=True)): + total_query = Q() + for item in group.metadatagroupitem_set.filter(enabled=True): + try: + value_query = Q(**{'value__%s' % item.operator: eval(item.expression, metadata_dict)}) + except Exception, e: + errors.append(e) + value_query = Q() + + if item.negated: + query = (Q(metadata_type__id=item.metadata_type.id) & ~value_query) + else: + query = (Q(metadata_type__id=item.metadata_type.id) & value_query) + + if item.inclusion == INCLUSION_AND: + total_query &= query + elif item.inclusion == INCLUSION_OR: + total_query |= query + document_id_list = DocumentMetadata.objects.filter(query).values_list('document', flat=True) + metadata_groups[group] = Document.objects.filter(Q(id__in=document_id_list) & ~Q(id=self.id)) or [] + return metadata_groups, errors def create_fs_links(self): if FILESYSTEM_FILESERVING_ENABLE: @@ -326,6 +357,7 @@ class MetadataGroup(models.Model): verbose_name=_(u'document type'), help_text=_(u'If left blank, all document types will be matched.')) name = models.CharField(max_length=32, verbose_name=_(u'name')) label = models.CharField(max_length=32, verbose_name=_(u'label')) + enabled = models.BooleanField(default=True, verbose_name=_(u'enabled')) def __unicode__(self): return self.label if self.label else self.name @@ -335,7 +367,6 @@ class MetadataGroup(models.Model): verbose_name_plural = _(u'metadata document groups') - INCLUSION_AND = '&' INCLUSION_OR = '|' @@ -344,24 +375,36 @@ INCLUSION_CHOICES = ( (INCLUSION_OR, _(u'or')), ) -OPERATOR_EQUAL = ' ' -OPERATOR_IS_NOT_EQUAL = '~' - OPERATOR_CHOCIES = ( - (OPERATOR_EQUAL, _(u'is equal')), - (OPERATOR_IS_NOT_EQUAL, _(u'is not equal')), + ('exact', _(u'is equal')), + ('iexact', _(u'is equal (case insensitive)')), + ('contains', _(u'contains')), + ('icontains', _(u'contains (case insensitive)')), + ('in', _(u'is in')), + ('gt', _(u'is greater than')), + ('gte', _(u'is greater than or equal')), + ('lt', _(u'is less than')), + ('lte', _(u'is less than or equal')), + ('startswith', _(u'starts with')), + ('istartswith', _(u'starts with (case insensitive)')), + ('endswith', _(u'ends with')), + ('iendswith', _(u'ends with (case insensitive)')), + ('regex', _(u'is in regular expression')), + ('iregex', _(u'is in regular expression (case insensitive)')), ) class MetadataGroupItem(models.Model): metadata_group = models.ForeignKey(MetadataGroup, verbose_name=_(u'metadata group')) - inclusion = models.CharField(default=INCLUSION_AND, max_length=16, choices=INCLUSION_CHOICES) + inclusion = models.CharField(default=INCLUSION_AND, max_length=16, choices=INCLUSION_CHOICES, help_text=_(u'The inclusion is ignored for the first item.')) metadata_type = models.ForeignKey(MetadataType, verbose_name=_(u'metadata type'), help_text=_(u'This represents the metadata of all other documents.')) operator = models.CharField(max_length=16, choices=OPERATOR_CHOCIES) - expression = models.CharField(max_length=64, + expression = models.CharField(max_length=128, verbose_name=_(u'expression'), help_text=_(u'This expression will be evaluated against the current seleted document. The document metadata is available as variables of the same name but with the "metadata_" prefix added their name.')) + negated = models.BooleanField(default=False, verbose_name=_(u'negated'), help_text=_(u'Inverts the logic of the operator.')) + enabled = models.BooleanField(default=True, verbose_name=_(u'enabled')) def __unicode__(self): - return '%s %s %s %s' % (self.get_inclusion_display(), self.metadata_type, self.get_operator_display(), self.expression) + return '[%s] %s %s %s %s %s' % ('x' if self.enabled else ' ', self.get_inclusion_display(), self.metadata_type, _(u'not') if self.negated else '', self.get_operator_display(), self.expression) class Meta: verbose_name = _(u'metadata group item') diff --git a/apps/documents/views.py b/apps/documents/views.py index a26a2bf1a8..47739dd3ac 100755 --- a/apps/documents/views.py +++ b/apps/documents/views.py @@ -32,6 +32,8 @@ from documents.conf.settings import FILESYSTEM_FILESERVING_ENABLE from documents.conf.settings import STAGING_FILES_PREVIEW_SIZE from documents.conf.settings import PREVIEW_SIZE from documents.conf.settings import THUMBNAIL_SIZE +from documents.conf.settings import GROUP_MAX_RESULTS +from documents.conf.settings import GROUP_SHOW_EMPTY from utils import save_metadata, save_metadata_list, decode_metadata_from_url @@ -192,12 +194,6 @@ def upload_document_with_type(request, document_type_id, multiple=True): return render_to_response('generic_form.html', context, context_instance=RequestContext(request)) -from django.db.models import Q - -from models import MetadataGroup - -from models import INCLUSION_AND, INCLUSION_OR, OPERATOR_EQUAL, OPERATOR_IS_NOT_EQUAL - def document_view(request, document_id): document = get_object_or_404(Document, pk=document_id) form = DocumentForm_view(instance=document, extra_fields=[ @@ -213,43 +209,12 @@ def document_view(request, document_id): {'label':_(u'UUID'), 'field':'uuid'}, ]) - metadata_groups = {} - if MetadataGroup.objects.all().count(): - metadata_dict = {} - for document_metadata in document.documentmetadata_set.all(): - metadata_dict['metadata_%s' % document_metadata.metadata_type.name] = document_metadata.value - - for group in MetadataGroup.objects.filter(Q(document_type=document.document_type) | Q(document_type=None)): - total_query = None - for count, item in enumerate(group.metadatagroupitem_set.all()): - try: - expression_result = eval(item.expression, metadata_dict) - - if item.operator == OPERATOR_EQUAL: - value_query = Q(documentmetadata__value=expression_result) - elif item.operator == OPERATOR_IS_NOT_EQUAL: - value_query = ~Q(documentmetadata__value=expression_result) - - query = (Q(documentmetadata__metadata_type__id=item.metadata_type.id) & value_query) - if count == 0: - total_query = query - else: - if item.inclusion == INCLUSION_AND: - total_query &= query - elif item.inclusion == INCLUSION_AND: - total_query |= query - except Exception, e: - if request.user.is_staff: - messages.warning(request, _(u'Metadata group query error: %s' % e)) - else: - pass - - - if total_query: - print 'total_query',total_query - metadata_groups[group] = Document.objects.filter(total_query) - print 'documents',Document.objects.filter(total_query) - + + metadata_groups, errors = document.get_metadata_groups() + if request.user.is_staff and errors: + for error in errors: + messages.warning(request, _(u'Metadata group query error: %s' % error)) + preview_form = DocumentPreviewForm(document=document) form_list = [ { @@ -284,13 +249,21 @@ def document_view(request, document_id): sidebar_groups = [] for group, data in metadata_groups.items(): - sidebar_groups.append({ - 'title':group.label, - 'name':'generic_list_subtemplate.html', - 'object_list':data, - 'hide_columns':True, - 'hide_header':True, - }) + if len(data) or GROUP_SHOW_EMPTY: + if len(data): + if len(data) > GROUP_MAX_RESULTS: + total_string = '(%s out of %s)' % (GROUP_MAX_RESULTS, len(data)) + else: + total_string = '(%s)' % len(data) + else: + total_string = '' + sidebar_groups.append({ + 'title':'%s %s' % (group.label, total_string), + 'name':'generic_list_subtemplate.html', + 'object_list':data[:GROUP_MAX_RESULTS], + 'hide_columns':True, + 'hide_header':True, + }) return render_to_response('generic_detail.html', { 'form_list':form_list, diff --git a/docs/Changelog.txt b/docs/Changelog.txt index 4f1bd9d9be..3bee993bda 100644 --- a/docs/Changelog.txt +++ b/docs/Changelog.txt @@ -3,3 +3,4 @@ * Show only document metadata in document list view. * If one document type exists, the create document wizard skips the first step. * Changed to a liquid css grid +* Added the ability to group documents by their metadata diff --git a/docs/TODO b/docs/TODO index 863c87fca0..28cffe5641 100755 --- a/docs/TODO +++ b/docs/TODO @@ -25,6 +25,7 @@ * Add css grids - DONE * If theres only one document type on db skip step 1 of wizard - DONE * Be able to delete staging file - DONE +* Group documents by metadata - DONE * Document list filtering by metadata * Filterform date filtering widget * Validate GET data before saving file @@ -56,7 +57,6 @@ * Add unpaper to pre OCR document cleanup * Support distributed OCR queues (RabbitMQ & Celery?) * DXF viewer - http://code.google.com/p/dxf-reader/source/browse/#svn%2Ftrunk -* Group documents by metadata * Support spreadsheets, wordprocessing docs using openoffice in server mode * WebDAV support * Handle ziped or rar archives diff --git a/settings.py b/settings.py index d372e6a294..c5eeae4564 100755 --- a/settings.py +++ b/settings.py @@ -191,6 +191,10 @@ LOGIN_EXEMPT_URLS = ( #DOCUMENTS_THUMBNAIL_SIZE = '50x50' #DOCUMENTS_DISPLAY_SIZE = '1024x768' +# Groups +#DOCUMENTS_GROUP_MAX_RESULTS = 20 +#DOCUMENTS_GROUP_SHOW_EMPTY = True + # Serving #DOCUMENTS_FILESYSTEM_FILESERVING_ENABLE = True #DOCUMENTS_FILESYSTEM_FILESERVING_PATH = u'/tmp/mayan/documents' @@ -202,6 +206,7 @@ LOGIN_EXEMPT_URLS = ( #CONVERTER_CONVERT_PATH = u'/usr/bin/convert' #CONVERTER_OCR_OPTIONS = u'-colorspace Gray -depth 8 -resample 200x200' #OCR_TESSERACT_PATH = u'/usr/bin/tesseract' + # Override SEARCH_SHOW_OBJECT_TYPE = False #======== End of configuration options =======