From 41f24bc596aaba3ae70ed69868db043bdd4b7826 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 13 Jan 2015 13:00:51 -0400 Subject: [PATCH 1/7] Use ugettext_lazy to translate language names. Issue #139 --- mayan/settings/base.py | 50 +++++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/mayan/settings/base.py b/mayan/settings/base.py index 07c209443b..678e02ff05 100644 --- a/mayan/settings/base.py +++ b/mayan/settings/base.py @@ -12,6 +12,8 @@ https://docs.djangoproject.com/en/1.6/ref/settings/ import os import sys +from django.utils.translation import ugettext_lazy as _ + _file_path = os.path.abspath(os.path.dirname(__file__)).split('/') BASE_DIR = '/'.join(_file_path[0:-2]) @@ -148,32 +150,30 @@ USE_TZ = True PROJECT_TITLE = 'Mayan EDMS' PROJECT_NAME = 'mayan' -ugettext = lambda s: s - LANGUAGES = ( - ('ar', ugettext('Arabic')), - ('bg', ugettext('Bulgarian')), - ('bs', ugettext('Bosnian (Bosnia and Herzegovina)')), - ('da', ugettext('Danish')), - ('de', ugettext('German (Germany)')), - ('en', ugettext('English')), - ('es', ugettext('Spanish')), - ('fa', ugettext('Persian')), - ('fr', ugettext('French')), - ('hu', ugettext('Hungarian')), - ('hr', ugettext('Croatian')), - ('id', ugettext('Indonesian')), - ('it', ugettext('Italian')), - ('nl', ugettext('Dutch (Nethherlands)')), - ('pl', ugettext('Polish')), - ('pt', ugettext('Portuguese')), - ('pt-br', ugettext('Portuguese (Brazil)')), - ('ro', ugettext('Romanian (Romania)')), - ('ru', ugettext('Russian')), - ('sl', ugettext('Slovenian')), - ('tr', ugettext('Turkish')), - ('vi', ugettext('Vietnamese (Viet Nam)')), - ('zh-cn', ugettext('Chinese (China)')), + ('ar', _('Arabic')), + ('bg', _('Bulgarian')), + ('bs', _('Bosnian (Bosnia and Herzegovina)')), + ('da', _('Danish')), + ('de', _('German (Germany)')), + ('en', _('English')), + ('es', _('Spanish')), + ('fa', _('Persian')), + ('fr', _('French')), + ('hu', _('Hungarian')), + ('hr', _('Croatian')), + ('id', _('Indonesian')), + ('it', _('Italian')), + ('nl', _('Dutch (Nethherlands)')), + ('pl', _('Polish')), + ('pt', _('Portuguese')), + ('pt-br', _('Portuguese (Brazil)')), + ('ro', _('Romanian (Romania)')), + ('ru', _('Russian')), + ('sl', _('Slovenian')), + ('tr', _('Turkish')), + ('vi', _('Vietnamese (Viet Nam)')), + ('zh-cn', _('Chinese (China)')), ) SITE_ID = 1 From 17a238dec7af54fd4b010972db9ebb6aa1210e63 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 13 Jan 2015 17:04:35 -0400 Subject: [PATCH 2/7] Don't delete the actual index definitions --- mayan/apps/document_indexing/tools.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/mayan/apps/document_indexing/tools.py b/mayan/apps/document_indexing/tools.py index 5f5dc27ba9..f0517e2488 100644 --- a/mayan/apps/document_indexing/tools.py +++ b/mayan/apps/document_indexing/tools.py @@ -10,9 +10,6 @@ def do_rebuild_all_indexes(): for instance_node in IndexInstanceNode.objects.all(): instance_node.delete() - for index in Index.objects.all(): - index.delete() - for document in Document.objects.all(): # TODO: Launch all concurrently as background tasks index_document(document) From 148ebffa5cd6aa145262a17a496a477c69d9f301 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 13 Jan 2015 17:59:24 -0400 Subject: [PATCH 3/7] Remove comment and unused import --- mayan/apps/document_indexing/tools.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mayan/apps/document_indexing/tools.py b/mayan/apps/document_indexing/tools.py index f0517e2488..71470eb25e 100644 --- a/mayan/apps/document_indexing/tools.py +++ b/mayan/apps/document_indexing/tools.py @@ -3,7 +3,7 @@ from __future__ import absolute_import from documents.models import Document from .api import index_document -from .models import Index, IndexInstanceNode +from .models import IndexInstanceNode def do_rebuild_all_indexes(): @@ -11,5 +11,4 @@ def do_rebuild_all_indexes(): instance_node.delete() for document in Document.objects.all(): - # TODO: Launch all concurrently as background tasks index_document(document) From 53197c40102910eb080e5b16c1f49a61c41af0a5 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 13 Jan 2015 18:01:42 -0400 Subject: [PATCH 4/7] Add locking to avoid index rebuild clashing with index updates or index node deletion --- mayan/apps/document_indexing/tasks.py | 62 +++++++++++++++++++-------- 1 file changed, 45 insertions(+), 17 deletions(-) diff --git a/mayan/apps/document_indexing/tasks.py b/mayan/apps/document_indexing/tasks.py index 29490b56cd..d0415d3622 100644 --- a/mayan/apps/document_indexing/tasks.py +++ b/mayan/apps/document_indexing/tasks.py @@ -11,32 +11,60 @@ logger = logging.getLogger(__name__) RETRY_DELAY = 20 # TODO: convert this into a config option -@app.task(ignore_result=True) -def task_delete_empty_index_nodes(): - delete_empty_index_nodes() +@app.task(bind=True, ignore_result=True) +def task_delete_empty_index_nodes(self): + try: + rebuild_lock = Lock.acquire_lock('document_indexing_task_do_rebuild_all_indexes') + except LockError as exception: + # A rebuild is happening, retry later + raise self.retry(exc=exception, countdown=RETRY_DELAY) + else: + try: + delete_empty_index_nodes() + finally: + rebuild_lock.release() @app.task(bind=True, ignore_result=True) def task_index_document(self, document_id): - # TODO: Add concurrent task control try: - lock = Lock.acquire_lock('document_indexing_task_update_index_document_%d' % document_id) + rebuild_lock = Lock.acquire_lock('document_indexing_task_do_rebuild_all_indexes') except LockError as exception: - # This document is being reindexed by another task, retry later + # A rebuild is happening, retry later raise self.retry(exc=exception, countdown=RETRY_DELAY) else: try: - document = Document.objects.get(pk=document_id) - except Document.DoesNotExist: - # Document was deleted before we could execute, abort about updating - pass + lock = Lock.acquire_lock('document_indexing_task_update_index_document_%d' % document_id) + except LockError as exception: + # This document is being reindexed by another task, retry later + raise self.retry(exc=exception, countdown=RETRY_DELAY) else: - index_document(document) + try: + document = Document.objects.get(pk=document_id) + except Document.DoesNotExist: + # Document was deleted before we could execute, abort about updating + pass + else: + index_document(document) + finally: + lock.release() + finally: + rebuild_lock.release() + + +@app.task(bind=True, ignore_result=True) +def task_do_rebuild_all_indexes(self): + if Lock.filter(name__startswith='document_indexing_task_update_index_document'): + # A document index update is happening, wait + raise self.retry(countdown=RETRY_DELAY) + + try: + lock = Lock.acquire_lock('document_indexing_task_do_rebuild_all_indexes') + except LockError as exception: + # Another rebuild is happening, retry later + raise self.retry(exc=exception, countdown=RETRY_DELAY) + else: + try: + do_rebuild_all_indexes() finally: lock.release() - - -@app.task(ignore_result=True) -def task_do_rebuild_all_indexes(): - # TODO: Find a way to rebuild after all pending updates are finished - do_rebuild_all_indexes() From adcf621ee189e546fe0118beb9e8735bb62c7e2d Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 14 Jan 2015 01:51:54 -0400 Subject: [PATCH 5/7] Preserve zoom level and rotation when moving thru document pages in the page detail view. Issue #138. --- mayan/apps/documents/links.py | 8 ++++---- mayan/apps/documents/views.py | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/mayan/apps/documents/links.py b/mayan/apps/documents/links.py index 68e518a647..5243442909 100644 --- a/mayan/apps/documents/links.py +++ b/mayan/apps/documents/links.py @@ -68,10 +68,10 @@ document_page_transformation_delete = {'text': _(u'Delete'), 'class': 'no-parent document_page_view = {'text': _(u'Page image'), 'class': 'no-parent-history', 'view': 'documents:document_page_view', 'args': 'page.pk', 'famfam': 'page_white_picture', 'permissions': [PERMISSION_DOCUMENT_VIEW]} document_page_text = {'text': _(u'Page text'), 'class': 'no-parent-history', 'view': 'documents:document_page_text', 'args': 'page.pk', 'famfam': 'page_white_text', 'permissions': [PERMISSION_DOCUMENT_VIEW]} document_page_edit = {'text': _(u'Edit page text'), 'class': 'no-parent-history', 'view': 'documents:document_page_edit', 'args': 'page.pk', 'famfam': 'page_white_edit', 'permissions': [PERMISSION_DOCUMENT_EDIT]} -document_page_navigation_next = {'text': _(u'Next page'), 'class': 'no-parent-history', 'view': 'documents:document_page_navigation_next', 'args': 'page.pk', 'famfam': 'resultset_next', 'permissions': [PERMISSION_DOCUMENT_VIEW], 'conditional_disable': is_last_page} -document_page_navigation_previous = {'text': _(u'Previous page'), 'class': 'no-parent-history', 'view': 'documents:document_page_navigation_previous', 'args': 'page.pk', 'famfam': 'resultset_previous', 'permissions': [PERMISSION_DOCUMENT_VIEW], 'conditional_disable': is_first_page} -document_page_navigation_first = {'text': _(u'First page'), 'class': 'no-parent-history', 'view': 'documents:document_page_navigation_first', 'args': 'page.pk', 'famfam': 'resultset_first', 'permissions': [PERMISSION_DOCUMENT_VIEW], 'conditional_disable': is_first_page} -document_page_navigation_last = {'text': _(u'Last page'), 'class': 'no-parent-history', 'view': 'documents:document_page_navigation_last', 'args': 'page.pk', 'famfam': 'resultset_last', 'permissions': [PERMISSION_DOCUMENT_VIEW], 'conditional_disable': is_last_page} +document_page_navigation_next = {'text': _(u'Next page'), 'class': 'no-parent-history', 'view': 'documents:document_page_navigation_next', 'args': 'page.pk', 'famfam': 'resultset_next', 'permissions': [PERMISSION_DOCUMENT_VIEW], 'conditional_disable': is_last_page, 'keep_query': True} +document_page_navigation_previous = {'text': _(u'Previous page'), 'class': 'no-parent-history', 'view': 'documents:document_page_navigation_previous', 'args': 'page.pk', 'famfam': 'resultset_previous', 'permissions': [PERMISSION_DOCUMENT_VIEW], 'conditional_disable': is_first_page, 'keep_query': True} +document_page_navigation_first = {'text': _(u'First page'), 'class': 'no-parent-history', 'view': 'documents:document_page_navigation_first', 'args': 'page.pk', 'famfam': 'resultset_first', 'permissions': [PERMISSION_DOCUMENT_VIEW], 'conditional_disable': is_first_page, 'keep_query': True} +document_page_navigation_last = {'text': _(u'Last page'), 'class': 'no-parent-history', 'view': 'documents:document_page_navigation_last', 'args': 'page.pk', 'famfam': 'resultset_last', 'permissions': [PERMISSION_DOCUMENT_VIEW], 'conditional_disable': is_last_page, 'keep_query': True} document_page_zoom_in = {'text': _(u'Zoom in'), 'class': 'no-parent-history', 'view': 'documents:document_page_zoom_in', 'args': 'page.pk', 'famfam': 'zoom_in', 'permissions': [PERMISSION_DOCUMENT_VIEW], 'conditional_disable': is_max_zoom} document_page_zoom_out = {'text': _(u'Zoom out'), 'class': 'no-parent-history', 'view': 'documents:document_page_zoom_out', 'args': 'page.pk', 'famfam': 'zoom_out', 'permissions': [PERMISSION_DOCUMENT_VIEW], 'conditional_disable': is_min_zoom} document_page_rotate_right = {'text': _(u'Rotate right'), 'class': 'no-parent-history', 'view': 'documents:document_page_rotate_right', 'args': 'page.pk', 'famfam': 'arrow_turn_right', 'permissions': [PERMISSION_DOCUMENT_VIEW]} diff --git a/mayan/apps/documents/views.py b/mayan/apps/documents/views.py index 7e5d854bd6..bb06d813fc 100644 --- a/mayan/apps/documents/views.py +++ b/mayan/apps/documents/views.py @@ -653,7 +653,7 @@ def document_page_navigation_next(request, document_page_id): return HttpResponseRedirect(request.META.get('HTTP_REFERER', reverse(settings.LOGIN_REDIRECT_URL))) else: document_page = get_object_or_404(document_page.siblings, page_number=document_page.page_number + 1) - return HttpResponseRedirect(reverse(view, args=[document_page.pk])) + return HttpResponseRedirect('{0}?{1}'.format(reverse(view, args=[document_page.pk]), request.GET.urlencode())) def document_page_navigation_previous(request, document_page_id): @@ -671,7 +671,7 @@ def document_page_navigation_previous(request, document_page_id): return HttpResponseRedirect(request.META.get('HTTP_REFERER', reverse(settings.LOGIN_REDIRECT_URL))) else: document_page = get_object_or_404(document_page.siblings, page_number=document_page.page_number - 1) - return HttpResponseRedirect(reverse(view, args=[document_page.pk])) + return HttpResponseRedirect('{0}?{1}'.format(reverse(view, args=[document_page.pk]), request.GET.urlencode())) def document_page_navigation_first(request, document_page_id): @@ -685,7 +685,7 @@ def document_page_navigation_first(request, document_page_id): view = resolve_to_name(urlparse.urlparse(request.META.get('HTTP_REFERER', reverse(settings.LOGIN_REDIRECT_URL))).path) - return HttpResponseRedirect(reverse(view, args=[document_page.pk])) + return HttpResponseRedirect('{0}?{1}'.format(reverse(view, args=[document_page.pk]), request.GET.urlencode())) def document_page_navigation_last(request, document_page_id): @@ -699,7 +699,7 @@ def document_page_navigation_last(request, document_page_id): view = resolve_to_name(urlparse.urlparse(request.META.get('HTTP_REFERER', reverse(settings.LOGIN_REDIRECT_URL))).path) - return HttpResponseRedirect(reverse(view, args=[document_page.pk])) + return HttpResponseRedirect('{0}?{1}'.format(reverse(view, args=[document_page.pk]), request.GET.urlencode())) def document_list_recent(request): From fd1f5d1dadf45817583d2ff56da5ca1800141872 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 14 Jan 2015 02:22:26 -0400 Subject: [PATCH 6/7] Make document language choices a configurable list. Issue #137. To override the default list of 600+ languages add a configuration entry to your settings/local.py like so: DOCUMENTS_LANGUAGE_CHOICES = [('eng', 'English'), ('deu', 'German')] to make the list translatable import ugettext_lazy and enclose the language name with _(): from django.utils.translation import ugettext_lazy as _ DOCUMENTS_LANGUAGE_CHOICES = [('eng', _('English')), ('deu', _('German'))] --- mayan/apps/documents/literals.py | 4 ---- mayan/apps/documents/models.py | 5 ++--- mayan/apps/documents/serializers.py | 2 +- mayan/apps/documents/settings.py | 4 ++++ 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/mayan/apps/documents/literals.py b/mayan/apps/documents/literals.py index bc8d5eee50..81f08edae6 100644 --- a/mayan/apps/documents/literals.py +++ b/mayan/apps/documents/literals.py @@ -1,5 +1,3 @@ -import pycountry - PICTURE_ERROR_SMALL = u'picture_error.png' PICTURE_ERROR_MEDIUM = u'1297211435_error.png' PICTURE_UNKNOWN_SMALL = u'1299549572_unknown2.png' @@ -7,6 +5,4 @@ PICTURE_UNKNOWN_MEDIUM = u'1299549805_unknown.png' DEFAULT_ZIP_FILENAME = u'document_bundle.zip' -LANGUAGE_CHOICES = [(i.bibliographic, i.name) for i in list(pycountry.languages)] - DOCUMENT_IMAGE_TASK_TIMEOUT = 20 diff --git a/mayan/apps/documents/models.py b/mayan/apps/documents/models.py index 9eb0c7ae1e..75699f09d9 100644 --- a/mayan/apps/documents/models.py +++ b/mayan/apps/documents/models.py @@ -32,12 +32,11 @@ from mimetype.api import get_mimetype from .events import event_document_create from .exceptions import NewDocumentVersionNotAllowed -from .literals import LANGUAGE_CHOICES from .managers import (DocumentManager, DocumentPageTransformationManager, DocumentTypeManager, RecentDocumentManager) from .runtime import storage_backend -from .settings import (CACHE_PATH, DISPLAY_SIZE, LANGUAGE, ZOOM_MAX_LEVEL, - ZOOM_MIN_LEVEL) +from .settings import (CACHE_PATH, DISPLAY_SIZE, LANGUAGE, LANGUAGE_CHOICES, + ZOOM_MAX_LEVEL, ZOOM_MIN_LEVEL) from .signals import post_version_upload, post_document_type_change HASH_FUNCTION = lambda x: hashlib.sha256(x).hexdigest() # document image cache name hash function diff --git a/mayan/apps/documents/serializers.py b/mayan/apps/documents/serializers.py index 34cd930465..b4303cb3c7 100644 --- a/mayan/apps/documents/serializers.py +++ b/mayan/apps/documents/serializers.py @@ -2,9 +2,9 @@ from __future__ import absolute_import from rest_framework import serializers -from .literals import LANGUAGE_CHOICES from .models import (Document, DocumentVersion, DocumentPage, DocumentType, RecentDocument) +from .settings import LANGUAGE_CHOICES class DocumentPageSerializer(serializers.HyperlinkedModelSerializer): diff --git a/mayan/apps/documents/settings.py b/mayan/apps/documents/settings.py index 9cfed38acc..3dbd660e8a 100644 --- a/mayan/apps/documents/settings.py +++ b/mayan/apps/documents/settings.py @@ -1,12 +1,15 @@ """Configuration options for the documents app""" import os +import pycountry from django.conf import settings from django.utils.translation import ugettext_lazy as _ from smart_settings.api import register_settings +LANGUAGE_CHOICES = [(i.bibliographic, i.name) for i in list(pycountry.languages)] + register_settings( namespace=u'documents', module=u'documents.settings', @@ -27,5 +30,6 @@ register_settings( # {'name': u'CACHE_PATH', 'global_name': u'DOCUMENTS_CACHE_PATH', 'default': os.path.join(settings.MEDIA_ROOT, 'image_cache'), 'exists': True}, {'name': u'LANGUAGE', 'global_name': u'DOCUMENTS_LANGUAGE', 'default': u'eng', 'description': _('Default documents language (in ISO639-2 format).')}, + {'name': u'LANGUAGE_CHOICES', 'global_name': u'DOCUMENTS_LANGUAGE_CHOICES', 'default': LANGUAGE_CHOICES, 'description': _('List of supported document languages.')}, ] ) From 8b8c915bd06988a21c10659b7fc79bc36e427a0d Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 14 Jan 2015 02:27:33 -0400 Subject: [PATCH 7/7] Make the default list of document languages translatable. Issue #139 --- mayan/apps/documents/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mayan/apps/documents/settings.py b/mayan/apps/documents/settings.py index 3dbd660e8a..e93af99faa 100644 --- a/mayan/apps/documents/settings.py +++ b/mayan/apps/documents/settings.py @@ -8,7 +8,7 @@ from django.utils.translation import ugettext_lazy as _ from smart_settings.api import register_settings -LANGUAGE_CHOICES = [(i.bibliographic, i.name) for i in list(pycountry.languages)] +LANGUAGE_CHOICES = [(i.bibliographic, _(i.name)) for i in list(pycountry.languages)] register_settings( namespace=u'documents',