diff --git a/apps/documents/__init__.py b/apps/documents/__init__.py index 6490162cda..7b42534141 100755 --- a/apps/documents/__init__.py +++ b/apps/documents/__init__.py @@ -48,7 +48,6 @@ document_preview = {'text':_('preview'), 'class':'fancybox', 'view':'document_pr document_download = {'text':_('download'), 'view':'document_download', 'args':'object.id', 'famfam':'page_save', 'permissions':{'namespace':'documents', 'permissions':[PERMISSION_DOCUMENT_DOWNLOAD]}} document_find_duplicates = {'text':_('find duplicates'), 'view':'document_find_duplicates', 'args':'object.id', 'famfam':'page_refresh', 'permissions':{'namespace':'documents', 'permissions':[PERMISSION_DOCUMENT_VIEW]}} document_find_all_duplicates = {'text':_('find all duplicates'), 'view':'document_find_all_duplicates', 'famfam':'page_refresh', 'permissions':{'namespace':'documents', 'permissions':[PERMISSION_DOCUMENT_VIEW]}} -document_recreate_all_links = {'text':_('recreate index links'), 'view':'document_recreate_all_links', 'famfam':'page_link', 'permissions':{'namespace':'documents', 'permissions':[PERMISSION_DOCUMENT_TOOLS]}} document_page_transformation_create = {'text':_('create new transformation'), 'view':'document_page_transformation_create', 'args':'object.id', 'famfam':'pencil_add', 'permissions':{'namespace':'documents', 'permissions':[PERMISSION_DOCUMENT_TRANSFORM]}} document_page_transformation_edit = {'text':_('edit'), 'view':'document_page_transformation_edit', 'args':'object.id', 'famfam':'pencil_go', 'permissions':{'namespace':'documents', 'permissions':[PERMISSION_DOCUMENT_TRANSFORM]}} diff --git a/apps/documents/admin.py b/apps/documents/admin.py index d384c6e3b2..16dc2d53c8 100755 --- a/apps/documents/admin.py +++ b/apps/documents/admin.py @@ -2,9 +2,11 @@ from django.contrib import admin from models import MetadataType, DocumentType, Document, \ DocumentTypeMetadataType, DocumentMetadata, DocumentTypeFilename, \ - MetadataIndex, DocumentMetadataIndex, DocumentPage, MetadataGroup, \ + MetadataIndex, DocumentPage, MetadataGroup, \ MetadataGroupItem, DocumentPageTransformation +from filesystem_serving.admin import DocumentMetadataIndexInline + class MetadataTypeAdmin(admin.ModelAdmin): list_display = ('name', 'title', 'default', 'lookup') @@ -40,15 +42,6 @@ class DocumentMetadataInline(admin.StackedInline): extra = 0 classes = ('collapse-open',) allow_add = False - readonly_fields = ('metadata_type', 'value') - - -class DocumentMetadataIndexInline(admin.StackedInline): - model = DocumentMetadataIndex - extra = 1 - classes = ('collapse-open',) - allow_add = True - readonly_fields = ('suffix', 'metadata_index', 'filename') class DocumentPageTransformationAdmin(admin.ModelAdmin): diff --git a/apps/documents/conf/settings.py b/apps/documents/conf/settings.py index 2ca12933b8..7e0fd9c6ec 100755 --- a/apps/documents/conf/settings.py +++ b/apps/documents/conf/settings.py @@ -60,9 +60,3 @@ DEFAULT_TRANSFORMATIONS = getattr(settings, 'DOCUMENTS_DEFAULT_TRANSFORMATIONS', #Groups GROUP_MAX_RESULTS = getattr(settings, 'DOCUMENTS_GROUP_MAX_RESULTS', 20) GROUP_SHOW_EMPTY = getattr(settings, 'DOCUMENTS_GROUP_SHOW_EMPTY', True) - -# Serving -FILESYSTEM_FILESERVING_ENABLE = getattr(settings, 'DOCUMENTS_FILESYSTEM_FILESERVING_ENABLE', True) -FILESYSTEM_FILESERVING_PATH = getattr(settings, 'DOCUMENTS_FILESYSTEM_FILESERVING_PATH', u'/tmp/mayan/documents') -FILESYSTEM_SLUGIFY_PATHS = getattr(settings, 'DOCUMENTS_SLUGIFY_PATHS', False) -FILESYSTEM_MAX_RENAME_COUNT = getattr(settings, 'DOCUMENTS_FILESYSTEM_MAX_RENAME_COUNT', 200) diff --git a/apps/documents/models.py b/apps/documents/models.py index 74515254e7..bc54f12edd 100755 --- a/apps/documents/models.py +++ b/apps/documents/models.py @@ -1,15 +1,11 @@ -import errno import os -import mimetypes from datetime import datetime import sys from python_magic import magic from django.conf import settings from django.db import models -from django.template.defaultfilters import slugify from django.utils.translation import ugettext_lazy as _ -from django.utils.translation import ugettext from django.db.models import Q @@ -22,16 +18,9 @@ from documents.conf.settings import UUID_FUNCTION from documents.conf.settings import PAGE_COUNT_FUNCTION from documents.conf.settings import STORAGE_BACKEND from documents.conf.settings import STORAGE_DIRECTORY_NAME -from documents.conf.settings import FILESYSTEM_FILESERVING_ENABLE -from documents.conf.settings import FILESYSTEM_FILESERVING_PATH -from documents.conf.settings import FILESYSTEM_SLUGIFY_PATHS -from documents.conf.settings import FILESYSTEM_MAX_RENAME_COUNT from documents.conf.settings import AVAILABLE_TRANSFORMATIONS from documents.conf.settings import DEFAULT_TRANSFORMATIONS -if FILESYSTEM_SLUGIFY_PATHS == False: - #Do not slugify path or filenames and extensions - slugify = lambda x:x def get_filename_from_uuid(instance, filename, directory=STORAGE_DIRECTORY_NAME): @@ -208,113 +197,6 @@ class Document(models.Model): page_transformation.arguments = transformation['arguments'] page_transformation.save() - - - def create_fs_links(self): - if FILESYSTEM_FILESERVING_ENABLE: - if not self.exists(): - raise Exception(ugettext(u'Not creating metadata indexing, document not found in document storage')) - metadata_dict = {'document':self} - metadata_dict.update(dict([(metadata.metadata_type.name, slugify(metadata.value)) for metadata in self.documentmetadata_set.all()])) - - for metadata_index in self.document_type.metadataindex_set.all(): - if metadata_index.enabled: - try: - fabricated_directory = eval(metadata_index.expression, metadata_dict) - target_directory = os.path.join(FILESYSTEM_FILESERVING_PATH, fabricated_directory) - try: - os.makedirs(target_directory) - except OSError, exc: - if exc.errno == errno.EEXIST: - pass - else: - raise OSError(ugettext(u'Unable to create metadata indexing directory: %s') % exc) - - - next_available_filename(self, metadata_index, target_directory, slugify(self.file_filename), slugify(self.file_extension)) - except NameError, exc: - #raise NameError(ugettext(u'Error in metadata indexing expression: %s') % exc) - #This should be a warning not an error - pass - - - def delete_fs_links(self): - if FILESYSTEM_FILESERVING_ENABLE: - for document_metadata_index in self.documentmetadataindex_set.all(): - try: - os.unlink(document_metadata_index.filename) - document_metadata_index.delete() - except OSError, exc: - if exc.errno == errno.ENOENT: - #No longer exits, so delete db entry anyway - document_metadata_index.delete() - else: - raise OSError(ugettext(u'Unable to delete metadata indexing symbolic link: %s') % exc) - - path, filename = os.path.split(document_metadata_index.filename) - - #Cleanup directory of dead stuff - #Delete siblings that are dead links - try: - for f in os.listdir(path): - filepath = os.path.join(path, f) - if os.path.islink(filepath): - #Get link's source - source = os.readlink(filepath) - if os.path.isabs(source): - if not os.path.exists(source): - #link's source is absolute and doesn't exit - os.unlink(filepath) - else: - os.unlink(os.path.join(path, filepath)) - elif os.path.isdir(filepath): - #is a directory, try to delete it - try: - os.removedirs(path) - except: - pass - except OSError, exc: - pass - - - #Remove the directory if it is empty - try: - os.removedirs(path) - except: - pass - - -def next_available_filename(document, metadata_index, path, filename, extension, suffix=0): - target = filename - if suffix: - target = '_'.join([filename, unicode(suffix)]) - filepath = os.path.join(path, os.extsep.join([target, extension])) - matches=DocumentMetadataIndex.objects.filter(filename=filepath) - if matches.count() == 0: - document_metadata_index = DocumentMetadataIndex( - document=document, metadata_index=metadata_index, - filename=filepath) - try: - os.symlink(document.file.path, filepath) - document_metadata_index.save() - except OSError, exc: - if exc.errno == errno.EEXIST: - #This link should not exist, try to delete it - try: - os.unlink(filepath) - #Try again with same suffix - return next_available_filename(document, metadata_index, path, filename, extension, suffix) - except Exception, exc: - raise Exception(ugettext(u'Unable to create symbolic link, filename clash: %(filepath)s; %(exc)s') % {'filepath':filepath, 'exc':exc}) - - else: - raise OSError(ugettext(u'Unable to create symbolic link: %(filepath)s; %(exc)s') % {'filepath':filepath, 'exc':exc}) - - return filepath - else: - if suffix > FILESYSTEM_MAX_RENAME_COUNT: - raise Exception(ugettext(u'Maximum rename count reached, not creating symbolic link')) - return next_available_filename(document, metadata_index, path, filename, extension, suffix+1) available_functions_string = (_(u' Available functions: %s') % ','.join(['%s()' % name for name, function in AVAILABLE_FUNCTIONS.items()])) if AVAILABLE_FUNCTIONS else '' @@ -368,20 +250,6 @@ class MetadataIndex(models.Model): verbose_name_plural = _(u'metadata indexes') -class DocumentMetadataIndex(models.Model): - document = models.ForeignKey(Document, verbose_name=_(u'document')) - metadata_index = models.ForeignKey(MetadataIndex, verbose_name=_(u'metadata index')) - filename = models.CharField(max_length=255, verbose_name=_(u'filename')) - suffix = models.PositiveIntegerField(default=0, verbose_name=_(u'suffix')) - - def __unicode__(self): - return unicode(self.filename) - - class Meta: - verbose_name = _(u'document metadata index') - verbose_name_plural = _(u'document metadata indexes') - - class DocumentMetadata(models.Model): document = models.ForeignKey(Document, verbose_name=_(u'document')) metadata_type = models.ForeignKey(MetadataType, verbose_name=_(u'metadata type')) diff --git a/apps/documents/urls.py b/apps/documents/urls.py index e418986c77..12596ef3bd 100755 --- a/apps/documents/urls.py +++ b/apps/documents/urls.py @@ -28,7 +28,6 @@ urlpatterns = patterns('documents.views', url(r'^document/(?P\d+)/download/$', 'document_download', (), 'document_download'), url(r'^document/(?P\d+)/create/siblings/$', 'document_create_sibling', {'multiple':True if ENABLE_SINGLE_DOCUMENT_UPLOAD == False else False}, 'document_create_sibling'), url(r'^document/(?P\d+)/find_duplicates/$', 'document_find_duplicates', (), 'document_find_duplicates'), - url(r'^recreate_all_links/$', 'document_recreate_all_links', (), 'document_recreate_all_links'), url(r'^duplicates/$', 'document_find_all_duplicates', (), 'document_find_all_duplicates'), url(r'^staging_file/(?P\w+)/preview/$', 'staging_file_preview', (), 'staging_file_preview'), diff --git a/apps/documents/utils.py b/apps/documents/utils.py index df43e50a13..08e9e11b30 100755 --- a/apps/documents/utils.py +++ b/apps/documents/utils.py @@ -111,29 +111,4 @@ def save_metadata(metadata_dict, document): #unquote_plus handles utf-8?!? #http://stackoverflow.com/questions/4382875/handling-iri-in-django document_metadata.value=unquote_plus(metadata_dict['value'])#.decode('utf-8') - document_metadata.save() - - -def recreate_links(raise_exception=True): - errors = [] - for document in Document.objects.all(): - try: - document.delete_fs_links() - except Exception, e: - print document - if raise_exception: - raise Exception(e) - else: - error.append(e) - - for document in Document.objects.all(): - try: - document.create_fs_links() - except Exception, e: - print document - if raise_exception: - raise Exception(e) - else: - error.append(e) - - return errors + document_metadata.save() diff --git a/apps/documents/views.py b/apps/documents/views.py index f22ce9d229..98f51f68f3 100755 --- a/apps/documents/views.py +++ b/apps/documents/views.py @@ -20,8 +20,10 @@ from permissions.api import check_permissions, Unauthorized from filetransfers.api import serve_file from converter.api import convert, in_image_cache, QUALITY_DEFAULT from converter import TRANFORMATION_CHOICES +from filesystem_serving.api import document_create_fs_links, document_delete_fs_links -from utils import from_descriptor_to_tempfile, recreate_links + +from utils import from_descriptor_to_tempfile from models import Document, DocumentMetadata, DocumentType, MetadataType, \ DocumentPage, DocumentPageTransformation @@ -37,7 +39,6 @@ from ocr.models import add_document_to_queue from documents.conf.settings import DELETE_STAGING_FILE_AFTER_UPLOAD from documents.conf.settings import USE_STAGING_DIRECTORY -from documents.conf.settings import FILESYSTEM_FILESERVING_ENABLE from documents.conf.settings import STAGING_FILES_PREVIEW_SIZE from documents.conf.settings import PREVIEW_SIZE from documents.conf.settings import THUMBNAIL_SIZE @@ -48,6 +49,8 @@ from documents.conf.settings import AUTOMATIC_OCR from documents.conf.settings import UNCOMPRESS_COMPRESSED_LOCAL_FILES from documents.conf.settings import UNCOMPRESS_COMPRESSED_STAGING_FILES +from filesystem_serving.conf.settings import FILESERVING_ENABLE + from documents import PERMISSION_DOCUMENT_CREATE, \ PERMISSION_DOCUMENT_CREATE, PERMISSION_DOCUMENT_PROPERTIES_EDIT, \ @@ -128,7 +131,7 @@ def _handle_save_document(request, document, form=None): save_metadata_list(decode_metadata_from_url(request.GET), document) try: - document.create_fs_links() + document_create_fs_links(document) except Exception, e: messages.error(request, e) @@ -300,7 +303,7 @@ def document_view(request, document_id): }, ] - if FILESYSTEM_FILESERVING_ENABLE: + if FILESERVING_ENABLE: subtemplates_dict.append({ 'name':'generic_list_subtemplate.html', 'title':_(u'index links'), @@ -370,7 +373,7 @@ def document_edit(request, document_id): form = DocumentForm_edit(request.POST, initial={'document_type':document.document_type}) if form.is_valid(): try: - document.delete_fs_links() + document_delete_fs_links(document) except Exception, e: messages.error(request, e) return HttpResponseRedirect(reverse('document_list')) @@ -386,7 +389,7 @@ def document_edit(request, document_id): messages.success(request, _(u'Document %s edited successfully.') % document) try: - document.create_fs_links() + document_create_fs_links(document) messages.success(request, _(u'Document filesystem links updated successfully.')) except Exception, e: messages.error(request, e) @@ -433,7 +436,7 @@ def document_edit_metadata(request, document_id): if formset.is_valid(): save_metadata_list(formset.cleaned_data, document) try: - document.delete_fs_links() + document_delete_fs_links(document) except Exception, e: messages.error(request, e) return HttpResponseRedirect(reverse('document_list')) @@ -441,7 +444,7 @@ def document_edit_metadata(request, document_id): messages.success(request, _(u'Metadata for document %s edited successfully.') % document) try: - document.create_fs_links() + document_create_fs_links(document) messages.success(request, _(u'Document filesystem links updated successfully.')) except Exception, e: messages.error(request, e) @@ -739,29 +742,3 @@ def document_find_all_duplicates(request): raise Http404(e) return _find_duplicate_list(request, include_source=False) - - -def document_recreate_all_links(request): - permissions = [PERMISSION_DOCUMENT_TOOLS] - try: - check_permissions(request.user, 'documents', permissions) - except Unauthorized, e: - raise Http404(e) - - previous = request.POST.get('previous', request.GET.get('previous', request.META.get('HTTP_REFERER', None))) - next = request.POST.get('next', request.GET.get('next', request.META.get('HTTP_REFERER', None))) - - if request.method != 'POST': - return render_to_response('generic_confirm.html', { - 'previous':previous, - 'next':next, - 'message':_(u'On large databases this operation may take some time to execute.'), - }, context_instance=RequestContext(request)) - else: - try: - recreate_links() - messages.success(request, _(u'Filesystem links re-creation completed successfully.')) - except Exception, e: - messages.error(request, _(u'Filesystem links re-creation error: %s') % e) - - return HttpResponseRedirect(next) diff --git a/apps/filesystem_serving/__init__.py b/apps/filesystem_serving/__init__.py new file mode 100644 index 0000000000..b52f872283 --- /dev/null +++ b/apps/filesystem_serving/__init__.py @@ -0,0 +1,14 @@ +from django.utils.translation import ugettext_lazy as _ + + +from permissions.api import register_permissions + + +FILESYSTEM_SERVING_RECREATE_LINKS = 'recreate_links' + +register_permissions('filesystem_serving', [ + {'name':FILESYSTEM_SERVING_RECREATE_LINKS, 'label':_(u'Recreate filesystem links.')}, +]) + + +filesystem_serving_recreate_all_links = {'text':_('recreate index links'), 'view':'recreate_all_links', 'famfam':'page_link', 'permissions':{'namespace':'filesystem_serving', 'permissions':[FILESYSTEM_SERVING_RECREATE_LINKS]}} diff --git a/apps/filesystem_serving/admin.py b/apps/filesystem_serving/admin.py new file mode 100644 index 0000000000..246f23d85a --- /dev/null +++ b/apps/filesystem_serving/admin.py @@ -0,0 +1,10 @@ +from django.contrib import admin + +from models import DocumentMetadataIndex + +class DocumentMetadataIndexInline(admin.StackedInline): + model = DocumentMetadataIndex + extra = 1 + classes = ('collapse-open',) + allow_add = True + readonly_fields = ('suffix', 'metadata_index', 'filename') diff --git a/apps/filesystem_serving/api.py b/apps/filesystem_serving/api.py new file mode 100644 index 0000000000..8da2c30b74 --- /dev/null +++ b/apps/filesystem_serving/api.py @@ -0,0 +1,155 @@ +import errno +import os +from django.template.defaultfilters import slugify +from django.utils.translation import ugettext_lazy as _ + + +from filesystem_serving.conf.settings import FILESERVING_ENABLE +from filesystem_serving.conf.settings import FILESERVING_PATH +from filesystem_serving.conf.settings import SLUGIFY_PATHS +from filesystem_serving.conf.settings import MAX_RENAME_COUNT + +from models import DocumentMetadataIndex, Document + +if SLUGIFY_PATHS == False: + #Do not slugify path or filenames and extensions + slugify = lambda x:x + + +def document_create_fs_links(document): + if FILESERVING_ENABLE: + if not document.exists(): + raise Exception(_(u'Not creating metadata indexing, document not found in document storage')) + metadata_dict = {'document':document} + metadata_dict.update(dict([(metadata.metadata_type.name, slugify(metadata.value)) for metadata in document.documentmetadata_set.all()])) + + for metadata_index in document.document_type.metadataindex_set.all(): + if metadata_index.enabled: + try: + fabricated_directory = eval(metadata_index.expression, metadata_dict) + target_directory = os.path.join(FILESERVING_PATH, fabricated_directory) + try: + os.makedirs(target_directory) + except OSError, exc: + if exc.errno == errno.EEXIST: + pass + else: + raise OSError(_(u'Unable to create metadata indexing directory: %s') % exc) + + + next_available_filename(document, metadata_index, target_directory, slugify(document.file_filename), slugify(document.file_extension)) + except NameError, exc: + raise NameError(_(u'Error in metadata indexing expression: %s') % exc) + #This should be a warning not an error + #pass + except Exception, exc: + raise Exception(_(u'Unable to create metadata indexing directory: %s') % exc) + + +def document_delete_fs_links(document): + if FILESERVING_ENABLE: + for document_metadata_index in document.documentmetadataindex_set.all(): + try: + os.unlink(document_metadata_index.filename) + document_metadata_index.delete() + except OSError, exc: + if exc.errno == errno.ENOENT: + #No longer exits, so delete db entry anyway + document_metadata_index.delete() + else: + raise OSError(_(u'Unable to delete metadata indexing symbolic link: %s') % exc) + + path, filename = os.path.split(document_metadata_index.filename) + + #Cleanup directory of dead stuff + #Delete siblings that are dead links + try: + for f in os.listdir(path): + filepath = os.path.join(path, f) + if os.path.islink(filepath): + #Get link's source + source = os.readlink(filepath) + if os.path.isabs(source): + if not os.path.exists(source): + #link's source is absolute and doesn't exit + os.unlink(filepath) + else: + os.unlink(os.path.join(path, filepath)) + elif os.path.isdir(filepath): + #is a directory, try to delete it + try: + os.removedirs(path) + except: + pass + except OSError, exc: + pass + + + #Remove the directory if it is empty + try: + os.removedirs(path) + except: + pass + + +def next_available_filename(document, metadata_index, path, filename, extension, suffix=0): + target = filename + if suffix: + target = '_'.join([filename, unicode(suffix)]) + filepath = os.path.join(path, os.extsep.join([target, extension])) + matches=DocumentMetadataIndex.objects.filter(filename=filepath) + if matches.count() == 0: + document_metadata_index = DocumentMetadataIndex( + document=document, metadata_index=metadata_index, + filename=filepath) + try: + os.symlink(document.file.path, filepath) + document_metadata_index.save() + except OSError, exc: + if exc.errno == errno.EEXIST: + #This link should not exist, try to delete it + try: + os.unlink(filepath) + #Try again with same suffix + return next_available_filename(document, metadata_index, path, filename, extension, suffix) + except Exception, exc: + raise Exception(_(u'Unable to create symbolic link, filename clash: %(filepath)s; %(exc)s') % {'filepath':filepath, 'exc':exc}) + + else: + raise OSError(_(u'Unable to create symbolic link: %(filepath)s; %(exc)s') % {'filepath':filepath, 'exc':exc}) + + return filepath + else: + if suffix > MAX_RENAME_COUNT: + raise Exception(_(u'Maximum rename count reached, not creating symbolic link')) + return next_available_filename(document, metadata_index, path, filename, extension, suffix+1) + + +#TODO: diferentiate between evaluation error and filesystem errors +def do_recreate_all_links(raise_exception=True): + errors = [] + warnings = [] + + for document in Document.objects.all(): + try: + document_delete_fs_links(document) + except NameError, e: + warnings.append('%s: %s' % (document, e)) + except Exception, e: + if raise_exception: + raise Exception(e) + else: + errors.append('%s: %s' % (document, e)) + + for document in Document.objects.all(): + try: + document_create_fs_links(document) + except NameError, e: + warnings.append('%s: %s' % (document, e)) + except Exception, e: + if raise_exception: + raise Exception(e) + else: + errors.append('%s: %s' % (document, e)) + + return errors, warnings diff --git a/apps/filesystem_serving/conf/__init__.py b/apps/filesystem_serving/conf/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/apps/filesystem_serving/conf/settings.py b/apps/filesystem_serving/conf/settings.py new file mode 100644 index 0000000000..205f64c819 --- /dev/null +++ b/apps/filesystem_serving/conf/settings.py @@ -0,0 +1,7 @@ +from django.conf import settings + +# Serving +FILESERVING_ENABLE = getattr(settings, 'FILESYSTEM_FILESERVING_ENABLE', True) +FILESERVING_PATH = getattr(settings, 'FILESYSTEM_FILESERVING_PATH', u'/tmp/mayan/documents') +SLUGIFY_PATHS = getattr(settings, 'FILESYSTEM_SLUGIFY_PATHS', False) +MAX_RENAME_COUNT = getattr(settings, 'FILESYSTEM_MAX_RENAME_COUNT', 200) diff --git a/apps/filesystem_serving/models.py b/apps/filesystem_serving/models.py new file mode 100644 index 0000000000..3e3294d5e3 --- /dev/null +++ b/apps/filesystem_serving/models.py @@ -0,0 +1,18 @@ +from django.db import models +from django.utils.translation import ugettext_lazy as _ + +from documents.models import Document, MetadataIndex + + +class DocumentMetadataIndex(models.Model): + document = models.ForeignKey(Document, verbose_name=_(u'document')) + metadata_index = models.ForeignKey(MetadataIndex, verbose_name=_(u'metadata index')) + filename = models.CharField(max_length=255, verbose_name=_(u'filename')) + suffix = models.PositiveIntegerField(default=0, verbose_name=_(u'suffix')) + + def __unicode__(self): + return unicode(self.filename) + + class Meta: + verbose_name = _(u'document metadata index') + verbose_name_plural = _(u'document metadata indexes') diff --git a/apps/filesystem_serving/tests.py b/apps/filesystem_serving/tests.py new file mode 100644 index 0000000000..2247054b35 --- /dev/null +++ b/apps/filesystem_serving/tests.py @@ -0,0 +1,23 @@ +""" +This file demonstrates two different styles of tests (one doctest and one +unittest). These will both pass when you run "manage.py test". + +Replace these with more appropriate tests for your application. +""" + +from django.test import TestCase + +class SimpleTest(TestCase): + def test_basic_addition(self): + """ + Tests that 1 + 1 always equals 2. + """ + self.failUnlessEqual(1 + 1, 2) + +__test__ = {"doctest": """ +Another way to test that 1 + 1 is equal to 2. + +>>> 1 + 1 == 2 +True +"""} + diff --git a/apps/filesystem_serving/urls.py b/apps/filesystem_serving/urls.py new file mode 100644 index 0000000000..7bd5f95cb0 --- /dev/null +++ b/apps/filesystem_serving/urls.py @@ -0,0 +1,5 @@ +from django.conf.urls.defaults import * + +urlpatterns = patterns('filesystem_serving.views', + url(r'^recreate_all_links/$', 'recreate_all_links', (), 'recreate_all_links'), +) diff --git a/apps/filesystem_serving/views.py b/apps/filesystem_serving/views.py new file mode 100644 index 0000000000..22f05a4817 --- /dev/null +++ b/apps/filesystem_serving/views.py @@ -0,0 +1,41 @@ +from django.utils.translation import ugettext_lazy as _ +from django.http import HttpResponse, HttpResponseRedirect, Http404 +from django.shortcuts import render_to_response +from django.template import RequestContext +from django.contrib import messages + + +from permissions.api import check_permissions, Unauthorized + + +from filesystem_serving import FILESYSTEM_SERVING_RECREATE_LINKS +from api import do_recreate_all_links + + +def recreate_all_links(request): + permissions = [FILESYSTEM_SERVING_RECREATE_LINKS] + try: + check_permissions(request.user, 'filesystem_serving', permissions) + except Unauthorized, e: + raise Http404(e) + + previous = request.POST.get('previous', request.GET.get('previous', request.META.get('HTTP_REFERER', None))) + next = request.POST.get('next', request.GET.get('next', request.META.get('HTTP_REFERER', None))) + + if request.method != 'POST': + return render_to_response('generic_confirm.html', { + 'previous':previous, + 'next':next, + 'message':_(u'On large databases this operation may take some time to execute.'), + }, context_instance=RequestContext(request)) + else: + try: + errors, warnings = do_recreate_all_links() + messages.success(request, _(u'Filesystem links re-creation completed successfully.')) + for warning in warnings: + messages.warning(request, warning) + + except Exception, e: + messages.error(request, _(u'Filesystem links re-creation error: %s') % e) + + return HttpResponseRedirect(next) diff --git a/apps/main/__init__.py b/apps/main/__init__.py index f24efbac1c..d899b96145 100755 --- a/apps/main/__init__.py +++ b/apps/main/__init__.py @@ -4,7 +4,8 @@ from common.api import register_menu from permissions import role_list -from documents import document_find_all_duplicates, document_recreate_all_links +from documents import document_find_all_duplicates +from filesystem_serving import filesystem_serving_recreate_all_links check_settings = {'text':_(u'settings'), 'view':'check_settings', 'famfam':'cog'} @@ -13,7 +14,7 @@ register_menu([ {'text':_(u'home'), 'view':'home', 'famfam':'house', 'position':0}, {'text':_(u'tools'), 'view':'tools_menu', 'links': [ - document_find_all_duplicates, document_recreate_all_links + document_find_all_duplicates, filesystem_serving_recreate_all_links ],'famfam':'wrench', 'name':'tools','position':7}, {'text':_(u'setup'), 'view':'check_settings', 'links': [ diff --git a/apps/main/views.py b/apps/main/views.py index af90c136aa..419921bc62 100755 --- a/apps/main/views.py +++ b/apps/main/views.py @@ -9,6 +9,7 @@ from common.conf import settings as common_settings from documents.conf import settings as documents_settings from converter.conf import settings as converter_settings from ocr.conf import settings as ocr_settings +from filesystem_serving.conf import settings as filesystem_serving_settings def home(request): @@ -30,14 +31,17 @@ def check_settings(request): {'name':'DOCUMENTS_PREVIEW_SIZE', 'value':documents_settings.PREVIEW_SIZE}, {'name':'DOCUMENTS_THUMBNAIL_SIZE', 'value':documents_settings.THUMBNAIL_SIZE}, {'name':'DOCUMENTS_DISPLAY_SIZE', 'value':documents_settings.DISPLAY_SIZE}, - {'name':'DOCUMENTS_FILESYSTEM_FILESERVING_ENABLE', 'value':documents_settings.FILESYSTEM_FILESERVING_ENABLE}, - {'name':'DOCUMENTS_FILESYSTEM_FILESERVING_PATH', 'value':documents_settings.FILESYSTEM_FILESERVING_PATH, 'exists':True}, - {'name':'DOCUMENTS_SLUGIFY_PATHS', 'value':documents_settings.FILESYSTEM_SLUGIFY_PATHS}, - {'name':'DOCUMENTS_FILESYSTEM_MAX_RENAME_COUNT', 'value':documents_settings.FILESYSTEM_MAX_RENAME_COUNT}, {'name':'DOCUMENTS_AUTOMATIC_OCR', 'value':documents_settings.AUTOMATIC_OCR}, {'name':'DOCUMENTS_ENABLE_SINGLE_DOCUMENT_UPLOAD', 'value':documents_settings.ENABLE_SINGLE_DOCUMENT_UPLOAD}, {'name':'DOCUMENTS_UNCOMPRESS_COMPRESSED_LOCAL_FILES', 'value':documents_settings.UNCOMPRESS_COMPRESSED_LOCAL_FILES}, {'name':'DOCUMENTS_UNCOMPRESS_COMPRESSED_STAGING_FILES', 'value':documents_settings.UNCOMPRESS_COMPRESSED_STAGING_FILES}, + + #Filesystem_serving + {'name':'FILESYSTEM_FILESERVING_ENABLE', 'value':filesystem_serving_settings.FILESERVING_ENABLE}, + {'name':'FILESYSTEM_FILESERVING_PATH', 'value':filesystem_serving_settings.FILESERVING_PATH, 'exists':True}, + {'name':'FILESYSTEM_SLUGIFY_PATHS', 'value':filesystem_serving_settings.SLUGIFY_PATHS}, + {'name':'FILESYSTEM_MAX_RENAME_COUNT', 'value':filesystem_serving_settings.MAX_RENAME_COUNT}, + #Common {'name':'COMMON_TEMPORARY_DIRECTORY', 'value':common_settings.TEMPORARY_DIRECTORY, 'exists':True}, diff --git a/docs/Changelog.txt b/docs/Changelog.txt index 085c9464db..b79bcdcb2b 100644 --- a/docs/Changelog.txt +++ b/docs/Changelog.txt @@ -20,3 +20,26 @@ * Added per document duplicate search and a tools menu option to seach all duplicated documents * Added document tool that deletes and re-creates all documents filesystem links * Increased document's and document metadata index filename field's size to 255 characters + +2011-Feb 26 +* Added document description to the field search list +* Sort OCR queued documents according to submitted date & time +* Document filesystem serving is now a separate app + - Step to update: + 1) rename the following settings: + DOCUMENTS_FILESYSTEM_FILESERVING_ENABLE + DOCUMENTS_FILESYSTEM_FILESERVING_PATH + DOCUMENTS_FILESYSTEM_SLUGIFY_PATHS + DOCUMENTS_FILESYSTEM_MAX_RENAME_COUNT + to: + FILESYSTEM_FILESERVING_ENABLE + FILESYSTEM_FILESERVING_PATH + FILESYSTEM_SLUGIFY_PATHS + FILESYSTEM_MAX_RENAME_COUNT + + 2) Do a ./manage.py syncdb + 3) Execute 'Recreate index links' locate in the tools menu + 4) Wait a few minutes + + Some warnings may be returned, but these are not fatal as they might + be related to missing metadata in some documents. diff --git a/settings.py b/settings.py index cc9f47f861..b2cf2a0146 100755 --- a/settings.py +++ b/settings.py @@ -131,6 +131,7 @@ INSTALLED_APPS = ( 'sentry', 'sentry.client', 'sentry.client.celery', + 'filesystem_serving', ) @@ -209,10 +210,10 @@ LOGIN_EXEMPT_URLS = ( #DOCUMENTS_GROUP_SHOW_EMPTY = True # Serving -#DOCUMENTS_FILESYSTEM_FILESERVING_ENABLE = True -#DOCUMENTS_FILESYSTEM_FILESERVING_PATH = u'/tmp/mayan/documents' -#DOCUMENTS_FILESYSTEM_SLUGIFY_PATHS = False -#DOCUMENTS_FILESYSTEM_MAX_RENAME_COUNT = 200 +#FILESYSTEM_FILESERVING_ENABLE = True +#FILESYSTEM_FILESERVING_PATH = u'/tmp/mayan/documents' +#FILESYSTEM_SLUGIFY_PATHS = False +#FILESYSTEM_MAX_RENAME_COUNT = 200 # Misc #COMMON_TEMPORARY_DIRECTORY = u'/tmp' diff --git a/urls.py b/urls.py index 7b4acac0c6..bedbd3e692 100755 --- a/urls.py +++ b/urls.py @@ -10,6 +10,7 @@ urlpatterns = patterns('', (r'^', include('common.urls')), (r'^', include('main.urls')), (r'^documents/', include('documents.urls')), + (r'^filesystem_serving/', include('filesystem_serving.urls')), (r'^search/', include('dynamic_search.urls')), (r'^ocr/', include('ocr.urls')), (r'^permissions/', include('permissions.urls')),