Extracted document filesystem serving into a separate app, see docs/Changelog.txt to update database accordingly

apps/filesystem_serving/__init__.py

@@ -0,0 +1,14 @@
+from django.utils.translation import ugettext_lazy as _
+
+
+from permissions.api import register_permissions
+
+
+FILESYSTEM_SERVING_RECREATE_LINKS = 'recreate_links'
+
+register_permissions('filesystem_serving', [
+    {'name':FILESYSTEM_SERVING_RECREATE_LINKS, 'label':_(u'Recreate filesystem links.')},
+])
+
+
+filesystem_serving_recreate_all_links = {'text':_('recreate index links'), 'view':'recreate_all_links', 'famfam':'page_link', 'permissions':{'namespace':'filesystem_serving', 'permissions':[FILESYSTEM_SERVING_RECREATE_LINKS]}}

apps/filesystem_serving/admin.py

@@ -0,0 +1,10 @@
+from django.contrib import admin
+
+from models import DocumentMetadataIndex
+
+class DocumentMetadataIndexInline(admin.StackedInline):
+    model = DocumentMetadataIndex
+    extra = 1
+    classes = ('collapse-open',)
+    allow_add = True
+    readonly_fields = ('suffix', 'metadata_index', 'filename')

apps/filesystem_serving/api.py

@@ -0,0 +1,155 @@
+import errno
+import os
+from django.template.defaultfilters import slugify
+from django.utils.translation import ugettext_lazy as _
+
+
+from filesystem_serving.conf.settings import FILESERVING_ENABLE
+from filesystem_serving.conf.settings import FILESERVING_PATH
+from filesystem_serving.conf.settings import SLUGIFY_PATHS
+from filesystem_serving.conf.settings import MAX_RENAME_COUNT
+
+from models import DocumentMetadataIndex, Document
+
+if SLUGIFY_PATHS == False:
+    #Do not slugify path or filenames and extensions
+    slugify = lambda x:x
+
+
+def document_create_fs_links(document):
+    if FILESERVING_ENABLE:
+        if not document.exists():
+            raise Exception(_(u'Not creating metadata indexing, document not found in document storage'))
+        metadata_dict = {'document':document}
+        metadata_dict.update(dict([(metadata.metadata_type.name, slugify(metadata.value)) for metadata in document.documentmetadata_set.all()]))
+            
+        for metadata_index in document.document_type.metadataindex_set.all():
+            if metadata_index.enabled:
+                try:
+                    fabricated_directory = eval(metadata_index.expression, metadata_dict)
+                    target_directory = os.path.join(FILESERVING_PATH, fabricated_directory)
+                    try:
+                        os.makedirs(target_directory)
+                    except OSError, exc:
+                        if exc.errno == errno.EEXIST:
+                            pass
+                        else: 
+                            raise OSError(_(u'Unable to create metadata indexing directory: %s') % exc)
+                   
+
+                    next_available_filename(document, metadata_index, target_directory, slugify(document.file_filename), slugify(document.file_extension))
+                except NameError, exc:
+                    raise NameError(_(u'Error in metadata indexing expression: %s') % exc)
+                    #This should be a warning not an error
+                    #pass
+                except Exception, exc:
+                    raise Exception(_(u'Unable to create metadata indexing directory: %s') % exc)
+
+
+def document_delete_fs_links(document):
+    if FILESERVING_ENABLE:
+        for document_metadata_index in document.documentmetadataindex_set.all():
+            try:
+                os.unlink(document_metadata_index.filename)
+                document_metadata_index.delete()
+            except OSError, exc:
+                if exc.errno == errno.ENOENT:
+                    #No longer exits, so delete db entry anyway
+                    document_metadata_index.delete()
+                else: 
+                    raise OSError(_(u'Unable to delete metadata indexing symbolic link: %s') % exc)
+        
+            path, filename = os.path.split(document_metadata_index.filename)
+            
+            #Cleanup directory of dead stuff
+            #Delete siblings that are dead links
+            try:
+                for f in os.listdir(path):
+                    filepath = os.path.join(path, f)
+                    if os.path.islink(filepath):
+                        #Get link's source
+                        source = os.readlink(filepath)
+                        if os.path.isabs(source):
+                            if not os.path.exists(source):
+                                #link's source is absolute and doesn't exit
+                                os.unlink(filepath)
+                        else:
+                            os.unlink(os.path.join(path, filepath))
+                    elif os.path.isdir(filepath):
+                        #is a directory, try to delete it
+                        try:
+                            os.removedirs(path)
+                        except:
+                            pass                            
+            except OSError, exc:
+                pass
+
+
+            #Remove the directory if it is empty
+            try:
+                os.removedirs(path)
+            except:
+                pass
+
+           
+def next_available_filename(document, metadata_index, path, filename, extension, suffix=0): 
+    target = filename
+    if suffix:
+        target = '_'.join([filename, unicode(suffix)])
+    filepath = os.path.join(path, os.extsep.join([target, extension]))
+    matches=DocumentMetadataIndex.objects.filter(filename=filepath)
+    if matches.count() == 0:
+        document_metadata_index = DocumentMetadataIndex(
+            document=document, metadata_index=metadata_index,
+            filename=filepath)
+        try:
+            os.symlink(document.file.path, filepath)
+            document_metadata_index.save()
+        except OSError, exc:
+            if exc.errno == errno.EEXIST:
+                #This link should not exist, try to delete it
+                try:
+                    os.unlink(filepath)
+                    #Try again with same suffix
+                    return next_available_filename(document, metadata_index, path, filename, extension, suffix)
+                except Exception, exc:
+                    raise Exception(_(u'Unable to create symbolic link, filename clash: %(filepath)s; %(exc)s') % {'filepath':filepath, 'exc':exc})    
+                
+            else:
+                raise OSError(_(u'Unable to create symbolic link: %(filepath)s; %(exc)s') % {'filepath':filepath, 'exc':exc})
+        
+        return filepath
+    else:
+        if suffix > MAX_RENAME_COUNT:
+            raise Exception(_(u'Maximum rename count reached, not creating symbolic link'))
+        return next_available_filename(document, metadata_index, path, filename, extension, suffix+1)
+
+
+#TODO: diferentiate between evaluation error and filesystem errors
+def do_recreate_all_links(raise_exception=True):
+    errors = []
+    warnings = []
+    
+    for document in Document.objects.all():
+        try:
+            document_delete_fs_links(document)
+        except NameError, e:
+            warnings.append('%s: %s' % (document, e))
+        except Exception, e:
+            if raise_exception:
+                raise Exception(e)
+            else:
+                errors.append('%s: %s' % (document, e))
+
+    for document in Document.objects.all():
+        try:
+            document_create_fs_links(document)
+        except NameError, e:
+            warnings.append('%s: %s' % (document, e))
+        except Exception, e:
+            if raise_exception:
+                raise Exception(e)
+            else:
+                errors.append('%s: %s' % (document, e))
+    
+    return errors, warnings

apps/filesystem_serving/conf/settings.py

@@ -0,0 +1,7 @@
+from django.conf import settings
+
+# Serving
+FILESERVING_ENABLE = getattr(settings, 'FILESYSTEM_FILESERVING_ENABLE', True)
+FILESERVING_PATH = getattr(settings, 'FILESYSTEM_FILESERVING_PATH', u'/tmp/mayan/documents')
+SLUGIFY_PATHS = getattr(settings, 'FILESYSTEM_SLUGIFY_PATHS', False)
+MAX_RENAME_COUNT = getattr(settings, 'FILESYSTEM_MAX_RENAME_COUNT', 200)

apps/filesystem_serving/models.py

@@ -0,0 +1,18 @@
+from django.db import models
+from django.utils.translation import ugettext_lazy as _
+
+from documents.models import Document, MetadataIndex
+
+
+class DocumentMetadataIndex(models.Model):
+    document = models.ForeignKey(Document, verbose_name=_(u'document'))
+    metadata_index = models.ForeignKey(MetadataIndex, verbose_name=_(u'metadata index'))
+    filename = models.CharField(max_length=255, verbose_name=_(u'filename'))
+    suffix = models.PositiveIntegerField(default=0, verbose_name=_(u'suffix'))
+
+    def __unicode__(self):
+        return unicode(self.filename)
+
+    class Meta:
+        verbose_name = _(u'document metadata index')
+        verbose_name_plural = _(u'document metadata indexes')

apps/filesystem_serving/tests.py

@@ -0,0 +1,23 @@
+"""
+This file demonstrates two different styles of tests (one doctest and one
+unittest). These will both pass when you run "manage.py test".
+
+Replace these with more appropriate tests for your application.
+"""
+
+from django.test import TestCase
+
+class SimpleTest(TestCase):
+    def test_basic_addition(self):
+        """
+        Tests that 1 + 1 always equals 2.
+        """
+        self.failUnlessEqual(1 + 1, 2)
+
+__test__ = {"doctest": """
+Another way to test that 1 + 1 is equal to 2.
+
+>>> 1 + 1 == 2
+True
+"""}
+

apps/filesystem_serving/urls.py

@@ -0,0 +1,5 @@
+from django.conf.urls.defaults import *
+
+urlpatterns = patterns('filesystem_serving.views',
+    url(r'^recreate_all_links/$', 'recreate_all_links', (), 'recreate_all_links'),
+)

apps/filesystem_serving/views.py

@@ -0,0 +1,41 @@
+from django.utils.translation import ugettext_lazy as _
+from django.http import HttpResponse, HttpResponseRedirect, Http404
+from django.shortcuts import render_to_response
+from django.template import RequestContext
+from django.contrib import messages
+
+
+from permissions.api import check_permissions, Unauthorized
+
+
+from filesystem_serving import FILESYSTEM_SERVING_RECREATE_LINKS
+from api import do_recreate_all_links
+
+
+def recreate_all_links(request):
+    permissions = [FILESYSTEM_SERVING_RECREATE_LINKS]
+    try:
+        check_permissions(request.user, 'filesystem_serving', permissions)
+    except Unauthorized, e:
+        raise Http404(e)
+
+    previous = request.POST.get('previous', request.GET.get('previous', request.META.get('HTTP_REFERER', None)))
+    next = request.POST.get('next', request.GET.get('next', request.META.get('HTTP_REFERER', None)))
+    
+    if request.method != 'POST':
+        return render_to_response('generic_confirm.html', {
+            'previous':previous,
+            'next':next,
+            'message':_(u'On large databases this operation may take some time to execute.'),
+        }, context_instance=RequestContext(request))
+    else:     
+        try:
+            errors, warnings = do_recreate_all_links()
+            messages.success(request, _(u'Filesystem links re-creation completed successfully.'))
+            for warning in warnings:
+                messages.warning(request, warning)
+                
+        except Exception, e:
+            messages.error(request, _(u'Filesystem links re-creation error: %s') % e)
+            
+        return HttpResponseRedirect(next)