From 0674dedd4a987348fa84d241a03471077b7780a6 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 8 Feb 2011 01:35:24 -0400 Subject: [PATCH] Completed refactoring of filesystem indexing --- apps/documents/conf/settings.py | 1 + apps/documents/models.py | 172 +++++++++++++++++++++++--------- apps/documents/staging.py | 2 +- apps/documents/views.py | 59 +++-------- settings.py | 1 + 5 files changed, 146 insertions(+), 89 deletions(-) diff --git a/apps/documents/conf/settings.py b/apps/documents/conf/settings.py index 5e0190b46d..9f353c7d42 100644 --- a/apps/documents/conf/settings.py +++ b/apps/documents/conf/settings.py @@ -33,3 +33,4 @@ STORAGE_DIRECTORY_NAME = getattr(settings, 'DOCUMENTS_STORAGE_DIRECTORY_NAME', ' FILESYSTEM_FILESERVING_ENABLE = getattr(settings, 'DOCUMENTS_FILESYSTEM_FILESERVING_ENABLE', True) FILESYSTEM_FILESERVING_PATH = getattr(settings, 'DOCUMENTS_FILESERVING_PATH', u'/tmp/mayan/documents') FILESYSTEM_SLUGIFY_PATHS = getattr(settings, 'DOCUMENTS_SLUGIFY_PATHS', False) +FILESYSTEM_MAX_RENAME_COUNT = getattr(settings, 'DOCUMENTS_FILESYSTEM_MAX_RENAME_COUNT', 200) diff --git a/apps/documents/models.py b/apps/documents/models.py index 40d17bd7c3..330bab9358 100644 --- a/apps/documents/models.py +++ b/apps/documents/models.py @@ -21,6 +21,7 @@ from documents.conf.settings import STORAGE_DIRECTORY_NAME from documents.conf.settings import FILESYSTEM_FILESERVING_ENABLE from documents.conf.settings import FILESYSTEM_FILESERVING_PATH from documents.conf.settings import FILESYSTEM_SLUGIFY_PATHS +from documents.conf.settings import FILESYSTEM_MAX_RENAME_COUNT if FILESYSTEM_SLUGIFY_PATHS == False: @@ -42,7 +43,7 @@ def populate_file_extension_and_mimetype(instance, filename): #remove prefix '.' instance.file_extension = extension[1:] - +''' def custom_eval(format, dictionary): try: #Do a normal substitution @@ -62,7 +63,7 @@ def custom_eval(format, dictionary): (exc_type, exc_info, tb) = sys.exc_info() print exc_info raise Exception(e) - +''' class DocumentType(models.Model): name = models.CharField(max_length=32, verbose_name=_(u'name')) @@ -117,58 +118,138 @@ class Document(models.Model): self.delete_fs_links() super(Document, self).delete(*args, **kwargs) - def calculate_fs_links(self): - metadata_dict = {'document':self} - metadata_dict.update(dict([(metadata.metadata_type.name, slugify(metadata.value)) for metadata in self.documentmetadata_set.all()])) - - for metadata_index in self.document_type.metadataindex_set.all(): - if metadata_index.enabled: - #print eval(metadata_index.expression, metadata_dict) - fabricated_directory = custom_eval(metadata_index.expression, metadata_dict) - target_directory = os.path.join(FILESYSTEM_FILESERVING_PATH, fabricated_directory) - #print target_directory - - - final_path = os.path.join(target_directory, os.extsep.join([slugify(self.file_filename), slugify(self.file_extension)])) - print final_path - #targets = [] - #for metadata in self.documentmetadata_set.all(): - # if metadata.metadata_type.documenttypemetadatatype_set.all()[0].create_directory_link: - # target_directory = os.path.join(FILESYSTEM_FILESERVING_PATH, slugify(metadata.metadata_type.name), slugify(metadata.value)) - # targets.append(os.path.join(target_directory, os.extsep.join([slugify(self.file_filename), slugify(self.file_extension)]))) - #return targets - def create_fs_links(self): if FILESYSTEM_FILESERVING_ENABLE: - for target in self.calculate_fs_links(): - try: - os.makedirs(os.path.dirname(target)) - except OSError, exc: - if exc.errno == errno.EEXIST: - pass - else: - raise OSError(ugettext(u'Unable to create metadata indexing directory: %s') % exc) - try: - os.symlink(os.path.abspath(self.file.path), target) - except OSError, exc: - if exc.errno == errno.EEXIST: - pass - else: - raise OSError(ugettext(u'Unable to create metadata indexing symbolic link: %s') % exc) + metadata_dict = {'document':self} + metadata_dict.update(dict([(metadata.metadata_type.name, slugify(metadata.value)) for metadata in self.documentmetadata_set.all()])) + + for metadata_index in self.document_type.metadataindex_set.all(): + if metadata_index.enabled: + fabricated_directory = eval(metadata_index.expression, metadata_dict) + target_directory = os.path.join(FILESYSTEM_FILESERVING_PATH, fabricated_directory) + try: + os.makedirs(target_directory) + except OSError, exc: + if exc.errno == errno.EEXIST: + pass + else: + raise OSError(ugettext(u'Unable to create metadata indexing directory: %s') % exc) + + + next_available_filename(self, metadata_index, target_directory, slugify(self.file_filename), slugify(self.file_extension)) + + #try: + # os.symlink(os.path.abspath(self.file.path), filepath) + #except OSError, exc: + # raise OSError(ugettext(u'Unable to create symbolic link: %s') % exc) + + ''' + try: + filepath = create_symlink(os.path.abspath(self.file.path), target_directory, slugify(self.file_filename), slugify(self.file_extension)) + document_metadata_index = DocumentMetadataIndex( + document=self, metadata_index=metadata_index, + filename=filepath) + document_metadata_index.save() + except Exception, e: + raise Exception(ugettext(u'Unable to create metadata indexing symbolic link: %s') % e) + ''' + + def delete_fs_links(self): if FILESYSTEM_FILESERVING_ENABLE: - for target in self.calculate_fs_links(): + for document_metadata_index in self.documentmetadataindex_set.all(): try: - os.unlink(target) + os.unlink(document_metadata_index.filename) + document_metadata_index.delete() except OSError, exc: if exc.errno == errno.ENOENT: - pass + #No longer exits, so delete db entry anyway + document_metadata_index.delete() else: raise OSError(ugettext(u'Unable to delete metadata indexing symbolic link: %s') % exc) - - + + path, filename = os.path.split(document_metadata_index.filename) + + #Cleanup directory of dead stuff + #Delete siblings that are dead links + try: + for f in os.listdir(path): + filepath = os.path.join(path, f) + if os.path.islink(filepath): + #Get link's source + source = os.readlink(filepath) + if os.path.isabs(source): + if not os.path.exists(source): + #link's source is absolute and doesn't exit + os.unlink(filepath) + else: + os.unlink(os.path.join(path, filepath)) + elif os.path.isdir(filepath): + #is a directory, try to delete it + try: + os.removedirs(path) + except: + pass + except OSError, exc: + pass + #Remove the directory if it is empty + try: + os.removedirs(path) + except: + pass + +def next_available_filename(document, metadata_index, path, filename, extension, suffix=0): + target = filename + if suffix: + target = '_'.join([filename, unicode(suffix)]) + filepath = os.path.join(path, os.extsep.join([target, extension])) + matches=DocumentMetadataIndex.objects.filter(filename=filepath) + if matches.count() == 0: + document_metadata_index = DocumentMetadataIndex( + document=document, metadata_index=metadata_index, + filename=filepath) + try: + os.symlink(os.path.abspath(document.file.path), filepath) + document_metadata_index.save() + except OSError, exc: + if exc.errno == errno.EEXIST: + #This link should not exist, try to delete it + try: + os.unlink(filepath) + #Try again with same suffix + return next_available_filename(document, metadata_index, path, filename, extension, suffix) + except Exception, exc: + raise Exception(ugettext(u'Unable to create symbolic link, filename clash: %s; %s') % (filepath, exc)) + + else: + raise OSError(ugettext(u'Unable to create symbolic link: %s; %s') % (filepath, exc)) + + return filepath + else: + if suffix > FILESYSTEM_MAX_RENAME_COUNT: + raise Exception(ugettext(u'Maximum rename count reached, not creating symbolic link')) + return next_available_filename(document, metadata_index, path, filename, extension, suffix+1) + +''' +def create_symlink(source, path, filename, extension, suffix=0): + try: + target = filename + if suffix: + target = '_'.join([filename, unicode(suffix)]) + filepath = os.path.join(path, os.extsep.join([target, extension])) + os.symlink(source, filepath) + return filepath + except OSError, exc: + if exc.errno == errno.EEXIST: + if suffix > FILESYSTEM_MAX_RENAME_COUNT: + raise Exception(ugettext(u'Maximum rename count reached, not creating symbolic link')) + return create_symlink(source, path, filename, extension, suffix+1) + else: + raise OSError(ugettext(u'Unable to create symbolic link: %s') % exc) +''' + available_functions_string = (_(u' Available functions: %s') % ','.join(['%s()' % name for name, function in AVAILABLE_FUNCTIONS.items()])) if AVAILABLE_FUNCTIONS else '' available_models_string = (_(u' Available models: %s') % ','.join([name for name, model in AVAILABLE_MODELS.items()])) if AVAILABLE_MODELS else '' @@ -224,8 +305,9 @@ class MetadataIndex(models.Model): class DocumentMetadataIndex(models.Model): document = models.ForeignKey(Document, verbose_name=_(u'document')) - metadata_indexing = models.ForeignKey(MetadataIndex, verbose_name=_(u'metadata indexing')) - filename = models.CharField(max_length=128) + metadata_index = models.ForeignKey(MetadataIndex, verbose_name=_(u'metadata index')) + filename = models.CharField(max_length=128, verbose_name=_(u'filename')) + suffix = models.PositiveIntegerField(default=0, verbose_name=_(u'suffix')) def __unicode__(self): return unicode(self.filename) diff --git a/apps/documents/staging.py b/apps/documents/staging.py index 0ceeb31d29..3df19d8c30 100644 --- a/apps/documents/staging.py +++ b/apps/documents/staging.py @@ -11,7 +11,7 @@ from django.utils.translation import ugettext from documents.conf.settings import STAGING_DIRECTORY HASH_FUNCTION = lambda x: hashlib.sha256(x).hexdigest() - +#TODO: Do benchmarks #func = lambda:[StagingFile.get_all() is None for i in range(100)] #t1=time.time();func();t2=time.time();print '%s took %0.3f ms' % (func.func_name, (t2-t1)*1000.0) diff --git a/apps/documents/views.py b/apps/documents/views.py index 9d84b5804b..200c75b345 100644 --- a/apps/documents/views.py +++ b/apps/documents/views.py @@ -20,6 +20,8 @@ from staging import StagingFile from documents.conf.settings import DELETE_STAGING_FILE_AFTER_UPLOAD from documents.conf.settings import USE_STAGING_DIRECTORY +from documents.conf.settings import FILESYSTEM_FILESERVING_ENABLE + def document_list(request): return object_list( @@ -178,8 +180,6 @@ def upload_document_with_type(request, document_type_id, multiple=True): def document_view(request, document_id): document = get_object_or_404(Document, pk=document_id) - ##############TEST############ - document.calculate_fs_links() form = DocumentForm_view(instance=document, extra_fields=[ {'label':_(u'Filename'), 'field':'file_filename'}, {'label':_(u'File extension'), 'field':'file_extension'}, @@ -191,10 +191,7 @@ def document_view(request, document_id): {'label':_(u'Exists in storage'), 'field':'exists'} ]) - return render_to_response('generic_detail.html', { - 'form':form, - 'object':document, - 'subtemplates_dict':[ + subtemplates_dict = [ { 'name':'generic_list_subtemplate.html', 'title':_(u'metadata'), @@ -202,7 +199,19 @@ def document_view(request, document_id): 'extra_columns':[{'name':_(u'value'), 'attribute':'value'}], 'hide_link':True, }, - ], + ] + + if FILESYSTEM_FILESERVING_ENABLE: + subtemplates_dict.append({ + 'name':'generic_list_subtemplate.html', + 'title':_(u'index links'), + 'object_list':document.documentmetadataindex_set.all(), + 'hide_link':True}) + + return render_to_response('generic_detail.html', { + 'form':form, + 'object':document, + 'subtemplates_dict':subtemplates_dict, }, context_instance=RequestContext(request)) @@ -257,39 +266,3 @@ def document_edit(request, document_id): 'object':document, }, context_instance=RequestContext(request)) - -''' -def document_create_from_staging(request, file_id, document_type_id, multiple=True): - if USE_STAGING_DIRECTORY: - document_type = get_object_or_404(DocumentType, pk=document_type_id) - staging_file = StagingFile.get(id=int(file_id)) - - try: - document = Document(file=staging_file.upload(), document_type=document_type) - document.save() - except Exception, e: - messages.error(request, e) - else: - url = urlparse(request.META['HTTP_REFERER']) - #Take the url parameter defining the metadata values and turn - # then into a dictionary - params = dict([part.split('=') for part in url[4].split('&')]) - _save_metadata(params, document) - messages.success(request, _(u'Staging file: %s, uploaded successfully.') % staging_file.filename) - try: - document.create_fs_links() - except Exception, e: - messages.error(request, e) - - if DELETE_STAGING_FILE_AFTER_UPLOAD: - try: - staging_file.delete() - messages.success(request, _(u'Staging file: %s, deleted successfully.') % staging_file.filename) - except Exception, e: - messages.error(request, e) - - if multiple: - return HttpResponseRedirect(request.META['HTTP_REFERER']) - else: - return HttpResponseRedirect(reverse('document_list')) -''' diff --git a/settings.py b/settings.py index c7befe6700..db1b85bb02 100644 --- a/settings.py +++ b/settings.py @@ -181,6 +181,7 @@ LOGIN_EXEMPT_URLS = ( #DOCUMENTS_FILESYSTEM_FILESERVING_ENABLE = True #DOCUMENTS_FILESYSTEM_FILESERVING_PATH = u'/tmp/mayan/documents' #DOCUMENTS_FILESYSTEM_SLUGIFY_PATHS = False +#DOCUMENTS_FILESYSTEM_MAX_RENAME_COUNT = 200 #======== End of configuration options ======= try: