matthias/mayan-edms
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Completed refactoring of filesystem indexing

Browse Source
This commit is contained in:
Roberto Rosario
2011-02-08 01:35:24 -04:00
parent 5b9369057b
commit 0674dedd4a
5 changed files with 146 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
apps/documents/conf/settings.py
View File

@@ -33,3 +33,4 @@ STORAGE_DIRECTORY_NAME = getattr(settings, 'DOCUMENTS_STORAGE_DIRECTORY_NAME', '
FILESYSTEM_FILESERVING_ENABLE = getattr(settings, 'DOCUMENTS_FILESYSTEM_FILESERVING_ENABLE', True)
FILESYSTEM_FILESERVING_PATH = getattr(settings, 'DOCUMENTS_FILESERVING_PATH', u'/tmp/mayan/documents')
FILESYSTEM_SLUGIFY_PATHS = getattr(settings, 'DOCUMENTS_SLUGIFY_PATHS', False)
FILESYSTEM_MAX_RENAME_COUNT = getattr(settings, 'DOCUMENTS_FILESYSTEM_MAX_RENAME_COUNT', 200)

172
apps/documents/models.py
View File

@@ -21,6 +21,7 @@ from documents.conf.settings import STORAGE_DIRECTORY_NAME
from documents.conf.settings import FILESYSTEM_FILESERVING_ENABLE
from documents.conf.settings import FILESYSTEM_FILESERVING_PATH
from documents.conf.settings import FILESYSTEM_SLUGIFY_PATHS
from documents.conf.settings import FILESYSTEM_MAX_RENAME_COUNT
if FILESYSTEM_SLUGIFY_PATHS == False:
@@ -42,7 +43,7 @@ def populate_file_extension_and_mimetype(instance, filename):
#remove prefix '.'
instance.file_extension = extension[1:]
'''
def custom_eval(format, dictionary):
try:
#Do a normal substitution
@@ -62,7 +63,7 @@ def custom_eval(format, dictionary):
(exc_type, exc_info, tb) = sys.exc_info()
print exc_info
raise Exception(e)
'''
class DocumentType(models.Model):
name = models.CharField(max_length=32, verbose_name=_(u'name'))
@@ -117,58 +118,138 @@ class Document(models.Model):
self.delete_fs_links()
super(Document, self).delete(*args, **kwargs)
def calculate_fs_links(self):
metadata_dict = {'document':self}
metadata_dict.update(dict([(metadata.metadata_type.name, slugify(metadata.value)) for metadata in self.documentmetadata_set.all()]))
for metadata_index in self.document_type.metadataindex_set.all():
if metadata_index.enabled:
#print eval(metadata_index.expression, metadata_dict)
fabricated_directory = custom_eval(metadata_index.expression, metadata_dict)
target_directory = os.path.join(FILESYSTEM_FILESERVING_PATH, fabricated_directory)
#print target_directory
final_path = os.path.join(target_directory, os.extsep.join([slugify(self.file_filename), slugify(self.file_extension)]))
print final_path
#targets = []
#for metadata in self.documentmetadata_set.all():
# if metadata.metadata_type.documenttypemetadatatype_set.all()[0].create_directory_link:
# target_directory = os.path.join(FILESYSTEM_FILESERVING_PATH, slugify(metadata.metadata_type.name), slugify(metadata.value))
# targets.append(os.path.join(target_directory, os.extsep.join([slugify(self.file_filename), slugify(self.file_extension)])))
#return targets
def create_fs_links(self):
if FILESYSTEM_FILESERVING_ENABLE:
for target in self.calculate_fs_links():
try:
os.makedirs(os.path.dirname(target))
except OSError, exc:
if exc.errno == errno.EEXIST:
pass
else:
raise OSError(ugettext(u'Unable to create metadata indexing directory: %s') % exc)
try:
os.symlink(os.path.abspath(self.file.path), target)
except OSError, exc:
if exc.errno == errno.EEXIST:
pass
else:
raise OSError(ugettext(u'Unable to create metadata indexing symbolic link: %s') % exc)
metadata_dict = {'document':self}
metadata_dict.update(dict([(metadata.metadata_type.name, slugify(metadata.value)) for metadata in self.documentmetadata_set.all()]))
for metadata_index in self.document_type.metadataindex_set.all():
if metadata_index.enabled:
fabricated_directory = eval(metadata_index.expression, metadata_dict)
target_directory = os.path.join(FILESYSTEM_FILESERVING_PATH, fabricated_directory)
try:
os.makedirs(target_directory)
except OSError, exc:
if exc.errno == errno.EEXIST:
pass
else:
raise OSError(ugettext(u'Unable to create metadata indexing directory: %s') % exc)
next_available_filename(self, metadata_index, target_directory, slugify(self.file_filename), slugify(self.file_extension))
#try:
# os.symlink(os.path.abspath(self.file.path), filepath)
#except OSError, exc:
# raise OSError(ugettext(u'Unable to create symbolic link: %s') % exc)
'''
try:
filepath = create_symlink(os.path.abspath(self.file.path), target_directory, slugify(self.file_filename), slugify(self.file_extension))
document_metadata_index = DocumentMetadataIndex(
document=self, metadata_index=metadata_index,
filename=filepath)
document_metadata_index.save()
except Exception, e:
raise Exception(ugettext(u'Unable to create metadata indexing symbolic link: %s') % e)
'''
def delete_fs_links(self):
if FILESYSTEM_FILESERVING_ENABLE:
for target in self.calculate_fs_links():
for document_metadata_index in self.documentmetadataindex_set.all():
try:
os.unlink(target)
os.unlink(document_metadata_index.filename)
document_metadata_index.delete()
except OSError, exc:
if exc.errno == errno.ENOENT:
pass
#No longer exits, so delete db entry anyway
document_metadata_index.delete()
else:
raise OSError(ugettext(u'Unable to delete metadata indexing symbolic link: %s') % exc)
path, filename = os.path.split(document_metadata_index.filename)
#Cleanup directory of dead stuff
#Delete siblings that are dead links
try:
for f in os.listdir(path):
filepath = os.path.join(path, f)
if os.path.islink(filepath):
#Get link's source
source = os.readlink(filepath)
if os.path.isabs(source):
if not os.path.exists(source):
#link's source is absolute and doesn't exit
os.unlink(filepath)
else:
os.unlink(os.path.join(path, filepath))
elif os.path.isdir(filepath):
#is a directory, try to delete it
try:
os.removedirs(path)
except:
pass
except OSError, exc:
pass
#Remove the directory if it is empty
try:
os.removedirs(path)
except:
pass
def next_available_filename(document, metadata_index, path, filename, extension, suffix=0):
target = filename
if suffix:
target = '_'.join([filename, unicode(suffix)])
filepath = os.path.join(path, os.extsep.join([target, extension]))
matches=DocumentMetadataIndex.objects.filter(filename=filepath)
if matches.count() == 0:
document_metadata_index = DocumentMetadataIndex(
document=document, metadata_index=metadata_index,
filename=filepath)
try:
os.symlink(os.path.abspath(document.file.path), filepath)
document_metadata_index.save()
except OSError, exc:
if exc.errno == errno.EEXIST:
#This link should not exist, try to delete it
try:
os.unlink(filepath)
#Try again with same suffix
return next_available_filename(document, metadata_index, path, filename, extension, suffix)
except Exception, exc:
raise Exception(ugettext(u'Unable to create symbolic link, filename clash: %s; %s') % (filepath, exc))
else:
raise OSError(ugettext(u'Unable to create symbolic link: %s; %s') % (filepath, exc))
return filepath
else:
if suffix > FILESYSTEM_MAX_RENAME_COUNT:
raise Exception(ugettext(u'Maximum rename count reached, not creating symbolic link'))
return next_available_filename(document, metadata_index, path, filename, extension, suffix+1)
'''
def create_symlink(source, path, filename, extension, suffix=0):
try:
target = filename
if suffix:
target = '_'.join([filename, unicode(suffix)])
filepath = os.path.join(path, os.extsep.join([target, extension]))
os.symlink(source, filepath)
return filepath
except OSError, exc:
if exc.errno == errno.EEXIST:
if suffix > FILESYSTEM_MAX_RENAME_COUNT:
raise Exception(ugettext(u'Maximum rename count reached, not creating symbolic link'))
return create_symlink(source, path, filename, extension, suffix+1)
else:
raise OSError(ugettext(u'Unable to create symbolic link: %s') % exc)
'''
available_functions_string = (_(u' Available functions: %s') % ','.join(['%s()' % name for name, function in AVAILABLE_FUNCTIONS.items()])) if AVAILABLE_FUNCTIONS else ''
available_models_string = (_(u' Available models: %s') % ','.join([name for name, model in AVAILABLE_MODELS.items()])) if AVAILABLE_MODELS else ''
@@ -224,8 +305,9 @@ class MetadataIndex(models.Model):
class DocumentMetadataIndex(models.Model):
document = models.ForeignKey(Document, verbose_name=_(u'document'))
metadata_indexing = models.ForeignKey(MetadataIndex, verbose_name=_(u'metadata indexing'))
filename = models.CharField(max_length=128)
metadata_index = models.ForeignKey(MetadataIndex, verbose_name=_(u'metadata index'))
filename = models.CharField(max_length=128, verbose_name=_(u'filename'))
suffix = models.PositiveIntegerField(default=0, verbose_name=_(u'suffix'))
def __unicode__(self):
return unicode(self.filename)

2
apps/documents/staging.py
View File

@@ -11,7 +11,7 @@ from django.utils.translation import ugettext
from documents.conf.settings import STAGING_DIRECTORY
HASH_FUNCTION = lambda x: hashlib.sha256(x).hexdigest()
#TODO: Do benchmarks
#func = lambda:[StagingFile.get_all() is None for i in range(100)]
#t1=time.time();func();t2=time.time();print '%s took %0.3f ms' % (func.func_name, (t2-t1)*1000.0)

59
apps/documents/views.py
View File

@@ -20,6 +20,8 @@ from staging import StagingFile
from documents.conf.settings import DELETE_STAGING_FILE_AFTER_UPLOAD
from documents.conf.settings import USE_STAGING_DIRECTORY
from documents.conf.settings import FILESYSTEM_FILESERVING_ENABLE
def document_list(request):
return object_list(
@@ -178,8 +180,6 @@ def upload_document_with_type(request, document_type_id, multiple=True):
def document_view(request, document_id):
document = get_object_or_404(Document, pk=document_id)
##############TEST############
document.calculate_fs_links()
form = DocumentForm_view(instance=document, extra_fields=[
{'label':_(u'Filename'), 'field':'file_filename'},
{'label':_(u'File extension'), 'field':'file_extension'},
@@ -191,10 +191,7 @@ def document_view(request, document_id):
{'label':_(u'Exists in storage'), 'field':'exists'}
])
return render_to_response('generic_detail.html', {
'form':form,
'object':document,
'subtemplates_dict':[
subtemplates_dict = [
{
'name':'generic_list_subtemplate.html',
'title':_(u'metadata'),
@@ -202,7 +199,19 @@ def document_view(request, document_id):
'extra_columns':[{'name':_(u'value'), 'attribute':'value'}],
'hide_link':True,
},
],
]
if FILESYSTEM_FILESERVING_ENABLE:
subtemplates_dict.append({
'name':'generic_list_subtemplate.html',
'title':_(u'index links'),
'object_list':document.documentmetadataindex_set.all(),
'hide_link':True})
return render_to_response('generic_detail.html', {
'form':form,
'object':document,
'subtemplates_dict':subtemplates_dict,
}, context_instance=RequestContext(request))
@@ -257,39 +266,3 @@ def document_edit(request, document_id):
'object':document,
}, context_instance=RequestContext(request))
'''
def document_create_from_staging(request, file_id, document_type_id, multiple=True):
if USE_STAGING_DIRECTORY:
document_type = get_object_or_404(DocumentType, pk=document_type_id)
staging_file = StagingFile.get(id=int(file_id))
try:
document = Document(file=staging_file.upload(), document_type=document_type)
document.save()
except Exception, e:
messages.error(request, e)
else:
url = urlparse(request.META['HTTP_REFERER'])
#Take the url parameter defining the metadata values and turn
# then into a dictionary
params = dict([part.split('=') for part in url[4].split('&')])
_save_metadata(params, document)
messages.success(request, _(u'Staging file: %s, uploaded successfully.') % staging_file.filename)
try:
document.create_fs_links()
except Exception, e:
messages.error(request, e)
if DELETE_STAGING_FILE_AFTER_UPLOAD:
try:
staging_file.delete()
messages.success(request, _(u'Staging file: %s, deleted successfully.') % staging_file.filename)
except Exception, e:
messages.error(request, e)
if multiple:
return HttpResponseRedirect(request.META['HTTP_REFERER'])
else:
return HttpResponseRedirect(reverse('document_list'))
'''

1
settings.py
View File

@@ -181,6 +181,7 @@ LOGIN_EXEMPT_URLS = (
#DOCUMENTS_FILESYSTEM_FILESERVING_ENABLE = True
#DOCUMENTS_FILESYSTEM_FILESERVING_PATH = u'/tmp/mayan/documents'
#DOCUMENTS_FILESYSTEM_SLUGIFY_PATHS = False
#DOCUMENTS_FILESYSTEM_MAX_RENAME_COUNT = 200
#======== End of configuration options =======
try: