Cleanups, add logging, move magic numbers to literals, add release method, improve file descriptor assignment.

This commit is contained in:
Roberto Rosario
2015-07-28 21:55:17 -04:00
parent e4fa3c4960
commit 22f823aca2
@@ -1,15 +1,13 @@
from __future__ import unicode_literals from __future__ import unicode_literals
import datetime import datetime
import logging
from collections import defaultdict
from errno import ENOENT from errno import ENOENT
from stat import S_IFDIR, S_IFLNK, S_IFREG import logging
from sys import argv, exit from optparse import make_option
from stat import S_IFDIR, S_IFREG
from time import time from time import time
from fuse import FUSE, FuseOSError, Operations, LoggingMixIn from fuse import FUSE, FuseOSError, Operations
import pytz import pytz
from django.core import management from django.core import management
@@ -19,12 +17,13 @@ from djcelery.models import IntervalSchedule, PeriodicTask
from document_indexing.models import Index, IndexInstanceNode from document_indexing.models import Index, IndexInstanceNode
from documents.models import Document from documents.models import Document
MAX_FILE_DESCRIPTOR = 65535
MIN_FILE_DESCRIPTOR = 0
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
class IndexFS(LoggingMixIn, Operations): class IndexFS(Operations):
@staticmethod def _path_to_node(self, path, directory_only=True):
def path_to_node(path, index, directory_only=True):
logger.debug('path: %s', path) logger.debug('path: %s', path)
logger.debug('directory_only: %s', directory_only) logger.debug('directory_only: %s', directory_only)
@@ -32,12 +31,12 @@ class IndexFS(LoggingMixIn, Operations):
logger.debug('parts: %s', parts) logger.debug('parts: %s', parts)
directory = index.instance_root node = self.index.instance_root
if len(parts) > 1 and parts[1] != '': if len(parts) > 1 and parts[1] != '':
for part in parts[1:]: for part in parts[1:]:
try: try:
directory = directory.children.get(value=part) node = node.children.get(value=part)
except IndexInstanceNode.DoesNotExist: except IndexInstanceNode.DoesNotExist:
logger.debug('%s does not exists', part) logger.debug('%s does not exists', part)
@@ -45,44 +44,50 @@ class IndexFS(LoggingMixIn, Operations):
return None return None
else: else:
try: try:
result = directory.documents.get(label=part) if node.index_template_node.link_documents:
logger.debug('path %s is a valid file path', path) result = node.documents.get(label=part)
return result logger.debug('path %s is a valid file path', path)
return result
else:
return None
except Document.DoesNotExist: except Document.DoesNotExist:
logger.debug('path %s is a file, but is not found', path) logger.debug('path %s is a file, but is not found', path)
return None return None
logger.debug('directory: %s', directory) logger.debug('node: %s', node)
#logger.debug('directory children: %s', get_children()) logger.debug('node is root: %s', node.is_root_node())
logger.debug('directory is root: %s', directory.is_root_node())
return directory return node
def __init__(self): def __init__(self, index_slug):
self.fd_count = 0 self.fd_count = MIN_FILE_DESCRIPTOR
self.fd = {} self.fd = {}
self.index = Index.objects.first() try:
self.index = Index.objects.get(slug=index_slug)
except Index.DoesNotExist:
print 'Unknown index.'
exit(1)
def getattr(self, path, fh=None): def getattr(self, path, fh=None):
logger.debug('path: %s, fh: %s', path, fh)
now = time() now = time()
result = IndexFS.path_to_node(path=path, index=self.index, directory_only=False) result = self._path_to_node(path=path, directory_only=False)
if not result: if not result:
raise FuseOSError(ENOENT) raise FuseOSError(ENOENT)
if isinstance(result, IndexInstanceNode): if isinstance(result, IndexInstanceNode):
return { return {
'st_mode': (S_IFDIR | 0755), 'st_ctime': now, 'st_mtime': now, 'st_mode': (S_IFDIR | 0555), 'st_ctime': now, 'st_mtime': now,
'st_atime': now, 'st_nlink': 2 'st_atime': now, 'st_nlink': 2
} }
else: else:
now = time()
return { return {
'st_mode': S_IFREG | 0755, 'st_mode': (S_IFREG | 0555),
'st_ctime': (result.date_added.replace(tzinfo=None) - result.date_added.utcoffset() - datetime.datetime(1970, 1, 1)).total_seconds(), 'st_ctime': (result.date_added.replace(tzinfo=None) - result.date_added.utcoffset() - datetime.datetime(1970, 1, 1)).total_seconds(),
'st_mtime': (result.latest_version.timestamp.replace(tzinfo=None) - result.latest_version.timestamp.utcoffset() - datetime.datetime(1970, 1, 1)).total_seconds(), 'st_mtime': (result.latest_version.timestamp.replace(tzinfo=None) - result.latest_version.timestamp.utcoffset() - datetime.datetime(1970, 1, 1)).total_seconds(),
'st_atime': now, 'st_nlink': 1, 'st_atime': now,
'st_size': result.size 'st_size': result.size
} }
@@ -90,55 +95,45 @@ class IndexFS(LoggingMixIn, Operations):
return '' return ''
def open(self, path, flags): def open(self, path, flags):
result = IndexFS.path_to_node(path=path, index=self.index, directory_only=False) result = self._path_to_node(path=path, directory_only=False)
if isinstance(result, Document): if isinstance(result, Document):
self.fd_count += 1 self.fd_count += 1
if self.fd_count > MAX_FILE_DESCRIPTOR:
self.fb_count = MIN_FILE_DESCRIPTOR
# TODO: implement _get_next_file_descriptor()
# TODO: don't provide a file descriptor already in use
self.fd[self.fd_count] = result.open() self.fd[self.fd_count] = result.open()
return self.fd_count return self.fd_count
else: else:
raiseFuseOSError(ENOENT) raiseFuseOSError(ENOENT)
def release(self, path, fh):
self.fd[fh] = None
del(self.fd[fh])
def read(self, path, size, offset, fh): def read(self, path, size, offset, fh):
return self.fd[self.fd_count].read(size) return self.fd[self.fd_count].read(size)
def readdir(self, path, fh): def readdir(self, path, fh):
logger.debug('path: %s', path) logger.debug('path: %s', path)
''' node = self._path_to_node(path=path, directory_only=True)
parts = path.split('/')
directory = self.index.instance_root if not node:
if len(parts) > 1 and parts[1] != '':
for part in parts[1:]:
try:
directory = directory.children.get(value=part)
except IndexInstanceNode.DoesNotExist:
logger.debug('%s does not exists', part)
raise FuseOSError(ENOENT)
logger.debug('directory: %s', directory)
'''
directory = IndexFS.path_to_node(path=path, index=self.index, directory_only=True)
if not directory:
raiseFuseOSError(ENOENT) raiseFuseOSError(ENOENT)
result = ['.', '..'] result = ['.', '..']
directories = directory.get_children().order_by('value') for child_node in node.get_children().values_list('value', flat=True):
if '/' not in child_node:
result.append(child_node)
for directory in directories: if node.index_template_node.link_documents:
value = directory.value.replace('/', '_') for document in node.documents.all():
result.append(value) if '/' not in document.label:
result.append(document.label)
if directory.index_template_node.link_documents:
for document in directory.documents.all():
value = document.label.replace('/', '_')
result.append(value)
return result return result
@@ -146,5 +141,21 @@ class IndexFS(LoggingMixIn, Operations):
class Command(management.BaseCommand): class Command(management.BaseCommand):
help = 'Mount an index as a FUSE filesystem.' help = 'Mount an index as a FUSE filesystem.'
option_list = management.BaseCommand.option_list + (
make_option(
'--index',
action='store',
dest='index',
help='Index to mirror at the mount point.'
),
make_option(
'--mountpoint',
action='store',
dest='mountpoint',
help='Filesystem location at which to mount the selected index.'
),
)
def handle(self, *args, **options): def handle(self, *args, **options):
fuse = FUSE(IndexFS(), '/tmp/mnt', foreground=True) fuse = FUSE(operations=IndexFS(index_slug=options['index']), mountpoint=options['mountpoint'], foreground=True)