matthias/mayan-edms
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Add shared cache class and add mounted index cache invalidation when document and index instance nodes are updated or deleted.

Browse Source 
Signed-off-by: Roberto Rosario <roberto.rosario.gonzalez@gmail.com>
This commit is contained in:
Roberto Rosario
2018-10-08 02:22:05 -04:00
parent c715d0fc8e
commit 7ff974382b
11 changed files with 201 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
HISTORY.rst
View File

@@ -29,6 +29,8 @@
* Restore use of the .store_body variable accidentally remove in
63a77d0235ffef3cd49924ba280879313c622682. Closes GitLab issue #519.
Thanks to TheOneValen @TheOneValen for the report.
* Add shared cache class and add mounted index cache invalidation when
document and index instance nodes are updated or deleted.
3.1.4 (2018-10-4)
=================

2
docs/releases/3.1.5.rst
View File

@@ -57,6 +57,8 @@ Other changes
* Restore use of the .store_body variable accidentally remove in
63a77d0235ffef3cd49924ba280879313c622682. Closes GitLab issue #519.
Thanks to TheOneValen @TheOneValen for the report.
* Add shared cache class and add mounted index cache invalidation when
document and index instance nodes are updated or deleted.
Removals
--------

38
mayan/apps/mirroring/apps.py
View File

@@ -1,10 +1,44 @@
from __future__ import unicode_literals
from django import apps
from django.apps import apps
from django.db.models.signals import pre_delete, pre_save
from django.utils.translation import ugettext_lazy as _
from common.apps import MayanAppConfig
class MirroringApp(apps.AppConfig):
from .handlers import handler_document_cache_delete, handler_node_cache_delete
class MirroringApp(MayanAppConfig):
has_tests = True
name = 'mirroring'
verbose_name = _('Mirroring')
def ready(self):
super(MirroringApp, self).ready()
Document = apps.get_model(app_label='documents', model_name='Document')
IndexInstanceNode = apps.get_model(
app_label='document_indexing', model_name='IndexInstanceNode'
)
pre_delete.connect(
handler_document_cache_delete,
dispatch_uid='mirroring_handler_document_cache_delete',
sender=Document
)
pre_delete.connect(
handler_node_cache_delete,
dispatch_uid='mirroring_handler_node_cache_delete',
sender=IndexInstanceNode
)
pre_save.connect(
handler_document_cache_delete,
dispatch_uid='mirroring_handler_document_cache_delete',
sender=Document
)
pre_save.connect(
handler_node_cache_delete,
dispatch_uid='mirroring_handler_node_cache_delete',
sender=IndexInstanceNode
)

82
mayan/apps/mirroring/caches.py Normal file
View File

@@ -0,0 +1,82 @@
from __future__ import unicode_literals
from django.core.cache import caches
from .settings import (
setting_document_lookup_cache_timeout, setting_node_lookup_cache_timeout
)
class IndexFilesystemCache(object):
@staticmethod
def get_document_key(document):
return 'document_pk_{}'.format(document.pk)
@staticmethod
def get_node_key(node):
return 'node_pk_{}'.format(node.pk)
@staticmethod
def get_path_key(path):
return 'path_{}'.format(path)
def __init__(self, name='default'):
self.cache = caches[name]
def clear_node(self, node):
node_key = IndexFilesystemCache.get_node_key(node=node)
path_cache = self.cache.get(key=node_key)
if path_cache:
path = path_cache.get('path')
if path:
self.clean_path(path=path)
self.cache.delete(key=node_key)
def clear_document(self, document):
document_key = IndexFilesystemCache.get_document_key(document=document)
path_cache = self.cache.get(key=document_key)
if path_cache:
path = path_cache.get('path')
if path:
self.clean_path(path=path)
self.cache.delete(key=document_key)
def clean_path(self, path):
self.cache.delete(
key=IndexFilesystemCache.get_path_key(path=path)
)
def get_path(self, path):
return self.cache.get(
key=IndexFilesystemCache.get_path_key(path=path)
)
def set_path(self, path, document=None, node=None):
# Must provide a document_pk or a node_pk
# Not both
# Providing both is also not correct
if document:
self.cache.set(
key=IndexFilesystemCache.get_path_key(path=path),
value={'document_pk': document.pk},
timeout=setting_document_lookup_cache_timeout.value
)
self.cache.set(
key=IndexFilesystemCache.get_document_key(document=document),
value={'path': path},
timeout=setting_document_lookup_cache_timeout.value
)
elif node:
self.cache.set(
key=IndexFilesystemCache.get_path_key(path=path),
value={'node_pk': node.pk},
timeout=setting_node_lookup_cache_timeout.value
)
self.cache.set(
key=IndexFilesystemCache.get_node_key(node=node),
value={'path': path},
timeout=setting_node_lookup_cache_timeout.value
)

29
mayan/apps/mirroring/classes.py → mayan/apps/mirroring/filesystems.py
View File

@@ -8,7 +8,6 @@ from time import time
from fuse import FuseOSError, Operations
from django.core.cache import caches
from django.core.exceptions import MultipleObjectsReturned
from django.db.models import Count, F, Func, Value
@@ -18,9 +17,8 @@ from documents.models import Document
from .literals import (
MAX_FILE_DESCRIPTOR, MIN_FILE_DESCRIPTOR, FILE_MODE, DIRECTORY_MODE
)
from .settings import (
setting_document_lookup_cache_timeout, setting_node_lookup_cache_timeout
)
from .runtime import cache
logger = logging.getLogger(__name__)
@@ -59,17 +57,17 @@ class IndexFilesystem(Operations):
node = self.index.instance_root
if len(parts) > 1 and parts[1] != '':
obj = self.cache.get(path)
path_cache = cache.get_path(path=path)
if obj:
node_pk = obj.get('node_pk')
if path_cache:
node_pk = path_cache.get('node_pk')
if node_pk:
if access_only:
return True
else:
return IndexInstanceNode.objects.get(pk=node_pk)
document_pk = obj.get('document_pk')
document_pk = path_cache.get('document_pk')
if document_pk:
if access_only:
return True
@@ -87,16 +85,13 @@ class IndexFilesystem(Operations):
else:
try:
if node.index_template_node.link_documents:
result = node.documents.get(label=part)
document = node.documents.get(label=part)
logger.debug(
'path %s is a valid file path', path
)
self.cache.set(
path, {'document_pk': result.pk},
setting_document_lookup_cache_timeout.value
)
cache.set_path(path=path, document=document)
return result
return document
else:
return None
except Document.DoesNotExist:
@@ -109,10 +104,7 @@ class IndexFilesystem(Operations):
except MultipleObjectsReturned:
return None
self.cache.set(
path, {'node_pk': node.pk},
setting_node_lookup_cache_timeout.value
)
cache.set_path(path=path, node=node)
logger.debug('node: %s', node)
logger.debug('node is root: %s', node.is_root_node())
@@ -122,7 +114,6 @@ class IndexFilesystem(Operations):
def __init__(self, index_slug):
self.file_descriptor_count = MIN_FILE_DESCRIPTOR
self.file_descriptors = {}
self.cache = caches['default']
try:
self.index = Index.objects.get(slug=index_slug)

11
mayan/apps/mirroring/handlers.py Normal file
View File

@@ -0,0 +1,11 @@
from __future__ import unicode_literals
from .runtime import cache
def handler_document_cache_delete(sender, **kwargs):
cache.clear_document(document=kwargs['instance'])
def handler_node_cache_delete(sender, **kwargs):
cache.clear_node(node=kwargs['instance'])

2
mayan/apps/mirroring/management/commands/mountindex.py
View File

@@ -7,7 +7,7 @@ from fuse import FUSE
from django.core import management
from django.core.management.base import CommandError
from ...classes import IndexFilesystem
from ...filesystems import IndexFilesystem
logger = logging.getLogger(__name__)

3
mayan/apps/mirroring/runtime.py Normal file
View File

@@ -0,0 +1,3 @@
from .caches import IndexFilesystemCache
cache = IndexFilesystemCache()

3
mayan/apps/mirroring/tests/literals.py
View File

@@ -1,4 +1,7 @@
from __future__ import absolute_import, unicode_literals
TEST_DOCUMENT_PK = 99
TEST_NODE_EXPRESSION = 'level_1'
TEST_NODE_EXPRESSION_MULTILINE = 'first\r\nsecond\r\nthird'
TEST_NODE_PK = 88
TEST_PATH = '/test/path'

49
mayan/apps/mirroring/tests/test_caches.py Normal file
View File

@@ -0,0 +1,49 @@
from __future__ import absolute_import, unicode_literals
from common.tests import BaseTestCase
from ..caches import IndexFilesystemCache
from .literals import TEST_DOCUMENT_PK, TEST_NODE_PK, TEST_PATH
class MockDocument(object):
pk = TEST_DOCUMENT_PK
class MockNode(object):
pk = TEST_NODE_PK
class IndexFilesystemCacheTestCase(BaseTestCase):
def setUp(self):
super(IndexFilesystemCacheTestCase, self).setUp()
self.cache = IndexFilesystemCache()
self.document = MockDocument()
self.node = MockNode()
def test_set_path_document(self):
self.cache.set_path(path=TEST_PATH, document=self.document)
self.assertEquals(
{'document_pk': TEST_DOCUMENT_PK},
self.cache.get_path(path=TEST_PATH)
)
def test_set_path_document_clear_document(self):
self.cache.set_path(path=TEST_PATH, document=self.document)
self.cache.clear_document(document=self.document)
self.assertEquals(None, self.cache.get_path(path=TEST_PATH))
def test_set_path_node(self):
self.cache.set_path(path=TEST_PATH, node=self.node)
self.assertEquals(
{'node_pk': TEST_NODE_PK},
self.cache.get_path(path=TEST_PATH)
)
def test_set_path_node_clear_node(self):
self.cache.set_path(path=TEST_PATH, node=self.node)
self.cache.clear_node(node=self.node)
self.assertEquals(None, self.cache.get_path(path=TEST_PATH))

4
mayan/apps/mirroring/tests/test_classes.py → mayan/apps/mirroring/tests/test_filesystems.py
View File

@@ -11,7 +11,7 @@ from documents.tests import DocumentTestMixin
from document_indexing.tests import DocumentIndexingTestMixin
from ..classes import IndexFilesystem
from ..filesystems import IndexFilesystem
from .literals import (
TEST_NODE_EXPRESSION, TEST_NODE_EXPRESSION_MULTILINE
@@ -19,7 +19,7 @@ from .literals import (
@override_settings(OCR_AUTO_OCR=False)
class IndexFSTestCase(DocumentIndexingTestMixin, DocumentTestMixin, BaseTestCase):
class IndexFilesystemTestCase(DocumentIndexingTestMixin, DocumentTestMixin, BaseTestCase):
auto_upload_document = False
def test_document_access(self):