diff --git a/HISTORY.rst b/HISTORY.rst index 5965d99131..57ecc7f660 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -29,6 +29,8 @@ * Restore use of the .store_body variable accidentally remove in 63a77d0235ffef3cd49924ba280879313c622682. Closes GitLab issue #519. Thanks to TheOneValen @TheOneValen for the report. +* Add shared cache class and add mounted index cache invalidation when + document and index instance nodes are updated or deleted. 3.1.4 (2018-10-4) ================= diff --git a/docs/releases/3.1.5.rst b/docs/releases/3.1.5.rst index c291d3f0c8..66d8a617f6 100644 --- a/docs/releases/3.1.5.rst +++ b/docs/releases/3.1.5.rst @@ -57,6 +57,8 @@ Other changes * Restore use of the .store_body variable accidentally remove in 63a77d0235ffef3cd49924ba280879313c622682. Closes GitLab issue #519. Thanks to TheOneValen @TheOneValen for the report. +* Add shared cache class and add mounted index cache invalidation when + document and index instance nodes are updated or deleted. Removals -------- diff --git a/mayan/apps/mirroring/apps.py b/mayan/apps/mirroring/apps.py index 428df7d1d7..5dc800661a 100644 --- a/mayan/apps/mirroring/apps.py +++ b/mayan/apps/mirroring/apps.py @@ -1,10 +1,44 @@ from __future__ import unicode_literals -from django import apps +from django.apps import apps +from django.db.models.signals import pre_delete, pre_save from django.utils.translation import ugettext_lazy as _ +from common.apps import MayanAppConfig -class MirroringApp(apps.AppConfig): +from .handlers import handler_document_cache_delete, handler_node_cache_delete + + +class MirroringApp(MayanAppConfig): has_tests = True name = 'mirroring' verbose_name = _('Mirroring') + + def ready(self): + super(MirroringApp, self).ready() + + Document = apps.get_model(app_label='documents', model_name='Document') + IndexInstanceNode = apps.get_model( + app_label='document_indexing', model_name='IndexInstanceNode' + ) + + pre_delete.connect( + handler_document_cache_delete, + dispatch_uid='mirroring_handler_document_cache_delete', + sender=Document + ) + pre_delete.connect( + handler_node_cache_delete, + dispatch_uid='mirroring_handler_node_cache_delete', + sender=IndexInstanceNode + ) + pre_save.connect( + handler_document_cache_delete, + dispatch_uid='mirroring_handler_document_cache_delete', + sender=Document + ) + pre_save.connect( + handler_node_cache_delete, + dispatch_uid='mirroring_handler_node_cache_delete', + sender=IndexInstanceNode + ) diff --git a/mayan/apps/mirroring/caches.py b/mayan/apps/mirroring/caches.py new file mode 100644 index 0000000000..aaef782938 --- /dev/null +++ b/mayan/apps/mirroring/caches.py @@ -0,0 +1,82 @@ +from __future__ import unicode_literals + +from django.core.cache import caches + +from .settings import ( + setting_document_lookup_cache_timeout, setting_node_lookup_cache_timeout +) + + +class IndexFilesystemCache(object): + @staticmethod + def get_document_key(document): + return 'document_pk_{}'.format(document.pk) + + @staticmethod + def get_node_key(node): + return 'node_pk_{}'.format(node.pk) + + @staticmethod + def get_path_key(path): + return 'path_{}'.format(path) + + def __init__(self, name='default'): + self.cache = caches[name] + + def clear_node(self, node): + node_key = IndexFilesystemCache.get_node_key(node=node) + path_cache = self.cache.get(key=node_key) + if path_cache: + path = path_cache.get('path') + if path: + self.clean_path(path=path) + + self.cache.delete(key=node_key) + + def clear_document(self, document): + document_key = IndexFilesystemCache.get_document_key(document=document) + path_cache = self.cache.get(key=document_key) + if path_cache: + path = path_cache.get('path') + if path: + self.clean_path(path=path) + + self.cache.delete(key=document_key) + + def clean_path(self, path): + self.cache.delete( + key=IndexFilesystemCache.get_path_key(path=path) + ) + + def get_path(self, path): + return self.cache.get( + key=IndexFilesystemCache.get_path_key(path=path) + ) + + def set_path(self, path, document=None, node=None): + # Must provide a document_pk or a node_pk + # Not both + # Providing both is also not correct + + if document: + self.cache.set( + key=IndexFilesystemCache.get_path_key(path=path), + value={'document_pk': document.pk}, + timeout=setting_document_lookup_cache_timeout.value + ) + self.cache.set( + key=IndexFilesystemCache.get_document_key(document=document), + value={'path': path}, + timeout=setting_document_lookup_cache_timeout.value + ) + elif node: + self.cache.set( + key=IndexFilesystemCache.get_path_key(path=path), + value={'node_pk': node.pk}, + timeout=setting_node_lookup_cache_timeout.value + ) + self.cache.set( + key=IndexFilesystemCache.get_node_key(node=node), + value={'path': path}, + timeout=setting_node_lookup_cache_timeout.value + ) diff --git a/mayan/apps/mirroring/classes.py b/mayan/apps/mirroring/filesystems.py similarity index 89% rename from mayan/apps/mirroring/classes.py rename to mayan/apps/mirroring/filesystems.py index c6f8fe5ce3..07582ce411 100644 --- a/mayan/apps/mirroring/classes.py +++ b/mayan/apps/mirroring/filesystems.py @@ -8,7 +8,6 @@ from time import time from fuse import FuseOSError, Operations -from django.core.cache import caches from django.core.exceptions import MultipleObjectsReturned from django.db.models import Count, F, Func, Value @@ -18,9 +17,8 @@ from documents.models import Document from .literals import ( MAX_FILE_DESCRIPTOR, MIN_FILE_DESCRIPTOR, FILE_MODE, DIRECTORY_MODE ) -from .settings import ( - setting_document_lookup_cache_timeout, setting_node_lookup_cache_timeout -) +from .runtime import cache + logger = logging.getLogger(__name__) @@ -59,17 +57,17 @@ class IndexFilesystem(Operations): node = self.index.instance_root if len(parts) > 1 and parts[1] != '': - obj = self.cache.get(path) + path_cache = cache.get_path(path=path) - if obj: - node_pk = obj.get('node_pk') + if path_cache: + node_pk = path_cache.get('node_pk') if node_pk: if access_only: return True else: return IndexInstanceNode.objects.get(pk=node_pk) - document_pk = obj.get('document_pk') + document_pk = path_cache.get('document_pk') if document_pk: if access_only: return True @@ -87,16 +85,13 @@ class IndexFilesystem(Operations): else: try: if node.index_template_node.link_documents: - result = node.documents.get(label=part) + document = node.documents.get(label=part) logger.debug( 'path %s is a valid file path', path ) - self.cache.set( - path, {'document_pk': result.pk}, - setting_document_lookup_cache_timeout.value - ) + cache.set_path(path=path, document=document) - return result + return document else: return None except Document.DoesNotExist: @@ -109,10 +104,7 @@ class IndexFilesystem(Operations): except MultipleObjectsReturned: return None - self.cache.set( - path, {'node_pk': node.pk}, - setting_node_lookup_cache_timeout.value - ) + cache.set_path(path=path, node=node) logger.debug('node: %s', node) logger.debug('node is root: %s', node.is_root_node()) @@ -122,7 +114,6 @@ class IndexFilesystem(Operations): def __init__(self, index_slug): self.file_descriptor_count = MIN_FILE_DESCRIPTOR self.file_descriptors = {} - self.cache = caches['default'] try: self.index = Index.objects.get(slug=index_slug) diff --git a/mayan/apps/mirroring/handlers.py b/mayan/apps/mirroring/handlers.py new file mode 100644 index 0000000000..9489a55a6e --- /dev/null +++ b/mayan/apps/mirroring/handlers.py @@ -0,0 +1,11 @@ +from __future__ import unicode_literals + +from .runtime import cache + + +def handler_document_cache_delete(sender, **kwargs): + cache.clear_document(document=kwargs['instance']) + + +def handler_node_cache_delete(sender, **kwargs): + cache.clear_node(node=kwargs['instance']) diff --git a/mayan/apps/mirroring/management/commands/mountindex.py b/mayan/apps/mirroring/management/commands/mountindex.py index d1965cfd1c..d360f26c17 100644 --- a/mayan/apps/mirroring/management/commands/mountindex.py +++ b/mayan/apps/mirroring/management/commands/mountindex.py @@ -7,7 +7,7 @@ from fuse import FUSE from django.core import management from django.core.management.base import CommandError -from ...classes import IndexFilesystem +from ...filesystems import IndexFilesystem logger = logging.getLogger(__name__) diff --git a/mayan/apps/mirroring/runtime.py b/mayan/apps/mirroring/runtime.py new file mode 100644 index 0000000000..9d66c1dc38 --- /dev/null +++ b/mayan/apps/mirroring/runtime.py @@ -0,0 +1,3 @@ +from .caches import IndexFilesystemCache + +cache = IndexFilesystemCache() diff --git a/mayan/apps/mirroring/tests/literals.py b/mayan/apps/mirroring/tests/literals.py index 83d80d2359..6231941d5e 100644 --- a/mayan/apps/mirroring/tests/literals.py +++ b/mayan/apps/mirroring/tests/literals.py @@ -1,4 +1,7 @@ from __future__ import absolute_import, unicode_literals +TEST_DOCUMENT_PK = 99 TEST_NODE_EXPRESSION = 'level_1' TEST_NODE_EXPRESSION_MULTILINE = 'first\r\nsecond\r\nthird' +TEST_NODE_PK = 88 +TEST_PATH = '/test/path' diff --git a/mayan/apps/mirroring/tests/test_caches.py b/mayan/apps/mirroring/tests/test_caches.py new file mode 100644 index 0000000000..4d5cceaa90 --- /dev/null +++ b/mayan/apps/mirroring/tests/test_caches.py @@ -0,0 +1,49 @@ +from __future__ import absolute_import, unicode_literals + +from common.tests import BaseTestCase + +from ..caches import IndexFilesystemCache + +from .literals import TEST_DOCUMENT_PK, TEST_NODE_PK, TEST_PATH + + +class MockDocument(object): + pk = TEST_DOCUMENT_PK + + +class MockNode(object): + pk = TEST_NODE_PK + + +class IndexFilesystemCacheTestCase(BaseTestCase): + def setUp(self): + super(IndexFilesystemCacheTestCase, self).setUp() + self.cache = IndexFilesystemCache() + self.document = MockDocument() + self.node = MockNode() + + def test_set_path_document(self): + self.cache.set_path(path=TEST_PATH, document=self.document) + self.assertEquals( + {'document_pk': TEST_DOCUMENT_PK}, + self.cache.get_path(path=TEST_PATH) + ) + + def test_set_path_document_clear_document(self): + self.cache.set_path(path=TEST_PATH, document=self.document) + self.cache.clear_document(document=self.document) + + self.assertEquals(None, self.cache.get_path(path=TEST_PATH)) + + def test_set_path_node(self): + self.cache.set_path(path=TEST_PATH, node=self.node) + self.assertEquals( + {'node_pk': TEST_NODE_PK}, + self.cache.get_path(path=TEST_PATH) + ) + + def test_set_path_node_clear_node(self): + self.cache.set_path(path=TEST_PATH, node=self.node) + self.cache.clear_node(node=self.node) + + self.assertEquals(None, self.cache.get_path(path=TEST_PATH)) diff --git a/mayan/apps/mirroring/tests/test_classes.py b/mayan/apps/mirroring/tests/test_filesystems.py similarity index 96% rename from mayan/apps/mirroring/tests/test_classes.py rename to mayan/apps/mirroring/tests/test_filesystems.py index 79914f0893..6a0409e9d4 100644 --- a/mayan/apps/mirroring/tests/test_classes.py +++ b/mayan/apps/mirroring/tests/test_filesystems.py @@ -11,7 +11,7 @@ from documents.tests import DocumentTestMixin from document_indexing.tests import DocumentIndexingTestMixin -from ..classes import IndexFilesystem +from ..filesystems import IndexFilesystem from .literals import ( TEST_NODE_EXPRESSION, TEST_NODE_EXPRESSION_MULTILINE @@ -19,7 +19,7 @@ from .literals import ( @override_settings(OCR_AUTO_OCR=False) -class IndexFSTestCase(DocumentIndexingTestMixin, DocumentTestMixin, BaseTestCase): +class IndexFilesystemTestCase(DocumentIndexingTestMixin, DocumentTestMixin, BaseTestCase): auto_upload_document = False def test_document_access(self):