From 88bc29e4d78f7467c7de2800ded9c5b21f9bbb00 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Thu, 25 Jul 2019 02:22:57 -0400 Subject: [PATCH] Update the file caching app - Add view to list available caches. - Add links to view and purge caches. - Add permissions. - Add events. - Add purge task. - Remove document image clear link and view. This is now handled by the file caching app. Signed-off-by: Roberto Rosario --- HISTORY.rst | 2 + docs/releases/3.3.rst | 2 + mayan/apps/documents/apps.py | 4 +- mayan/apps/documents/icons.py | 2 - mayan/apps/documents/links.py | 12 +--- mayan/apps/documents/queues.py | 4 -- mayan/apps/documents/tasks.py | 11 ---- mayan/apps/documents/urls.py | 6 +- mayan/apps/documents/views/misc_views.py | 17 +----- mayan/apps/file_caching/apps.py | 71 ++++++++++++++++++++++++ mayan/apps/file_caching/events.py | 19 +++++++ mayan/apps/file_caching/icons.py | 11 ++++ mayan/apps/file_caching/links.py | 22 ++++++++ mayan/apps/file_caching/models.py | 60 ++++++++++++++++++-- mayan/apps/file_caching/permissions.py | 14 +++++ mayan/apps/file_caching/queues.py | 10 ++++ mayan/apps/file_caching/tasks.py | 25 +++++++++ mayan/apps/file_caching/urls.py | 20 +++++++ mayan/apps/file_caching/views.py | 53 ++++++++++++++++++ 19 files changed, 309 insertions(+), 56 deletions(-) create mode 100644 mayan/apps/file_caching/events.py create mode 100644 mayan/apps/file_caching/icons.py create mode 100644 mayan/apps/file_caching/links.py create mode 100644 mayan/apps/file_caching/permissions.py create mode 100644 mayan/apps/file_caching/queues.py create mode 100644 mayan/apps/file_caching/tasks.py create mode 100644 mayan/apps/file_caching/urls.py create mode 100644 mayan/apps/file_caching/views.py diff --git a/HISTORY.rst b/HISTORY.rst index fa09d0aa14..942e3fa898 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -61,6 +61,8 @@ regardless of MIME type. - Remove task inspection from task manager app. - Move pagination navigation inside the toolbar. +- Remove document image clear link and view. + This is now handled by the file caching app. 3.2.6 (2019-07-10) ================== diff --git a/docs/releases/3.3.rst b/docs/releases/3.3.rst index 1daaceae32..3ff4c6d77e 100644 --- a/docs/releases/3.3.rst +++ b/docs/releases/3.3.rst @@ -76,6 +76,8 @@ Changes - Add platform template to return queues for a worker. - Remove task inspection from task manager app. - Move pagination navigation inside the toolbar. +- Remove document image clear link and view. + This is now handled by the file caching app. Removals diff --git a/mayan/apps/documents/apps.py b/mayan/apps/documents/apps.py index 25a943294b..aec4a06e21 100644 --- a/mayan/apps/documents/apps.py +++ b/mayan/apps/documents/apps.py @@ -47,7 +47,7 @@ from .handlers import ( handler_remove_empty_duplicates_lists, handler_scan_duplicates_for ) from .links import ( - link_clear_image_cache, link_document_clear_transformations, + link_document_clear_transformations, link_document_clone_transformations, link_document_delete, link_document_document_type_edit, link_document_download, link_document_duplicates_list, link_document_edit, @@ -377,7 +377,7 @@ class DocumentsApp(MayanAppConfig): menu_setup.bind_links(links=(link_document_type_setup,)) menu_tools.bind_links( - links=(link_clear_image_cache, link_duplicated_document_scan) + links=(link_duplicated_document_scan,) ) # Document type links diff --git a/mayan/apps/documents/icons.py b/mayan/apps/documents/icons.py index 8b9b4067d9..fe745fe538 100644 --- a/mayan/apps/documents/icons.py +++ b/mayan/apps/documents/icons.py @@ -12,8 +12,6 @@ icon_document_type = Icon( icon_menu_documents = Icon(driver_name='fontawesome', symbol='book') -icon_clear_image_cache = Icon(driver_name='fontawesome', symbol='file-image') - icon_dashboard_document_types = icon_document_type icon_dashboard_documents_in_trash = Icon( driver_name='fontawesome', symbol='trash-alt' diff --git a/mayan/apps/documents/links.py b/mayan/apps/documents/links.py index 98920e1dc1..17ed67313e 100644 --- a/mayan/apps/documents/links.py +++ b/mayan/apps/documents/links.py @@ -8,7 +8,7 @@ from mayan.apps.converter.permissions import ( from mayan.apps.navigation.classes import Link from .icons import ( - icon_clear_image_cache, icon_document_list_recent_access, + icon_document_list_recent_access, icon_recent_added_document_list, icon_document_page_navigation_first, icon_document_page_navigation_last, icon_document_page_navigation_next, icon_document_page_navigation_previous, icon_document_page_return, @@ -264,16 +264,6 @@ link_document_list_deleted = Link( text=_('Trash can'), view='documents:document_list_deleted' ) -# Tools -link_clear_image_cache = Link( - icon_class=icon_clear_image_cache, - description=_( - 'Clear the graphics representations used to speed up the documents\' ' - 'display and interactive transformations results.' - ), permissions=(permission_document_tools,), - text=_('Clear document image cache'), - view='documents:document_clear_image_cache' -) link_trash_can_empty = Link( permissions=(permission_empty_trash,), text=_('Empty trash'), view='documents:trash_can_empty' diff --git a/mayan/apps/documents/queues.py b/mayan/apps/documents/queues.py index 57f4c4b32a..5ac0fa5f01 100644 --- a/mayan/apps/documents/queues.py +++ b/mayan/apps/documents/queues.py @@ -61,10 +61,6 @@ queue_documents_periodic.add_task_type( schedule=timedelta(seconds=DELETE_STALE_STUBS_INTERVAL), ) -queue_tools.add_task_type( - dotted_path='mayan.apps.documents.tasks.task_clear_image_cache', - label=_('Clear image cache') -) queue_tools.add_task_type( dotted_path='mayan.apps.documents.tasks.task_scan_duplicates_all', label=_('Duplicated document scan') diff --git a/mayan/apps/documents/tasks.py b/mayan/apps/documents/tasks.py index 857eb38e66..98e08d1e25 100644 --- a/mayan/apps/documents/tasks.py +++ b/mayan/apps/documents/tasks.py @@ -41,17 +41,6 @@ def task_check_trash_periods(): DocumentType.objects.check_trash_periods() -@app.task(ignore_result=True) -def task_clear_image_cache(): - Document = apps.get_model( - app_label='documents', model_name='Document' - ) - - logger.info('Starting document cache invalidation') - Document.objects.invalidate_cache() - logger.info('Finished document cache invalidation') - - @app.task(ignore_result=True) def task_delete_document(trashed_document_id): DeletedDocument = apps.get_model( diff --git a/mayan/apps/documents/urls.py b/mayan/apps/documents/urls.py index 6d954eebb0..e36181662e 100644 --- a/mayan/apps/documents/urls.py +++ b/mayan/apps/documents/urls.py @@ -13,7 +13,7 @@ from .api_views import ( APIRecentDocumentListView ) from .views import ( - ClearImageCacheView, DocumentDocumentTypeEditView, DocumentDownloadFormView, + DocumentDocumentTypeEditView, DocumentDownloadFormView, DocumentDownloadView, DocumentDuplicatesListView, DocumentEditView, DocumentListView, DocumentPageListView, DocumentPageNavigationFirst, DocumentPageNavigationLast, DocumentPageNavigationNext, @@ -272,10 +272,6 @@ urlpatterns = [ view=DocumentTransformationsClearView.as_view(), name='document_multiple_clear_transformations' ), - url( - regex=r'^cache/clear/$', view=ClearImageCacheView.as_view(), - name='document_clear_image_cache' - ), url( regex=r'^page/(?P\d+)/$', view=DocumentPageView.as_view(), name='document_page_view' diff --git a/mayan/apps/documents/views/misc_views.py b/mayan/apps/documents/views/misc_views.py index 0f2bf4fab8..5500f53951 100644 --- a/mayan/apps/documents/views/misc_views.py +++ b/mayan/apps/documents/views/misc_views.py @@ -8,25 +8,12 @@ from django.utils.translation import ugettext_lazy as _ from mayan.apps.common.generics import ConfirmView from ..permissions import permission_document_tools -from ..tasks import task_clear_image_cache, task_scan_duplicates_all +from ..tasks import task_scan_duplicates_all -__all__ = ('ClearImageCacheView', 'ScanDuplicatedDocuments') +__all__ = ('ScanDuplicatedDocuments',) logger = logging.getLogger(__name__) -class ClearImageCacheView(ConfirmView): - extra_context = { - 'title': _('Clear the document image cache?') - } - view_permission = permission_document_tools - - def view_action(self): - task_clear_image_cache.apply_async() - messages.success( - self.request, _('Document cache clearing queued successfully.') - ) - - class ScanDuplicatedDocuments(ConfirmView): extra_context = { 'title': _('Scan for duplicated documents?') diff --git a/mayan/apps/file_caching/apps.py b/mayan/apps/file_caching/apps.py index bb333be6e0..9d505c304b 100644 --- a/mayan/apps/file_caching/apps.py +++ b/mayan/apps/file_caching/apps.py @@ -1,8 +1,79 @@ from __future__ import unicode_literals +from django.utils.translation import ugettext_lazy as _ + +from mayan.apps.acls.classes import ModelPermission +from mayan.apps.acls.links import link_acl_list +from mayan.apps.acls.permissions import permission_acl_edit, permission_acl_view from mayan.apps.common.apps import MayanAppConfig +from mayan.apps.common.menus import ( + menu_list_facet, menu_multi_item, menu_object, menu_secondary, menu_tools +) +from mayan.apps.events.classes import ModelEventType +from mayan.apps.events.links import ( + link_events_for_object, link_object_event_types_user_subcriptions_list +) +from mayan.apps.navigation.classes import SourceColumn + +from .events import event_cache_edited, event_cache_purged +from .links import ( + link_caches_list, link_cache_multiple_purge, link_cache_purge +) +from .permissions import permission_cache_purge, permission_cache_view class FileCachingConfig(MayanAppConfig): + app_namespace = 'file_caching' + app_url = 'file_caching' has_tests = False name = 'mayan.apps.file_caching' + verbose_name = _('File caching') + + def ready(self): + super(FileCachingConfig, self).ready() + from actstream import registry + + Cache = self.get_model(model_name='Cache') + + ModelEventType.register( + event_types=(event_cache_edited, event_cache_purged,), + model=Cache + ) + + ModelPermission.register( + model=Cache, permissions=( + permission_acl_edit, permission_acl_view, + permission_cache_purge, permission_cache_view + ) + ) + + SourceColumn(attribute='name', source=Cache) + SourceColumn(attribute='label', source=Cache) + SourceColumn(attribute='storage_instance_path', source=Cache) + SourceColumn(attribute='get_maximum_size_display', source=Cache) + SourceColumn(attribute='get_total_size_display', source=Cache) + + menu_list_facet.bind_links( + links=( + link_acl_list, link_events_for_object, + link_object_event_types_user_subcriptions_list, + ), sources=(Cache,) + ) + + menu_object.bind_links( + links=(link_cache_purge,), + sources=(Cache,) + ) + menu_multi_item.bind_links( + links=(link_cache_multiple_purge,), + sources=(Cache,) + ) + menu_secondary.bind_links( + links=(link_caches_list,), sources=( + Cache, + ) + ) + + menu_tools.bind_links(links=(link_caches_list,)) + + registry.register(Cache) diff --git a/mayan/apps/file_caching/events.py b/mayan/apps/file_caching/events.py new file mode 100644 index 0000000000..529710d64b --- /dev/null +++ b/mayan/apps/file_caching/events.py @@ -0,0 +1,19 @@ +from __future__ import absolute_import, unicode_literals + +from django.utils.translation import ugettext_lazy as _ + +from mayan.apps.events.classes import EventTypeNamespace + +namespace = EventTypeNamespace( + label=_('File caching'), name='file_caching' +) + +event_cache_created = namespace.add_event_type( + label=_('Cache created'), name='cache_created' +) +event_cache_edited = namespace.add_event_type( + label=_('Cache edited'), name='cache_edited' +) +event_cache_purged = namespace.add_event_type( + label=_('Cache purge'), name='cache_purged' +) diff --git a/mayan/apps/file_caching/icons.py b/mayan/apps/file_caching/icons.py new file mode 100644 index 0000000000..94b4287456 --- /dev/null +++ b/mayan/apps/file_caching/icons.py @@ -0,0 +1,11 @@ +from __future__ import absolute_import, unicode_literals + +from mayan.apps.appearance.classes import Icon + +icon_file_caching = Icon( + driver_name='fontawesome', symbol='warehouse' +) +icon_cache_purge = Icon( + driver_name='fontawesome-dual', primary_symbol='warehouse', + secondary_symbol='check' +) diff --git a/mayan/apps/file_caching/links.py b/mayan/apps/file_caching/links.py new file mode 100644 index 0000000000..b0bc57cae4 --- /dev/null +++ b/mayan/apps/file_caching/links.py @@ -0,0 +1,22 @@ +from __future__ import unicode_literals + +from django.utils.translation import ugettext_lazy as _ + +from mayan.apps.navigation.classes import Link + +from .icons import icon_cache_purge, icon_file_caching +from .permissions import permission_cache_purge, permission_cache_view + +link_caches_list = Link( + icon_class=icon_file_caching, permissions=(permission_cache_view,), + text=_('File caches'), view='file_caching:cache_list' +) +link_cache_purge = Link( + icon_class=icon_cache_purge, kwargs={'cache_id': 'resolved_object.id'}, + permissions=(permission_cache_purge,), text=_('Purge cache'), + view='file_caching:cache_purge' +) +link_cache_multiple_purge = Link( + icon_class=icon_cache_purge, text=_('Purge cache'), + view='file_caching:cache_multiple_purge' +) diff --git a/mayan/apps/file_caching/models.py b/mayan/apps/file_caching/models.py index 6b8e6bb3cc..8ea22ee4aa 100644 --- a/mayan/apps/file_caching/models.py +++ b/mayan/apps/file_caching/models.py @@ -6,6 +6,7 @@ import logging from django.core.files.base import ContentFile from django.db import models, transaction from django.db.models import Sum +from django.template.defaultfilters import filesizeformat from django.utils.encoding import python_2_unicode_compatible from django.utils.functional import cached_property from django.utils.module_loading import import_string @@ -14,18 +15,31 @@ from django.utils.translation import ugettext_lazy as _ from mayan.apps.lock_manager.exceptions import LockError from mayan.apps.lock_manager.runtime import locking_backend +from .events import ( + event_cache_created, event_cache_edited, event_cache_purged +) + logger = logging.getLogger(__name__) @python_2_unicode_compatible class Cache(models.Model): name = models.CharField( - max_length=128, unique=True, verbose_name=_('Name') + help_text=_('Internal name of the cache.'), max_length=128, + unique=True, verbose_name=_('Name') + ) + label = models.CharField( + help_text=_('A short text describing the cache.'), max_length=128, + verbose_name=_('Label') + ) + maximum_size = models.PositiveIntegerField( + help_text=_('Maximum size of the cache in bytes.'), + verbose_name=_('Maximum size') ) - label = models.CharField(max_length=128, verbose_name=_('Label')) - maximum_size = models.PositiveIntegerField(verbose_name=_('Maximum size')) storage_instance_path = models.CharField( - max_length=255, unique=True, verbose_name=_('Storage instance path') + help_text=_( + 'Dotted path to the actual storage class used for the cache.' + ), max_length=255, unique=True, verbose_name=_('Storage instance path') ) class Meta: @@ -38,21 +52,55 @@ class Cache(models.Model): def get_files(self): return CachePartitionFile.objects.filter(partition__cache__id=self.pk) + def get_maximum_size_display(self): + return filesizeformat(bytes_=self.maximum_size) + + get_maximum_size_display.short_description = _('Maximum size') + def get_total_size(self): + """ + Return the actual usage of the cache. + """ return self.get_files().aggregate( file_size__sum=Sum('file_size') )['file_size__sum'] or 0 + def get_total_size_display(self): + return filesizeformat(bytes_=self.get_total_size()) + + get_total_size_display.short_description = _('Total size') + def prune(self): + """ + Deletes files until the total size of the cache is below the allowed + maximum size of the cache. + """ while self.get_total_size() > self.maximum_size: self.get_files().earliest().delete() - def purge(self): + def purge(self, _user=None): + """ + Deletes the entire cache. + """ for partition in self.partitions.all(): partition.purge() + event_cache_purged.commit(actor=_user, target=self) + def save(self, *args, **kwargs): - result = super(Cache, self).save(*args, **kwargs) + _user = kwargs.pop('_user', None) + with transaction.atomic(): + is_new = not self.pk + result = super(Cache, self).save(*args, **kwargs) + if is_new: + event_cache_created.commit( + actor=_user, target=self + ) + else: + event_cache_edited.commit( + actor=_user, target=self + ) + self.prune() return result diff --git a/mayan/apps/file_caching/permissions.py b/mayan/apps/file_caching/permissions.py new file mode 100644 index 0000000000..4c2609ee10 --- /dev/null +++ b/mayan/apps/file_caching/permissions.py @@ -0,0 +1,14 @@ +from __future__ import absolute_import, unicode_literals + +from django.utils.translation import ugettext_lazy as _ + +from mayan.apps.permissions import PermissionNamespace + +namespace = PermissionNamespace(label=_('File caching'), name='file_caching') + +permission_cache_purge = namespace.add_permission( + label=_('Purge a file cache'), name='file_caching_cache_purge' +) +permission_cache_view = namespace.add_permission( + label=_('View a file cache'), name='file_caching_cache_view' +) diff --git a/mayan/apps/file_caching/queues.py b/mayan/apps/file_caching/queues.py new file mode 100644 index 0000000000..77d8af40cb --- /dev/null +++ b/mayan/apps/file_caching/queues.py @@ -0,0 +1,10 @@ +from __future__ import absolute_import, unicode_literals + +from django.utils.translation import ugettext_lazy as _ + +from mayan.apps.common.queues import queue_tools + +queue_tools.add_task_type( + dotted_path='mayan.apps.file_caching.tasks.task_cache_purge', + label=_('Purge a file cache') +) diff --git a/mayan/apps/file_caching/tasks.py b/mayan/apps/file_caching/tasks.py new file mode 100644 index 0000000000..06696a1c9d --- /dev/null +++ b/mayan/apps/file_caching/tasks.py @@ -0,0 +1,25 @@ +from __future__ import unicode_literals + +import logging + +from django.apps import apps +from django.contrib.auth import get_user_model + +from mayan.celery import app + +logger = logging.getLogger(__name__) + + +@app.task(ignore_result=True) +def task_cache_purge(cache_id, user_id=None): + Cache = apps.get_model( + app_label='file_caching', model_name='Cache' + ) + User = get_user_model() + + cache = Cache.objects.get(pk=cache_id) + user = User.objects.get(pk=user_id) + + logger.info('Starting cache id %s purge', cache) + cache.purge(_user=user) + logger.info('Finished cache id %s purge', cache) diff --git a/mayan/apps/file_caching/urls.py b/mayan/apps/file_caching/urls.py new file mode 100644 index 0000000000..4ce37409f6 --- /dev/null +++ b/mayan/apps/file_caching/urls.py @@ -0,0 +1,20 @@ +from __future__ import unicode_literals + +from django.conf.urls import url + +from .views import CacheListView, CachePurgeView + +urlpatterns = [ + url( + regex=r'^caches/$', + name='cache_list', view=CacheListView.as_view() + ), + url( + regex=r'^caches/(?P\d+)/purge/$', + name='cache_purge', view=CachePurgeView.as_view() + ), + url( + regex=r'^caches/multiple/purge/$', name='cache_multiple_purge', + view=CachePurgeView.as_view() + ), +] diff --git a/mayan/apps/file_caching/views.py b/mayan/apps/file_caching/views.py new file mode 100644 index 0000000000..bf0d86ef7e --- /dev/null +++ b/mayan/apps/file_caching/views.py @@ -0,0 +1,53 @@ +from __future__ import absolute_import, unicode_literals + +from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import ungettext + +from mayan.apps.common.generics import ( + MultipleObjectConfirmActionView, SingleObjectListView +) + +from .models import Cache +from .permissions import permission_cache_purge, permission_cache_view + +from .tasks import task_cache_purge + + +class CacheListView(SingleObjectListView): + model = Cache + permission = permission_cache_view + + def get_extra_context(self): + return { + 'hide_object': True, + 'title': _('File caches list') + } + + +class CachePurgeView(MultipleObjectConfirmActionView): + model = Cache + object_permission = permission_cache_purge + pk_url_kwarg = 'cache_id' + success_message_singular = '%(count)d cache submitted for purging.' + success_message_plural = '%(count)d caches submitted for purging.' + + def get_extra_context(self): + queryset = self.object_list + + result = { + 'title': ungettext( + singular='Submit the selected cache for purging?', + plural='Submit the selected caches for purging?', + number=queryset.count() + ) + } + + if queryset.count() == 1: + result['object'] = queryset.first() + + return result + + def object_action(self, form, instance): + task_cache_purge.apply_async( + kwargs={'cache_id': instance.pk, 'user_id': self.request.user.pk} + )