Convert workflow previews app to use file caching

Signed-off-by: Roberto Rosario <roberto.rosario@mayan-edms.com>

mayan/apps/document_states/api_views.py

@@ -27,7 +27,6 @@ from .serializers import (
 )
 
 from .settings import settings_workflow_image_cache_time
-from .storages import storage_workflowimagecache
 from .tasks import task_generate_workflow_image
 
 
@@ -204,7 +203,8 @@ class APIWorkflowImageView(generics.RetrieveAPIView):
         )
 
         cache_filename = task.get(timeout=WORKFLOW_IMAGE_TASK_TIMEOUT)
-        with storage_workflowimagecache.open(cache_filename) as file_object:
+        cache_file = self.get_object().cache_partition.get_file(filename=cache_filename)
+        with cache_file.open() as file_object:
             response = HttpResponse(file_object.read(), content_type='image')
             if '_hash' in request.GET:
                 patch_cache_control(

mayan/apps/document_states/apps.py

@@ -1,7 +1,7 @@
 from __future__ import unicode_literals
 
 from django.apps import apps
-from django.db.models.signals import post_save
+from django.db.models.signals import post_migrate, post_save
 from django.utils.translation import ugettext_lazy as _
 
 from mayan.apps.acls.classes import ModelPermission
@@ -25,7 +25,8 @@ from .classes import DocumentStateHelper, WorkflowAction
 from .events import event_workflow_created, event_workflow_edited
 from .dependencies import *  # NOQA
 from .handlers import (
-    handler_index_document, handler_launch_workflow, handler_trigger_transition
+    handler_create_workflow_image_cache, handler_index_document,
+    handler_launch_workflow, handler_trigger_transition
 )
 from .html_widgets import WorkflowLogExtraDataWidget, widget_transition_events
 from .links import (
@@ -452,6 +453,10 @@ class DocumentStatesApp(MayanAppConfig):
 
         # Index updating
 
+        post_migrate.connect(
+            dispatch_uid='workflows_handler_create_workflow_image_cache',
+            receiver=handler_create_workflow_image_cache,
+        )
         post_save.connect(
             dispatch_uid='workflows_handler_index_document_save',
             receiver=handler_index_document,

mayan/apps/document_states/handlers.py

@@ -6,6 +6,22 @@ from django.utils.translation import ugettext_lazy as _
 from mayan.apps.document_indexing.tasks import task_index_document
 from mayan.apps.events.classes import EventType
 
+from .literals import (
+    WORKFLOW_IMAGE_CACHE_STORAGE_INSTANCE_PATH, WORKFLOW_IMAGE_CACHE_NAME
+)
+from .settings import setting_workflow_image_cache_maximum_size
+
+
+def handler_create_workflow_image_cache(sender, **kwargs):
+    Cache = apps.get_model(app_label='file_caching', model_name='Cache')
+    Cache.objects.update_or_create(
+        defaults={
+            'label': _('Workflow images'),
+            'storage_instance_path': WORKFLOW_IMAGE_CACHE_STORAGE_INSTANCE_PATH,
+            'maximum_size': setting_workflow_image_cache_maximum_size.value,
+        }, name=WORKFLOW_IMAGE_CACHE_NAME,
+    )
+
 
 def handler_index_document(sender, **kwargs):
     task_index_document.apply_async(

mayan/apps/document_states/literals.py

@@ -2,6 +2,8 @@ from __future__ import unicode_literals
 
 from django.utils.translation import ugettext_lazy as _
 
+DEFAULT_WORKFLOW_IMAGE_CACHE_MAXIMUM_SIZE = 50 * 2 ** 20  # 50 Megabytes
+
 FIELD_TYPE_CHOICE_CHAR = 1
 FIELD_TYPE_CHOICE_INTEGER = 2
 FIELD_TYPE_CHOICES = (
@@ -30,4 +32,6 @@ WORKFLOW_ACTION_WHEN_CHOICES = (
     (WORKFLOW_ACTION_ON_ENTRY, _('On entry')),
     (WORKFLOW_ACTION_ON_EXIT, _('On exit')),
 )
+WORKFLOW_IMAGE_CACHE_NAME = 'workflow_images'
+WORKFLOW_IMAGE_CACHE_STORAGE_INSTANCE_PATH = 'mayan.apps.document_states.storages.storage_workflowimagecache'
 WORKFLOW_IMAGE_TASK_TIMEOUT = 60

mayan/apps/document_states/models.py

@@ -7,16 +7,17 @@ import logging
 from furl import furl
 from graphviz import Digraph
 
+from django.apps import apps
 from django.conf import settings
 from django.core import serializers
 from django.core.exceptions import PermissionDenied, ValidationError
-from django.core.files.base import ContentFile
 from django.db import IntegrityError, models, transaction
 from django.db.models import F, Max, Q
 from django.urls import reverse
 from django.utils.encoding import (
     force_bytes, force_text, python_2_unicode_compatible
 )
+from django.utils.functional import cached_property
 from django.utils.module_loading import import_string
 from django.utils.translation import ugettext_lazy as _
 
@@ -31,11 +32,11 @@ from .error_logs import error_log_state_actions
 from .events import event_workflow_created, event_workflow_edited
 from .literals import (
     FIELD_TYPE_CHOICES, WIDGET_CLASS_CHOICES, WORKFLOW_ACTION_WHEN_CHOICES,
-    WORKFLOW_ACTION_ON_ENTRY, WORKFLOW_ACTION_ON_EXIT
+    WORKFLOW_ACTION_ON_ENTRY, WORKFLOW_ACTION_ON_EXIT,
+    WORKFLOW_IMAGE_CACHE_NAME
 )
 from .managers import WorkflowManager
 from .permissions import permission_workflow_transition
-from .storages import storage_workflowimagecache
 
 logger = logging.getLogger(__name__)
 
@@ -72,18 +73,36 @@ class Workflow(models.Model):
     def __str__(self):
         return self.label
 
-    def generate_image(self):
+    @cached_property
-        cache_filename = '{}-{}'.format(self.id, self.get_hash())
+    def cache(self):
-        image = self.render()
+        Cache = apps.get_model(app_label='file_caching', model_name='Cache')
+        return Cache.objects.get(name=WORKFLOW_IMAGE_CACHE_NAME)
 
-        # Since open "wb+" doesn't create files, check if the file
+    @cached_property
-        # exists, if not then create it
+    def cache_partition(self):
-        if not storage_workflowimagecache.exists(cache_filename):
+        partition, created = self.cache.partitions.get_or_create(
-            storage_workflowimagecache.save(
+            name='{}'.format(self.pk)
-                name=cache_filename, content=ContentFile(content='')
+        )
+        return partition
+
+    def delete(self, *args, **kwargs):
+        self.cache_partition.delete()
+        return super(Workflow, self).delete(*args, **kwargs)
+
+    def generate_image(self):
+        cache_filename = '{}'.format(self.get_hash())
+
+        if self.cache_partition.get_file(filename=cache_filename):
+            logger.debug(
+                'workflow cache file "%s" found', cache_filename
+            )
+        else:
+            logger.debug(
+                'workflow cache file "%s" not found', cache_filename
             )
 
-        with storage_workflowimagecache.open(cache_filename, mode='wb+') as file_object:
+            image = self.render()
+            with self.cache_partition.create_file(filename=cache_filename) as file_object:
                 file_object.write(image)
 
         return cache_filename
@@ -107,6 +126,8 @@ class Workflow(models.Model):
             Workflow.objects.filter(pk=self.pk)
         ) + list(
             WorkflowState.objects.filter(workflow__pk=self.pk)
+        ) + list(
+            WorkflowStateAction.objects.filter(state__workflow__pk=self.pk)
         ) + list(
             WorkflowTransition.objects.filter(workflow__pk=self.pk)
         )

mayan/apps/document_states/settings.py

@@ -7,8 +7,20 @@ from django.utils.translation import ugettext_lazy as _
 
 from mayan.apps.smart_settings.classes import Namespace
 
+from .literals import DEFAULT_WORKFLOW_IMAGE_CACHE_MAXIMUM_SIZE
+from .utils import callback_update_workflow_image_cache_size
+
 namespace = Namespace(label=_('Workflows'), name='document_states')
 
+setting_workflow_image_cache_maximum_size = namespace.add_setting(
+    global_name='WORKFLOW_IMAGE_CACHE_MAXIMUM_SIZE',
+    default=DEFAULT_WORKFLOW_IMAGE_CACHE_MAXIMUM_SIZE,
+    help_text=_(
+        'The threshold at which the WORKFLOW_IMAGE_CACHE_STORAGE_BACKEND will '
+        'start deleting the oldest workflow image cache files. Specify the '
+        'size in bytes.'
+    ), post_edit_function=callback_update_workflow_image_cache_size
+)
 settings_workflow_image_cache_time = namespace.add_setting(
     global_name='WORKFLOWS_IMAGE_CACHE_TIME', default='31556926',
     help_text=_(

mayan/apps/document_states/utils.py

@@ -0,0 +1,12 @@
+from __future__ import unicode_literals
+
+from django.apps import apps
+
+from .literals import WORKFLOW_IMAGE_CACHE_NAME
+
+
+def callback_update_workflow_image_cache_size(setting):
+    Cache = apps.get_model(app_label='file_caching', model_name='Cache')
+    cache = Cache.objects.get(name=WORKFLOW_IMAGE_CACHE_NAME)
+    cache.maximum_size = setting.value
+    cache.save()