Appearance: Fix form CSS media rendering

Fix the way the form CSS contained in the media attribute
is rendered. This is now an interator and not a single value.
Replace the current method with a for loop.

Signed-off-by: Roberto Rosario <roberto.rosario.gonzalez@gmail.com>

HISTORY.rst

@@ -97,6 +97,9 @@
   cache invalidation is tied to index updates. This makes the
   timeout less relevant. The purpose of the cache timeout is
   now avoid runaway memory usage.
+- Refactored the workflow preview generation to work as an
+  background task API service. Solves GitLab issue #532.
+  A new task queue named "document_states_fast" was created.
 
 3.1.9 (2018-11-01)
 ==================

docker/etc/supervisor/mayan.conf

@@ -22,7 +22,7 @@ user = root
 [program:mayan-worker-fast]
 autorestart = false
 autostart = true
-command = nice -n 1 /bin/bash -c "${MAYAN_BIN} celery --settings=${MAYAN_SETTINGS_MODULE} worker -Ofair -l ERROR -Q converter,sources_fast -n mayan-worker-fast.%%h ${MAYAN_WORKER_FAST_CONCURRENCY}"
+command = nice -n 1 /bin/bash -c "${MAYAN_BIN} celery --settings=${MAYAN_SETTINGS_MODULE} worker -Ofair -l ERROR -Q converter,document_states_fast,sources_fast -n mayan-worker-fast.%%h ${MAYAN_WORKER_FAST_CONCURRENCY}"
 killasgroup = true
 numprocs = 1
 priority = 998

docs/chapters/deploying.rst

@@ -107,7 +107,7 @@ Create the supervisor file at ``/etc/supervisor/conf.d/mayan.conf``:
     [program:mayan-worker-fast]
     autorestart = true
     autostart = true
-    command = nice -n 1 /opt/mayan-edms/bin/mayan-edms.py celery worker -Ofair -l ERROR -Q converter,sources_fast -n mayan-worker-fast.%%h --concurrency=1
+    command = nice -n 1 /opt/mayan-edms/bin/mayan-edms.py celery worker -Ofair -l ERROR -Q converter,document_states_fast,sources_fast -n mayan-worker-fast.%%h --concurrency=1
     killasgroup = true
     numprocs = 1
     priority = 998
@@ -276,7 +276,7 @@ Create the supervisor file at ``/etc/supervisor/conf.d/mayan.conf``:
     [program:mayan-worker-fast]
     autorestart = true
     autostart = true
-    command = nice -n 1 /opt/mayan-edms/bin/mayan-edms.py celery worker -Ofair -l ERROR -Q converter,sources_fast -n mayan-worker-fast.%%h
+    command = nice -n 1 /opt/mayan-edms/bin/mayan-edms.py celery worker -Ofair -l ERROR -Q converter,document_states_fast,sources_fast -n mayan-worker-fast.%%h
     killasgroup = true
     numprocs = 1
     priority = 998

mayan/apps/appearance/templates/appearance/generic_form_instance.html

@@ -4,7 +4,9 @@
 
 {% load appearance_tags %}
 
-{{ form.media.render_css|safe }}
+{% for asset in form.media.render_css %}
+    {{ asset|safe }}
+{% endfor %}
 
 {% for group, errors in form.errors.items %}
     {% for error in errors %}
@@ -38,83 +40,82 @@
     {% endfor %}
     </tr>
 {% else %}
-
     {% for field in form.hidden_fields %}
         {{ field }}
     {% endfor %}
-        {% for field in form.visible_fields %}
-            <div class="form-group {% if field.errors %}has-error{% endif %}">
-                {# We display the label then the field for all except checkboxes #}
-                {% if field|widget_type != 'checkboxinput' and not field.field.widget.attrs.hidden %}
-                    {% if not hide_labels %}{{ field.label_tag }}{% if field.field.required and not read_only %} ({% trans 'required' %}){% endif %}{% endif %}
-                {% endif %}
-                {% if field|widget_type == 'checkboxinput' %}
-                    <div class="checkbox">
-                        <label>
-                            <input {% if field.value %}checked="checked"{% endif %} name="{% if form.prefix %}{{ form.prefix }}-{% endif %}{{ field.name }}" type="checkbox">
-                            {% if not hide_labels %}{{ field.label }}{% if field.field.required and not read_only %} ({% trans 'required' %}){% endif %}{% endif %}
-                        </label>
-                    </div>
-                {% elif field|widget_type == 'emailinput' %}
-                    {% if read_only %}
-                        {{ field.value }}
-                    {% else %}
-                        {% render_field field class+="form-control" %}
-                    {% endif %}
-                {% elif field|widget_type == 'textinput' %}
-                    {% if read_only %}
-                        {{ field.value }}
-                    {% else %}
-                        {% render_field field class+="form-control" %}
-                    {% endif %}
-                {% elif field|widget_type == 'textarea' %}
-                    {% if read_only %}
-                        {{ field.value }}
-                    {% else %}
-                        {% render_field field class+="form-control" %}
-                    {% endif %}
-                {% elif field|widget_type == 'select' %}
-                    {% if read_only %}
-                        {{ field|get_choice_value }}
-                    {% else %}
-                        {% render_field field class+="form-control" %}
-                    {% endif %}
-                {% elif field|widget_type == 'selectmultiple' %}
-                    {% if read_only %}
-                        {{ field|get_choice_value }}
-                    {% else %}
-                        {% render_field field class+="form-control" %}
-                    {% endif %}
-                {% elif field|widget_type == 'clearablefileinput' %}
-                    {# Don't add 'form-control' class to filebrowse fields #}
-                    {% if field.errors %}
-                        {% render_field field class+="form-control" %}
-                    {% else %}
-                        {% render_field field class+="" %}
-                    {% endif %}
-                {% elif field|widget_type == 'radioselect' %}
-                    <div class="radio">
-                        {% render_field field %}
-                    </div>
-                {% elif field|widget_type == 'checkboxselectmultiple' %}
-                    {% for option in field %}
-                        <div class="checkbox">
-                            {{ option }}
-                        </div>
-                    {% endfor %}
-                {% elif field|widget_type == 'datetimeinput' or field|widget_type == 'dateinput' %}
-                    {% if read_only %}
-                        {{ field.value }}
-                    {% else %}
-                        {% render_field field class+="form-control" %}
-                    {% endif %}
+    {% for field in form.visible_fields %}
+        <div class="form-group {% if field.errors %}has-error{% endif %}">
+            {# We display the label then the field for all except checkboxes #}
+            {% if field|widget_type != 'checkboxinput' and not field.field.widget.attrs.hidden %}
+                {% if not hide_labels %}{{ field.label_tag }}{% if field.field.required and not read_only %} ({% trans 'required' %}){% endif %}{% endif %}
+            {% endif %}
+            {% if field|widget_type == 'checkboxinput' %}
+                <div class="checkbox">
+                    <label>
+                        <input {% if field.value %}checked="checked"{% endif %} name="{% if form.prefix %}{{ form.prefix }}-{% endif %}{{ field.name }}" type="checkbox">
+                        {% if not hide_labels %}{{ field.label }}{% if field.field.required and not read_only %} ({% trans 'required' %}){% endif %}{% endif %}
+                    </label>
+                </div>
+            {% elif field|widget_type == 'emailinput' %}
+                {% if read_only %}
+                    {{ field.value }}
                 {% else %}
                     {% render_field field class+="form-control" %}
                 {% endif %}
+            {% elif field|widget_type == 'textinput' %}
+                {% if read_only %}
+                    {{ field.value }}
+                {% else %}
+                    {% render_field field class+="form-control" %}
+                {% endif %}
+            {% elif field|widget_type == 'textarea' %}
+                {% if read_only %}
+                    {{ field.value }}
+                {% else %}
+                    {% render_field field class+="form-control" %}
+                {% endif %}
+            {% elif field|widget_type == 'select' %}
+                {% if read_only %}
+                    {{ field|get_choice_value }}
+                {% else %}
+                    {% render_field field class+="form-control" %}
+                {% endif %}
+            {% elif field|widget_type == 'selectmultiple' %}
+                {% if read_only %}
+                    {{ field|get_choice_value }}
+                {% else %}
+                    {% render_field field class+="form-control" %}
+                {% endif %}
+            {% elif field|widget_type == 'clearablefileinput' %}
+                {# Don't add 'form-control' class to filebrowse fields #}
+                {% if field.errors %}
+                    {% render_field field class+="form-control" %}
+                {% else %}
+                    {% render_field field class+="" %}
+                {% endif %}
+            {% elif field|widget_type == 'radioselect' %}
+                <div class="radio">
+                    {% render_field field %}
+                </div>
+            {% elif field|widget_type == 'checkboxselectmultiple' %}
+                {% for option in field %}
+                    <div class="checkbox">
+                        {{ option }}
+                    </div>
+                {% endfor %}
+            {% elif field|widget_type == 'datetimeinput' or field|widget_type == 'dateinput' %}
+                {% if read_only %}
+                    {{ field.value }}
+                {% else %}
+                    {% render_field field class+="form-control" %}
+                {% endif %}
+            {% else %}
+                {% render_field field class+="form-control" %}
+            {% endif %}
 
-                {% if field.help_text %}<p class="help-block">{{ field.help_text|safe }}</p>{% endif %}
-            </div>
-        {% endfor %}
+            {% if field.help_text %}<p class="help-block">{{ field.help_text|safe }}</p>{% endif %}
+        </div>
+    {% endfor %}
 {% endif %}
 
 <script>

mayan/apps/document_states/api_views.py

@@ -1,5 +1,6 @@
 from __future__ import absolute_import, unicode_literals
 
+from django.http import HttpResponse
 from django.shortcuts import get_object_or_404
 
 from rest_framework import generics
@@ -10,6 +11,7 @@ from documents.permissions import permission_document_type_view
 from rest_api.filters import MayanObjectPermissionsFilter
 from rest_api.permissions import MayanPermission
 
+from .literals import WORKFLOW_IMAGE_TASK_TIMEOUT
 from .models import Workflow
 from .permissions import (
     permission_workflow_create, permission_workflow_delete,
@@ -22,6 +24,8 @@ from .serializers import (
     WritableWorkflowInstanceLogEntrySerializer, WritableWorkflowSerializer,
     WritableWorkflowTransitionSerializer
 )
+from .storages import storage_workflowimagecache
+from .tasks import task_generate_workflow_image
 
 
 class APIDocumentTypeWorkflowListView(generics.ListAPIView):
@@ -223,6 +227,35 @@ class APIWorkflowView(generics.RetrieveUpdateDestroyAPIView):
             return WritableWorkflowSerializer
 
 
+class APIWorkflowImageView(generics.RetrieveAPIView):
+    """
+    get: Returns an image representation of the selected workflow.
+    """
+    filter_backends = (MayanObjectPermissionsFilter,)
+    mayan_object_permissions = {
+        'GET': (permission_workflow_view,),
+    }
+    queryset = Workflow.objects.all()
+
+    def get_serializer(self, *args, **kwargs):
+        return None
+
+    def get_serializer_class(self):
+        return None
+
+    def retrieve(self, request, *args, **kwargs):
+        task = task_generate_workflow_image.apply_async(
+            kwargs=dict(
+                document_state_id=self.get_object().pk,
+            )
+        )
+
+        cache_filename = task.get(timeout=WORKFLOW_IMAGE_TASK_TIMEOUT)
+        with storage_workflowimagecache.open(cache_filename) as file_object:
+            response = HttpResponse(file_object.read(), content_type='image')
+            return response
+
+
 # Workflow state views
 
 

mayan/apps/document_states/apps.py

@@ -248,12 +248,23 @@ class DocumentStatesApp(MayanAppConfig):
                 ),
             )
         )
+        app.conf.CELERY_QUEUES.extend(
+            (
+                Queue(
+                    'document_states_fast', Exchange('document_states_fast'),
+                    routing_key='document_states_fast'
+                ),
+            )
+        )
 
         app.conf.CELERY_ROUTES.update(
             {
-                'document_states.tasks.task_launch_all_workflows': {
+                'document_states.tasks.task_generate_document_state_image': {
                     'queue': 'document_states'
                 },
+                'document_states.tasks.task_launch_all_workflows': {
+                    'queue': 'document_states_fast'
+                },
             }
         )
 

mayan/apps/document_states/forms.py

@@ -178,7 +178,7 @@ class WorkflowInstanceTransitionForm(forms.Form):
 
 
 class WorkflowPreviewForm(forms.Form):
-    preview = forms.CharField(widget=WorkflowImageWidget())
+    preview = forms.IntegerField(widget=WorkflowImageWidget())
 
     def __init__(self, *args, **kwargs):
         instance = kwargs.pop('instance', None)

mayan/apps/document_states/literals.py

@@ -9,3 +9,4 @@ WORKFLOW_ACTION_WHEN_CHOICES = (
     (WORKFLOW_ACTION_ON_ENTRY, _('On entry')),
     (WORKFLOW_ACTION_ON_EXIT, _('On exit')),
 )
+WORKFLOW_IMAGE_TASK_TIMEOUT = 60

mayan/apps/document_states/models.py

@@ -6,6 +6,7 @@ import logging
 from graphviz import Digraph
 
 from django.conf import settings
+from django.core.files.base import ContentFile
 from django.core.exceptions import PermissionDenied, ValidationError
 from django.db import IntegrityError, models
 from django.db.models import F, Max, Q
@@ -26,6 +27,7 @@ from .literals import (
 )
 from .managers import WorkflowManager
 from .permissions import permission_workflow_transition
+from .storages import storage_workflowimagecache
 
 logger = logging.getLogger(__name__)
 
@@ -62,6 +64,22 @@ class Workflow(models.Model):
     def __str__(self):
         return self.label
 
+    def generate_image(self):
+        cache_filename = '{}'.format(self.id)
+        image = self.render()
+
+        # Since open "wb+" doesn't create files, check if the file
+        # exists, if not then create it
+        if not storage_workflowimagecache.exists(cache_filename):
+            storage_workflowimagecache.save(
+                name=cache_filename, content=ContentFile(content='')
+            )
+
+        with storage_workflowimagecache.open(cache_filename, 'wb+') as file_object:
+            file_object.write(image)
+
+        return cache_filename
+
     def get_document_types_not_in_workflow(self):
         return DocumentType.objects.exclude(pk__in=self.document_types.all())
 

mayan/apps/document_states/queues.py

@@ -4,10 +4,19 @@ from django.utils.translation import ugettext_lazy as _
 
 from task_manager.classes import CeleryQueue
 
+
 queue_document_states = CeleryQueue(
     name='document_states', label=_('Document states')
 )
+queue_document_states_fast = CeleryQueue(
+    name='document_states_fast', label=_('Document states fast')
+)
+
 queue_document_states.add_task_type(
     name='document_states.tasks.task_launch_all_workflows',
     label=_('Launch all workflows')
 )
+queue_document_states_fast.add_task_type(
+    name='document_states.tasks.task_generate_document_state_image',
+    label=_('Generate workflow previews')
+)

mayan/apps/document_states/serializers.py

@@ -174,6 +174,7 @@ class WorkflowSerializer(serializers.HyperlinkedModelSerializer):
     document_types_url = serializers.HyperlinkedIdentityField(
         view_name='rest_api:workflow-document-type-list'
     )
+    image_url = serializers.SerializerMethodField()
     states = WorkflowStateSerializer(many=True, required=False)
     transitions = WorkflowTransitionSerializer(many=True, required=False)
 
@@ -182,11 +183,18 @@ class WorkflowSerializer(serializers.HyperlinkedModelSerializer):
             'url': {'view_name': 'rest_api:workflow-detail'},
         }
         fields = (
-            'document_types_url', 'id', 'internal_name', 'label', 'states',
-            'transitions', 'url'
+            'document_types_url', 'image_url', 'id', 'internal_name', 'label',
+            'states', 'transitions', 'url'
         )
         model = Workflow
 
+    def get_image_url(self, instance):
+        return reverse(
+            'rest_api:workflow-image', args=(
+                instance.pk,
+            ), request=self.context['request'], format=self.context['format']
+        )
+
 
 class WorkflowInstanceLogEntrySerializer(serializers.ModelSerializer):
     document_workflow_url = serializers.SerializerMethodField()

mayan/apps/document_states/settings.py

@@ -0,0 +1,25 @@
+from __future__ import unicode_literals
+
+import os
+
+from django.conf import settings
+from django.utils.translation import ugettext_lazy as _
+
+from smart_settings import Namespace
+
+namespace = Namespace(name='document_states', label=_('Workflows'))
+
+setting_workflowimagecache_storage = namespace.add_setting(
+    global_name='WORKFLOWS_IMAGE_CACHE_STORAGE_BACKEND',
+    default='django.core.files.storage.FileSystemStorage', help_text=_(
+        'Path to the Storage subclass to use when storing the cached '
+        'workflow image files.'
+    )
+)
+setting_workflowimagecache_storage_arguments = namespace.add_setting(
+    global_name='WORKFLOWS_IMAGE_CACHE_STORAGE_BACKEND_ARGUMENTS',
+    default={'location': os.path.join(settings.MEDIA_ROOT, 'workflows')},
+    help_text=_(
+        'Arguments to pass to the WORKFLOWS_IMAGE_CACHE_STORAGE_BACKEND.'
+    )
+)

mayan/apps/document_states/storages.py

@@ -0,0 +1,12 @@
+from __future__ import unicode_literals
+
+from django.utils.module_loading import import_string
+
+from .settings import (
+    setting_workflowimagecache_storage,
+    setting_workflowimagecache_storage_arguments
+)
+
+storage_workflowimagecache = import_string(
+    dotted_path=setting_workflowimagecache_storage.value
+)(**setting_workflowimagecache_storage_arguments.value)

mayan/apps/document_states/tasks.py

@@ -9,6 +9,17 @@ from mayan.celery import app
 logger = logging.getLogger(__name__)
 
 
+@app.task()
+def task_generate_workflow_image(document_state_id):
+    Workflow = apps.get_model(
+        app_label='document_states', model_name='Workflow'
+    )
+
+    workflow = Workflow.objects.get(pk=document_state_id)
+
+    return workflow.generate_image()
+
+
 @app.task(ignore_result=True)
 def task_launch_all_workflows():
     Document = apps.get_model(app_label='documents', model_name='Document')

mayan/apps/document_states/templates/document_states/forms/widgets/workflow_image.html

@@ -0,0 +1 @@
+<img class="img-responsive" name="{{ widget.name }}" {% include "django/forms/widgets/attrs.html" %} src="{% url 'rest_api:workflow-image' widget.value.pk %}" style="margin:auto;" />

mayan/apps/document_states/urls.py

@@ -4,9 +4,10 @@ from django.conf.urls import url
 
 from .api_views import (
     APIDocumentTypeWorkflowListView, APIWorkflowDocumentTypeList,
-    APIWorkflowDocumentTypeView, APIWorkflowInstanceListView,
-    APIWorkflowInstanceView, APIWorkflowInstanceLogEntryListView,
-    APIWorkflowListView, APIWorkflowStateListView, APIWorkflowStateView,
+    APIWorkflowDocumentTypeView, APIWorkflowImageView,
+    APIWorkflowInstanceListView, APIWorkflowInstanceView,
+    APIWorkflowInstanceLogEntryListView, APIWorkflowListView,
+    APIWorkflowStateListView, APIWorkflowStateView,
     APIWorkflowTransitionListView, APIWorkflowTransitionView, APIWorkflowView
 )
 from .views import (
@@ -188,6 +189,10 @@ api_urls = [
         r'^workflows/(?P<pk>[0-9]+)/$', APIWorkflowView.as_view(),
         name='workflow-detail'
     ),
+    url(
+        r'^workflows/(?P<pk>[0-9]+)/image/$',
+        APIWorkflowImageView.as_view(), name='workflow-image'
+    ),
     url(
         r'^workflows/(?P<pk>[0-9]+)/document_types/$',
         APIWorkflowDocumentTypeList.as_view(),

mayan/apps/document_states/widgets.py

@@ -1,8 +1,7 @@
 from __future__ import unicode_literals
 
 from django import forms
-from django.urls import reverse
-from django.utils.html import format_html_join, mark_safe
+from django.utils.html import format_html_join
 
 
 def widget_transition_events(transition):
@@ -15,19 +14,10 @@ def widget_transition_events(transition):
     )
 
 
-def widget_workflow_diagram(workflow):
-    return mark_safe(
-        '<img class="img-responsive" src="{}" style="margin:auto;">'.format(
-            reverse('document_states:workflow_image', args=(workflow.pk,))
-        )
-    )
-
-
 class WorkflowImageWidget(forms.widgets.Widget):
-    def render(self, name, value, attrs=None):
-        if value:
-            output = []
-            output.append(widget_workflow_diagram(value))
-            return mark_safe(''.join(output))
-        else:
-            return ''
+    template_name = 'document_states/forms/widgets/workflow_image.html'
+
+    def format_value(self, value):
+        if value == '' or value is None:
+            return None
+        return value