Experimental platform app

Signed-off-by: Roberto Rosario <Roberto.Rosario@mayan-edms.com>

mayan/apps/checkouts/queues.py

@@ -5,11 +5,13 @@ from datetime import timedelta
 from django.utils.translation import ugettext_lazy as _
 
 from mayan.apps.task_manager.classes import CeleryQueue
+from mayan.apps.task_manager.workers import worker_medium
 
 from .literals import CHECK_EXPIRED_CHECK_OUTS_INTERVAL
 
 queue_checkouts_periodic = CeleryQueue(
-    label=_('Checkouts periodic'), name='checkouts_periodic', transient=True
+    label=_('Checkouts periodic'), name='checkouts_periodic', transient=True,
+    worker=worker_medium
 )
 queue_checkouts_periodic.add_task_type(
     label=_('Check expired checkouts'),

mayan/apps/common/queues.py

@@ -5,15 +5,17 @@ from datetime import timedelta
 from django.utils.translation import ugettext_lazy as _
 
 from mayan.apps.task_manager.classes import CeleryQueue
+from mayan.apps.task_manager.workers import worker_medium, worker_slow
 
 from .literals import DELETE_STALE_UPLOADS_INTERVAL
 
 queue_default = CeleryQueue(
-    default_queue=True, label=_('Default'), name='default'
+    default_queue=True, label=_('Default'), name='default', worker=worker_medium
 )
-queue_tools = CeleryQueue(label=_('Tools'), name='tools')
+queue_tools = CeleryQueue(label=_('Tools'), name='tools', worker=worker_slow)
 queue_common_periodic = CeleryQueue(
-    label=_('Common periodic'), name='common_periodic', transient=True
+    label=_('Common periodic'), name='common_periodic', transient=True,
+    worker=worker_slow
 )
 queue_common_periodic.add_task_type(
     dotted_path='mayan.apps.common.tasks.task_delete_stale_uploads',

mayan/apps/document_indexing/queues.py

@@ -4,8 +4,9 @@ from django.utils.translation import ugettext_lazy as _
 
 from mayan.apps.common.queues import queue_tools
 from mayan.apps.task_manager.classes import CeleryQueue
+from mayan.apps.task_manager.workers import worker_medium
 
-queue_indexing = CeleryQueue(label=_('Indexing'), name='indexing')
+queue_indexing = CeleryQueue(label=_('Indexing'), name='indexing', worker=worker_medium)
 
 queue_indexing.add_task_type(
     label=_('Delete empty index nodes'),

mayan/apps/document_parsing/queues.py

@@ -3,8 +3,9 @@ from __future__ import unicode_literals
 from django.utils.translation import ugettext_lazy as _
 
 from mayan.apps.task_manager.classes import CeleryQueue
+from mayan.apps.task_manager.workers import worker_slow
 
-queue_ocr = CeleryQueue(name='parsing', label=_('Parsing'))
+queue_ocr = CeleryQueue(name='parsing', label=_('Parsing'), worker=worker_slow)
 queue_ocr.add_task_type(
     dotted_path='mayan.apps.document_parsing.tasks.task_parse_document_version',
     label=_('Document version parsing')

mayan/apps/document_signatures/queues.py

@@ -4,8 +4,9 @@ from django.utils.translation import ugettext_lazy as _
 
 from mayan.apps.common.queues import queue_tools
 from mayan.apps.task_manager.classes import CeleryQueue
+from mayan.apps.task_manager.workers import worker_medium
 
-queue_signatures = CeleryQueue(label=_('Signatures'), name='signatures')
+queue_signatures = CeleryQueue(label=_('Signatures'), name='signatures', worker=worker_medium)
 
 queue_signatures.add_task_type(
     label=_('Verify key signatures'),

mayan/apps/document_states/queues.py

@@ -3,9 +3,10 @@ from __future__ import absolute_import, unicode_literals
 from django.utils.translation import ugettext_lazy as _
 
 from mayan.apps.task_manager.classes import CeleryQueue
+from mayan.apps.task_manager.workers import worker_slow
 
 queue_document_states = CeleryQueue(
-    name='document_states', label=_('Document states')
+    name='document_states', label=_('Document states'), worker=worker_slow
 )
 queue_document_states.add_task_type(
     dotted_path='mayan.apps.document_states.tasks.task_launch_all_workflows',

mayan/apps/documents/queues.py

@@ -6,6 +6,7 @@ from django.utils.translation import ugettext_lazy as _
 
 from mayan.apps.common.queues import queue_tools
 from mayan.apps.task_manager.classes import CeleryQueue
+from mayan.apps.task_manager.workers import worker_fast, worker_medium
 
 from .literals import (
     CHECK_DELETE_PERIOD_INTERVAL, CHECK_TRASH_PERIOD_INTERVAL,
@@ -13,16 +14,16 @@ from .literals import (
 )
 
 queue_converter = CeleryQueue(
-    name='converter', label=_('Converter'), transient=True
+    name='converter', label=_('Converter'), transient=True, worker=worker_fast
 )
 queue_documents_periodic = CeleryQueue(
-    name='documents_periodic', label=_('Documents periodic'), transient=True
+    name='documents_periodic', label=_('Documents periodic'), transient=True, worker=worker_medium
 )
 queue_uploads = CeleryQueue(
-    name='uploads', label=_('Uploads')
+    name='uploads', label=_('Uploads'), worker=worker_medium
 )
 queue_documents = CeleryQueue(
-    name='documents', label=_('Documents')
+    name='documents', label=_('Documents'), worker=worker_medium
 )
 
 queue_converter.add_task_type(

mayan/apps/mailer/queues.py

@@ -3,8 +3,9 @@ from __future__ import absolute_import, unicode_literals
 from django.utils.translation import ugettext_lazy as _
 
 from mayan.apps.task_manager.classes import CeleryQueue
+from mayan.apps.task_manager.workers import worker_slow
 
-queue_mailing = CeleryQueue(label=_('Mailing'), name='mailing')
+queue_mailing = CeleryQueue(label=_('Mailing'), name='mailing', worker=worker_slow)
 queue_mailing.add_task_type(
     label=_('Send document'),
     dotted_path='mayan.apps.mailer.tasks.task_send_document'

mayan/apps/mayan_statistics/queues.py

@@ -3,9 +3,10 @@ from __future__ import unicode_literals
 from django.utils.translation import ugettext_lazy as _
 
 from mayan.apps.task_manager.classes import CeleryQueue
+from mayan.apps.task_manager.workers import worker_slow
 
 queue_statistics = CeleryQueue(
-    label=_('Statistics'), name='statistics', transient=True
+    label=_('Statistics'), name='statistics', transient=True, worker=worker_slow
 )
 
 queue_statistics.add_task_type(

mayan/apps/metadata/queues.py

@@ -3,9 +3,10 @@ from __future__ import absolute_import, unicode_literals
 from django.utils.translation import ugettext_lazy as _
 
 from mayan.apps.task_manager.classes import CeleryQueue
+from mayan.apps.task_manager.workers import worker_medium
 
 queue_metadata = CeleryQueue(
-    label=_('Metadata'), name='metadata'
+    label=_('Metadata'), name='metadata', worker=worker_medium
 )
 queue_metadata.add_task_type(
     label=_('Remove metadata type'),

mayan/apps/ocr/queues.py

@@ -3,8 +3,9 @@ from __future__ import unicode_literals
 from django.utils.translation import ugettext_lazy as _
 
 from mayan.apps.task_manager.classes import CeleryQueue
+from mayan.apps.task_manager.workers import worker_slow
 
-queue_ocr = CeleryQueue(name='ocr', label=_('OCR'))
+queue_ocr = CeleryQueue(name='ocr', label=_('OCR'), worker=worker_slow)
 queue_ocr.add_task_type(
     dotted_path='mayan.apps.ocr.tasks.task_do_ocr',
     label=_('Document version OCR')

mayan/apps/platform/__init__.py

@@ -0,0 +1,3 @@
+from __future__ import unicode_literals
+
+default_app_config = 'mayan.apps.platform.apps.PlatformApp'

mayan/apps/platform/apps.py

@@ -0,0 +1,101 @@
+from __future__ import absolute_import, unicode_literals
+
+from django.apps import apps
+from django.db.models.signals import m2m_changed, pre_delete
+from django.utils.translation import ugettext_lazy as _
+
+from mayan.apps.acls.classes import ModelPermission
+from mayan.apps.acls.links import link_acl_list
+from mayan.apps.acls.permissions import permission_acl_edit, permission_acl_view
+from mayan.apps.common.apps import MayanAppConfig
+from mayan.apps.common.classes import ModelField
+from mayan.apps.common.menus import (
+    menu_facet, menu_list_facet, menu_main, menu_multi_item, menu_object,
+    menu_secondary
+)
+from mayan.apps.documents.search import document_page_search, document_search
+from mayan.apps.events.classes import ModelEventType
+from mayan.apps.events.links import (
+    link_events_for_object, link_object_event_types_user_subcriptions_list,
+)
+from mayan.apps.events.permissions import permission_events_view
+from mayan.apps.navigation.classes import SourceColumn
+
+"""
+from .dependencies import *  # NOQA
+from .events import (
+    event_tag_attach, event_tag_created, event_tag_edited, event_tag_remove
+)
+from .handlers import handler_index_document, handler_tag_pre_delete
+from .html_widgets import widget_document_tags
+from .links import (
+    link_document_tag_list, link_document_multiple_attach_multiple_tag,
+    link_document_multiple_tag_multiple_remove,
+    link_document_tag_multiple_remove, link_document_tag_multiple_attach, link_tag_create,
+    link_tag_delete, link_tag_edit, link_tag_list,
+    link_tag_multiple_delete, link_tag_document_list
+)
+from .menus import menu_tags
+from .methods import method_document_get_tags
+from .permissions import (
+    permission_tag_attach, permission_tag_delete, permission_tag_edit,
+    permission_tag_remove, permission_tag_view
+)
+from .search import tag_search  # NOQA
+"""
+#from mayan.apps.common.classes import Template
+from mayan.apps.task_manager.classes import Worker
+
+
+class PlatformTemplate(object):
+    _registry = {}
+    context = {}
+
+    @classmethod
+    def all(cls):
+        return cls._registry.values()
+
+    @classmethod
+    def get(cls, name):
+        return cls._registry[name]
+
+    def __init__(self, name, template_name):
+        self.name = name
+        self.template_name = template_name
+        self.__class__._registry[name] = self
+
+    def get_context(self):
+        return self.context
+
+    def render(self):
+        from django.template import loader
+        return loader.render_to_string(
+            template_name=self.template_name,
+            context=self.get_context()
+        )
+
+
+
+class PlatformTemplateSupervisord(PlatformTemplate):
+    def get_context(self):
+        return {'workers': Worker.all()}
+
+
+class PlatformApp(MayanAppConfig):
+    app_namespace = 'platform'
+    app_url = 'platform'
+    has_rest_api = False
+    has_tests = False
+    name = 'mayan.apps.platform'
+    verbose_name = _('Platform')
+
+    def ready(self):
+        super(PlatformApp, self).ready()
+
+        t = PlatformTemplateSupervisord(
+            name='platform_supervisord',
+            template_name='platform/supervisord.html'
+        )
+
+        print ("#@#@", t.render())
+        print("!!!!", Worker.all())

mayan/apps/platform/classes.py

@@ -0,0 +1 @@
+

mayan/apps/platform/templates/platform/supervisord.html

@@ -0,0 +1,52 @@
+[program:mayan-gunicorn]
+autorestart = false
+autostart = true
+command = /bin/bash -c "${MAYAN_GUNICORN_BIN} -w ${MAYAN_GUNICORN_WORKERS} mayan.wsgi --max-requests 500 --max-requests-jitter 50 --worker-class gevent --bind 0.0.0.0:8000  --env DJANGO_SETTINGS_MODULE=${MAYAN_SETTINGS_MODULE}" --timeout 120
+redirect_stderr = true
+stderr_logfile = /dev/fd/2
+stderr_logfile_maxbytes = 0
+stdout_logfile = /dev/fd/1
+stdout_logfile_maxbytes = 0
+user = mayan
+
+[program:redis]
+autorestart = false
+autostart = true
+command = /bin/bash -c "if [ ${MAYAN_BROKER_URL} == ${MAYAN_DEFAULT_BROKER_URL} ] && [ ${MAYAN_CELERY_RESULT_BACKEND} == ${MAYAN_DEFAULT_CELERY_RESULT_BACKEND} ];then /usr/bin/redis-server /etc/redis/;fi"
+stderr_logfile = /dev/fd/2
+stderr_logfile_maxbytes = 0
+stdout_logfile = /dev/fd/1
+stdout_logfile_maxbytes = 0
+user = root
+
+{% for worker in workers %}
+[program:mayan-worker-{{ worker.name }}]
+autorestart = false
+autostart = true
+command = nice -n {{ worker.nice_level }} /bin/bash -c "${MAYAN_BIN} celery --settings=${MAYAN_SETTINGS_MODULE} worker -Ofair -l ERROR -Q {% for queue in worker.queues %}{{ queue.name }}{% if not forloop.last %},{% endif %}{% endfor %} -n mayan-worker-{{ worker.name }}.%%h ${MAYAN_WORKER_{{ worker.name|upper }}_CONCURRENCY}"
+killasgroup = true
+numprocs = 1
+priority = 998
+startsecs = 10
+stderr_logfile = /dev/fd/2
+stderr_logfile_maxbytes = 0
+stdout_logfile = /dev/fd/1
+stdout_logfile_maxbytes = 0
+stopwaitsecs = 1
+user = mayan
+{% endfor %}
+
+[program:mayan-celery-beat]
+autorestart = false
+autostart = true
+command = nice -n 1 /bin/bash -c "${MAYAN_BIN} celery --settings=${MAYAN_SETTINGS_MODULE} beat --pidfile= -l ERROR"
+killasgroup = true
+numprocs = 1
+priority = 998
+stderr_logfile = /dev/fd/2
+stderr_logfile_maxbytes = 0
+stdout_logfile = /dev/fd/1
+stdout_logfile_maxbytes = 0
+startsecs = 10
+stopwaitsecs = 1
+user = mayan

mayan/apps/sources/queues.py

@@ -3,15 +3,18 @@ from __future__ import unicode_literals
 from django.utils.translation import ugettext_lazy as _
 
 from mayan.apps.task_manager.classes import CeleryQueue
+from mayan.apps.task_manager.workers import worker_fast, worker_medium
 
 queue_sources = CeleryQueue(
-    label=_('Sources'), name='sources'
+    label=_('Sources'), name='sources', worker=worker_medium
 )
 queue_sources_periodic = CeleryQueue(
-    label=_('Sources periodic'), name='sources_periodic', transient=True
+    label=_('Sources periodic'), name='sources_periodic', transient=True,
+    worker=worker_medium
 )
 queue_sources_fast = CeleryQueue(
-    label=_('Sources fast'), name='sources_fast', transient=True
+    label=_('Sources fast'), name='sources_fast', transient=True,
+    worker=worker_fast
 )
 
 queue_sources_fast.add_task_type(

mayan/apps/task_manager/classes.py

@@ -93,13 +93,14 @@ class CeleryQueue(object):
         for instance in cls.all():
             instance._update_celery()
 
-    def __init__(self, name, label, default_queue=False, transient=False):
+    def __init__(self, name, label, worker, default_queue=False, transient=False):
         self.name = name
         self.label = label
         self.default_queue = default_queue
         self.transient = transient
         self.task_types = []
         self.__class__._registry[name] = self
+        worker.queues.append(self)
 
     def __str__(self):
         return force_text(self.label)
@@ -164,3 +165,18 @@ class CeleryQueue(object):
                         },
                     }
                 )
+
+
+class Worker(object):
+    _registry = {}
+
+    @classmethod
+    def all(cls):
+        return cls._registry.values()
+
+    def __init__(self, name, label=None, nice_level=0):
+        self.name = name
+        self.label = label
+        self.nice_level = nice_level
+        self.queues = []
+        self.__class__._registry[name] = self

mayan/apps/task_manager/workers.py

@@ -0,0 +1,7 @@
+from __future__ import absolute_import, unicode_literals
+
+from .classes import Worker
+
+worker_fast = Worker(name='fast', nice_level=1)
+worker_medium = Worker(name='medium', nice_level=18)
+worker_slow = Worker(name='slow', nice_level=19)