matthias/mayan-edms
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Experimental platform app

Browse Source 
Signed-off-by: Roberto Rosario <Roberto.Rosario@mayan-edms.com>
This commit is contained in:
Roberto Rosario
2019-05-13 02:52:31 -04:00
parent 082981e25f
commit 501acbfb0f
18 changed files with 216 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
mayan/apps/checkouts/queues.py
View File

@@ -5,11 +5,13 @@ from datetime import timedelta
from django.utils.translation import ugettext_lazy as _
from mayan.apps.task_manager.classes import CeleryQueue
from mayan.apps.task_manager.workers import worker_medium
from .literals import CHECK_EXPIRED_CHECK_OUTS_INTERVAL
queue_checkouts_periodic = CeleryQueue(
label=_('Checkouts periodic'), name='checkouts_periodic', transient=True
label=_('Checkouts periodic'), name='checkouts_periodic', transient=True,
worker=worker_medium
)
queue_checkouts_periodic.add_task_type(
label=_('Check expired checkouts'),

8
mayan/apps/common/queues.py
View File

@@ -5,15 +5,17 @@ from datetime import timedelta
from django.utils.translation import ugettext_lazy as _
from mayan.apps.task_manager.classes import CeleryQueue
from mayan.apps.task_manager.workers import worker_medium, worker_slow
from .literals import DELETE_STALE_UPLOADS_INTERVAL
queue_default = CeleryQueue(
default_queue=True, label=_('Default'), name='default'
default_queue=True, label=_('Default'), name='default', worker=worker_medium
)
queue_tools = CeleryQueue(label=_('Tools'), name='tools')
queue_tools = CeleryQueue(label=_('Tools'), name='tools', worker=worker_slow)
queue_common_periodic = CeleryQueue(
label=_('Common periodic'), name='common_periodic', transient=True
label=_('Common periodic'), name='common_periodic', transient=True,
worker=worker_slow
)
queue_common_periodic.add_task_type(
dotted_path='mayan.apps.common.tasks.task_delete_stale_uploads',

3
mayan/apps/document_indexing/queues.py
View File

@@ -4,8 +4,9 @@ from django.utils.translation import ugettext_lazy as _
from mayan.apps.common.queues import queue_tools
from mayan.apps.task_manager.classes import CeleryQueue
from mayan.apps.task_manager.workers import worker_medium
queue_indexing = CeleryQueue(label=_('Indexing'), name='indexing')
queue_indexing = CeleryQueue(label=_('Indexing'), name='indexing', worker=worker_medium)
queue_indexing.add_task_type(
label=_('Delete empty index nodes'),

3
mayan/apps/document_parsing/queues.py
View File

@@ -3,8 +3,9 @@ from __future__ import unicode_literals
from django.utils.translation import ugettext_lazy as _
from mayan.apps.task_manager.classes import CeleryQueue
from mayan.apps.task_manager.workers import worker_slow
queue_ocr = CeleryQueue(name='parsing', label=_('Parsing'))
queue_ocr = CeleryQueue(name='parsing', label=_('Parsing'), worker=worker_slow)
queue_ocr.add_task_type(
dotted_path='mayan.apps.document_parsing.tasks.task_parse_document_version',
label=_('Document version parsing')

3
mayan/apps/document_signatures/queues.py
View File

@@ -4,8 +4,9 @@ from django.utils.translation import ugettext_lazy as _
from mayan.apps.common.queues import queue_tools
from mayan.apps.task_manager.classes import CeleryQueue
from mayan.apps.task_manager.workers import worker_medium
queue_signatures = CeleryQueue(label=_('Signatures'), name='signatures')
queue_signatures = CeleryQueue(label=_('Signatures'), name='signatures', worker=worker_medium)
queue_signatures.add_task_type(
label=_('Verify key signatures'),

3
mayan/apps/document_states/queues.py
View File

@@ -3,9 +3,10 @@ from __future__ import absolute_import, unicode_literals
from django.utils.translation import ugettext_lazy as _
from mayan.apps.task_manager.classes import CeleryQueue
from mayan.apps.task_manager.workers import worker_slow
queue_document_states = CeleryQueue(
name='document_states', label=_('Document states')
name='document_states', label=_('Document states'), worker=worker_slow
)
queue_document_states.add_task_type(
dotted_path='mayan.apps.document_states.tasks.task_launch_all_workflows',

9
mayan/apps/documents/queues.py
View File

@@ -6,6 +6,7 @@ from django.utils.translation import ugettext_lazy as _
from mayan.apps.common.queues import queue_tools
from mayan.apps.task_manager.classes import CeleryQueue
from mayan.apps.task_manager.workers import worker_fast, worker_medium
from .literals import (
CHECK_DELETE_PERIOD_INTERVAL, CHECK_TRASH_PERIOD_INTERVAL,
@@ -13,16 +14,16 @@ from .literals import (
)
queue_converter = CeleryQueue(
name='converter', label=_('Converter'), transient=True
name='converter', label=_('Converter'), transient=True, worker=worker_fast
)
queue_documents_periodic = CeleryQueue(
name='documents_periodic', label=_('Documents periodic'), transient=True
name='documents_periodic', label=_('Documents periodic'), transient=True, worker=worker_medium
)
queue_uploads = CeleryQueue(
name='uploads', label=_('Uploads')
name='uploads', label=_('Uploads'), worker=worker_medium
)
queue_documents = CeleryQueue(
name='documents', label=_('Documents')
name='documents', label=_('Documents'), worker=worker_medium
)
queue_converter.add_task_type(

3
mayan/apps/mailer/queues.py
View File

@@ -3,8 +3,9 @@ from __future__ import absolute_import, unicode_literals
from django.utils.translation import ugettext_lazy as _
from mayan.apps.task_manager.classes import CeleryQueue
from mayan.apps.task_manager.workers import worker_slow
queue_mailing = CeleryQueue(label=_('Mailing'), name='mailing')
queue_mailing = CeleryQueue(label=_('Mailing'), name='mailing', worker=worker_slow)
queue_mailing.add_task_type(
label=_('Send document'),
dotted_path='mayan.apps.mailer.tasks.task_send_document'

3
mayan/apps/mayan_statistics/queues.py
View File

@@ -3,9 +3,10 @@ from __future__ import unicode_literals
from django.utils.translation import ugettext_lazy as _
from mayan.apps.task_manager.classes import CeleryQueue
from mayan.apps.task_manager.workers import worker_slow
queue_statistics = CeleryQueue(
label=_('Statistics'), name='statistics', transient=True
label=_('Statistics'), name='statistics', transient=True, worker=worker_slow
)
queue_statistics.add_task_type(

3
mayan/apps/metadata/queues.py
View File

@@ -3,9 +3,10 @@ from __future__ import absolute_import, unicode_literals
from django.utils.translation import ugettext_lazy as _
from mayan.apps.task_manager.classes import CeleryQueue
from mayan.apps.task_manager.workers import worker_medium
queue_metadata = CeleryQueue(
label=_('Metadata'), name='metadata'
label=_('Metadata'), name='metadata', worker=worker_medium
)
queue_metadata.add_task_type(
label=_('Remove metadata type'),

3
mayan/apps/ocr/queues.py
View File

@@ -3,8 +3,9 @@ from __future__ import unicode_literals
from django.utils.translation import ugettext_lazy as _
from mayan.apps.task_manager.classes import CeleryQueue
from mayan.apps.task_manager.workers import worker_slow
queue_ocr = CeleryQueue(name='ocr', label=_('OCR'))
queue_ocr = CeleryQueue(name='ocr', label=_('OCR'), worker=worker_slow)
queue_ocr.add_task_type(
dotted_path='mayan.apps.ocr.tasks.task_do_ocr',
label=_('Document version OCR')

3
mayan/apps/platform/__init__.py Normal file
View File

@@ -0,0 +1,3 @@
from __future__ import unicode_literals
default_app_config = 'mayan.apps.platform.apps.PlatformApp'

101
mayan/apps/platform/apps.py Normal file
View File

@@ -0,0 +1,101 @@
from __future__ import absolute_import, unicode_literals
from django.apps import apps
from django.db.models.signals import m2m_changed, pre_delete
from django.utils.translation import ugettext_lazy as _
from mayan.apps.acls.classes import ModelPermission
from mayan.apps.acls.links import link_acl_list
from mayan.apps.acls.permissions import permission_acl_edit, permission_acl_view
from mayan.apps.common.apps import MayanAppConfig
from mayan.apps.common.classes import ModelField
from mayan.apps.common.menus import (
menu_facet, menu_list_facet, menu_main, menu_multi_item, menu_object,
menu_secondary
)
from mayan.apps.documents.search import document_page_search, document_search
from mayan.apps.events.classes import ModelEventType
from mayan.apps.events.links import (
link_events_for_object, link_object_event_types_user_subcriptions_list,
)
from mayan.apps.events.permissions import permission_events_view
from mayan.apps.navigation.classes import SourceColumn
"""
from .dependencies import * # NOQA
from .events import (
event_tag_attach, event_tag_created, event_tag_edited, event_tag_remove
)
from .handlers import handler_index_document, handler_tag_pre_delete
from .html_widgets import widget_document_tags
from .links import (
link_document_tag_list, link_document_multiple_attach_multiple_tag,
link_document_multiple_tag_multiple_remove,
link_document_tag_multiple_remove, link_document_tag_multiple_attach, link_tag_create,
link_tag_delete, link_tag_edit, link_tag_list,
link_tag_multiple_delete, link_tag_document_list
)
from .menus import menu_tags
from .methods import method_document_get_tags
from .permissions import (
permission_tag_attach, permission_tag_delete, permission_tag_edit,
permission_tag_remove, permission_tag_view
)
from .search import tag_search # NOQA
"""
#from mayan.apps.common.classes import Template
from mayan.apps.task_manager.classes import Worker
class PlatformTemplate(object):
_registry = {}
context = {}
@classmethod
def all(cls):
return cls._registry.values()
@classmethod
def get(cls, name):
return cls._registry[name]
def __init__(self, name, template_name):
self.name = name
self.template_name = template_name
self.__class__._registry[name] = self
def get_context(self):
return self.context
def render(self):
from django.template import loader
return loader.render_to_string(
template_name=self.template_name,
context=self.get_context()
)
class PlatformTemplateSupervisord(PlatformTemplate):
def get_context(self):
return {'workers': Worker.all()}
class PlatformApp(MayanAppConfig):
app_namespace = 'platform'
app_url = 'platform'
has_rest_api = False
has_tests = False
name = 'mayan.apps.platform'
verbose_name = _('Platform')
def ready(self):
super(PlatformApp, self).ready()
t = PlatformTemplateSupervisord(
name='platform_supervisord',
template_name='platform/supervisord.html'
)
print ("#@#@", t.render())
print("!!!!", Worker.all())

1
mayan/apps/platform/classes.py Normal file
View File

@@ -0,0 +1 @@

52
mayan/apps/platform/templates/platform/supervisord.html Normal file
View File

@@ -0,0 +1,52 @@
[program:mayan-gunicorn]
autorestart = false
autostart = true
command = /bin/bash -c "${MAYAN_GUNICORN_BIN} -w ${MAYAN_GUNICORN_WORKERS} mayan.wsgi --max-requests 500 --max-requests-jitter 50 --worker-class gevent --bind 0.0.0.0:8000 --env DJANGO_SETTINGS_MODULE=${MAYAN_SETTINGS_MODULE}" --timeout 120
redirect_stderr = true
stderr_logfile = /dev/fd/2
stderr_logfile_maxbytes = 0
stdout_logfile = /dev/fd/1
stdout_logfile_maxbytes = 0
user = mayan
[program:redis]
autorestart = false
autostart = true
command = /bin/bash -c "if [ ${MAYAN_BROKER_URL} == ${MAYAN_DEFAULT_BROKER_URL} ] && [ ${MAYAN_CELERY_RESULT_BACKEND} == ${MAYAN_DEFAULT_CELERY_RESULT_BACKEND} ];then /usr/bin/redis-server /etc/redis/;fi"
stderr_logfile = /dev/fd/2
stderr_logfile_maxbytes = 0
stdout_logfile = /dev/fd/1
stdout_logfile_maxbytes = 0
user = root
{% for worker in workers %}
[program:mayan-worker-{{ worker.name }}]
autorestart = false
autostart = true
command = nice -n {{ worker.nice_level }} /bin/bash -c "${MAYAN_BIN} celery --settings=${MAYAN_SETTINGS_MODULE} worker -Ofair -l ERROR -Q {% for queue in worker.queues %}{{ queue.name }}{% if not forloop.last %},{% endif %}{% endfor %} -n mayan-worker-{{ worker.name }}.%%h ${MAYAN_WORKER_{{ worker.name|upper }}_CONCURRENCY}"
killasgroup = true
numprocs = 1
priority = 998
startsecs = 10
stderr_logfile = /dev/fd/2
stderr_logfile_maxbytes = 0
stdout_logfile = /dev/fd/1
stdout_logfile_maxbytes = 0
stopwaitsecs = 1
user = mayan
{% endfor %}
[program:mayan-celery-beat]
autorestart = false
autostart = true
command = nice -n 1 /bin/bash -c "${MAYAN_BIN} celery --settings=${MAYAN_SETTINGS_MODULE} beat --pidfile= -l ERROR"
killasgroup = true
numprocs = 1
priority = 998
stderr_logfile = /dev/fd/2
stderr_logfile_maxbytes = 0
stdout_logfile = /dev/fd/1
stdout_logfile_maxbytes = 0
startsecs = 10
stopwaitsecs = 1
user = mayan

9
mayan/apps/sources/queues.py
View File

@@ -3,15 +3,18 @@ from __future__ import unicode_literals
from django.utils.translation import ugettext_lazy as _
from mayan.apps.task_manager.classes import CeleryQueue
from mayan.apps.task_manager.workers import worker_fast, worker_medium
queue_sources = CeleryQueue(
label=_('Sources'), name='sources'
label=_('Sources'), name='sources', worker=worker_medium
)
queue_sources_periodic = CeleryQueue(
label=_('Sources periodic'), name='sources_periodic', transient=True
label=_('Sources periodic'), name='sources_periodic', transient=True,
worker=worker_medium
)
queue_sources_fast = CeleryQueue(
label=_('Sources fast'), name='sources_fast', transient=True
label=_('Sources fast'), name='sources_fast', transient=True,
worker=worker_fast
)
queue_sources_fast.add_task_type(

18
mayan/apps/task_manager/classes.py
View File

@@ -93,13 +93,14 @@ class CeleryQueue(object):
for instance in cls.all():
instance._update_celery()
def __init__(self, name, label, default_queue=False, transient=False):
def __init__(self, name, label, worker, default_queue=False, transient=False):
self.name = name
self.label = label
self.default_queue = default_queue
self.transient = transient
self.task_types = []
self.__class__._registry[name] = self
worker.queues.append(self)
def __str__(self):
return force_text(self.label)
@@ -164,3 +165,18 @@ class CeleryQueue(object):
},
}
)
class Worker(object):
_registry = {}
@classmethod
def all(cls):
return cls._registry.values()
def __init__(self, name, label=None, nice_level=0):
self.name = name
self.label = label
self.nice_level = nice_level
self.queues = []
self.__class__._registry[name] = self

7
mayan/apps/task_manager/workers.py Normal file
View File

@@ -0,0 +1,7 @@
from __future__ import absolute_import, unicode_literals
from .classes import Worker
worker_fast = Worker(name='fast', nice_level=1)
worker_medium = Worker(name='medium', nice_level=18)
worker_slow = Worker(name='slow', nice_level=19)