diff --git a/mayan/apps/checkouts/queues.py b/mayan/apps/checkouts/queues.py index 323f0d4d5f..b98d7b1037 100644 --- a/mayan/apps/checkouts/queues.py +++ b/mayan/apps/checkouts/queues.py @@ -5,11 +5,13 @@ from datetime import timedelta from django.utils.translation import ugettext_lazy as _ from mayan.apps.task_manager.classes import CeleryQueue +from mayan.apps.task_manager.workers import worker_medium from .literals import CHECK_EXPIRED_CHECK_OUTS_INTERVAL queue_checkouts_periodic = CeleryQueue( - label=_('Checkouts periodic'), name='checkouts_periodic', transient=True + label=_('Checkouts periodic'), name='checkouts_periodic', transient=True, + worker=worker_medium ) queue_checkouts_periodic.add_task_type( label=_('Check expired checkouts'), diff --git a/mayan/apps/common/queues.py b/mayan/apps/common/queues.py index 9612710e2f..1763897342 100644 --- a/mayan/apps/common/queues.py +++ b/mayan/apps/common/queues.py @@ -5,15 +5,17 @@ from datetime import timedelta from django.utils.translation import ugettext_lazy as _ from mayan.apps.task_manager.classes import CeleryQueue +from mayan.apps.task_manager.workers import worker_medium, worker_slow from .literals import DELETE_STALE_UPLOADS_INTERVAL queue_default = CeleryQueue( - default_queue=True, label=_('Default'), name='default' + default_queue=True, label=_('Default'), name='default', worker=worker_medium ) -queue_tools = CeleryQueue(label=_('Tools'), name='tools') +queue_tools = CeleryQueue(label=_('Tools'), name='tools', worker=worker_slow) queue_common_periodic = CeleryQueue( - label=_('Common periodic'), name='common_periodic', transient=True + label=_('Common periodic'), name='common_periodic', transient=True, + worker=worker_slow ) queue_common_periodic.add_task_type( dotted_path='mayan.apps.common.tasks.task_delete_stale_uploads', diff --git a/mayan/apps/document_indexing/queues.py b/mayan/apps/document_indexing/queues.py index e000909136..c07e3cf3f9 100644 --- a/mayan/apps/document_indexing/queues.py +++ b/mayan/apps/document_indexing/queues.py @@ -4,8 +4,9 @@ from django.utils.translation import ugettext_lazy as _ from mayan.apps.common.queues import queue_tools from mayan.apps.task_manager.classes import CeleryQueue +from mayan.apps.task_manager.workers import worker_medium -queue_indexing = CeleryQueue(label=_('Indexing'), name='indexing') +queue_indexing = CeleryQueue(label=_('Indexing'), name='indexing', worker=worker_medium) queue_indexing.add_task_type( label=_('Delete empty index nodes'), diff --git a/mayan/apps/document_parsing/queues.py b/mayan/apps/document_parsing/queues.py index fed6d834a7..013604df70 100644 --- a/mayan/apps/document_parsing/queues.py +++ b/mayan/apps/document_parsing/queues.py @@ -3,8 +3,9 @@ from __future__ import unicode_literals from django.utils.translation import ugettext_lazy as _ from mayan.apps.task_manager.classes import CeleryQueue +from mayan.apps.task_manager.workers import worker_slow -queue_ocr = CeleryQueue(name='parsing', label=_('Parsing')) +queue_ocr = CeleryQueue(name='parsing', label=_('Parsing'), worker=worker_slow) queue_ocr.add_task_type( dotted_path='mayan.apps.document_parsing.tasks.task_parse_document_version', label=_('Document version parsing') diff --git a/mayan/apps/document_signatures/queues.py b/mayan/apps/document_signatures/queues.py index 839c72f8a0..7f109676a2 100644 --- a/mayan/apps/document_signatures/queues.py +++ b/mayan/apps/document_signatures/queues.py @@ -4,8 +4,9 @@ from django.utils.translation import ugettext_lazy as _ from mayan.apps.common.queues import queue_tools from mayan.apps.task_manager.classes import CeleryQueue +from mayan.apps.task_manager.workers import worker_medium -queue_signatures = CeleryQueue(label=_('Signatures'), name='signatures') +queue_signatures = CeleryQueue(label=_('Signatures'), name='signatures', worker=worker_medium) queue_signatures.add_task_type( label=_('Verify key signatures'), diff --git a/mayan/apps/document_states/queues.py b/mayan/apps/document_states/queues.py index 4b077b703c..6b269d5b99 100644 --- a/mayan/apps/document_states/queues.py +++ b/mayan/apps/document_states/queues.py @@ -3,9 +3,10 @@ from __future__ import absolute_import, unicode_literals from django.utils.translation import ugettext_lazy as _ from mayan.apps.task_manager.classes import CeleryQueue +from mayan.apps.task_manager.workers import worker_slow queue_document_states = CeleryQueue( - name='document_states', label=_('Document states') + name='document_states', label=_('Document states'), worker=worker_slow ) queue_document_states.add_task_type( dotted_path='mayan.apps.document_states.tasks.task_launch_all_workflows', diff --git a/mayan/apps/documents/queues.py b/mayan/apps/documents/queues.py index 460b2cd0a2..57f4c4b32a 100644 --- a/mayan/apps/documents/queues.py +++ b/mayan/apps/documents/queues.py @@ -6,6 +6,7 @@ from django.utils.translation import ugettext_lazy as _ from mayan.apps.common.queues import queue_tools from mayan.apps.task_manager.classes import CeleryQueue +from mayan.apps.task_manager.workers import worker_fast, worker_medium from .literals import ( CHECK_DELETE_PERIOD_INTERVAL, CHECK_TRASH_PERIOD_INTERVAL, @@ -13,16 +14,16 @@ from .literals import ( ) queue_converter = CeleryQueue( - name='converter', label=_('Converter'), transient=True + name='converter', label=_('Converter'), transient=True, worker=worker_fast ) queue_documents_periodic = CeleryQueue( - name='documents_periodic', label=_('Documents periodic'), transient=True + name='documents_periodic', label=_('Documents periodic'), transient=True, worker=worker_medium ) queue_uploads = CeleryQueue( - name='uploads', label=_('Uploads') + name='uploads', label=_('Uploads'), worker=worker_medium ) queue_documents = CeleryQueue( - name='documents', label=_('Documents') + name='documents', label=_('Documents'), worker=worker_medium ) queue_converter.add_task_type( diff --git a/mayan/apps/mailer/queues.py b/mayan/apps/mailer/queues.py index 23906b02cd..127899df1b 100644 --- a/mayan/apps/mailer/queues.py +++ b/mayan/apps/mailer/queues.py @@ -3,8 +3,9 @@ from __future__ import absolute_import, unicode_literals from django.utils.translation import ugettext_lazy as _ from mayan.apps.task_manager.classes import CeleryQueue +from mayan.apps.task_manager.workers import worker_slow -queue_mailing = CeleryQueue(label=_('Mailing'), name='mailing') +queue_mailing = CeleryQueue(label=_('Mailing'), name='mailing', worker=worker_slow) queue_mailing.add_task_type( label=_('Send document'), dotted_path='mayan.apps.mailer.tasks.task_send_document' diff --git a/mayan/apps/mayan_statistics/queues.py b/mayan/apps/mayan_statistics/queues.py index e529f59f24..ca6aebc345 100644 --- a/mayan/apps/mayan_statistics/queues.py +++ b/mayan/apps/mayan_statistics/queues.py @@ -3,9 +3,10 @@ from __future__ import unicode_literals from django.utils.translation import ugettext_lazy as _ from mayan.apps.task_manager.classes import CeleryQueue +from mayan.apps.task_manager.workers import worker_slow queue_statistics = CeleryQueue( - label=_('Statistics'), name='statistics', transient=True + label=_('Statistics'), name='statistics', transient=True, worker=worker_slow ) queue_statistics.add_task_type( diff --git a/mayan/apps/metadata/queues.py b/mayan/apps/metadata/queues.py index ba86bbf12d..f28a195e9d 100644 --- a/mayan/apps/metadata/queues.py +++ b/mayan/apps/metadata/queues.py @@ -3,9 +3,10 @@ from __future__ import absolute_import, unicode_literals from django.utils.translation import ugettext_lazy as _ from mayan.apps.task_manager.classes import CeleryQueue +from mayan.apps.task_manager.workers import worker_medium queue_metadata = CeleryQueue( - label=_('Metadata'), name='metadata' + label=_('Metadata'), name='metadata', worker=worker_medium ) queue_metadata.add_task_type( label=_('Remove metadata type'), diff --git a/mayan/apps/ocr/queues.py b/mayan/apps/ocr/queues.py index be89a399e1..dc3a3d81fe 100644 --- a/mayan/apps/ocr/queues.py +++ b/mayan/apps/ocr/queues.py @@ -3,8 +3,9 @@ from __future__ import unicode_literals from django.utils.translation import ugettext_lazy as _ from mayan.apps.task_manager.classes import CeleryQueue +from mayan.apps.task_manager.workers import worker_slow -queue_ocr = CeleryQueue(name='ocr', label=_('OCR')) +queue_ocr = CeleryQueue(name='ocr', label=_('OCR'), worker=worker_slow) queue_ocr.add_task_type( dotted_path='mayan.apps.ocr.tasks.task_do_ocr', label=_('Document version OCR') diff --git a/mayan/apps/platform/__init__.py b/mayan/apps/platform/__init__.py new file mode 100644 index 0000000000..4f2efbdd2a --- /dev/null +++ b/mayan/apps/platform/__init__.py @@ -0,0 +1,3 @@ +from __future__ import unicode_literals + +default_app_config = 'mayan.apps.platform.apps.PlatformApp' diff --git a/mayan/apps/platform/apps.py b/mayan/apps/platform/apps.py new file mode 100644 index 0000000000..320b65972d --- /dev/null +++ b/mayan/apps/platform/apps.py @@ -0,0 +1,101 @@ +from __future__ import absolute_import, unicode_literals + +from django.apps import apps +from django.db.models.signals import m2m_changed, pre_delete +from django.utils.translation import ugettext_lazy as _ + +from mayan.apps.acls.classes import ModelPermission +from mayan.apps.acls.links import link_acl_list +from mayan.apps.acls.permissions import permission_acl_edit, permission_acl_view +from mayan.apps.common.apps import MayanAppConfig +from mayan.apps.common.classes import ModelField +from mayan.apps.common.menus import ( + menu_facet, menu_list_facet, menu_main, menu_multi_item, menu_object, + menu_secondary +) +from mayan.apps.documents.search import document_page_search, document_search +from mayan.apps.events.classes import ModelEventType +from mayan.apps.events.links import ( + link_events_for_object, link_object_event_types_user_subcriptions_list, +) +from mayan.apps.events.permissions import permission_events_view +from mayan.apps.navigation.classes import SourceColumn + +""" +from .dependencies import * # NOQA +from .events import ( + event_tag_attach, event_tag_created, event_tag_edited, event_tag_remove +) +from .handlers import handler_index_document, handler_tag_pre_delete +from .html_widgets import widget_document_tags +from .links import ( + link_document_tag_list, link_document_multiple_attach_multiple_tag, + link_document_multiple_tag_multiple_remove, + link_document_tag_multiple_remove, link_document_tag_multiple_attach, link_tag_create, + link_tag_delete, link_tag_edit, link_tag_list, + link_tag_multiple_delete, link_tag_document_list +) +from .menus import menu_tags +from .methods import method_document_get_tags +from .permissions import ( + permission_tag_attach, permission_tag_delete, permission_tag_edit, + permission_tag_remove, permission_tag_view +) +from .search import tag_search # NOQA +""" +#from mayan.apps.common.classes import Template +from mayan.apps.task_manager.classes import Worker + + +class PlatformTemplate(object): + _registry = {} + context = {} + + @classmethod + def all(cls): + return cls._registry.values() + + @classmethod + def get(cls, name): + return cls._registry[name] + + def __init__(self, name, template_name): + self.name = name + self.template_name = template_name + self.__class__._registry[name] = self + + def get_context(self): + return self.context + + def render(self): + from django.template import loader + return loader.render_to_string( + template_name=self.template_name, + context=self.get_context() + ) + + + +class PlatformTemplateSupervisord(PlatformTemplate): + def get_context(self): + return {'workers': Worker.all()} + + +class PlatformApp(MayanAppConfig): + app_namespace = 'platform' + app_url = 'platform' + has_rest_api = False + has_tests = False + name = 'mayan.apps.platform' + verbose_name = _('Platform') + + def ready(self): + super(PlatformApp, self).ready() + + t = PlatformTemplateSupervisord( + name='platform_supervisord', + template_name='platform/supervisord.html' + ) + + print ("#@#@", t.render()) + print("!!!!", Worker.all()) diff --git a/mayan/apps/platform/classes.py b/mayan/apps/platform/classes.py new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/mayan/apps/platform/classes.py @@ -0,0 +1 @@ + diff --git a/mayan/apps/platform/templates/platform/supervisord.html b/mayan/apps/platform/templates/platform/supervisord.html new file mode 100644 index 0000000000..d11cf3cdc6 --- /dev/null +++ b/mayan/apps/platform/templates/platform/supervisord.html @@ -0,0 +1,52 @@ +[program:mayan-gunicorn] +autorestart = false +autostart = true +command = /bin/bash -c "${MAYAN_GUNICORN_BIN} -w ${MAYAN_GUNICORN_WORKERS} mayan.wsgi --max-requests 500 --max-requests-jitter 50 --worker-class gevent --bind 0.0.0.0:8000 --env DJANGO_SETTINGS_MODULE=${MAYAN_SETTINGS_MODULE}" --timeout 120 +redirect_stderr = true +stderr_logfile = /dev/fd/2 +stderr_logfile_maxbytes = 0 +stdout_logfile = /dev/fd/1 +stdout_logfile_maxbytes = 0 +user = mayan + +[program:redis] +autorestart = false +autostart = true +command = /bin/bash -c "if [ ${MAYAN_BROKER_URL} == ${MAYAN_DEFAULT_BROKER_URL} ] && [ ${MAYAN_CELERY_RESULT_BACKEND} == ${MAYAN_DEFAULT_CELERY_RESULT_BACKEND} ];then /usr/bin/redis-server /etc/redis/;fi" +stderr_logfile = /dev/fd/2 +stderr_logfile_maxbytes = 0 +stdout_logfile = /dev/fd/1 +stdout_logfile_maxbytes = 0 +user = root + +{% for worker in workers %} +[program:mayan-worker-{{ worker.name }}] +autorestart = false +autostart = true +command = nice -n {{ worker.nice_level }} /bin/bash -c "${MAYAN_BIN} celery --settings=${MAYAN_SETTINGS_MODULE} worker -Ofair -l ERROR -Q {% for queue in worker.queues %}{{ queue.name }}{% if not forloop.last %},{% endif %}{% endfor %} -n mayan-worker-{{ worker.name }}.%%h ${MAYAN_WORKER_{{ worker.name|upper }}_CONCURRENCY}" +killasgroup = true +numprocs = 1 +priority = 998 +startsecs = 10 +stderr_logfile = /dev/fd/2 +stderr_logfile_maxbytes = 0 +stdout_logfile = /dev/fd/1 +stdout_logfile_maxbytes = 0 +stopwaitsecs = 1 +user = mayan +{% endfor %} + +[program:mayan-celery-beat] +autorestart = false +autostart = true +command = nice -n 1 /bin/bash -c "${MAYAN_BIN} celery --settings=${MAYAN_SETTINGS_MODULE} beat --pidfile= -l ERROR" +killasgroup = true +numprocs = 1 +priority = 998 +stderr_logfile = /dev/fd/2 +stderr_logfile_maxbytes = 0 +stdout_logfile = /dev/fd/1 +stdout_logfile_maxbytes = 0 +startsecs = 10 +stopwaitsecs = 1 +user = mayan diff --git a/mayan/apps/sources/queues.py b/mayan/apps/sources/queues.py index 56eb0b36f0..45351faa36 100644 --- a/mayan/apps/sources/queues.py +++ b/mayan/apps/sources/queues.py @@ -3,15 +3,18 @@ from __future__ import unicode_literals from django.utils.translation import ugettext_lazy as _ from mayan.apps.task_manager.classes import CeleryQueue +from mayan.apps.task_manager.workers import worker_fast, worker_medium queue_sources = CeleryQueue( - label=_('Sources'), name='sources' + label=_('Sources'), name='sources', worker=worker_medium ) queue_sources_periodic = CeleryQueue( - label=_('Sources periodic'), name='sources_periodic', transient=True + label=_('Sources periodic'), name='sources_periodic', transient=True, + worker=worker_medium ) queue_sources_fast = CeleryQueue( - label=_('Sources fast'), name='sources_fast', transient=True + label=_('Sources fast'), name='sources_fast', transient=True, + worker=worker_fast ) queue_sources_fast.add_task_type( diff --git a/mayan/apps/task_manager/classes.py b/mayan/apps/task_manager/classes.py index 4b484c61fa..e36f1ef342 100644 --- a/mayan/apps/task_manager/classes.py +++ b/mayan/apps/task_manager/classes.py @@ -93,13 +93,14 @@ class CeleryQueue(object): for instance in cls.all(): instance._update_celery() - def __init__(self, name, label, default_queue=False, transient=False): + def __init__(self, name, label, worker, default_queue=False, transient=False): self.name = name self.label = label self.default_queue = default_queue self.transient = transient self.task_types = [] self.__class__._registry[name] = self + worker.queues.append(self) def __str__(self): return force_text(self.label) @@ -164,3 +165,18 @@ class CeleryQueue(object): }, } ) + + +class Worker(object): + _registry = {} + + @classmethod + def all(cls): + return cls._registry.values() + + def __init__(self, name, label=None, nice_level=0): + self.name = name + self.label = label + self.nice_level = nice_level + self.queues = [] + self.__class__._registry[name] = self diff --git a/mayan/apps/task_manager/workers.py b/mayan/apps/task_manager/workers.py new file mode 100644 index 0000000000..de370627de --- /dev/null +++ b/mayan/apps/task_manager/workers.py @@ -0,0 +1,7 @@ +from __future__ import absolute_import, unicode_literals + +from .classes import Worker + +worker_fast = Worker(name='fast', nice_level=1) +worker_medium = Worker(name='medium', nice_level=18) +worker_slow = Worker(name='slow', nice_level=19)