diff --git a/apps/clustering/__init__.py b/apps/clustering/__init__.py new file mode 100644 index 0000000000..4149c12347 --- /dev/null +++ b/apps/clustering/__init__.py @@ -0,0 +1,18 @@ +from __future__ import absolute_import + +from django.utils.translation import ugettext_lazy as _ + +from scheduler.api import register_interval_job +from navigation.api import bind_links +from project_tools.api import register_tool + +from .tasks import refresh_node +from .links import tool_link, node_list +from .models import Node + +NODE_REFRESH_INTERVAL = 1 + +register_interval_job('refresh_node', _(u'Update a node\'s properties.'), refresh_node, seconds=NODE_REFRESH_INTERVAL) + +register_tool(tool_link) +bind_links([Node, 'node_list'], [node_list], menu_name='secondary_menu') diff --git a/apps/clustering/admin.py b/apps/clustering/admin.py new file mode 100644 index 0000000000..ad92b0a549 --- /dev/null +++ b/apps/clustering/admin.py @@ -0,0 +1,13 @@ +from __future__ import absolute_import + +from django.contrib import admin +from django.utils.translation import ugettext_lazy as _ + +from .models import Node + + +class NodeAdmin(admin.ModelAdmin): + list_display = ('hostname', 'cpuload', 'heartbeat', 'memory_usage') + + +admin.site.register(Node, NodeAdmin) diff --git a/apps/clustering/links.py b/apps/clustering/links.py new file mode 100644 index 0000000000..606103a6de --- /dev/null +++ b/apps/clustering/links.py @@ -0,0 +1,10 @@ +from __future__ import absolute_import + +from django.utils.translation import ugettext_lazy as _ + +from navigation.api import Link + +from .permissions import (PERMISSION_NODES_VIEW) + +tool_link = Link(text=_(u'clustering'), view='node_list', icon='server.png', permissions=[PERMISSION_NODES_VIEW]) # children_view_regex=[r'^index_setup', r'^template_node']) +node_list = Link(text=_(u'node list'), view='node_list', sprite='server', permissions=[PERMISSION_NODES_VIEW]) diff --git a/apps/clustering/migrations/0001_initial.py b/apps/clustering/migrations/0001_initial.py new file mode 100644 index 0000000000..7932f6ade5 --- /dev/null +++ b/apps/clustering/migrations/0001_initial.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding model 'Node' + db.create_table('clustering_node', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('hostname', self.gf('django.db.models.fields.CharField')(max_length=255)), + ('cpuload', self.gf('django.db.models.fields.PositiveIntegerField')(default=0, blank=True)), + ('heartbeat', self.gf('django.db.models.fields.DateTimeField')(default=datetime.datetime(2012, 7, 30, 0, 0), blank=True)), + ('memory_usage', self.gf('django.db.models.fields.FloatField')(blank=True)), + )) + db.send_create_signal('clustering', ['Node']) + + + def backwards(self, orm): + # Deleting model 'Node' + db.delete_table('clustering_node') + + + models = { + 'clustering.node': { + 'Meta': {'object_name': 'Node'}, + 'cpuload': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0', 'blank': 'True'}), + 'heartbeat': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime(2012, 7, 30, 0, 0)', 'blank': 'True'}), + 'hostname': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'memory_usage': ('django.db.models.fields.FloatField', [], {'blank': 'True'}) + } + } + + complete_apps = ['clustering'] \ No newline at end of file diff --git a/apps/clustering/migrations/__init__.py b/apps/clustering/migrations/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/apps/clustering/models.py b/apps/clustering/models.py new file mode 100644 index 0000000000..67ded6cf04 --- /dev/null +++ b/apps/clustering/models.py @@ -0,0 +1,44 @@ +from __future__ import absolute_import + +import os +import datetime +import platform + +import psutil + +from django.db import models, IntegrityError, transaction +from django.db import close_connection +from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import ugettext + + +class NodeManager(models.Manager): + def myself(self): + node, created = self.model.objects.get_or_create(hostname=platform.node(), defaults={'memory_usage': 100}) + node.refresh() + return node + + +class Node(models.Model): + hostname = models.CharField(max_length=255, verbose_name=_(u'hostname')) + cpuload = models.PositiveIntegerField(blank=True, default=0, verbose_name=_(u'cpu load')) + heartbeat = models.DateTimeField(blank=True, default=datetime.datetime.now(), verbose_name=_(u'last heartbeat check')) + memory_usage = models.FloatField(blank=True, verbose_name=_(u'memory usage')) + + objects = NodeManager() + + def __unicode__(self): + return self.hostname + + def refresh(self): + self.cpuload = psutil.cpu_percent() + self.memory_usage = psutil.phymem_usage().percent + self.save() + + def save(self, *args, **kwargs): + self.heartbeat = datetime.datetime.now() + return super(Node, self).save(*args, **kwargs) + + class Meta: + verbose_name = _(u'node') + verbose_name_plural = _(u'nodes') diff --git a/apps/clustering/permissions.py b/apps/clustering/permissions.py new file mode 100644 index 0000000000..6065936140 --- /dev/null +++ b/apps/clustering/permissions.py @@ -0,0 +1,8 @@ +from __future__ import absolute_import + +from django.utils.translation import ugettext_lazy as _ + +from permissions.models import PermissionNamespace, Permission + +namespace = PermissionNamespace('clustering', _(u'Clustering')) +PERMISSION_NODES_VIEW = Permission.objects.register(namespace, 'nodes_view', _(u'View the nodes in a Mayan cluster')) diff --git a/apps/clustering/tasks.py b/apps/clustering/tasks.py new file mode 100644 index 0000000000..aa9f01b53e --- /dev/null +++ b/apps/clustering/tasks.py @@ -0,0 +1,17 @@ +from __future__ import absolute_import + +import logging + +from lock_manager.decorators import simple_locking + +from .models import Node + +LOCK_EXPIRE = 10 + +logger = logging.getLogger(__name__) + + +@simple_locking('refresh_node', 10) +def refresh_node(): + logger.debug('starting') + node = Node.objects.myself() # Automatically calls the refresh() method too diff --git a/apps/clustering/urls.py b/apps/clustering/urls.py new file mode 100644 index 0000000000..e43cf0041d --- /dev/null +++ b/apps/clustering/urls.py @@ -0,0 +1,6 @@ +from django.conf.urls.defaults import patterns, url + + +urlpatterns = patterns('clustering.views', + url(r'^node/list/$', 'node_list', (), 'node_list'), +) diff --git a/apps/clustering/views.py b/apps/clustering/views.py new file mode 100644 index 0000000000..efaafe1ad5 --- /dev/null +++ b/apps/clustering/views.py @@ -0,0 +1,67 @@ +from __future__ import absolute_import + +from django.shortcuts import render_to_response +from django.template import RequestContext +from django.utils.translation import ugettext_lazy as _ +from django.shortcuts import get_object_or_404 +from django.db.models.loading import get_model +from django.http import Http404 +from django.core.exceptions import PermissionDenied + +from permissions.models import Permission +from common.utils import encapsulate +from acls.models import AccessEntry + +from .models import Node +from .permissions import PERMISSION_NODES_VIEW + + +def node_list(request): + Permission.objects.check_permissions(request.user, [PERMISSION_NODES_VIEW]) + + context = { + 'object_list': Node.objects.all(), + 'title': _(u'nodes'), + 'extra_columns': [ + { + 'name': _(u'hostname'), + 'attribute': 'hostname', + }, + { + 'name': _(u'cpu load'), + 'attribute': 'cpuload', + }, + { + 'name': _(u'heartbeat'), + 'attribute': 'heartbeat', + }, + { + 'name': _(u'memory usage'), + 'attribute': 'memory_usage', + }, + + ], + 'hide_object': True, + } + + return render_to_response('generic_list.html', context, + context_instance=RequestContext(request)) + + +def node_workers(request, node_pk): + node = get_object_or_404(Node, pk=node_pk) + + try: + Permission.objects.check_permissions(request.user, [PERMISSION_NODES_VIEW]) + except PermissionDenied: + AccessEntry.objects.check_access(PERMISSION_NODES_VIEW, request.user, node) + + context = { + 'object_list': node.workers.all(), + 'title': _(u'workers for node: %s') % node, + 'object': node, + 'hide_object': True, + } + + return render_to_response('generic_list.html', context, + context_instance=RequestContext(request)) diff --git a/settings.py b/settings.py index c627ac33f6..aa9dc1e814 100644 --- a/settings.py +++ b/settings.py @@ -162,6 +162,7 @@ INSTALLED_APPS = ( 'converter', 'user_management', 'mimetype', + 'clustering', 'scheduler', 'job_processor', # Mayan EDMS diff --git a/urls.py b/urls.py index 4a5ebe61ec..82b10cc5d1 100644 --- a/urls.py +++ b/urls.py @@ -36,10 +36,12 @@ urlpatterns = patterns('', (r'^checkouts/', include('checkouts.urls')), (r'^installation/', include('installation.urls')), (r'^scheduler/', include('scheduler.urls')), + (r'^job_processing/', include('job_processor.urls')), (r'^bootstrap/', include('bootstrap.urls')), (r'^diagnostics/', include('diagnostics.urls')), (r'^maintenance/', include('maintenance.urls')), (r'^statistics/', include('statistics.urls')), + (r'^clustering/', include('clustering.urls')), )