Add clustering app

apps/clustering/__init__.py

@@ -0,0 +1,18 @@
+from __future__ import absolute_import
+
+from django.utils.translation import ugettext_lazy as _
+
+from scheduler.api import register_interval_job
+from navigation.api import bind_links
+from project_tools.api import register_tool
+
+from .tasks import refresh_node
+from .links import tool_link, node_list
+from .models import Node
+
+NODE_REFRESH_INTERVAL = 1
+
+register_interval_job('refresh_node', _(u'Update a node\'s properties.'), refresh_node, seconds=NODE_REFRESH_INTERVAL)
+
+register_tool(tool_link)
+bind_links([Node, 'node_list'], [node_list], menu_name='secondary_menu')

apps/clustering/admin.py

@@ -0,0 +1,13 @@
+from __future__ import absolute_import
+
+from django.contrib import admin
+from django.utils.translation import ugettext_lazy as _
+
+from .models import Node
+
+
+class NodeAdmin(admin.ModelAdmin):
+    list_display = ('hostname', 'cpuload', 'heartbeat', 'memory_usage')
+
+
+admin.site.register(Node, NodeAdmin)

apps/clustering/links.py

@@ -0,0 +1,10 @@
+from __future__ import absolute_import
+
+from django.utils.translation import ugettext_lazy as _
+
+from navigation.api import Link
+
+from .permissions import (PERMISSION_NODES_VIEW)
+
+tool_link = Link(text=_(u'clustering'), view='node_list', icon='server.png', permissions=[PERMISSION_NODES_VIEW])  # children_view_regex=[r'^index_setup', r'^template_node'])
+node_list = Link(text=_(u'node list'), view='node_list', sprite='server', permissions=[PERMISSION_NODES_VIEW])

apps/clustering/migrations/0001_initial.py

@@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding model 'Node'
+        db.create_table('clustering_node', (
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('hostname', self.gf('django.db.models.fields.CharField')(max_length=255)),
+            ('cpuload', self.gf('django.db.models.fields.PositiveIntegerField')(default=0, blank=True)),
+            ('heartbeat', self.gf('django.db.models.fields.DateTimeField')(default=datetime.datetime(2012, 7, 30, 0, 0), blank=True)),
+            ('memory_usage', self.gf('django.db.models.fields.FloatField')(blank=True)),
+        ))
+        db.send_create_signal('clustering', ['Node'])
+
+
+    def backwards(self, orm):
+        # Deleting model 'Node'
+        db.delete_table('clustering_node')
+
+
+    models = {
+        'clustering.node': {
+            'Meta': {'object_name': 'Node'},
+            'cpuload': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0', 'blank': 'True'}),
+            'heartbeat': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime(2012, 7, 30, 0, 0)', 'blank': 'True'}),
+            'hostname': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'memory_usage': ('django.db.models.fields.FloatField', [], {'blank': 'True'})
+        }
+    }
+
+    complete_apps = ['clustering']

apps/clustering/models.py

@@ -0,0 +1,44 @@
+from __future__ import absolute_import
+
+import os
+import datetime
+import platform
+
+import psutil
+
+from django.db import models, IntegrityError, transaction
+from django.db import close_connection
+from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import ugettext
+
+
+class NodeManager(models.Manager):
+    def myself(self):
+        node, created = self.model.objects.get_or_create(hostname=platform.node(), defaults={'memory_usage': 100})
+        node.refresh()
+        return node
+
+
+class Node(models.Model):
+    hostname = models.CharField(max_length=255, verbose_name=_(u'hostname'))
+    cpuload = models.PositiveIntegerField(blank=True, default=0, verbose_name=_(u'cpu load'))
+    heartbeat = models.DateTimeField(blank=True, default=datetime.datetime.now(), verbose_name=_(u'last heartbeat check'))
+    memory_usage = models.FloatField(blank=True, verbose_name=_(u'memory usage'))
+    
+    objects = NodeManager()
+
+    def __unicode__(self):
+        return self.hostname
+        
+    def refresh(self):
+        self.cpuload = psutil.cpu_percent()
+        self.memory_usage = psutil.phymem_usage().percent
+        self.save()
+   
+    def save(self, *args, **kwargs):
+        self.heartbeat = datetime.datetime.now()
+        return super(Node, self).save(*args, **kwargs)
+
+    class Meta:
+        verbose_name = _(u'node')
+        verbose_name_plural = _(u'nodes')

apps/clustering/permissions.py

@@ -0,0 +1,8 @@
+from __future__ import absolute_import
+
+from django.utils.translation import ugettext_lazy as _
+
+from permissions.models import PermissionNamespace, Permission
+
+namespace = PermissionNamespace('clustering', _(u'Clustering'))
+PERMISSION_NODES_VIEW = Permission.objects.register(namespace, 'nodes_view', _(u'View the nodes in a Mayan cluster'))

apps/clustering/tasks.py

@@ -0,0 +1,17 @@
+from __future__ import absolute_import
+
+import logging
+
+from lock_manager.decorators import simple_locking
+
+from .models import Node
+
+LOCK_EXPIRE = 10
+
+logger = logging.getLogger(__name__)
+
+
+@simple_locking('refresh_node', 10)
+def refresh_node():
+    logger.debug('starting')
+    node = Node.objects.myself()  # Automatically calls the refresh() method too

apps/clustering/urls.py

@@ -0,0 +1,6 @@
+from django.conf.urls.defaults import patterns, url
+
+
+urlpatterns = patterns('clustering.views',
+    url(r'^node/list/$', 'node_list', (), 'node_list'),
+)

apps/clustering/views.py

@@ -0,0 +1,67 @@
+from __future__ import absolute_import
+
+from django.shortcuts import render_to_response
+from django.template import RequestContext
+from django.utils.translation import ugettext_lazy as _
+from django.shortcuts import get_object_or_404
+from django.db.models.loading import get_model
+from django.http import Http404
+from django.core.exceptions import PermissionDenied
+
+from permissions.models import Permission
+from common.utils import encapsulate
+from acls.models import AccessEntry
+
+from .models import Node
+from .permissions import PERMISSION_NODES_VIEW
+
+
+def node_list(request):
+    Permission.objects.check_permissions(request.user, [PERMISSION_NODES_VIEW])
+
+    context = {
+        'object_list': Node.objects.all(),
+        'title': _(u'nodes'),
+        'extra_columns': [
+            {
+                'name': _(u'hostname'),
+                'attribute': 'hostname',
+            },
+            {
+                'name': _(u'cpu load'),
+                'attribute': 'cpuload',
+            },
+            {
+                'name': _(u'heartbeat'),
+                'attribute': 'heartbeat',
+            },
+            {
+                'name': _(u'memory usage'),
+                'attribute': 'memory_usage',
+            },
+
+        ],
+        'hide_object': True,
+    }
+
+    return render_to_response('generic_list.html', context,
+        context_instance=RequestContext(request))
+
+
+def node_workers(request, node_pk):
+    node = get_object_or_404(Node, pk=node_pk)
+
+    try:
+        Permission.objects.check_permissions(request.user, [PERMISSION_NODES_VIEW])
+    except PermissionDenied:
+        AccessEntry.objects.check_access(PERMISSION_NODES_VIEW, request.user, node)
+
+    context = {
+        'object_list': node.workers.all(),
+        'title': _(u'workers for node: %s') % node,
+        'object': node,
+        'hide_object': True,
+    }
+
+    return render_to_response('generic_list.html', context,
+        context_instance=RequestContext(request))

settings.py

@@ -162,6 +162,7 @@ INSTALLED_APPS = (
     'converter',
     'user_management',
     'mimetype',
+    'clustering',
     'scheduler',
     'job_processor',
     # Mayan EDMS

urls.py

@@ -36,10 +36,12 @@ urlpatterns = patterns('',
     (r'^checkouts/', include('checkouts.urls')),
     (r'^installation/', include('installation.urls')),
     (r'^scheduler/', include('scheduler.urls')),
+    (r'^job_processing/', include('job_processor.urls')),
     (r'^bootstrap/', include('bootstrap.urls')),
     (r'^diagnostics/', include('diagnostics.urls')),
     (r'^maintenance/', include('maintenance.urls')),
     (r'^statistics/', include('statistics.urls')),
+    (r'^clustering/', include('clustering.urls')),
 )