diff --git a/apps/clustering/__init__.py b/apps/clustering/__init__.py index 66a8549c26..ffbf7e2648 100644 --- a/apps/clustering/__init__.py +++ b/apps/clustering/__init__.py @@ -1,19 +1,31 @@ from __future__ import absolute_import from django.utils.translation import ugettext_lazy as _ +from django.db import transaction, DatabaseError from scheduler.api import LocalScheduler from navigation.api import bind_links from project_tools.api import register_tool +from project_setup.api import register_setup from .tasks import node_heartbeat, house_keeping -from .links import tool_link, node_list +from .links import tool_link, node_list, clustering_config_edit, setup_link from .models import Node, ClusteringConfig -clustering_scheduler = LocalScheduler('clustering', _(u'Clustering')) -clustering_scheduler.add_interval_job('node_heartbeat', _(u'Update a node\'s properties.'), node_heartbeat, seconds=ClusteringConfig.get().node_heartbeat_interval) -clustering_scheduler.add_interval_job('house_keeping', _(u'Check for unresponsive nodes in the cluster list.'), house_keeping, seconds=1) -clustering_scheduler.start() +@transaction.commit_on_success +def add_clustering_jobs(): + clustering_scheduler = LocalScheduler('clustering', _(u'Clustering')) + try: + clustering_scheduler.add_interval_job('node_heartbeat', _(u'Update a node\'s properties.'), node_heartbeat, seconds=ClusteringConfig.get().node_heartbeat_interval) + clustering_scheduler.add_interval_job('house_keeping', _(u'Check for unresponsive nodes in the cluster list.'), house_keeping, seconds=1) + except DatabaseError: + transaction.rollback() + clustering_scheduler.start() + + +add_clustering_jobs() register_tool(tool_link) +register_setup(setup_link) bind_links([Node, 'node_list'], [node_list], menu_name='secondary_menu') +bind_links(['clustering_config_edit'], [clustering_config_edit], menu_name='secondary_menu') diff --git a/apps/clustering/forms.py b/apps/clustering/forms.py new file mode 100644 index 0000000000..e0e38e4755 --- /dev/null +++ b/apps/clustering/forms.py @@ -0,0 +1,10 @@ +from __future__ import absolute_import + +from django import forms + +from .models import ClusteringConfig + + +class ClusteringConfigForm(forms.ModelForm): + class Meta: + model = ClusteringConfig diff --git a/apps/clustering/links.py b/apps/clustering/links.py index 606103a6de..009a2aa584 100644 --- a/apps/clustering/links.py +++ b/apps/clustering/links.py @@ -4,7 +4,9 @@ from django.utils.translation import ugettext_lazy as _ from navigation.api import Link -from .permissions import (PERMISSION_NODES_VIEW) +from .permissions import PERMISSION_NODES_VIEW, PERMISSION_EDIT_CLUSTER_CONFIGURATION tool_link = Link(text=_(u'clustering'), view='node_list', icon='server.png', permissions=[PERMISSION_NODES_VIEW]) # children_view_regex=[r'^index_setup', r'^template_node']) node_list = Link(text=_(u'node list'), view='node_list', sprite='server', permissions=[PERMISSION_NODES_VIEW]) +clustering_config_edit = Link(text=_(u'edit cluster configuration'), view='clustering_config_edit', sprite='server_edit', permissions=[PERMISSION_EDIT_CLUSTER_CONFIGURATION]) +setup_link = Link(text=_(u'cluster configuration'), view='clustering_config_edit', icon='server.png', permissions=[PERMISSION_EDIT_CLUSTER_CONFIGURATION]) diff --git a/apps/clustering/literals.py b/apps/clustering/literals.py new file mode 100644 index 0000000000..99c97f50d2 --- /dev/null +++ b/apps/clustering/literals.py @@ -0,0 +1,3 @@ +DEFAULT_NODE_HEARTBEAT_INTERVAL = 10 +DEFAULT_NODE_HEARTBEAT_TIMEOUT = 60 +DEFAULT_DEAD_NODE_REMOVAL_INTERVAL = 10 diff --git a/apps/clustering/migrations/0004_auto__del_field_clusteringconfig_node_time_to_live__add_field_clusteri.py b/apps/clustering/migrations/0004_auto__del_field_clusteringconfig_node_time_to_live__add_field_clusteri.py new file mode 100644 index 0000000000..f31ce93bd8 --- /dev/null +++ b/apps/clustering/migrations/0004_auto__del_field_clusteringconfig_node_time_to_live__add_field_clusteri.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Deleting field 'ClusteringConfig.node_time_to_live' + db.delete_column('clustering_clusteringconfig', 'node_time_to_live') + + # Adding field 'ClusteringConfig.node_heartbeat_timeout' + db.add_column('clustering_clusteringconfig', 'node_heartbeat_timeout', + self.gf('django.db.models.fields.PositiveIntegerField')(default=5), + keep_default=False) + + + def backwards(self, orm): + # Adding field 'ClusteringConfig.node_time_to_live' + db.add_column('clustering_clusteringconfig', 'node_time_to_live', + self.gf('django.db.models.fields.PositiveIntegerField')(default=5), + keep_default=False) + + # Deleting field 'ClusteringConfig.node_heartbeat_timeout' + db.delete_column('clustering_clusteringconfig', 'node_heartbeat_timeout') + + + models = { + 'clustering.clusteringconfig': { + 'Meta': {'object_name': 'ClusteringConfig'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'lock_id': ('django.db.models.fields.CharField', [], {'default': '1', 'unique': 'True', 'max_length': '1'}), + 'node_heartbeat_interval': ('django.db.models.fields.PositiveIntegerField', [], {'default': '1'}), + 'node_heartbeat_timeout': ('django.db.models.fields.PositiveIntegerField', [], {'default': '5'}) + }, + 'clustering.node': { + 'Meta': {'object_name': 'Node'}, + 'cpuload': ('django.db.models.fields.FloatField', [], {'default': '0.0', 'blank': 'True'}), + 'heartbeat': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime(2012, 8, 1, 0, 0)', 'blank': 'True'}), + 'hostname': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'memory_usage': ('django.db.models.fields.FloatField', [], {'default': '0.0', 'blank': 'True'}) + } + } + + complete_apps = ['clustering'] \ No newline at end of file diff --git a/apps/clustering/migrations/0005_auto__add_field_clusteringconfig_dead_node_removal_interval.py b/apps/clustering/migrations/0005_auto__add_field_clusteringconfig_dead_node_removal_interval.py new file mode 100644 index 0000000000..a31695a24e --- /dev/null +++ b/apps/clustering/migrations/0005_auto__add_field_clusteringconfig_dead_node_removal_interval.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding field 'ClusteringConfig.dead_node_removal_interval' + db.add_column('clustering_clusteringconfig', 'dead_node_removal_interval', + self.gf('django.db.models.fields.PositiveIntegerField')(default=10), + keep_default=False) + + + def backwards(self, orm): + # Deleting field 'ClusteringConfig.dead_node_removal_interval' + db.delete_column('clustering_clusteringconfig', 'dead_node_removal_interval') + + + models = { + 'clustering.clusteringconfig': { + 'Meta': {'object_name': 'ClusteringConfig'}, + 'dead_node_removal_interval': ('django.db.models.fields.PositiveIntegerField', [], {'default': '10'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'lock_id': ('django.db.models.fields.CharField', [], {'default': '1', 'unique': 'True', 'max_length': '1'}), + 'node_heartbeat_interval': ('django.db.models.fields.PositiveIntegerField', [], {'default': '10'}), + 'node_heartbeat_timeout': ('django.db.models.fields.PositiveIntegerField', [], {'default': '60'}) + }, + 'clustering.node': { + 'Meta': {'object_name': 'Node'}, + 'cpuload': ('django.db.models.fields.FloatField', [], {'default': '0.0', 'blank': 'True'}), + 'heartbeat': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime(2012, 8, 1, 0, 0)', 'blank': 'True'}), + 'hostname': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'memory_usage': ('django.db.models.fields.FloatField', [], {'default': '0.0', 'blank': 'True'}) + } + } + + complete_apps = ['clustering'] \ No newline at end of file diff --git a/apps/clustering/models.py b/apps/clustering/models.py index b083829f74..3715184310 100644 --- a/apps/clustering/models.py +++ b/apps/clustering/models.py @@ -6,15 +6,14 @@ import platform import psutil -from django.db import models, IntegrityError, transaction -from django.db import close_connection -from django.utils.translation import ugettext_lazy as _ -from django.utils.translation import ugettext +from django.core.exceptions import ValidationError +from django.db import models +from django.utils.translation import ugettext, ugettext_lazy as _ from common.models import Singleton -DEFAULT_NODE_TTL = 5 -DEFAULT_NODE_HEARTBEAT_INTERVAL = 1 +from .literals import (DEFAULT_NODE_HEARTBEAT_INTERVAL, DEFAULT_NODE_HEARTBEAT_TIMEOUT, + DEFAULT_DEAD_NODE_REMOVAL_INTERVAL) class NodeManager(models.Manager): @@ -63,7 +62,7 @@ class Node(models.Model): class ClusteringConfigManager(models.Manager): def dead_nodes(self): - return Node.objects.filter(heartbeat__lt=datetime.datetime.now() - datetime.timedelta(seconds=self.model.get().node_time_to_live)) + return Node.objects.filter(heartbeat__lt=datetime.datetime.now() - datetime.timedelta(seconds=self.model.get().node_heartbeat_timeout)) def delete_dead_nodes(self): self.dead_nodes().delete() @@ -76,14 +75,18 @@ class ClusteringConfigManager(models.Manager): class ClusteringConfig(Singleton): - node_time_to_live = models.PositiveIntegerField(verbose_name=(u'time to live (in seconds)'), default=DEFAULT_NODE_TTL) # After this time a worker is considered dead - node_heartbeat_interval = models.PositiveIntegerField(verbose_name=(u'heartbeat interval'), default=DEFAULT_NODE_HEARTBEAT_INTERVAL) - # TODO: add validation, interval cannot be greater than TTL + node_heartbeat_interval = models.PositiveIntegerField(verbose_name=(u'node heartbeat interval (in seconds)'), help_text=_(u'Interval of time for the node\'s heartbeat update to the cluster.'), default=DEFAULT_NODE_HEARTBEAT_INTERVAL) + node_heartbeat_timeout = models.PositiveIntegerField(verbose_name=(u'node heartbeat timeout (in seconds)'), help_text=_(u'After this amount of time a node without heartbeat updates is considered dead and removed from the cluster node list.'), default=DEFAULT_NODE_HEARTBEAT_TIMEOUT) + dead_node_removal_interval = models.PositiveIntegerField(verbose_name=(u'dead node check and removal interval (in seconds)'), help_text=_(u'Interval of time to check the cluster for unresponsive nodes and remove them from the cluster.'), default=DEFAULT_DEAD_NODE_REMOVAL_INTERVAL) objects = ClusteringConfigManager() def __unicode__(self): return ugettext('clustering config') + def clean(self): + if self.node_heartbeat_interval > self.node_heartbeat_timeout: + raise ValidationError(_(u'Heartbeat interval cannot be greater than heartbeat timeout or else nodes will always be rated as "dead"')) + class Meta: verbose_name = verbose_name_plural = _(u'clustering config') diff --git a/apps/clustering/permissions.py b/apps/clustering/permissions.py index 6065936140..60c34a78dd 100644 --- a/apps/clustering/permissions.py +++ b/apps/clustering/permissions.py @@ -6,3 +6,4 @@ from permissions.models import PermissionNamespace, Permission namespace = PermissionNamespace('clustering', _(u'Clustering')) PERMISSION_NODES_VIEW = Permission.objects.register(namespace, 'nodes_view', _(u'View the nodes in a Mayan cluster')) +PERMISSION_EDIT_CLUSTER_CONFIGURATION = Permission.objects.register(namespace, 'cluster_config', _(u'Edit the configuration of a Mayan cluster')) diff --git a/apps/clustering/urls.py b/apps/clustering/urls.py index e43cf0041d..fbeeaa1984 100644 --- a/apps/clustering/urls.py +++ b/apps/clustering/urls.py @@ -3,4 +3,5 @@ from django.conf.urls.defaults import patterns, url urlpatterns = patterns('clustering.views', url(r'^node/list/$', 'node_list', (), 'node_list'), + url(r'^edit/$', 'clustering_config_edit', (), 'clustering_config_edit'), ) diff --git a/apps/clustering/views.py b/apps/clustering/views.py index e00f7b4da2..675299916f 100644 --- a/apps/clustering/views.py +++ b/apps/clustering/views.py @@ -3,17 +3,19 @@ from __future__ import absolute_import from django.shortcuts import render_to_response from django.template import RequestContext from django.utils.translation import ugettext_lazy as _ -from django.shortcuts import get_object_or_404 +from django.shortcuts import get_object_or_404, HttpResponseRedirect from django.db.models.loading import get_model from django.http import Http404 from django.core.exceptions import PermissionDenied +from django.contrib import messages from permissions.models import Permission from common.utils import encapsulate from acls.models import AccessEntry -from .models import Node -from .permissions import PERMISSION_NODES_VIEW +from .forms import ClusteringConfigForm +from .models import Node, ClusteringConfig +from .permissions import PERMISSION_NODES_VIEW, PERMISSION_EDIT_CLUSTER_CONFIGURATION def node_list(request): @@ -65,3 +67,34 @@ def node_workers(request, node_pk): return render_to_response('generic_list.html', context, context_instance=RequestContext(request)) + + +def clustering_config_edit(request): + Permission.objects.check_permissions(request.user, [PERMISSION_EDIT_CLUSTER_CONFIGURATION]) + + cluster_config = ClusteringConfig.get() + + post_action_redirect = None + + previous = request.POST.get('previous', request.GET.get('previous', request.META.get('HTTP_REFERER', '/'))) + next = request.POST.get('next', request.GET.get('next', post_action_redirect if post_action_redirect else request.META.get('HTTP_REFERER', '/'))) + + + if request.method == 'POST': + form = ClusteringConfigForm(data=request.POST) + if form.is_valid(): + try: + form.save() + except Exception, exc: + messages.error(request, _(u'Error trying to edit cluster configuration; %s') % exc) + else: + messages.success(request, _(u'Cluster configuration edited successfully.')) + return HttpResponseRedirect(next) + else: + form = ClusteringConfigForm(instance=cluster_config) + + return render_to_response('generic_form.html', { + 'form': form, + 'object': cluster_config, + 'title': _(u'Edit cluster configuration') + }, context_instance=RequestContext(request))