107 lines
4.1 KiB
Python
107 lines
4.1 KiB
Python
from __future__ import absolute_import
|
|
|
|
import os
|
|
import datetime
|
|
import platform
|
|
|
|
import psutil
|
|
|
|
from django.core.exceptions import ValidationError
|
|
from django.db import models
|
|
from django.utils.translation import ugettext, ugettext_lazy as _
|
|
|
|
from common.models import Singleton
|
|
|
|
from .literals import (DEFAULT_NODE_HEARTBEAT_INTERVAL, DEFAULT_NODE_HEARTBEAT_TIMEOUT,
|
|
DEFAULT_DEAD_NODE_REMOVAL_INTERVAL, NODE_STATE_HEALTHY, NODE_STATE_CHOICES, NODE_STATE_DEAD,
|
|
DEFAULT_NODE_CPU_LOAD, DEFAULT_NODE_MEMORY_USAGE)
|
|
from .signals import node_died
|
|
|
|
|
|
class NodeManager(models.Manager):
|
|
def myself(self):
|
|
node, created = self.model.objects.get_or_create(hostname=platform.node())
|
|
node.refresh()
|
|
if created:
|
|
# Store the refresh data because is a new instance
|
|
node.save()
|
|
return node
|
|
|
|
def live_nodes(self):
|
|
return self.model.objects.filter(state=NODE_STATE_HEALTHY)
|
|
|
|
|
|
class Node(models.Model):
|
|
hostname = models.CharField(max_length=255, verbose_name=_(u'hostname'), unique=True)
|
|
cpuload = models.FloatField(blank=True, default=DEFAULT_NODE_CPU_LOAD, verbose_name=_(u'cpu load'))
|
|
heartbeat = models.DateTimeField(blank=True, default=datetime.datetime.now(), verbose_name=_(u'last heartbeat check'))
|
|
memory_usage = models.FloatField(blank=True, default=DEFAULT_NODE_MEMORY_USAGE, verbose_name=_(u'memory usage'))
|
|
state = models.CharField(max_length=4,
|
|
choices=NODE_STATE_CHOICES,
|
|
default=NODE_STATE_DEAD,
|
|
verbose_name=_(u'state'))
|
|
|
|
objects = NodeManager()
|
|
|
|
@classmethod
|
|
def platform_info(cls):
|
|
return {
|
|
'cpuload': psutil.cpu_percent(),
|
|
'memory_usage': psutil.phymem_usage().percent
|
|
}
|
|
|
|
def __unicode__(self):
|
|
return self.hostname
|
|
|
|
def refresh(self):
|
|
if self.hostname == platform.node():
|
|
# Make sure we can only update ourselves
|
|
info = Node.platform_info()
|
|
self.cpuload = info['cpuload']
|
|
self.memory_usage = info['memory_usage']
|
|
|
|
def is_healthy(self):
|
|
return self.state == NODE_STATE_HEALTHY
|
|
|
|
def mark_as_dead(self):
|
|
self.state=NODE_STATE_DEAD
|
|
node_died.send(sender=self, node=self)
|
|
self.save()
|
|
|
|
def send_heartbeat(self):
|
|
self.refresh()
|
|
self.state=NODE_STATE_HEALTHY
|
|
self.heartbeat = datetime.datetime.now()
|
|
self.save()
|
|
|
|
class Meta:
|
|
verbose_name = _(u'node')
|
|
verbose_name_plural = _(u'nodes')
|
|
|
|
|
|
class ClusteringConfigManager(models.Manager):
|
|
def dead_nodes(self):
|
|
return Node.objects.filter(state=NODE_STATE_HEALTHY).filter(heartbeat__lt=datetime.datetime.now() - datetime.timedelta(seconds=self.model.get().node_heartbeat_timeout))
|
|
|
|
def check_dead_nodes(self):
|
|
for node in self.dead_nodes():
|
|
node.mark_as_dead()
|
|
|
|
|
|
class ClusteringConfig(Singleton):
|
|
node_heartbeat_interval = models.PositiveIntegerField(verbose_name=(u'node heartbeat interval (in seconds)'), help_text=_(u'Interval of time for the node\'s heartbeat update to the cluster.'), default=DEFAULT_NODE_HEARTBEAT_INTERVAL)
|
|
node_heartbeat_timeout = models.PositiveIntegerField(verbose_name=(u'node heartbeat timeout (in seconds)'), help_text=_(u'After this amount of time a node without heartbeat updates is considered dead and removed from the cluster node list.'), default=DEFAULT_NODE_HEARTBEAT_TIMEOUT)
|
|
dead_node_removal_interval = models.PositiveIntegerField(verbose_name=(u'dead node check and removal interval (in seconds)'), help_text=_(u'Interval of time to check the cluster for unresponsive nodes and remove them from the cluster.'), default=DEFAULT_DEAD_NODE_REMOVAL_INTERVAL)
|
|
|
|
cluster = ClusteringConfigManager()
|
|
|
|
def __unicode__(self):
|
|
return ugettext('clustering config')
|
|
|
|
#def clean(self):
|
|
# if self.node_heartbeat_interval > self.node_heartbeat_timeout:
|
|
# raise ValidationError(_(u'Heartbeat interval cannot be greater than heartbeat timeout or else nodes will always be rated as "dead"'))
|
|
|
|
class Meta:
|
|
verbose_name = verbose_name_plural = _(u'clustering config')
|