Add job queue priority support

apps/job_processor/tasks.py

@@ -1,6 +1,7 @@
 from __future__ import absolute_import
 
 import logging
+import psutil
 
 from lock_manager import Lock, LockError
 from lock_manager.decorators import simple_locking
@@ -13,7 +14,7 @@ from .literals import JOB_QUEUE_STATE_STARTED
 LOCK_EXPIRE = 10
 MAX_CPU_LOAD = 90.0
 MAX_MEMORY_USAGE = 90.0
-NODE_MAX_WORKERS = 1
+NODE_MAX_WORKERS = len(psutil.cpu_percent(interval=0.1, percpu=True))  # Get CPU/cores count
 
 logger = logging.getLogger(__name__)
 
@@ -27,21 +28,19 @@ def job_queue_poll():
         lock = Lock.acquire_lock(lock_id, LOCK_EXPIRE)
     except LockError:
         pass
-    except Exception:
-        lock.release()
-        raise
     else:
         node = Node.objects.myself()
-        if node.cpuload < MAX_CPU_LOAD and node.memory_usage < MAX_MEMORY_USAGE and node.worker_set.count()<=NODE_MAX_WORKERS:
+        if node.cpuload < MAX_CPU_LOAD and node.memory_usage < MAX_MEMORY_USAGE and node.worker_set.count() < NODE_MAX_WORKERS:
             for job_queue in JobQueue.objects.filter(state=JOB_QUEUE_STATE_STARTED):
                 try:
                     job_item = job_queue.get_oldest_pending_job()
                     job_item.run()
+                    break;
                 except JobQueueNoPendingJobs:
                     logger.debug('no pending jobs for job queue: %s' % job_queue)
         else:
             logger.debug('CPU load or memory usage over limit')
-
+    finally:
         lock.release()