Add locking to avoid index rebuild clashing with index updates or index node deletion

mayan/apps/document_indexing/tasks.py

@@ -11,14 +11,28 @@ logger = logging.getLogger(__name__)
 RETRY_DELAY = 20  # TODO: convert this into a config option
 
 
-@app.task(ignore_result=True)
+@app.task(bind=True, ignore_result=True)
-def task_delete_empty_index_nodes():
+def task_delete_empty_index_nodes(self):
+    try:
+        rebuild_lock = Lock.acquire_lock('document_indexing_task_do_rebuild_all_indexes')
+    except LockError as exception:
+        # A rebuild is happening, retry later
+        raise self.retry(exc=exception, countdown=RETRY_DELAY)
+    else:
+        try:
             delete_empty_index_nodes()
+        finally:
+            rebuild_lock.release()
 
 
 @app.task(bind=True, ignore_result=True)
 def task_index_document(self, document_id):
-    # TODO: Add concurrent task control
+    try:
+        rebuild_lock = Lock.acquire_lock('document_indexing_task_do_rebuild_all_indexes')
+    except LockError as exception:
+        # A rebuild is happening, retry later
+        raise self.retry(exc=exception, countdown=RETRY_DELAY)
+    else:
         try:
             lock = Lock.acquire_lock('document_indexing_task_update_index_document_%d' % document_id)
         except LockError as exception:
@@ -34,9 +48,23 @@ def task_index_document(self, document_id):
                 index_document(document)
             finally:
                 lock.release()
+        finally:
+            rebuild_lock.release()
 
 
-@app.task(ignore_result=True)
+@app.task(bind=True, ignore_result=True)
-def task_do_rebuild_all_indexes():
+def task_do_rebuild_all_indexes(self):
-    # TODO: Find a way to rebuild after all pending updates are finished
+    if Lock.filter(name__startswith='document_indexing_task_update_index_document'):
+        # A document index update is happening, wait
+        raise self.retry(countdown=RETRY_DELAY)
+
+    try:
+        lock = Lock.acquire_lock('document_indexing_task_do_rebuild_all_indexes')
+    except LockError as exception:
+        # Another rebuild is happening, retry later
+        raise self.retry(exc=exception, countdown=RETRY_DELAY)
+    else:
+        try:
             do_rebuild_all_indexes()
+        finally:
+            lock.release()