diff --git a/HISTORY.rst b/HISTORY.rst index 9400d6e27d..ace468110c 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -147,6 +147,7 @@ - Update rendering of the readonly multiselect widget to conform to Django's updated field class interface. - Add warning when using SQLite as the database backend. - Use Mailgun's flanker library to process the email sources. +- Add locking for interval sources. This reduces the chance of repeated documents from long running email downloads. 2.7.3 (2017-09-11) ================== diff --git a/docs/releases/3.0.rst b/docs/releases/3.0.rst index f60cff9637..15b6dd1bb2 100644 --- a/docs/releases/3.0.rst +++ b/docs/releases/3.0.rst @@ -466,6 +466,7 @@ Other changes worth mentioning - Make error messages persistent and increase the timeout of warning to 10 seconds. - Improve rendering of the details form. - Update rendering of the readonly multiselect widget to conform to Django's updated field class interface. +- Add locking for interval sources. This reduces the chance of repeated documents from long running email downloads. Removals diff --git a/mayan/apps/sources/literals.py b/mayan/apps/sources/literals.py index aab8d4b02b..25c8e3ba2b 100644 --- a/mayan/apps/sources/literals.py +++ b/mayan/apps/sources/literals.py @@ -59,6 +59,7 @@ SOURCE_CHOICES = ( (SOURCE_CHOICE_EMAIL_IMAP, _('IMAP email')), ) +DEFAULT_SOURCE_LOCK_EXPIRE = 600 DEFAULT_INTERVAL = 600 DEFAULT_METADATA_ATTACHMENT_NAME = 'metadata.yaml' DEFAULT_POP3_TIMEOUT = 60 diff --git a/mayan/apps/sources/models.py b/mayan/apps/sources/models.py index e74357894b..867ed2e5a4 100644 --- a/mayan/apps/sources/models.py +++ b/mayan/apps/sources/models.py @@ -551,10 +551,6 @@ class EmailBaseModel(IntervalBaseModel): } ) - # TODO: Add lock to avoid running more than once concurrent same document - # download - # TODO: Use message ID for lock - @staticmethod def process_message(source, message_text): counter = 1 diff --git a/mayan/apps/sources/tasks.py b/mayan/apps/sources/tasks.py index af1b73f57f..2bd413090c 100644 --- a/mayan/apps/sources/tasks.py +++ b/mayan/apps/sources/tasks.py @@ -10,8 +10,12 @@ from django.utils.translation import ugettext_lazy as _ from mayan.celery import app from common.compressed_files import CompressedFile, NotACompressedFile +from lock_manager import LockError +from lock_manager.runtime import locking_backend -from .literals import DEFAULT_SOURCE_TASK_RETRY_DELAY +from .literals import ( + DEFAULT_SOURCE_LOCK_EXPIRE, DEFAULT_SOURCE_TASK_RETRY_DELAY +) logger = logging.getLogger(__name__) @@ -22,10 +26,19 @@ def task_check_interval_source(source_id): app_label='sources', model_name='Source' ) - source = Source.objects.get_subclass(pk=source_id) - if source.enabled: + lock_id = 'task_check_interval_source-%d' % source_id + try: + logger.debug('trying to acquire lock: %s', lock_id) + lock = locking_backend.acquire_lock(lock_id, DEFAULT_SOURCE_LOCK_EXPIRE) + except LockError: + logger.debug('unable to obtain lock: %s' % lock_id) + else: + logger.debug('acquired lock: %s', lock_id) + try: - source.check_source() + source = Source.objects.get_subclass(pk=source_id) + if source.enabled: + source.check_source() except Exception as exception: logger.error('Error processing source: %s; %s', source, exception) source.logs.create( @@ -33,6 +46,8 @@ def task_check_interval_source(source_id): ) else: source.logs.all().delete() + finally: + lock.release() @app.task(bind=True, default_retry_delay=DEFAULT_SOURCE_TASK_RETRY_DELAY, ignore_result=True) diff --git a/mayan/apps/sources/tests/test_models.py b/mayan/apps/sources/tests/test_models.py index 0d3c9bbf3c..ede17cc8ed 100644 --- a/mayan/apps/sources/tests/test_models.py +++ b/mayan/apps/sources/tests/test_models.py @@ -23,71 +23,6 @@ from .literals import ( ) -@override_settings(OCR_AUTO_OCR=False) -class UploadDocumentTestCase(BaseTestCase): - """ - Test creating documents - """ - - def setUp(self): - super(UploadDocumentTestCase, self).setUp() - self.document_type = DocumentType.objects.create( - label=TEST_DOCUMENT_TYPE_LABEL - ) - - def tearDown(self): - self.document_type.delete() - super(UploadDocumentTestCase, self).tearDown() - - def test_issue_gh_163(self): - """ - Non-ASCII chars in document name failing in upload via watch folder - gh-issue #163 https://github.com/mayan-edms/mayan-edms/issues/163 - """ - - temporary_directory = mkdtemp() - shutil.copy(TEST_NON_ASCII_DOCUMENT_PATH, temporary_directory) - - watch_folder = WatchFolderSource.objects.create( - document_type=self.document_type, folder_path=temporary_directory, - uncompress=SOURCE_UNCOMPRESS_CHOICE_Y - ) - watch_folder.check_source() - - self.assertEqual(Document.objects.count(), 1) - - document = Document.objects.first() - - self.assertEqual(document.exists(), True) - self.assertEqual(document.size, 17436) - - self.assertEqual(document.file_mimetype, 'image/png') - self.assertEqual(document.file_mime_encoding, 'binary') - self.assertEqual(document.label, TEST_NON_ASCII_DOCUMENT_FILENAME) - self.assertEqual(document.page_count, 1) - - # Test Non-ASCII named documents inside Non-ASCII named compressed file - - shutil.copy( - TEST_NON_ASCII_COMPRESSED_DOCUMENT_PATH, temporary_directory - ) - - watch_folder.check_source() - document = Document.objects.all()[1] - - self.assertEqual(Document.objects.count(), 2) - - self.assertEqual(document.exists(), True) - self.assertEqual(document.size, 17436) - - self.assertEqual(document.file_mimetype, 'image/png') - self.assertEqual(document.file_mime_encoding, 'binary') - self.assertEqual(document.label, TEST_NON_ASCII_DOCUMENT_FILENAME) - self.assertEqual(document.page_count, 1) - - shutil.rmtree(temporary_directory) - - @override_settings(OCR_AUTO_OCR=False) class CompressedUploadsTestCase(BaseTestCase): def setUp(self): @@ -189,3 +124,67 @@ class EmailFilenameDecodingTestCase(BaseTestCase): '' ), ) + + +@override_settings(OCR_AUTO_OCR=False) +class UploadDocumentTestCase(BaseTestCase): + """ + Test creating documents + """ + def setUp(self): + super(UploadDocumentTestCase, self).setUp() + self.document_type = DocumentType.objects.create( + label=TEST_DOCUMENT_TYPE_LABEL + ) + + def tearDown(self): + self.document_type.delete() + super(UploadDocumentTestCase, self).tearDown() + + def test_issue_gh_163(self): + """ + Non-ASCII chars in document name failing in upload via watch folder + gh-issue #163 https://github.com/mayan-edms/mayan-edms/issues/163 + """ + + temporary_directory = mkdtemp() + shutil.copy(TEST_NON_ASCII_DOCUMENT_PATH, temporary_directory) + + watch_folder = WatchFolderSource.objects.create( + document_type=self.document_type, folder_path=temporary_directory, + uncompress=SOURCE_UNCOMPRESS_CHOICE_Y + ) + watch_folder.check_source() + + self.assertEqual(Document.objects.count(), 1) + + document = Document.objects.first() + + self.assertEqual(document.exists(), True) + self.assertEqual(document.size, 17436) + + self.assertEqual(document.file_mimetype, 'image/png') + self.assertEqual(document.file_mime_encoding, 'binary') + self.assertEqual(document.label, TEST_NON_ASCII_DOCUMENT_FILENAME) + self.assertEqual(document.page_count, 1) + + # Test Non-ASCII named documents inside Non-ASCII named compressed file + + shutil.copy( + TEST_NON_ASCII_COMPRESSED_DOCUMENT_PATH, temporary_directory + ) + + watch_folder.check_source() + document = Document.objects.all()[1] + + self.assertEqual(Document.objects.count(), 2) + + self.assertEqual(document.exists(), True) + self.assertEqual(document.size, 17436) + + self.assertEqual(document.file_mimetype, 'image/png') + self.assertEqual(document.file_mime_encoding, 'binary') + self.assertEqual(document.label, TEST_NON_ASCII_DOCUMENT_FILENAME) + self.assertEqual(document.page_count, 1) + + shutil.rmtree(temporary_directory)