diff --git a/HISTORY.rst b/HISTORY.rst index 1fd8e03268..bb00dee4a8 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -82,7 +82,14 @@ - Use dropzone for document version upload form. - Allow the "Execute document tools" permission to be granted via ACL. - +- Add support for custom IMAP search criteria. +- Add support for executing custom IMAP STORE commands + on processed messages. +- Add support to execute the IMAP expunge command after each + processed message. +- Add support for specifing a destination IMAP mailbox for + processed messages. GitLab issue #399. Thanks to + Robert Schöftner (@robert.schoeftner). 3.2.8 (2019-10-01) ================== diff --git a/docs/releases/3.3.rst b/docs/releases/3.3.rst index 47878877f9..27018030d6 100644 --- a/docs/releases/3.3.rst +++ b/docs/releases/3.3.rst @@ -89,6 +89,14 @@ Changes - Unify all line endings to be Linux style. - Add support for changing the system messages position. GitLab issue #640. Thanks to Matthias Urhahn (@d4rken). +- Add support for custom IMAP search criteria. +- Add support for executing custom IMAP STORE commands + on processed messages. +- Add support for specifing a destination IMAP mailbox for + processed messages. GitLab issue #399. Thanks to + Robert Schöftner (@robert.schoeftner). +- Add support to execute the IMAP expunge command after each + processed message. Removals -------- @@ -200,6 +208,7 @@ Backward incompatible changes Bugs fixed or issues closed --------------------------- +- :gitlab-issue:`399` Archive of processed e-mails - :gitlab-issue:`526` RuntimeWarning: Never call result.get() within a task! - :gitlab-issue:`532` Workflow preview isn't updated right after transitions are modified - :gitlab-issue:`540` hint-outdated/update documentation diff --git a/mayan/apps/mailer/tests/test_workflow_actions.py b/mayan/apps/mailer/tests/test_workflow_actions.py index ce5e4408f2..e59c8c827a 100644 --- a/mayan/apps/mailer/tests/test_workflow_actions.py +++ b/mayan/apps/mailer/tests/test_workflow_actions.py @@ -86,7 +86,9 @@ class EmailActionTemplateTestCase(MetadataTypeTestMixin, MailerTestMixin, Workfl action.execute(context={'document': self.test_document}) self.assertEqual(len(mail.outbox), 1) self.assertEqual(mail.outbox[0].from_email, TEST_EMAIL_FROM_ADDRESS) - self.assertEqual(mail.outbox[0].to, [TEST_EMAIL_ADDRESS]) + self.assertEqual( + mail.outbox[0].to, [self.test_document.metadata.first().value] + ) def test_email_action_subject_template(self): self._create_test_metadata_type() @@ -107,6 +109,9 @@ class EmailActionTemplateTestCase(MetadataTypeTestMixin, MailerTestMixin, Workfl self.assertEqual(len(mail.outbox), 1) self.assertEqual(mail.outbox[0].from_email, TEST_EMAIL_FROM_ADDRESS) self.assertEqual(mail.outbox[0].to, [TEST_EMAIL_ADDRESS]) + self.assertEqual( + mail.outbox[0].subject, self.test_document.metadata.first().value + ) def test_email_action_body_template(self): self._create_test_metadata_type() diff --git a/mayan/apps/mailer/views.py b/mayan/apps/mailer/views.py index b2477c422e..bdae6df508 100644 --- a/mayan/apps/mailer/views.py +++ b/mayan/apps/mailer/views.py @@ -1,5 +1,6 @@ from __future__ import absolute_import, unicode_literals +from django.contrib import messages from django.http import Http404, HttpResponseRedirect from django.shortcuts import get_object_or_404 from django.template import RequestContext @@ -248,6 +249,9 @@ class UserMailerTestView(FormView): def form_valid(self, form): self.get_object().test(to=form.cleaned_data['email']) + messages.success( + message=_('Test email sent.'), request=self.request + ) return super(UserMailerTestView, self).form_valid(form=form) def get_extra_context(self): diff --git a/mayan/apps/sources/forms.py b/mayan/apps/sources/forms.py index e555967cd9..748d9f7458 100644 --- a/mayan/apps/sources/forms.py +++ b/mayan/apps/sources/forms.py @@ -120,7 +120,10 @@ class EmailSetupBaseForm(forms.ModelForm): class IMAPEmailSetupForm(EmailSetupBaseForm): class Meta(EmailSetupBaseForm.Meta): - fields = EmailSetupBaseForm.Meta.fields + ('mailbox',) + fields = EmailSetupBaseForm.Meta.fields + ( + 'mailbox', 'search_criteria', 'store_commands', + 'mailbox_destination', 'execute_expunge' + ) model = IMAPEmail diff --git a/mayan/apps/sources/literals.py b/mayan/apps/sources/literals.py index ee01737193..39a00b1371 100644 --- a/mayan/apps/sources/literals.py +++ b/mayan/apps/sources/literals.py @@ -3,6 +3,8 @@ from __future__ import unicode_literals from django.utils.translation import ugettext_lazy as _ DEFAULT_IMAP_MAILBOX = 'INBOX' +DEFAULT_IMAP_SEARCH_CRITERIA = 'NOT DELETED' +DEFAULT_IMAP_STORE_COMMANDS = '+FLAGS (\\Deleted)' DEFAULT_INTERVAL = 600 DEFAULT_METADATA_ATTACHMENT_NAME = 'metadata.yaml' DEFAULT_POP3_TIMEOUT = 60 diff --git a/mayan/apps/sources/migrations/0021_auto_20190629_0648.py b/mayan/apps/sources/migrations/0021_auto_20190629_0648.py index e804cce4e0..6c5c123462 100644 --- a/mayan/apps/sources/migrations/0021_auto_20190629_0648.py +++ b/mayan/apps/sources/migrations/0021_auto_20190629_0648.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.20 on 2019-06-29 06:48 from __future__ import unicode_literals from django.db import migrations, models @@ -15,6 +13,12 @@ class Migration(migrations.Migration): migrations.AlterField( model_name='emailbasemodel', name='metadata_attachment_name', - field=models.CharField(default='metadata.yaml', help_text='Name of the attachment that will contains the metadata type names and value pairs to be assigned to the rest of the downloaded attachments.', max_length=128, verbose_name='Metadata attachment name'), + field=models.CharField( + default='metadata.yaml', help_text='Name of the attachment ' + 'that will contains the metadata type names and value ' + 'pairs to be assigned to the rest of the downloaded ' + 'attachments.', max_length=128, + verbose_name='Metadata attachment name' + ), ), ] diff --git a/mayan/apps/sources/migrations/0022_auto_20191022_0737.py b/mayan/apps/sources/migrations/0022_auto_20191022_0737.py new file mode 100644 index 0000000000..5ac87c62ed --- /dev/null +++ b/mayan/apps/sources/migrations/0022_auto_20191022_0737.py @@ -0,0 +1,54 @@ +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ('sources', '0021_auto_20190629_0648'), + ] + + operations = [ + migrations.AddField( + model_name='imapemail', + name='execute_expunge', + field=models.BooleanField( + default=True, help_text='Execute the IMAP expunge command ' + 'after processing each email message.', + verbose_name='Execute expunge' + ), + ), + migrations.AddField( + model_name='imapemail', + name='mailbox_destination', + field=models.CharField( + blank=True, help_text='IMAP Mailbox to which processed ' + 'messages will be copied.', max_length=96, null=True, + verbose_name='Destination mailbox' + ), + ), + migrations.AddField( + model_name='imapemail', + name='store_commands', + field=models.TextField( + blank=True, default='+FLAGS (\\Deleted)', + help_text='IMAP STORE command to execute on messages ' + 'after they are processed. One command per line. Use ' + 'the commands specified in ' + 'https://tools.ietf.org/html/rfc2060.html#section-6.4.6 or ' + 'the custom commands for your IMAP server.', null=True, + verbose_name='Store commands' + ), + ), + migrations.AddField( + model_name='imapemail', + name='search_criteria', + field=models.TextField( + blank=True, default='NOT DELETED', help_text='Criteria to ' + 'use when searching for messages to process. Use the ' + 'format specified in ' + 'https://tools.ietf.org/html/rfc2060.html#section-6.4.4', + null=True, verbose_name='Search criteria' + ), + ), + ] diff --git a/mayan/apps/sources/models/email_sources.py b/mayan/apps/sources/models/email_sources.py index fa8ba913f8..fe5a5336dd 100644 --- a/mayan/apps/sources/models/email_sources.py +++ b/mayan/apps/sources/models/email_sources.py @@ -15,8 +15,10 @@ from mayan.apps.documents.models import Document from mayan.apps.metadata.api import set_bulk_metadata from mayan.apps.metadata.models import MetadataType +from ..exceptions import SourceException from ..literals import ( - DEFAULT_IMAP_MAILBOX, DEFAULT_METADATA_ATTACHMENT_NAME, + DEFAULT_IMAP_MAILBOX, DEFAULT_IMAP_SEARCH_CRITERIA, + DEFAULT_IMAP_STORE_COMMANDS, DEFAULT_METADATA_ATTACHMENT_NAME, DEFAULT_POP3_TIMEOUT, SOURCE_CHOICE_EMAIL_IMAP, SOURCE_CHOICE_EMAIL_POP3, SOURCE_UNCOMPRESS_CHOICE_N, SOURCE_UNCOMPRESS_CHOICE_Y, ) @@ -214,6 +216,32 @@ class IMAPEmail(EmailBaseModel): help_text=_('IMAP Mailbox from which to check for messages.'), max_length=64, verbose_name=_('Mailbox') ) + search_criteria = models.TextField( + blank=True, default=DEFAULT_IMAP_SEARCH_CRITERIA, help_text=_( + 'Criteria to use when searching for messages to process. ' + 'Use the format specified in ' + 'https://tools.ietf.org/html/rfc2060.html#section-6.4.4' + ), null=True, verbose_name=_('Search criteria') + ) + store_commands = models.TextField( + blank=True, default=DEFAULT_IMAP_STORE_COMMANDS, help_text=_( + 'IMAP STORE command to execute on messages after they are ' + 'processed. One command per line. Use the commands specified in ' + 'https://tools.ietf.org/html/rfc2060.html#section-6.4.6 or ' + 'the custom commands for your IMAP server.' + ), null=True, verbose_name=_('Store commands') + ) + execute_expunge = models.BooleanField( + default=True, help_text=_( + 'Execute the IMAP expunge command after processing each email ' + 'message.' + ), verbose_name=_('Execute expunge') + ) + mailbox_destination = models.CharField( + blank=True, help_text=_( + 'IMAP Mailbox to which processed messages will be copied.' + ), max_length=96, null=True, verbose_name=_('Destination mailbox') + ) objects = models.Manager() @@ -235,27 +263,59 @@ class IMAPEmail(EmailBaseModel): mailbox.login(user=self.username, password=self.password) mailbox.select(mailbox=self.mailbox) - status, data = mailbox.search(None, 'NOT', 'DELETED') - if data: - messages_info = data[0].split() - logger.debug('messages count: %s', len(messages_info)) + try: + status, data = mailbox.uid( + 'SEARCH', None, *self.search_criteria.strip().split() + ) + except Exception as exception: + raise SourceException( + 'Error executing search command; {}'.format(exception) + ) - for message_number in messages_info: - logger.debug('message_number: %s', message_number) - status, data = mailbox.fetch( - message_set=message_number, message_parts='(RFC822)' - ) + if data: + # data is a space separated sequence of message uids + uids = data[0].split() + logger.debug('messages count: %s', len(uids)) + logger.debug('message uids: %s', uids) + + for uid in uids: + logger.debug('message uid: %s', uid) + status, data = mailbox.uid('FETCH', uid, '(RFC822)') EmailBaseModel.process_message( source=self, message_text=data[0][1] ) if not test: - mailbox.store( - message_set=message_number, command='+FLAGS', - flags=r'\Deleted' - ) + if self.store_commands: + for command in self.store_commands.split('\n'): + try: + args = [uid] + args.extend(command.strip().split(' ')) + mailbox.uid('STORE', *args) + except Exception as exception: + raise SourceException( + 'Error executing IMAP store command "{}" ' + 'on message uid {}; {}'.format( + command, uid, exception + ) + ) + + if self.mailbox_destination: + try: + mailbox.uid( + 'COPY', uid, self.mailbox_destination + ) + except Exception as exception: + raise SourceException( + 'Error copying message uid {} to mailbox {}; ' + '{}'.format( + uid, self.mailbox_destination, exception + ) + ) + + if self.execute_expunge: + mailbox.expunge() - mailbox.expunge() mailbox.close() mailbox.logout() diff --git a/mayan/apps/sources/tests/test_models.py b/mayan/apps/sources/tests/test_models.py index 188b086931..914712577b 100644 --- a/mayan/apps/sources/tests/test_models.py +++ b/mayan/apps/sources/tests/test_models.py @@ -260,44 +260,136 @@ class EmailBaseTestCase(GenericDocumentTestCase): ) +class MockIMAPMessage(object): + flags = [] + uid = None + + def __init__(self): + self.uid = '999' + + def get_flags(self): + return ' '.join(self.flags) + + +class MockIMAPMailbox(object): + messages = [] + + def __init__(self, name='INBOX'): + self.name = name + + +class MockIMAPServer(object): + def __init__(self): + self.mailboxes = { + 'INBOX': MockIMAPMailbox(name='INBOX') + } + self.mailboxes['INBOX'].messages.append(MockIMAPMessage()) + self.mailbox_selected = None + + def login(self, user, password): + return ('OK', ['{} authenticated (Success)'.format(user)]) + + def select(self, mailbox='INBOX', readonly=False): + self.mailbox_selected = self.mailboxes[mailbox] + + return ( + 'OK', [ + len(self.mailbox_selected.messages) + ] + ) + + def search(self, charset, *criteria): + """ + 7.2.5. SEARCH Response + Contents: zero or more numbers + The SEARCH response occurs as a result of a SEARCH or UID SEARCH + command. The number(s) refer to those messages that match the + search criteria. For SEARCH, these are message sequence numbers; + for UID SEARCH, these are unique identifiers. Each number is + delimited by a space. + + Example: S: * SEARCH 2 3 6 + """ + results = [ + self.mailbox_selected.messages[0] + ] + + message_sequences = [] + for result in results: + message_sequences.append( + force_text( + self.mailbox_selected.messages.index(result) + ) + ) + + return ('OK', ' '.join(message_sequences)) + + def fetch(self, message_set, message_parts): + results = [] + for message_number in message_set.split(): + message = self.mailbox_selected.messages[int(message_number) - 1] + if '\\Seen' not in message.flags: + message.flags.append('\\Seen') + + results.append( + ( + '{} (RFC822 {{4800}}'.format(message_number), + TEST_EMAIL_BASE64_FILENAME, + ' FLAGS ({}))'.format(message.get_flags()) + ) + ) + return ('OK', results) + + def store(self, message_set, command, flags): + results = [] + + for message_number in message_set.split(): + message = self.mailbox_selected.messages[int(message_number) - 1] + + if command == 'FLAGS': + message.flags = flags.split() + elif command == '+FLAGS': + for flag in flags.split(): + if flag not in message.flags: + message.flags.append(flag) + elif command == '-FLAGS': + for flag in flags.split(): + if flag in message.flags: + message.flags.remove(flag) + + results.append( + '{} (FLAGS ({}))'.format(message_number, message.get_flags()) + ) + + return ('OK', results) + + def expunge(self): + result = [] + + for message in self.mailbox_selected.messages: + if '\\Deleted' in message.flags: + result.append( + force_text( + self.mailbox_selected.messages.index(message) + ) + ) + self.mailbox_selected.messages.remove(message) + + return ('OK', ' '.join(result)) + + def close(self): + return ('OK', ['Returned to authenticated state. (Success)']) + + def logout(self): + return ('BYE', ['LOGOUT Requested']) + + class IMAPSourceTestCase(GenericDocumentTestCase): auto_upload_document = False - class MockIMAPServer(object): - def login(self, user, password): - return ('OK', ['{} authenticated (Success)'.format(user)]) - - def select(self, mailbox='INBOX', readonly=False): - return ('OK', ['1']) - - def search(self, charset, *criteria): - return ('OK', ['1']) - - def fetch(self, message_set, message_parts): - return ( - 'OK', [ - ( - '1 (RFC822 {4800}', - TEST_EMAIL_BASE64_FILENAME - ), ' FLAGS (\\Seen))' - ] - ) - - def store(self, message_set, command, flags): - return ('OK', ['1 (FLAGS (\\Seen \\Deleted))']) - - def expunge(self): - return ('OK', ['1']) - - def close(self): - return ('OK', ['Returned to authenticated state. (Success)']) - - def logout(self): - return ('BYE', ['LOGOUT Requested']) - @mock.patch('imaplib.IMAP4_SSL', autospec=True) def test_download_document(self, mock_imaplib): - mock_imaplib.return_value = IMAPSourceTestCase.MockIMAPServer() + mock_imaplib.return_value = MockIMAPServer() self.source = IMAPEmail.objects.create( document_type=self.test_document_type, label='', host='', password='', username=''