matthias/mayan-edms
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

IMAP source improvements

Browse Source 
- Support multiple STORE commands. Defaults to +FLAGS (\Deleted)
  to conserve current behavior.
- Support custom search criteria. Defaults to NOT DELETED
  to converse current behavior.
- Support enabling/disabling IMAP expunge command
  after each message. Defaults to True to conserve
  current behavior.
- Increase functionality of the MockIMAPServer

Signed-off-by: Roberto Rosario <roberto.rosario@mayan-edms.com>
This commit is contained in:
Roberto Rosario
2019-10-24 04:01:51 -04:00
parent 1b6468522a
commit 4dea4129db
10 changed files with 294 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
HISTORY.rst
View File

@@ -82,7 +82,14 @@
- Use dropzone for document version upload form.
- Allow the "Execute document tools" permission to be
granted via ACL.
- Add support for custom IMAP search criteria.
- Add support for executing custom IMAP STORE commands
on processed messages.
- Add support to execute the IMAP expunge command after each
processed message.
- Add support for specifing a destination IMAP mailbox for
processed messages. GitLab issue #399. Thanks to
Robert Schöftner (@robert.schoeftner).
3.2.8 (2019-10-01)
==================

9
docs/releases/3.3.rst
View File

@@ -89,6 +89,14 @@ Changes
- Unify all line endings to be Linux style.
- Add support for changing the system messages position.
GitLab issue #640. Thanks to Matthias Urhahn (@d4rken).
- Add support for custom IMAP search criteria.
- Add support for executing custom IMAP STORE commands
on processed messages.
- Add support for specifing a destination IMAP mailbox for
processed messages. GitLab issue #399. Thanks to
Robert Schöftner (@robert.schoeftner).
- Add support to execute the IMAP expunge command after each
processed message.
Removals
--------
@@ -200,6 +208,7 @@ Backward incompatible changes
Bugs fixed or issues closed
---------------------------
- :gitlab-issue:`399` Archive of processed e-mails
- :gitlab-issue:`526` RuntimeWarning: Never call result.get() within a task!
- :gitlab-issue:`532` Workflow preview isn't updated right after transitions are modified
- :gitlab-issue:`540` hint-outdated/update documentation

7
mayan/apps/mailer/tests/test_workflow_actions.py
View File

@@ -86,7 +86,9 @@ class EmailActionTemplateTestCase(MetadataTypeTestMixin, MailerTestMixin, Workfl
action.execute(context={'document': self.test_document})
self.assertEqual(len(mail.outbox), 1)
self.assertEqual(mail.outbox[0].from_email, TEST_EMAIL_FROM_ADDRESS)
self.assertEqual(mail.outbox[0].to, [TEST_EMAIL_ADDRESS])
self.assertEqual(
mail.outbox[0].to, [self.test_document.metadata.first().value]
)
def test_email_action_subject_template(self):
self._create_test_metadata_type()
@@ -107,6 +109,9 @@ class EmailActionTemplateTestCase(MetadataTypeTestMixin, MailerTestMixin, Workfl
self.assertEqual(len(mail.outbox), 1)
self.assertEqual(mail.outbox[0].from_email, TEST_EMAIL_FROM_ADDRESS)
self.assertEqual(mail.outbox[0].to, [TEST_EMAIL_ADDRESS])
self.assertEqual(
mail.outbox[0].subject, self.test_document.metadata.first().value
)
def test_email_action_body_template(self):
self._create_test_metadata_type()

4
mayan/apps/mailer/views.py
View File

@@ -1,5 +1,6 @@
from __future__ import absolute_import, unicode_literals
from django.contrib import messages
from django.http import Http404, HttpResponseRedirect
from django.shortcuts import get_object_or_404
from django.template import RequestContext
@@ -248,6 +249,9 @@ class UserMailerTestView(FormView):
def form_valid(self, form):
self.get_object().test(to=form.cleaned_data['email'])
messages.success(
message=_('Test email sent.'), request=self.request
)
return super(UserMailerTestView, self).form_valid(form=form)
def get_extra_context(self):

5
mayan/apps/sources/forms.py
View File

@@ -120,7 +120,10 @@ class EmailSetupBaseForm(forms.ModelForm):
class IMAPEmailSetupForm(EmailSetupBaseForm):
class Meta(EmailSetupBaseForm.Meta):
fields = EmailSetupBaseForm.Meta.fields + ('mailbox',)
fields = EmailSetupBaseForm.Meta.fields + (
'mailbox', 'search_criteria', 'store_commands',
'mailbox_destination', 'execute_expunge'
)
model = IMAPEmail

2
mayan/apps/sources/literals.py
View File

@@ -3,6 +3,8 @@ from __future__ import unicode_literals
from django.utils.translation import ugettext_lazy as _
DEFAULT_IMAP_MAILBOX = 'INBOX'
DEFAULT_IMAP_SEARCH_CRITERIA = 'NOT DELETED'
DEFAULT_IMAP_STORE_COMMANDS = '+FLAGS (\\Deleted)'
DEFAULT_INTERVAL = 600
DEFAULT_METADATA_ATTACHMENT_NAME = 'metadata.yaml'
DEFAULT_POP3_TIMEOUT = 60

10
mayan/apps/sources/migrations/0021_auto_20190629_0648.py
View File

@@ -1,5 +1,3 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.11.20 on 2019-06-29 06:48
from __future__ import unicode_literals
from django.db import migrations, models
@@ -15,6 +13,12 @@ class Migration(migrations.Migration):
migrations.AlterField(
model_name='emailbasemodel',
name='metadata_attachment_name',
field=models.CharField(default='metadata.yaml', help_text='Name of the attachment that will contains the metadata type names and value pairs to be assigned to the rest of the downloaded attachments.', max_length=128, verbose_name='Metadata attachment name'),
field=models.CharField(
default='metadata.yaml', help_text='Name of the attachment '
'that will contains the metadata type names and value '
'pairs to be assigned to the rest of the downloaded '
'attachments.', max_length=128,
verbose_name='Metadata attachment name'
),
),
]

54
mayan/apps/sources/migrations/0022_auto_20191022_0737.py Normal file
View File

@@ -0,0 +1,54 @@
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('sources', '0021_auto_20190629_0648'),
]
operations = [
migrations.AddField(
model_name='imapemail',
name='execute_expunge',
field=models.BooleanField(
default=True, help_text='Execute the IMAP expunge command '
'after processing each email message.',
verbose_name='Execute expunge'
),
),
migrations.AddField(
model_name='imapemail',
name='mailbox_destination',
field=models.CharField(
blank=True, help_text='IMAP Mailbox to which processed '
'messages will be copied.', max_length=96, null=True,
verbose_name='Destination mailbox'
),
),
migrations.AddField(
model_name='imapemail',
name='store_commands',
field=models.TextField(
blank=True, default='+FLAGS (\\Deleted)',
help_text='IMAP STORE command to execute on messages '
'after they are processed. One command per line. Use '
'the commands specified in '
'https://tools.ietf.org/html/rfc2060.html#section-6.4.6 or '
'the custom commands for your IMAP server.', null=True,
verbose_name='Store commands'
),
),
migrations.AddField(
model_name='imapemail',
name='search_criteria',
field=models.TextField(
blank=True, default='NOT DELETED', help_text='Criteria to '
'use when searching for messages to process. Use the '
'format specified in '
'https://tools.ietf.org/html/rfc2060.html#section-6.4.4',
null=True, verbose_name='Search criteria'
),
),
]

90
mayan/apps/sources/models/email_sources.py
View File

@@ -15,8 +15,10 @@ from mayan.apps.documents.models import Document
from mayan.apps.metadata.api import set_bulk_metadata
from mayan.apps.metadata.models import MetadataType
from ..exceptions import SourceException
from ..literals import (
DEFAULT_IMAP_MAILBOX, DEFAULT_METADATA_ATTACHMENT_NAME,
DEFAULT_IMAP_MAILBOX, DEFAULT_IMAP_SEARCH_CRITERIA,
DEFAULT_IMAP_STORE_COMMANDS, DEFAULT_METADATA_ATTACHMENT_NAME,
DEFAULT_POP3_TIMEOUT, SOURCE_CHOICE_EMAIL_IMAP, SOURCE_CHOICE_EMAIL_POP3,
SOURCE_UNCOMPRESS_CHOICE_N, SOURCE_UNCOMPRESS_CHOICE_Y,
)
@@ -214,6 +216,32 @@ class IMAPEmail(EmailBaseModel):
help_text=_('IMAP Mailbox from which to check for messages.'),
max_length=64, verbose_name=_('Mailbox')
)
search_criteria = models.TextField(
blank=True, default=DEFAULT_IMAP_SEARCH_CRITERIA, help_text=_(
'Criteria to use when searching for messages to process. '
'Use the format specified in '
'https://tools.ietf.org/html/rfc2060.html#section-6.4.4'
), null=True, verbose_name=_('Search criteria')
)
store_commands = models.TextField(
blank=True, default=DEFAULT_IMAP_STORE_COMMANDS, help_text=_(
'IMAP STORE command to execute on messages after they are '
'processed. One command per line. Use the commands specified in '
'https://tools.ietf.org/html/rfc2060.html#section-6.4.6 or '
'the custom commands for your IMAP server.'
), null=True, verbose_name=_('Store commands')
)
execute_expunge = models.BooleanField(
default=True, help_text=_(
'Execute the IMAP expunge command after processing each email '
'message.'
), verbose_name=_('Execute expunge')
)
mailbox_destination = models.CharField(
blank=True, help_text=_(
'IMAP Mailbox to which processed messages will be copied.'
), max_length=96, null=True, verbose_name=_('Destination mailbox')
)
objects = models.Manager()
@@ -235,27 +263,59 @@ class IMAPEmail(EmailBaseModel):
mailbox.login(user=self.username, password=self.password)
mailbox.select(mailbox=self.mailbox)
status, data = mailbox.search(None, 'NOT', 'DELETED')
if data:
messages_info = data[0].split()
logger.debug('messages count: %s', len(messages_info))
try:
status, data = mailbox.uid(
'SEARCH', None, *self.search_criteria.strip().split()
)
except Exception as exception:
raise SourceException(
'Error executing search command; {}'.format(exception)
)
for message_number in messages_info:
logger.debug('message_number: %s', message_number)
status, data = mailbox.fetch(
message_set=message_number, message_parts='(RFC822)'
)
if data:
# data is a space separated sequence of message uids
uids = data[0].split()
logger.debug('messages count: %s', len(uids))
logger.debug('message uids: %s', uids)
for uid in uids:
logger.debug('message uid: %s', uid)
status, data = mailbox.uid('FETCH', uid, '(RFC822)')
EmailBaseModel.process_message(
source=self, message_text=data[0][1]
)
if not test:
mailbox.store(
message_set=message_number, command='+FLAGS',
flags=r'\Deleted'
)
if self.store_commands:
for command in self.store_commands.split('\n'):
try:
args = [uid]
args.extend(command.strip().split(' '))
mailbox.uid('STORE', *args)
except Exception as exception:
raise SourceException(
'Error executing IMAP store command "{}" '
'on message uid {}; {}'.format(
command, uid, exception
)
)
if self.mailbox_destination:
try:
mailbox.uid(
'COPY', uid, self.mailbox_destination
)
except Exception as exception:
raise SourceException(
'Error copying message uid {} to mailbox {}; '
'{}'.format(
uid, self.mailbox_destination, exception
)
)
if self.execute_expunge:
mailbox.expunge()
mailbox.expunge()
mailbox.close()
mailbox.logout()

158
mayan/apps/sources/tests/test_models.py
View File

@@ -260,44 +260,136 @@ class EmailBaseTestCase(GenericDocumentTestCase):
)
class MockIMAPMessage(object):
flags = []
uid = None
def __init__(self):
self.uid = '999'
def get_flags(self):
return ' '.join(self.flags)
class MockIMAPMailbox(object):
messages = []
def __init__(self, name='INBOX'):
self.name = name
class MockIMAPServer(object):
def __init__(self):
self.mailboxes = {
'INBOX': MockIMAPMailbox(name='INBOX')
}
self.mailboxes['INBOX'].messages.append(MockIMAPMessage())
self.mailbox_selected = None
def login(self, user, password):
return ('OK', ['{} authenticated (Success)'.format(user)])
def select(self, mailbox='INBOX', readonly=False):
self.mailbox_selected = self.mailboxes[mailbox]
return (
'OK', [
len(self.mailbox_selected.messages)
]
)
def search(self, charset, *criteria):
"""
7.2.5. SEARCH Response
Contents: zero or more numbers
The SEARCH response occurs as a result of a SEARCH or UID SEARCH
command. The number(s) refer to those messages that match the
search criteria. For SEARCH, these are message sequence numbers;
for UID SEARCH, these are unique identifiers. Each number is
delimited by a space.
Example: S: * SEARCH 2 3 6
"""
results = [
self.mailbox_selected.messages[0]
]
message_sequences = []
for result in results:
message_sequences.append(
force_text(
self.mailbox_selected.messages.index(result)
)
)
return ('OK', ' '.join(message_sequences))
def fetch(self, message_set, message_parts):
results = []
for message_number in message_set.split():
message = self.mailbox_selected.messages[int(message_number) - 1]
if '\\Seen' not in message.flags:
message.flags.append('\\Seen')
results.append(
(
'{} (RFC822 {{4800}}'.format(message_number),
TEST_EMAIL_BASE64_FILENAME,
' FLAGS ({}))'.format(message.get_flags())
)
)
return ('OK', results)
def store(self, message_set, command, flags):
results = []
for message_number in message_set.split():
message = self.mailbox_selected.messages[int(message_number) - 1]
if command == 'FLAGS':
message.flags = flags.split()
elif command == '+FLAGS':
for flag in flags.split():
if flag not in message.flags:
message.flags.append(flag)
elif command == '-FLAGS':
for flag in flags.split():
if flag in message.flags:
message.flags.remove(flag)
results.append(
'{} (FLAGS ({}))'.format(message_number, message.get_flags())
)
return ('OK', results)
def expunge(self):
result = []
for message in self.mailbox_selected.messages:
if '\\Deleted' in message.flags:
result.append(
force_text(
self.mailbox_selected.messages.index(message)
)
)
self.mailbox_selected.messages.remove(message)
return ('OK', ' '.join(result))
def close(self):
return ('OK', ['Returned to authenticated state. (Success)'])
def logout(self):
return ('BYE', ['LOGOUT Requested'])
class IMAPSourceTestCase(GenericDocumentTestCase):
auto_upload_document = False
class MockIMAPServer(object):
def login(self, user, password):
return ('OK', ['{} authenticated (Success)'.format(user)])
def select(self, mailbox='INBOX', readonly=False):
return ('OK', ['1'])
def search(self, charset, *criteria):
return ('OK', ['1'])
def fetch(self, message_set, message_parts):
return (
'OK', [
(
'1 (RFC822 {4800}',
TEST_EMAIL_BASE64_FILENAME
), ' FLAGS (\\Seen))'
]
)
def store(self, message_set, command, flags):
return ('OK', ['1 (FLAGS (\\Seen \\Deleted))'])
def expunge(self):
return ('OK', ['1'])
def close(self):
return ('OK', ['Returned to authenticated state. (Success)'])
def logout(self):
return ('BYE', ['LOGOUT Requested'])
@mock.patch('imaplib.IMAP4_SSL', autospec=True)
def test_download_document(self, mock_imaplib):
mock_imaplib.return_value = IMAPSourceTestCase.MockIMAPServer()
mock_imaplib.return_value = MockIMAPServer()
self.source = IMAPEmail.objects.create(
document_type=self.test_document_type, label='', host='',
password='', username=''