From 59eb6202fe7158bb7e01a097399035292d2b9a4d Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Mon, 20 Oct 2014 02:22:19 -0400 Subject: [PATCH 1/7] Initial commit to support receiving documents via email --- mayan/apps/sources/classes.py | 15 ++ mayan/apps/sources/literals.py | 12 +- ...albasemodel__add_pop3email__add_emailba.py | 148 ++++++++++++++ mayan/apps/sources/models.py | 181 +++++++++++++++++- mayan/settings/base.py | 2 + 5 files changed, 350 insertions(+), 8 deletions(-) create mode 100644 mayan/apps/sources/migrations/0005_auto__add_imapemail__add_intervalbasemodel__add_pop3email__add_emailba.py diff --git a/mayan/apps/sources/classes.py b/mayan/apps/sources/classes.py index ab04b2d675..de8d8d5e82 100644 --- a/mayan/apps/sources/classes.py +++ b/mayan/apps/sources/classes.py @@ -10,6 +10,21 @@ from converter.api import convert from mimetype.api import get_mimetype +class PseudoFile(File): + def __init__(self, file, name): + self.name = name + self.file = file + self.file.seek(0, os.SEEK_END) + self.size = self.file.tell() + self.file.seek(0) + + +class Attachment(File): + def __init__(self, part, name): + self.name = name + self.file = PseudoFile(StringIO(part.get_payload(decode=True)), name=name) + + class StagingFile(object): """ Simple class to extend the File class to add preview capabilities diff --git a/mayan/apps/sources/literals.py b/mayan/apps/sources/literals.py index 260c9160e4..e16b6d7db7 100644 --- a/mayan/apps/sources/literals.py +++ b/mayan/apps/sources/literals.py @@ -18,15 +18,25 @@ SOURCE_INTERACTIVE_UNCOMPRESS_CHOICES = ( SOURCE_CHOICE_WEB_FORM = 'webform' SOURCE_CHOICE_STAGING = 'staging' SOURCE_CHOICE_WATCH = 'watch' +SOURCE_CHOICE_EMAIL_POP3 = 'pop3' +SOURCE_CHOICE_EMAIL_IMAP = 'imap' SOURCE_CHOICES = ( (SOURCE_CHOICE_WEB_FORM, _(u'Web form')), (SOURCE_CHOICE_STAGING, _(u'Server staging folder')), (SOURCE_CHOICE_WATCH, _(u'Server watch folder')), + (SOURCE_CHOICE_EMAIL_POP3, _(u'POP3 email')), + (SOURCE_CHOICE_EMAIL_IMAP, _(u'IMAP email')), ) +# TODO: remove PLURALS SOURCE_CHOICES_PLURAL = ( (SOURCE_CHOICE_WEB_FORM, _(u'Web forms')), (SOURCE_CHOICE_STAGING, _(u'Server staging folders')), (SOURCE_CHOICE_WATCH, _(u'Server watch folders')), -) + (SOURCE_CHOICE_EMAIL_POP3, _(u'POP3 emails')), + (SOURCE_CHOICE_EMAIL_IMAP, _(u'IMAP emails')),) + +DEFAULT_INTERVAL = 60 +DEFAULT_POP3_TIMEOUT = 60 +DEFAULT_IMAP_MAILBOX = 'INBOX' diff --git a/mayan/apps/sources/migrations/0005_auto__add_imapemail__add_intervalbasemodel__add_pop3email__add_emailba.py b/mayan/apps/sources/migrations/0005_auto__add_imapemail__add_intervalbasemodel__add_pop3email__add_emailba.py new file mode 100644 index 0000000000..24ec380847 --- /dev/null +++ b/mayan/apps/sources/migrations/0005_auto__add_imapemail__add_intervalbasemodel__add_pop3email__add_emailba.py @@ -0,0 +1,148 @@ +# -*- coding: utf-8 -*- +from south.utils import datetime_utils as datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding model 'IMAPEmail' + db.create_table(u'sources_imapemail', ( + (u'emailbasemodel_ptr', self.gf('django.db.models.fields.related.OneToOneField')(to=orm['sources.EmailBaseModel'], unique=True, primary_key=True)), + ('mailbox', self.gf('django.db.models.fields.CharField')(default='INBOX', max_length=64)), + )) + db.send_create_signal(u'sources', ['IMAPEmail']) + + # Adding model 'IntervalBaseModel' + db.create_table(u'sources_intervalbasemodel', ( + (u'outofprocesssource_ptr', self.gf('django.db.models.fields.related.OneToOneField')(to=orm['sources.OutOfProcessSource'], unique=True, primary_key=True)), + ('interval', self.gf('django.db.models.fields.PositiveIntegerField')(default=60)), + ('document_type', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['documents.DocumentType'], null=True, blank=True)), + ('uncompress', self.gf('django.db.models.fields.CharField')(max_length=1)), + )) + db.send_create_signal(u'sources', ['IntervalBaseModel']) + + # Adding model 'POP3Email' + db.create_table(u'sources_pop3email', ( + (u'emailbasemodel_ptr', self.gf('django.db.models.fields.related.OneToOneField')(to=orm['sources.EmailBaseModel'], unique=True, primary_key=True)), + ('timeout', self.gf('django.db.models.fields.PositiveIntegerField')(default=60)), + )) + db.send_create_signal(u'sources', ['POP3Email']) + + # Adding model 'EmailBaseModel' + db.create_table(u'sources_emailbasemodel', ( + (u'intervalbasemodel_ptr', self.gf('django.db.models.fields.related.OneToOneField')(to=orm['sources.IntervalBaseModel'], unique=True, primary_key=True)), + ('host', self.gf('django.db.models.fields.CharField')(max_length=128)), + ('ssl', self.gf('django.db.models.fields.BooleanField')()), + ('port', self.gf('django.db.models.fields.PositiveIntegerField')(null=True, blank=True)), + ('username', self.gf('django.db.models.fields.CharField')(max_length=96)), + ('password', self.gf('django.db.models.fields.CharField')(max_length=96)), + )) + db.send_create_signal(u'sources', ['EmailBaseModel']) + + + def backwards(self, orm): + # Deleting model 'IMAPEmail' + db.delete_table(u'sources_imapemail') + + # Deleting model 'IntervalBaseModel' + db.delete_table(u'sources_intervalbasemodel') + + # Deleting model 'POP3Email' + db.delete_table(u'sources_pop3email') + + # Deleting model 'EmailBaseModel' + db.delete_table(u'sources_emailbasemodel') + + + models = { + u'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + u'documents.documenttype': { + 'Meta': {'ordering': "['name']", 'object_name': 'DocumentType'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '32'}) + }, + u'sources.emailbasemodel': { + 'Meta': {'ordering': "('title',)", 'object_name': 'EmailBaseModel', '_ormbases': [u'sources.IntervalBaseModel']}, + 'host': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + u'intervalbasemodel_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': u"orm['sources.IntervalBaseModel']", 'unique': 'True', 'primary_key': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '96'}), + 'port': ('django.db.models.fields.PositiveIntegerField', [], {'null': 'True', 'blank': 'True'}), + 'ssl': ('django.db.models.fields.BooleanField', [], {}), + 'username': ('django.db.models.fields.CharField', [], {'max_length': '96'}) + }, + u'sources.imapemail': { + 'Meta': {'ordering': "('title',)", 'object_name': 'IMAPEmail', '_ormbases': [u'sources.EmailBaseModel']}, + u'emailbasemodel_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': u"orm['sources.EmailBaseModel']", 'unique': 'True', 'primary_key': 'True'}), + 'mailbox': ('django.db.models.fields.CharField', [], {'default': "'INBOX'", 'max_length': '64'}) + }, + u'sources.interactivesource': { + 'Meta': {'ordering': "('title',)", 'object_name': 'InteractiveSource', '_ormbases': [u'sources.Source']}, + u'source_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': u"orm['sources.Source']", 'unique': 'True', 'primary_key': 'True'}) + }, + u'sources.intervalbasemodel': { + 'Meta': {'ordering': "('title',)", 'object_name': 'IntervalBaseModel', '_ormbases': [u'sources.OutOfProcessSource']}, + 'document_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['documents.DocumentType']", 'null': 'True', 'blank': 'True'}), + 'interval': ('django.db.models.fields.PositiveIntegerField', [], {'default': '60'}), + u'outofprocesssource_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': u"orm['sources.OutOfProcessSource']", 'unique': 'True', 'primary_key': 'True'}), + 'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}) + }, + u'sources.outofprocesssource': { + 'Meta': {'ordering': "('title',)", 'object_name': 'OutOfProcessSource', '_ormbases': [u'sources.Source']}, + u'source_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': u"orm['sources.Source']", 'unique': 'True', 'primary_key': 'True'}) + }, + u'sources.pop3email': { + 'Meta': {'ordering': "('title',)", 'object_name': 'POP3Email', '_ormbases': [u'sources.EmailBaseModel']}, + u'emailbasemodel_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': u"orm['sources.EmailBaseModel']", 'unique': 'True', 'primary_key': 'True'}), + 'timeout': ('django.db.models.fields.PositiveIntegerField', [], {'default': '60'}) + }, + u'sources.source': { + 'Meta': {'ordering': "('title',)", 'object_name': 'Source'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + u'sources.sourcetransformation': { + 'Meta': {'ordering': "('order',)", 'object_name': 'SourceTransformation'}, + 'arguments': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'object_id': ('django.db.models.fields.PositiveIntegerField', [], {}), + 'order': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0', 'null': 'True', 'db_index': 'True', 'blank': 'True'}), + 'transformation': ('django.db.models.fields.CharField', [], {'max_length': '128'}) + }, + u'sources.stagingfoldersource': { + 'Meta': {'ordering': "('title',)", 'object_name': 'StagingFolderSource', '_ormbases': [u'sources.InteractiveSource']}, + 'delete_after_upload': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'folder_path': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + u'interactivesource_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': u"orm['sources.InteractiveSource']", 'unique': 'True', 'primary_key': 'True'}), + 'preview_height': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}), + 'preview_width': ('django.db.models.fields.IntegerField', [], {}), + 'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}) + }, + u'sources.watchfoldersource': { + 'Meta': {'ordering': "('title',)", 'object_name': 'WatchFolderSource', '_ormbases': [u'sources.OutOfProcessSource']}, + 'delete_after_upload': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'folder_path': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'interval': ('django.db.models.fields.PositiveIntegerField', [], {}), + u'outofprocesssource_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': u"orm['sources.OutOfProcessSource']", 'unique': 'True', 'primary_key': 'True'}), + 'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}) + }, + u'sources.webformsource': { + 'Meta': {'ordering': "('title',)", 'object_name': 'WebFormSource', '_ormbases': [u'sources.InteractiveSource']}, + u'interactivesource_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': u"orm['sources.InteractiveSource']", 'unique': 'True', 'primary_key': 'True'}), + 'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}) + } + } + + complete_apps = ['sources'] \ No newline at end of file diff --git a/mayan/apps/sources/models.py b/mayan/apps/sources/models.py index 07e516a3fc..6292575c3b 100644 --- a/mayan/apps/sources/models.py +++ b/mayan/apps/sources/models.py @@ -1,6 +1,7 @@ from __future__ import absolute_import from ast import literal_eval +import json import logging import os @@ -15,15 +16,18 @@ from model_utils.managers import InheritanceManager from common.compressed_files import CompressedFile, NotACompressedFile from converter.api import get_available_transformations_choices from converter.literals import DIMENSION_SEPARATOR -from documents.models import Document +from djcelery.models import PeriodicTask, IntervalSchedule +from documents.models import Document, DocumentType from metadata.api import save_metadata_list -from .classes import StagingFile -from .literals import (SOURCE_CHOICES, SOURCE_CHOICES_PLURAL, - SOURCE_CHOICE_STAGING, SOURCE_CHOICE_WATCH, - SOURCE_CHOICE_WEB_FORM, +from .classes import Attachment, StagingFile +from .literals import (DEFAULT_INTERVAL, DEFAULT_POP3_TIMEOUT, + DEFAULT_IMAP_MAILBOX, SOURCE_CHOICES, + SOURCE_CHOICES_PLURAL, SOURCE_CHOICE_STAGING, + SOURCE_CHOICE_WATCH, SOURCE_CHOICE_WEB_FORM, SOURCE_INTERACTIVE_UNCOMPRESS_CHOICES, - SOURCE_UNCOMPRESS_CHOICES) + SOURCE_UNCOMPRESS_CHOICES, SOURCE_CHOICE_EMAIL_IMAP, + SOURCE_CHOICE_EMAIL_POP3) from .managers import SourceTransformationManager logger = logging.getLogger(__name__) @@ -190,8 +194,171 @@ class OutOfProcessSource(Source): verbose_name_plural = _(u'Out of process') +class IntervalBaseModel(OutOfProcessSource): + interval = models.PositiveIntegerField(default=DEFAULT_INTERVAL, verbose_name=_('Interval'), help_text=_('Interval in seconds between document downloads from this source.')) + document_type = models.ForeignKey(DocumentType, null=True, blank=True, verbose_name=_('Document type'), help_text=_('Assign a document type to documents uploaded from this source.')) + uncompress = models.CharField(max_length=1, choices=SOURCE_UNCOMPRESS_CHOICES, verbose_name=_('Uncompress'), help_text=_('Whether to expand or not, compressed archives.')) + + def save(self, *args, **kwargs): + new_source = not self.pk + super(IntervalBaseModel, self).save(*args, **kwargs) + periodic_task_name = 'check_interval_source-%i' % self.pk + if new_source: + interval_instance = IntervalSchedule.objects.create(every=self.interval) + PeriodicTask.objects.create( + name=periodic_task_name, + interval=interval_instance, + task='sources.tasks.task_check_interval_source', + queue='mailing', + args=json.dump({'source_id': self.pk}) + ) + else: + periodic_task = PeriodicTask.objects.get(name=periodic_task_name) + periodic_task.interval.every = self.interval + periodic_task.interval.save() + periodic_task.save() + + def delete(self, *args, **kwargs): + super(IntervalBaseModel, self).delete(*args, **kwargs) + periodic_task_name = 'check_interval_source-%i' % self.pk + periodic_task = PeriodicTask.objects.get(name=periodic_task_name) + interval_instance = periodic_task.interval + periodic_task.delete() + interval_instance.delete() + + class Meta: + verbose_name = _('Interval source') + verbose_name_plural = _('Interval sources') + + +class EmailBaseModel(IntervalBaseModel): + host = models.CharField(max_length=128, verbose_name=_('Host')) + ssl = models.BooleanField(verbose_name=_('SSL')) + port = models.PositiveIntegerField(blank=True, null=True, verbose_name=_('Port'), help_text=_('Typical choices are 110 for POP3, 995 for POP3 over SSL, 143 for IMAP, 993 for IMAP over SSL.')) + username = models.CharField(max_length=96, verbose_name=_('Username')) + password = models.CharField(max_length=96, verbose_name=_('Password')) + + # From: http://bookmarks.honewatson.com/2009/08/11/python-gmail-imaplib-search-subject-get-attachments/ + @staticmethod + def process_message(source, message): + email = message_from_string(message) + counter = 1 + + for part in email.walk(): + disposition = part.get('Content-Disposition', 'none') + logger.debug('Disposition: %s' % disposition) + + if disposition.startswith('attachment'): + raw_filename = part.get_filename() + + if raw_filename: + filename = collapse_rfc2231_value(raw_filename) + else: + filename = _('attachment-%i') % counter + counter += 1 + + logger.debug('filename: %s' % filename) + + document_file = Attachment(part, name=filename) + source.upload_file(document_file, expand=(source.uncompress == SOURCE_UNCOMPRESS_CHOICE_Y), document_type=source.document_type) + + class Meta: + verbose_name = _('Email source') + verbose_name_plural = _('Email sources') + + +class POP3Email(EmailBaseModel): + source_type = SOURCE_CHOICE_EMAIL_POP3 + + timeout = models.PositiveIntegerField(default=DEFAULT_POP3_TIMEOUT, verbose_name=_('Timeout')) + + def fetch_mail(self): + try: + logger.debug('Starting POP3 email fetch') + logger.debug('host: %s' % self.host) + logger.debug('ssl: %s' % self.ssl) + + if self.ssl: + mailbox = poplib.POP3_SSL(self.host, self.port) + else: + mailbox = poplib.POP3(self.host, self.port, timeout=POP3_TIMEOUT) + + mailbox.getwelcome() + mailbox.user(self.username) + mailbox.pass_(self.password) + messages_info = mailbox.list() + + logger.debug('messages_info:') + logger.debug(messages_info) + logger.debug('messages count: %s' % len(messages_info[1])) + + for message_info in messages_info[1]: + message_number, message_size = message_info.split() + logger.debug('message_number: %s' % message_number) + logger.debug('message_size: %s' % message_size) + + complete_message = '\n'.join(mailbox.retr(message_number)[1]) + + EmailBaseModel.process_message(source=self, message=complete_message) + mailbox.dele(message_number) + + mailbox.quit() + #SourceLog.objects.save_status(source=self, status='Successful connection.') + + except Exception as exception: + logger.error('Unhandled exception: %s' % exception) + #SourceLog.objects.save_status(source=self, status='Error: %s' % exc) + + class Meta: + verbose_name = _('POP email') + verbose_name_plural = _('POP email') + + +class IMAPEmail(EmailBaseModel): + source_type = SOURCE_CHOICE_EMAIL_IMAP + + mailbox = models.CharField(max_length=64, default=DEFAULT_IMAP_MAILBOX, verbose_name=_('Mailbox'), help_text=_('Mail from which to check for messages with attached documents.')) + + # http://www.doughellmann.com/PyMOTW/imaplib/ + def fetch_mail(self): + try: + logger.debug('Starting IMAP email fetch') + logger.debug('host: %s' % self.host) + logger.debug('ssl: %s' % self.ssl) + + if self.ssl: + mailbox = imaplib.IMAP4_SSL(self.host, self.port) + else: + mailbox = imaplib.IMAP4(self.host, self.port) + + mailbox.login(self.username, self.password) + mailbox.select(self.mailbox) + + status, data = mailbox.search(None, 'NOT', 'DELETED') + if data: + messages_info = data[0].split() + logger.debug('messages count: %s' % len(messages_info)) + + for message_number in messages_info: + logger.debug('message_number: %s' % message_number) + status, data = mailbox.fetch(message_number, '(RFC822)') + EmailBaseModel.process_message(source=self, message=data[0][1]) + mailbox.store(message_number, '+FLAGS', '\\Deleted') + + mailbox.expunge() + mailbox.close() + mailbox.logout() + #SourceLog.objects.save_status(source=self, status='Successful connection.') + except Exception as exception: + logger.error('Unhandled exception: %s' % exc) + #SourceLog.objects.save_status(source=self, status='Error: %s' % exc) + + class Meta: + verbose_name = _('IMAP email') + verbose_name_plural = _('IMAP email') + + class WatchFolderSource(OutOfProcessSource): - is_interactive = False source_type = SOURCE_CHOICE_WATCH folder_path = models.CharField(max_length=255, verbose_name=_(u'Folder path'), help_text=_(u'Server side filesystem path.')) diff --git a/mayan/settings/base.py b/mayan/settings/base.py index 871ffc94b5..26c1dd29e9 100644 --- a/mayan/settings/base.py +++ b/mayan/settings/base.py @@ -49,6 +49,7 @@ INSTALLED_APPS = ( # 3rd party 'compressor', 'corsheaders', + 'djcelery', 'filetransfers', 'mptt', 'rest_framework', @@ -266,6 +267,7 @@ REST_FRAMEWORK = { CELERY_TIMEZONE = 'UTC' CELERY_ENABLE_UTC = True CELERY_ALWAYS_EAGER = True +CELERYBEAT_SCHEDULER = 'djcelery.schedulers.DatabaseScheduler' # ------------ CORS ------------ CORS_ORIGIN_ALLOW_ALL = True # ------ Django REST Swagger ----- From 8fd22d8f630f783b87b90bc69550789984357788 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Mon, 20 Oct 2014 02:35:44 -0400 Subject: [PATCH 2/7] Fix URL regex, source_type no longer required --- mayan/apps/sources/urls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mayan/apps/sources/urls.py b/mayan/apps/sources/urls.py index 34adabb6a5..13edaa5f4d 100644 --- a/mayan/apps/sources/urls.py +++ b/mayan/apps/sources/urls.py @@ -15,7 +15,7 @@ urlpatterns = patterns('sources.views', url(r'^upload/document/new/interactive/(?P\d+)/$', 'upload_interactive', (), 'upload_interactive'), url(r'^upload/document/new/interactive/$', 'upload_interactive', (), 'upload_interactive'), - url(r'^upload/document/(?P\d+)/version/interactive/(?P\w+)/(?P\d+)/$', 'upload_interactive', (), 'upload_version'), + url(r'^upload/document/(?P\d+)/version/interactive/(?P\d+)/$', 'upload_interactive', (), 'upload_version'), url(r'^upload/document/(?P\d+)/version/interactive/$', 'upload_interactive', (), 'upload_version'), # Setup views From cf214801f5adbc507f446e5e72bd4f13d301c310 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Mon, 20 Oct 2014 02:36:19 -0400 Subject: [PATCH 3/7] Add missing view namespace --- mayan/apps/sources/views.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mayan/apps/sources/views.py b/mayan/apps/sources/views.py index 4ed676eb2e..ef0ed99ef0 100644 --- a/mayan/apps/sources/views.py +++ b/mayan/apps/sources/views.py @@ -111,6 +111,7 @@ def upload_interactive(request, source_id=None, document_pk=None): context = {} + # TODO: use InteractiveSource.objects.count() instead if results[SOURCE_CHOICE_WEB_FORM].count() == 0 and results[SOURCE_CHOICE_STAGING].count() == 0: source_setup_link = mark_safe('%s' % (reverse('sources:setup_web_form_list'), ugettext(u'Here'))) subtemplates_list.append( @@ -285,7 +286,7 @@ def upload_interactive(request, source_id=None, document_pk=None): 'subtemplates_list': subtemplates_list, 'temporary_navigation_links': { 'form_header': { - 'upload_version': { + 'sources:upload_version': { 'links': results['tab_links'] }, 'sources:upload_interactive': { From 713394396364274a79b32f0c34b7327e478df785 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Mon, 20 Oct 2014 17:48:45 -0400 Subject: [PATCH 4/7] Issue #65, complete set of code to implement receiving documents via email (experimental) --- mayan/apps/sources/__init__.py | 23 +++++---- mayan/apps/sources/forms.py | 14 +++++- mayan/apps/sources/links.py | 5 +- mayan/apps/sources/models.py | 13 ++--- mayan/apps/sources/tasks.py | 6 +++ mayan/apps/sources/urls.py | 5 +- mayan/apps/sources/views.py | 87 ++++++++++++++++++++++------------ 7 files changed, 103 insertions(+), 50 deletions(-) diff --git a/mayan/apps/sources/__init__.py b/mayan/apps/sources/__init__.py index fbbba9ce0b..d0ef0458d9 100644 --- a/mayan/apps/sources/__init__.py +++ b/mayan/apps/sources/__init__.py @@ -10,6 +10,7 @@ from rest_api.classes import APIEndPoint from .classes import StagingFile from .links import (document_create_multiple, document_create_siblings, + setup_imap_email_list, setup_pop3_email_list, setup_sources, setup_web_form_list, setup_source_create, setup_source_delete, setup_source_edit, setup_source_transformation_create, @@ -18,8 +19,8 @@ from .links import (document_create_multiple, document_create_siblings, setup_source_transformation_list, setup_staging_folder_list, setup_watch_folder_list, staging_file_delete, upload_version) -from .models import (SourceTransformation, StagingFolderSource, - WatchFolderSource, WebFormSource) +from .models import (IMAPEmail, POP3Email, SourceTransformation, + StagingFolderSource, WatchFolderSource, WebFormSource) from .urls import api_urls from .widgets import staging_file_thumbnail @@ -27,17 +28,23 @@ register_links([StagingFile], [staging_file_delete]) register_links(SourceTransformation, [setup_source_transformation_edit, setup_source_transformation_delete]) -register_links(['sources:setup_web_form_list', 'sources:setup_staging_folder_list', 'sources:setup_watch_folder_list', 'sources:setup_source_create'], [setup_web_form_list, setup_staging_folder_list], menu_name='form_header') +register_links(['sources:setup_imap_email_list', 'sources:setup_pop3_email_list', 'sources:setup_web_form_list', 'sources:setup_staging_folder_list', 'sources:setup_watch_folder_list', 'sources:setup_source_create'], [setup_web_form_list, setup_staging_folder_list, setup_pop3_email_list, setup_imap_email_list], menu_name='form_header') -register_links(WebFormSource, [setup_web_form_list, setup_staging_folder_list], menu_name='form_header') +register_links(WebFormSource, [setup_web_form_list, setup_staging_folder_list, setup_pop3_email_list, setup_imap_email_list], menu_name='form_header') register_links(WebFormSource, [setup_source_transformation_list, setup_source_edit, setup_source_delete]) -register_links(['sources:setup_web_form_list', 'sources:setup_staging_folder_list', 'sources:setup_watch_folder_list', 'sources:setup_source_edit', 'sources:setup_source_delete', 'sources:setup_source_create'], [setup_sources, setup_source_create], menu_name='sidebar') +register_links(['sources:setup_imap_email_list', 'sources:setup_pop3_email_list', 'sources:setup_web_form_list', 'sources:setup_staging_folder_list', 'sources:setup_watch_folder_list', 'sources:setup_source_edit', 'sources:setup_source_delete', 'sources:setup_source_create'], [setup_sources, setup_source_create], menu_name='sidebar') -register_links(StagingFolderSource, [setup_web_form_list, setup_staging_folder_list], menu_name='form_header') +register_links(StagingFolderSource, [setup_web_form_list, setup_staging_folder_list, setup_pop3_email_list, setup_imap_email_list], menu_name='form_header') register_links(StagingFolderSource, [setup_source_transformation_list, setup_source_edit, setup_source_delete]) -register_links(WatchFolderSource, [setup_web_form_list, setup_staging_folder_list, setup_watch_folder_list], menu_name='form_header') +register_links(POP3Email, [setup_web_form_list, setup_staging_folder_list, setup_pop3_email_list, setup_imap_email_list], menu_name='form_header') +register_links(POP3Email, [setup_source_transformation_list, setup_source_edit, setup_source_delete]) + +register_links(IMAPEmail, [setup_web_form_list, setup_staging_folder_list, setup_pop3_email_list, setup_imap_email_list], menu_name='form_header') +register_links(IMAPEmail, [setup_source_transformation_list, setup_source_edit, setup_source_delete]) + +register_links(WatchFolderSource, [setup_web_form_list, setup_staging_folder_list, setup_pop3_email_list, setup_imap_email_list], menu_name='form_header') register_links(WatchFolderSource, [setup_source_transformation_list, setup_source_edit, setup_source_delete]) # Document version @@ -45,7 +52,7 @@ register_links(['documents:document_version_list', 'documents:upload_version', ' register_links(['sources:setup_source_transformation_create', 'sources:setup_source_transformation_edit', 'sources:setup_source_transformation_delete', 'sources:setup_source_transformation_list'], [setup_source_transformation_create], menu_name='sidebar') -source_views = ['sources:setup_web_form_list', 'sources:setup_staging_folder_list', 'sources:setup_watch_folder_list', 'sources:setup_source_edit', 'sources:setup_source_delete', 'sources:setup_source_create', 'sources:setup_source_transformation_list', 'sources:setup_source_transformation_edit', 'sources:setup_source_transformation_delete', 'sources:setup_source_transformation_create'] +source_views = ['sources:setup_imap_email_list', 'sources:setup_pop3_email_list', 'sources:setup_web_form_list', 'sources:setup_staging_folder_list', 'sources:setup_watch_folder_list', 'sources:setup_source_edit', 'sources:setup_source_delete', 'sources:setup_source_create', 'sources:setup_source_transformation_list', 'sources:setup_source_transformation_edit', 'sources:setup_source_transformation_delete', 'sources:setup_source_transformation_create'] register_model_list_columns(StagingFile, [ { diff --git a/mayan/apps/sources/forms.py b/mayan/apps/sources/forms.py index 938e168384..ac2104fbd7 100644 --- a/mayan/apps/sources/forms.py +++ b/mayan/apps/sources/forms.py @@ -8,8 +8,8 @@ from django.utils.translation import ugettext_lazy as _ from documents.forms import DocumentForm -from .models import (SourceTransformation, StagingFolderSource, WebFormSource, - WatchFolderSource) +from .models import (IMAPEmail, POP3Email, SourceTransformation, + StagingFolderSource, WebFormSource, WatchFolderSource) from .utils import validate_whitelist_blacklist logger = logging.getLogger(__name__) @@ -85,6 +85,16 @@ class StagingFolderSetupForm(forms.ModelForm): model = StagingFolderSource +class POP3EmailSetupForm(forms.ModelForm): + class Meta: + model = POP3Email + + +class IMAPEmailSetupForm(forms.ModelForm): + class Meta: + model = IMAPEmail + + class WatchFolderSetupForm(forms.ModelForm): class Meta: model = WatchFolderSource diff --git a/mayan/apps/sources/links.py b/mayan/apps/sources/links.py index 8327bc48e8..730b9228a8 100644 --- a/mayan/apps/sources/links.py +++ b/mayan/apps/sources/links.py @@ -5,7 +5,8 @@ from django.utils.translation import ugettext_lazy as _ from documents.permissions import (PERMISSION_DOCUMENT_CREATE, PERMISSION_DOCUMENT_NEW_VERSION) -from .models import StagingFolderSource, WatchFolderSource, WebFormSource +from .models import (IMAPEmail, POP3Email, StagingFolderSource, + WatchFolderSource, WebFormSource) from .permissions import (PERMISSION_SOURCES_SETUP_CREATE, PERMISSION_SOURCES_SETUP_DELETE, PERMISSION_SOURCES_SETUP_EDIT, @@ -20,6 +21,8 @@ setup_sources = {'text': _(u'Sources'), 'view': 'sources:setup_web_form_list', ' setup_web_form_list = {'text': _(u'Web forms'), 'view': 'sources:setup_web_form_list', 'famfam': 'application_form', 'icon': 'application_form.png', 'children_classes': [WebFormSource], 'permissions': [PERMISSION_SOURCES_SETUP_VIEW]} setup_staging_folder_list = {'text': _(u'Staging folders'), 'view': 'sources:setup_staging_folder_list', 'famfam': 'folder_camera', 'children_classes': [StagingFolderSource], 'permissions': [PERMISSION_SOURCES_SETUP_VIEW]} setup_watch_folder_list = {'text': _(u'Watch folders'), 'view': 'sources:setup_watch_folder_list', 'famfam': 'folder_magnify', 'children_classes': [WatchFolderSource], 'permissions': [PERMISSION_SOURCES_SETUP_VIEW]} +setup_pop3_email_list = {'text': _(u'POP3 emails'), 'view': 'sources:setup_pop3_email_list', 'famfam': 'email', 'children_classes': [POP3Email], 'permissions': [PERMISSION_SOURCES_SETUP_VIEW]} +setup_imap_email_list = {'text': _(u'IMAP emails'), 'view': 'sources:setup_imap_email_list', 'famfam': 'email', 'children_classes': [IMAPEmail], 'permissions': [PERMISSION_SOURCES_SETUP_VIEW]} setup_source_edit = {'text': _(u'Edit'), 'view': 'sources:setup_source_edit', 'args': ['source.pk'], 'famfam': 'application_form_edit', 'permissions': [PERMISSION_SOURCES_SETUP_EDIT]} setup_source_delete = {'text': _(u'Delete'), 'view': 'sources:setup_source_delete', 'args': ['source.pk'], 'famfam': 'application_form_delete', 'permissions': [PERMISSION_SOURCES_SETUP_DELETE]} diff --git a/mayan/apps/sources/models.py b/mayan/apps/sources/models.py index 6292575c3b..9b204388d2 100644 --- a/mayan/apps/sources/models.py +++ b/mayan/apps/sources/models.py @@ -204,13 +204,13 @@ class IntervalBaseModel(OutOfProcessSource): super(IntervalBaseModel, self).save(*args, **kwargs) periodic_task_name = 'check_interval_source-%i' % self.pk if new_source: - interval_instance = IntervalSchedule.objects.create(every=self.interval) + interval_instance = IntervalSchedule.objects.create(every=self.interval, period='seconds') PeriodicTask.objects.create( name=periodic_task_name, interval=interval_instance, task='sources.tasks.task_check_interval_source', queue='mailing', - args=json.dump({'source_id': self.pk}) + kwargs=json.dumps({'source_id': self.pk}) ) else: periodic_task = PeriodicTask.objects.get(name=periodic_task_name) @@ -219,8 +219,8 @@ class IntervalBaseModel(OutOfProcessSource): periodic_task.save() def delete(self, *args, **kwargs): - super(IntervalBaseModel, self).delete(*args, **kwargs) periodic_task_name = 'check_interval_source-%i' % self.pk + super(IntervalBaseModel, self).delete(*args, **kwargs) periodic_task = PeriodicTask.objects.get(name=periodic_task_name) interval_instance = periodic_task.interval periodic_task.delete() @@ -303,11 +303,9 @@ class POP3Email(EmailBaseModel): mailbox.dele(message_number) mailbox.quit() - #SourceLog.objects.save_status(source=self, status='Successful connection.') - except Exception as exception: logger.error('Unhandled exception: %s' % exception) - #SourceLog.objects.save_status(source=self, status='Error: %s' % exc) + # TODO: Add user notification class Meta: verbose_name = _('POP email') @@ -348,10 +346,9 @@ class IMAPEmail(EmailBaseModel): mailbox.expunge() mailbox.close() mailbox.logout() - #SourceLog.objects.save_status(source=self, status='Successful connection.') except Exception as exception: logger.error('Unhandled exception: %s' % exc) - #SourceLog.objects.save_status(source=self, status='Error: %s' % exc) + # TODO: Add user notification class Meta: verbose_name = _('IMAP email') diff --git a/mayan/apps/sources/tasks.py b/mayan/apps/sources/tasks.py index 98cf19e9ad..0dc6c6258d 100644 --- a/mayan/apps/sources/tasks.py +++ b/mayan/apps/sources/tasks.py @@ -13,6 +13,12 @@ from .models import Source logger = logging.getLogger(__name__) +@app.task(ignore_result=True) +def task_check_interval_source(source_id): + source = Source.objects.get_subclass(pk=source_id) + source.fetch_mail() + + @app.task(ignore_result=True) def task_upload_document(source_id, file_path, filename=None, use_file_name=False, document_type_id=None, expand=False, metadata_dict_list=None, user_id=None, document_id=None, new_version_data=None, command_line=False, description=None): source = Source.objects.get_subclass(pk=source_id) diff --git a/mayan/apps/sources/urls.py b/mayan/apps/sources/urls.py index 13edaa5f4d..382cff78f4 100644 --- a/mayan/apps/sources/urls.py +++ b/mayan/apps/sources/urls.py @@ -5,7 +5,8 @@ from django.conf.urls import patterns, url from .api_views import (APIDocumentCreateView, APIStagingSourceFileView, APIStagingSourceFileImageView, APIStagingSourceListView, APIStagingSourceView) -from .literals import (SOURCE_CHOICE_STAGING, SOURCE_CHOICE_WATCH, +from .literals import (SOURCE_CHOICE_EMAIL_POP3, SOURCE_CHOICE_EMAIL_IMAP, + SOURCE_CHOICE_STAGING, SOURCE_CHOICE_WATCH, SOURCE_CHOICE_WEB_FORM) from .wizards import DocumentCreateWizard @@ -23,6 +24,8 @@ urlpatterns = patterns('sources.views', url(r'^setup/interactive/%s/list/$' % SOURCE_CHOICE_WEB_FORM, 'setup_source_list', {'source_type': SOURCE_CHOICE_WEB_FORM}, 'setup_web_form_list'), url(r'^setup/interactive/%s/list/$' % SOURCE_CHOICE_STAGING, 'setup_source_list', {'source_type': SOURCE_CHOICE_STAGING}, 'setup_staging_folder_list'), url(r'^setup/interactive/%s/list/$' % SOURCE_CHOICE_WATCH, 'setup_source_list', {'source_type': SOURCE_CHOICE_WATCH}, 'setup_watch_folder_list'), + url(r'^setup/interactive/%s/list/$' % SOURCE_CHOICE_EMAIL_POP3, 'setup_source_list', {'source_type': SOURCE_CHOICE_EMAIL_POP3}, 'setup_pop3_email_list'), + url(r'^setup/interactive/%s/list/$' % SOURCE_CHOICE_EMAIL_IMAP, 'setup_source_list', {'source_type': SOURCE_CHOICE_EMAIL_IMAP}, 'setup_imap_email_list'), url(r'^setup/interactive/(?P\w+)/list/$', 'setup_source_list', (), 'setup_source_list'), url(r'^setup/interactive/(?P\d+)/edit/$', 'setup_source_edit', (), 'setup_source_edit'), diff --git a/mayan/apps/sources/views.py b/mayan/apps/sources/views.py index ef0ed99ef0..819e3c9a4c 100644 --- a/mayan/apps/sources/views.py +++ b/mayan/apps/sources/views.py @@ -23,14 +23,16 @@ from documents.permissions import (PERMISSION_DOCUMENT_CREATE, from metadata.api import decode_metadata_from_url, metadata_repr_as_list from permissions.models import Permission -from .forms import (StagingDocumentForm, StagingFolderSetupForm, - SourceTransformationForm, SourceTransformationForm_create, - WatchFolderSetupForm, WebFormForm, WebFormSetupForm) -from .literals import (SOURCE_CHOICE_STAGING, SOURCE_CHOICE_WATCH, +from .forms import (POP3EmailSetupForm, IMAPEmailSetupForm, StagingDocumentForm, + StagingFolderSetupForm, SourceTransformationForm, + SourceTransformationForm_create, WatchFolderSetupForm, + WebFormForm, WebFormSetupForm) +from .literals import (SOURCE_CHOICE_EMAIL_IMAP, SOURCE_CHOICE_EMAIL_POP3, + SOURCE_CHOICE_STAGING, SOURCE_CHOICE_WATCH, SOURCE_CHOICE_WEB_FORM, SOURCE_UNCOMPRESS_CHOICE_ASK, SOURCE_UNCOMPRESS_CHOICE_Y) -from .models import (Source, StagingFolderSource, SourceTransformation, - WatchFolderSource, WebFormSource) +from .models import (IMAPEmail, POP3Email, Source, StagingFolderSource, + SourceTransformation, WatchFolderSource, WebFormSource) from .permissions import (PERMISSION_SOURCES_SETUP_CREATE, PERMISSION_SOURCES_SETUP_DELETE, PERMISSION_SOURCES_SETUP_EDIT, @@ -38,6 +40,32 @@ from .permissions import (PERMISSION_SOURCES_SETUP_CREATE, from .tasks import task_upload_document +def get_class(source_type): + if source_type == SOURCE_CHOICE_WEB_FORM: + return WebFormSource + elif source_type == SOURCE_CHOICE_STAGING: + return StagingFolderSource + elif source_type == SOURCE_CHOICE_WATCH: + return WatchFolderSource + elif source_type == SOURCE_CHOICE_EMAIL_POP3: + return POP3Email + elif source_type == SOURCE_CHOICE_EMAIL_IMAP: + return IMAPEmail + + +def get_form_class(source_type): + if source_type == SOURCE_CHOICE_WEB_FORM: + return WebFormSetupForm + elif source_type == SOURCE_CHOICE_STAGING: + return StagingFolderSetupForm + elif source_type == SOURCE_CHOICE_WATCH: + return WatchFolderSetupForm + elif source_type == SOURCE_CHOICE_EMAIL_POP3: + return POP3EmailSetupForm + elif source_type == SOURCE_CHOICE_EMAIL_IMAP: + return IMAPEmailSetupForm + + def document_create_siblings(request, document_id): Permission.objects.check_permissions(request.user, [PERMISSION_DOCUMENT_CREATE]) @@ -86,10 +114,20 @@ def get_active_tab_links(document=None): for staging_folder in staging_folders: tab_links.append(get_tab_link_for_source(staging_folder, document)) + pop3_emails = POP3Email.objects.filter(enabled=True) + for source_instance in pop3_emails: + tab_links.append(get_tab_link_for_source(source_instance, document)) + + imap_emails = IMAPEmail.objects.filter(enabled=True) + for source_instance in imap_emails: + tab_links.append(get_tab_link_for_source(source_instance, document)) + return { 'tab_links': tab_links, SOURCE_CHOICE_WEB_FORM: web_forms, - SOURCE_CHOICE_STAGING: staging_folders + SOURCE_CHOICE_STAGING: staging_folders, + SOURCE_CHOICE_EMAIL_POP3: pop3_emails, + SOURCE_CHOICE_EMAIL_IMAP: imap_emails } @@ -370,13 +408,9 @@ def staging_file_delete(request, staging_folder_pk, encoded_filename): def setup_source_list(request, source_type): Permission.objects.check_permissions(request.user, [PERMISSION_SOURCES_SETUP_VIEW]) - if source_type == SOURCE_CHOICE_WEB_FORM: - cls = WebFormSource - elif source_type == SOURCE_CHOICE_STAGING: - cls = StagingFolderSource - elif source_type == SOURCE_CHOICE_WATCH: - cls = WatchFolderSource + cls = get_class(source_type) + # TODO: remove plurals context = { 'object_list': cls.objects.all(), 'title': cls.class_fullname_plural(), @@ -393,12 +427,7 @@ def setup_source_edit(request, source_id): Permission.objects.check_permissions(request.user, [PERMISSION_SOURCES_SETUP_EDIT]) source = get_object_or_404(Source.objects.select_subclasses(), pk=source_id) - if isinstance(source, WebFormSource): - form_class = WebFormSetupForm - elif isinstance(source, StagingFolderSource): - form_class = StagingFolderSetupForm - elif isinstance(source, WatchFolderSource): - form_class = WatchFolderSetupForm + form_class = get_form_class(source.source_type) next = request.POST.get('next', request.GET.get('next', request.META.get('HTTP_REFERER', reverse(settings.LOGIN_REDIRECT_URL)))) @@ -439,6 +468,12 @@ def setup_source_delete(request, source_id): elif isinstance(source, WatchFolderSource): form_icon = u'folder_delete.png' redirect_view = 'sources:setup_watch_folder_list' + elif isinstance(source, POP3Email): + form_icon = u'folder_delete.png' + redirect_view = 'sources:setup_pop3_email_list' + elif isinstance(source, IMAPEmail): + form_icon = u'folder_delete.png' + redirect_view = 'sources:setup_imap_email_list' previous = request.POST.get('previous', request.GET.get('previous', request.META.get('HTTP_REFERER', redirect_view))) @@ -450,8 +485,7 @@ def setup_source_delete(request, source_id): messages.error(request, _(u'Error deleting source "%(source)s": %(error)s') % { 'source': source, 'error': exception }) - - return HttpResponseRedirect(redirect_view) + return HttpResponseRedirect(reverse(redirect_view)) context = { 'title': _(u'Are you sure you wish to delete the source: %s?') % source.fullname(), @@ -471,15 +505,8 @@ def setup_source_delete(request, source_id): def setup_source_create(request, source_type): Permission.objects.check_permissions(request.user, [PERMISSION_SOURCES_SETUP_CREATE]) - if source_type == SOURCE_CHOICE_WEB_FORM: - cls = WebFormSource - form_class = WebFormSetupForm - elif source_type == SOURCE_CHOICE_STAGING: - cls = WebFormSource - form_class = StagingFolderSetupForm - elif source_type == SOURCE_CHOICE_WATCH: - cls = WebFormSource - form_class = WatchFolderSetupForm + cls = get_class(source_type) + form_class = get_form_class(source_type) if request.method == 'POST': form = form_class(data=request.POST) From e9d53809df02b4e76ae7435a72e925b7e6c5d0b3 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Mon, 20 Oct 2014 21:56:57 -0400 Subject: [PATCH 5/7] Add missing import --- mayan/apps/sources/classes.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mayan/apps/sources/classes.py b/mayan/apps/sources/classes.py index de8d8d5e82..f3b0da0219 100644 --- a/mayan/apps/sources/classes.py +++ b/mayan/apps/sources/classes.py @@ -4,6 +4,11 @@ import base64 import os import urllib +try: + from cStringIO import StringIO +except ImportError: + from StringIO import StringIO + from django.core.files import File from converter.api import convert From eaf1e55faba5ac8ff249a8e35078c2761eafd64b Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Mon, 20 Oct 2014 21:57:19 -0400 Subject: [PATCH 6/7] Improve periodic task creation and interval reuse --- mayan/apps/sources/models.py | 61 ++++++++++++++++++++++-------------- 1 file changed, 38 insertions(+), 23 deletions(-) diff --git a/mayan/apps/sources/models.py b/mayan/apps/sources/models.py index 9b204388d2..dc9d0c3f39 100644 --- a/mayan/apps/sources/models.py +++ b/mayan/apps/sources/models.py @@ -1,9 +1,13 @@ from __future__ import absolute_import from ast import literal_eval +from email.Utils import collapse_rfc2231_value +from email import message_from_string import json +import imaplib import logging import os +import poplib from django.contrib.contenttypes import generic from django.contrib.contenttypes.models import ContentType @@ -26,8 +30,8 @@ from .literals import (DEFAULT_INTERVAL, DEFAULT_POP3_TIMEOUT, SOURCE_CHOICES_PLURAL, SOURCE_CHOICE_STAGING, SOURCE_CHOICE_WATCH, SOURCE_CHOICE_WEB_FORM, SOURCE_INTERACTIVE_UNCOMPRESS_CHOICES, - SOURCE_UNCOMPRESS_CHOICES, SOURCE_CHOICE_EMAIL_IMAP, - SOURCE_CHOICE_EMAIL_POP3) + SOURCE_UNCOMPRESS_CHOICES, SOURCE_UNCOMPRESS_CHOICE_Y, + SOURCE_CHOICE_EMAIL_IMAP, SOURCE_CHOICE_EMAIL_POP3) from .managers import SourceTransformationManager logger = logging.getLogger(__name__) @@ -36,6 +40,7 @@ logger = logging.getLogger(__name__) class Source(models.Model): title = models.CharField(max_length=64, verbose_name=_(u'Title')) enabled = models.BooleanField(default=True, verbose_name=_(u'Enabled')) + # TODO: remove whitelist and blacklists whitelist = models.TextField(blank=True, verbose_name=_(u'Whitelist'), editable=False) blacklist = models.TextField(blank=True, verbose_name=_(u'Blacklist'), editable=False) @@ -199,32 +204,41 @@ class IntervalBaseModel(OutOfProcessSource): document_type = models.ForeignKey(DocumentType, null=True, blank=True, verbose_name=_('Document type'), help_text=_('Assign a document type to documents uploaded from this source.')) uncompress = models.CharField(max_length=1, choices=SOURCE_UNCOMPRESS_CHOICES, verbose_name=_('Uncompress'), help_text=_('Whether to expand or not, compressed archives.')) + def _get_periodic_task_name(self, pk=None): + return 'check_interval_source-%i' % (pk or self.pk) + + def _delete_periodic_task(self, pk=None): + periodic_task = PeriodicTask.objects.get(name=self._get_periodic_task_name(pk)) + + interval_instance = periodic_task.interval + + if tuple(interval_instance.periodictask_set.values_list('id', flat=True)) == (periodic_task.pk,): + # Only delete the interval if nobody else is using it + interval_instance.delete() + else: + periodic_task.delete() + def save(self, *args, **kwargs): new_source = not self.pk super(IntervalBaseModel, self).save(*args, **kwargs) - periodic_task_name = 'check_interval_source-%i' % self.pk - if new_source: - interval_instance = IntervalSchedule.objects.create(every=self.interval, period='seconds') - PeriodicTask.objects.create( - name=periodic_task_name, - interval=interval_instance, - task='sources.tasks.task_check_interval_source', - queue='mailing', - kwargs=json.dumps({'source_id': self.pk}) - ) - else: - periodic_task = PeriodicTask.objects.get(name=periodic_task_name) - periodic_task.interval.every = self.interval - periodic_task.interval.save() - periodic_task.save() + + if not new_source: + self._delete_periodic_task() + + interval_instance, created = IntervalSchedule.objects.get_or_create(every=self.interval, period='seconds') + # Create a new interval or reuse someone else's + PeriodicTask.objects.create( + name=self._get_periodic_task_name(), + interval=interval_instance, + task='sources.tasks.task_check_interval_source', + queue='mailing', + kwargs=json.dumps({'source_id': self.pk}) + ) def delete(self, *args, **kwargs): - periodic_task_name = 'check_interval_source-%i' % self.pk + pk = self.pk super(IntervalBaseModel, self).delete(*args, **kwargs) - periodic_task = PeriodicTask.objects.get(name=periodic_task_name) - interval_instance = periodic_task.interval - periodic_task.delete() - interval_instance.delete() + self._delete_periodic_task(pk) class Meta: verbose_name = _('Interval source') @@ -239,6 +253,7 @@ class EmailBaseModel(IntervalBaseModel): password = models.CharField(max_length=96, verbose_name=_('Password')) # From: http://bookmarks.honewatson.com/2009/08/11/python-gmail-imaplib-search-subject-get-attachments/ + # TODO: Add lock to avoid being running more than once concurrent @staticmethod def process_message(source, message): email = message_from_string(message) @@ -347,7 +362,7 @@ class IMAPEmail(EmailBaseModel): mailbox.close() mailbox.logout() except Exception as exception: - logger.error('Unhandled exception: %s' % exc) + logger.error('Unhandled exception: %s' % exception) # TODO: Add user notification class Meta: From 2575e0849941a922ee2c934bee709b1963745460 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Mon, 20 Oct 2014 21:57:39 -0400 Subject: [PATCH 7/7] Only fetch when source is enabled --- mayan/apps/sources/tasks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mayan/apps/sources/tasks.py b/mayan/apps/sources/tasks.py index 0dc6c6258d..5516e7b4fb 100644 --- a/mayan/apps/sources/tasks.py +++ b/mayan/apps/sources/tasks.py @@ -16,7 +16,8 @@ logger = logging.getLogger(__name__) @app.task(ignore_result=True) def task_check_interval_source(source_id): source = Source.objects.get_subclass(pk=source_id) - source.fetch_mail() + if source.enabled: + source.fetch_mail() @app.task(ignore_result=True)