Merge branch 'feature/email_receive' into development

mayan/apps/sources/__init__.py

@@ -10,6 +10,7 @@ from rest_api.classes import APIEndPoint
 
 from .classes import StagingFile
 from .links import (document_create_multiple, document_create_siblings,
+                    setup_imap_email_list, setup_pop3_email_list,
                     setup_sources, setup_web_form_list, setup_source_create,
                     setup_source_delete, setup_source_edit,
                     setup_source_transformation_create,
@@ -18,8 +19,8 @@ from .links import (document_create_multiple, document_create_siblings,
                     setup_source_transformation_list,
                     setup_staging_folder_list, setup_watch_folder_list,
                     staging_file_delete, upload_version)
-from .models import (SourceTransformation, StagingFolderSource,
+from .models import (IMAPEmail, POP3Email, SourceTransformation,
-                     WatchFolderSource, WebFormSource)
+                     StagingFolderSource, WatchFolderSource, WebFormSource)
 from .urls import api_urls
 from .widgets import staging_file_thumbnail
 
@@ -27,17 +28,23 @@ register_links([StagingFile], [staging_file_delete])
 
 register_links(SourceTransformation, [setup_source_transformation_edit, setup_source_transformation_delete])
 
-register_links(['sources:setup_web_form_list', 'sources:setup_staging_folder_list', 'sources:setup_watch_folder_list', 'sources:setup_source_create'], [setup_web_form_list, setup_staging_folder_list], menu_name='form_header')
+register_links(['sources:setup_imap_email_list', 'sources:setup_pop3_email_list', 'sources:setup_web_form_list', 'sources:setup_staging_folder_list', 'sources:setup_watch_folder_list', 'sources:setup_source_create'], [setup_web_form_list, setup_staging_folder_list, setup_pop3_email_list, setup_imap_email_list], menu_name='form_header')
 
-register_links(WebFormSource, [setup_web_form_list, setup_staging_folder_list], menu_name='form_header')
+register_links(WebFormSource, [setup_web_form_list, setup_staging_folder_list, setup_pop3_email_list, setup_imap_email_list], menu_name='form_header')
 register_links(WebFormSource, [setup_source_transformation_list, setup_source_edit, setup_source_delete])
 
-register_links(['sources:setup_web_form_list', 'sources:setup_staging_folder_list', 'sources:setup_watch_folder_list', 'sources:setup_source_edit', 'sources:setup_source_delete', 'sources:setup_source_create'], [setup_sources, setup_source_create], menu_name='sidebar')
+register_links(['sources:setup_imap_email_list', 'sources:setup_pop3_email_list', 'sources:setup_web_form_list', 'sources:setup_staging_folder_list', 'sources:setup_watch_folder_list', 'sources:setup_source_edit', 'sources:setup_source_delete', 'sources:setup_source_create'], [setup_sources, setup_source_create], menu_name='sidebar')
 
-register_links(StagingFolderSource, [setup_web_form_list, setup_staging_folder_list], menu_name='form_header')
+register_links(StagingFolderSource, [setup_web_form_list, setup_staging_folder_list, setup_pop3_email_list, setup_imap_email_list], menu_name='form_header')
 register_links(StagingFolderSource, [setup_source_transformation_list, setup_source_edit, setup_source_delete])
 
-register_links(WatchFolderSource, [setup_web_form_list, setup_staging_folder_list, setup_watch_folder_list], menu_name='form_header')
+register_links(POP3Email, [setup_web_form_list, setup_staging_folder_list, setup_pop3_email_list, setup_imap_email_list], menu_name='form_header')
+register_links(POP3Email, [setup_source_transformation_list, setup_source_edit, setup_source_delete])
+
+register_links(IMAPEmail, [setup_web_form_list, setup_staging_folder_list, setup_pop3_email_list, setup_imap_email_list], menu_name='form_header')
+register_links(IMAPEmail, [setup_source_transformation_list, setup_source_edit, setup_source_delete])
+
+register_links(WatchFolderSource, [setup_web_form_list, setup_staging_folder_list, setup_pop3_email_list, setup_imap_email_list], menu_name='form_header')
 register_links(WatchFolderSource, [setup_source_transformation_list, setup_source_edit, setup_source_delete])
 
 # Document version
@@ -45,7 +52,7 @@ register_links(['documents:document_version_list', 'documents:upload_version', '
 
 register_links(['sources:setup_source_transformation_create', 'sources:setup_source_transformation_edit', 'sources:setup_source_transformation_delete', 'sources:setup_source_transformation_list'], [setup_source_transformation_create], menu_name='sidebar')
 
-source_views = ['sources:setup_web_form_list', 'sources:setup_staging_folder_list', 'sources:setup_watch_folder_list', 'sources:setup_source_edit', 'sources:setup_source_delete', 'sources:setup_source_create', 'sources:setup_source_transformation_list', 'sources:setup_source_transformation_edit', 'sources:setup_source_transformation_delete', 'sources:setup_source_transformation_create']
+source_views = ['sources:setup_imap_email_list', 'sources:setup_pop3_email_list', 'sources:setup_web_form_list', 'sources:setup_staging_folder_list', 'sources:setup_watch_folder_list', 'sources:setup_source_edit', 'sources:setup_source_delete', 'sources:setup_source_create', 'sources:setup_source_transformation_list', 'sources:setup_source_transformation_edit', 'sources:setup_source_transformation_delete', 'sources:setup_source_transformation_create']
 
 register_model_list_columns(StagingFile, [
     {

mayan/apps/sources/classes.py

@@ -4,12 +4,32 @@ import base64
 import os
 import urllib
 
+try:
+    from cStringIO import StringIO
+except ImportError:
+    from StringIO import StringIO
+
 from django.core.files import File
 
 from converter.api import convert
 from mimetype.api import get_mimetype
 
 
+class PseudoFile(File):
+    def __init__(self, file, name):
+        self.name = name
+        self.file = file
+        self.file.seek(0, os.SEEK_END)
+        self.size = self.file.tell()
+        self.file.seek(0)
+
+
+class Attachment(File):
+    def __init__(self, part, name):
+        self.name = name
+        self.file = PseudoFile(StringIO(part.get_payload(decode=True)), name=name)
+
+
 class StagingFile(object):
     """
     Simple class to extend the File class to add preview capabilities

mayan/apps/sources/forms.py

@@ -8,8 +8,8 @@ from django.utils.translation import ugettext_lazy as _
 
 from documents.forms import DocumentForm
 
-from .models import (SourceTransformation, StagingFolderSource, WebFormSource,
+from .models import (IMAPEmail, POP3Email, SourceTransformation,
-                     WatchFolderSource)
+                     StagingFolderSource, WebFormSource, WatchFolderSource)
 from .utils import validate_whitelist_blacklist
 
 logger = logging.getLogger(__name__)
@@ -85,6 +85,16 @@ class StagingFolderSetupForm(forms.ModelForm):
         model = StagingFolderSource
 
 
+class POP3EmailSetupForm(forms.ModelForm):
+    class Meta:
+        model = POP3Email
+
+
+class IMAPEmailSetupForm(forms.ModelForm):
+    class Meta:
+        model = IMAPEmail
+
+
 class WatchFolderSetupForm(forms.ModelForm):
     class Meta:
         model = WatchFolderSource

mayan/apps/sources/links.py

@@ -5,7 +5,8 @@ from django.utils.translation import ugettext_lazy as _
 from documents.permissions import (PERMISSION_DOCUMENT_CREATE,
                                    PERMISSION_DOCUMENT_NEW_VERSION)
 
-from .models import StagingFolderSource, WatchFolderSource, WebFormSource
+from .models import (IMAPEmail, POP3Email, StagingFolderSource,
+                     WatchFolderSource, WebFormSource)
 from .permissions import (PERMISSION_SOURCES_SETUP_CREATE,
                           PERMISSION_SOURCES_SETUP_DELETE,
                           PERMISSION_SOURCES_SETUP_EDIT,
@@ -20,6 +21,8 @@ setup_sources = {'text': _(u'Sources'), 'view': 'sources:setup_web_form_list', '
 setup_web_form_list = {'text': _(u'Web forms'), 'view': 'sources:setup_web_form_list', 'famfam': 'application_form', 'icon': 'application_form.png', 'children_classes': [WebFormSource], 'permissions': [PERMISSION_SOURCES_SETUP_VIEW]}
 setup_staging_folder_list = {'text': _(u'Staging folders'), 'view': 'sources:setup_staging_folder_list', 'famfam': 'folder_camera', 'children_classes': [StagingFolderSource], 'permissions': [PERMISSION_SOURCES_SETUP_VIEW]}
 setup_watch_folder_list = {'text': _(u'Watch folders'), 'view': 'sources:setup_watch_folder_list', 'famfam': 'folder_magnify', 'children_classes': [WatchFolderSource], 'permissions': [PERMISSION_SOURCES_SETUP_VIEW]}
+setup_pop3_email_list = {'text': _(u'POP3 emails'), 'view': 'sources:setup_pop3_email_list', 'famfam': 'email', 'children_classes': [POP3Email], 'permissions': [PERMISSION_SOURCES_SETUP_VIEW]}
+setup_imap_email_list = {'text': _(u'IMAP emails'), 'view': 'sources:setup_imap_email_list', 'famfam': 'email', 'children_classes': [IMAPEmail], 'permissions': [PERMISSION_SOURCES_SETUP_VIEW]}
 
 setup_source_edit = {'text': _(u'Edit'), 'view': 'sources:setup_source_edit', 'args': ['source.pk'], 'famfam': 'application_form_edit', 'permissions': [PERMISSION_SOURCES_SETUP_EDIT]}
 setup_source_delete = {'text': _(u'Delete'), 'view': 'sources:setup_source_delete', 'args': ['source.pk'], 'famfam': 'application_form_delete', 'permissions': [PERMISSION_SOURCES_SETUP_DELETE]}

mayan/apps/sources/literals.py

@@ -18,15 +18,25 @@ SOURCE_INTERACTIVE_UNCOMPRESS_CHOICES = (
 SOURCE_CHOICE_WEB_FORM = 'webform'
 SOURCE_CHOICE_STAGING = 'staging'
 SOURCE_CHOICE_WATCH = 'watch'
+SOURCE_CHOICE_EMAIL_POP3 = 'pop3'
+SOURCE_CHOICE_EMAIL_IMAP = 'imap'
 
 SOURCE_CHOICES = (
     (SOURCE_CHOICE_WEB_FORM, _(u'Web form')),
     (SOURCE_CHOICE_STAGING, _(u'Server staging folder')),
     (SOURCE_CHOICE_WATCH, _(u'Server watch folder')),
+    (SOURCE_CHOICE_EMAIL_POP3, _(u'POP3 email')),
+    (SOURCE_CHOICE_EMAIL_IMAP, _(u'IMAP email')),
 )
 
+# TODO: remove PLURALS
 SOURCE_CHOICES_PLURAL = (
     (SOURCE_CHOICE_WEB_FORM, _(u'Web forms')),
     (SOURCE_CHOICE_STAGING, _(u'Server staging folders')),
     (SOURCE_CHOICE_WATCH, _(u'Server watch folders')),
-)
+    (SOURCE_CHOICE_EMAIL_POP3, _(u'POP3 emails')),
+    (SOURCE_CHOICE_EMAIL_IMAP, _(u'IMAP emails')),)
+
+DEFAULT_INTERVAL = 60
+DEFAULT_POP3_TIMEOUT = 60
+DEFAULT_IMAP_MAILBOX = 'INBOX'

mayan/apps/sources/migrations/0005_auto__add_imapemail__add_intervalbasemodel__add_pop3email__add_emailba.py

@@ -0,0 +1,148 @@
+# -*- coding: utf-8 -*-
+from south.utils import datetime_utils as datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding model 'IMAPEmail'
+        db.create_table(u'sources_imapemail', (
+            (u'emailbasemodel_ptr', self.gf('django.db.models.fields.related.OneToOneField')(to=orm['sources.EmailBaseModel'], unique=True, primary_key=True)),
+            ('mailbox', self.gf('django.db.models.fields.CharField')(default='INBOX', max_length=64)),
+        ))
+        db.send_create_signal(u'sources', ['IMAPEmail'])
+
+        # Adding model 'IntervalBaseModel'
+        db.create_table(u'sources_intervalbasemodel', (
+            (u'outofprocesssource_ptr', self.gf('django.db.models.fields.related.OneToOneField')(to=orm['sources.OutOfProcessSource'], unique=True, primary_key=True)),
+            ('interval', self.gf('django.db.models.fields.PositiveIntegerField')(default=60)),
+            ('document_type', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['documents.DocumentType'], null=True, blank=True)),
+            ('uncompress', self.gf('django.db.models.fields.CharField')(max_length=1)),
+        ))
+        db.send_create_signal(u'sources', ['IntervalBaseModel'])
+
+        # Adding model 'POP3Email'
+        db.create_table(u'sources_pop3email', (
+            (u'emailbasemodel_ptr', self.gf('django.db.models.fields.related.OneToOneField')(to=orm['sources.EmailBaseModel'], unique=True, primary_key=True)),
+            ('timeout', self.gf('django.db.models.fields.PositiveIntegerField')(default=60)),
+        ))
+        db.send_create_signal(u'sources', ['POP3Email'])
+
+        # Adding model 'EmailBaseModel'
+        db.create_table(u'sources_emailbasemodel', (
+            (u'intervalbasemodel_ptr', self.gf('django.db.models.fields.related.OneToOneField')(to=orm['sources.IntervalBaseModel'], unique=True, primary_key=True)),
+            ('host', self.gf('django.db.models.fields.CharField')(max_length=128)),
+            ('ssl', self.gf('django.db.models.fields.BooleanField')()),
+            ('port', self.gf('django.db.models.fields.PositiveIntegerField')(null=True, blank=True)),
+            ('username', self.gf('django.db.models.fields.CharField')(max_length=96)),
+            ('password', self.gf('django.db.models.fields.CharField')(max_length=96)),
+        ))
+        db.send_create_signal(u'sources', ['EmailBaseModel'])
+
+
+    def backwards(self, orm):
+        # Deleting model 'IMAPEmail'
+        db.delete_table(u'sources_imapemail')
+
+        # Deleting model 'IntervalBaseModel'
+        db.delete_table(u'sources_intervalbasemodel')
+
+        # Deleting model 'POP3Email'
+        db.delete_table(u'sources_pop3email')
+
+        # Deleting model 'EmailBaseModel'
+        db.delete_table(u'sources_emailbasemodel')
+
+
+    models = {
+        u'contenttypes.contenttype': {
+            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        },
+        u'documents.documenttype': {
+            'Meta': {'ordering': "['name']", 'object_name': 'DocumentType'},
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '32'})
+        },
+        u'sources.emailbasemodel': {
+            'Meta': {'ordering': "('title',)", 'object_name': 'EmailBaseModel', '_ormbases': [u'sources.IntervalBaseModel']},
+            'host': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            u'intervalbasemodel_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': u"orm['sources.IntervalBaseModel']", 'unique': 'True', 'primary_key': 'True'}),
+            'password': ('django.db.models.fields.CharField', [], {'max_length': '96'}),
+            'port': ('django.db.models.fields.PositiveIntegerField', [], {'null': 'True', 'blank': 'True'}),
+            'ssl': ('django.db.models.fields.BooleanField', [], {}),
+            'username': ('django.db.models.fields.CharField', [], {'max_length': '96'})
+        },
+        u'sources.imapemail': {
+            'Meta': {'ordering': "('title',)", 'object_name': 'IMAPEmail', '_ormbases': [u'sources.EmailBaseModel']},
+            u'emailbasemodel_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': u"orm['sources.EmailBaseModel']", 'unique': 'True', 'primary_key': 'True'}),
+            'mailbox': ('django.db.models.fields.CharField', [], {'default': "'INBOX'", 'max_length': '64'})
+        },
+        u'sources.interactivesource': {
+            'Meta': {'ordering': "('title',)", 'object_name': 'InteractiveSource', '_ormbases': [u'sources.Source']},
+            u'source_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': u"orm['sources.Source']", 'unique': 'True', 'primary_key': 'True'})
+        },
+        u'sources.intervalbasemodel': {
+            'Meta': {'ordering': "('title',)", 'object_name': 'IntervalBaseModel', '_ormbases': [u'sources.OutOfProcessSource']},
+            'document_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['documents.DocumentType']", 'null': 'True', 'blank': 'True'}),
+            'interval': ('django.db.models.fields.PositiveIntegerField', [], {'default': '60'}),
+            u'outofprocesssource_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': u"orm['sources.OutOfProcessSource']", 'unique': 'True', 'primary_key': 'True'}),
+            'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'})
+        },
+        u'sources.outofprocesssource': {
+            'Meta': {'ordering': "('title',)", 'object_name': 'OutOfProcessSource', '_ormbases': [u'sources.Source']},
+            u'source_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': u"orm['sources.Source']", 'unique': 'True', 'primary_key': 'True'})
+        },
+        u'sources.pop3email': {
+            'Meta': {'ordering': "('title',)", 'object_name': 'POP3Email', '_ormbases': [u'sources.EmailBaseModel']},
+            u'emailbasemodel_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': u"orm['sources.EmailBaseModel']", 'unique': 'True', 'primary_key': 'True'}),
+            'timeout': ('django.db.models.fields.PositiveIntegerField', [], {'default': '60'})
+        },
+        u'sources.source': {
+            'Meta': {'ordering': "('title',)", 'object_name': 'Source'},
+            'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+            'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'})
+        },
+        u'sources.sourcetransformation': {
+            'Meta': {'ordering': "('order',)", 'object_name': 'SourceTransformation'},
+            'arguments': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'object_id': ('django.db.models.fields.PositiveIntegerField', [], {}),
+            'order': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0', 'null': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'transformation': ('django.db.models.fields.CharField', [], {'max_length': '128'})
+        },
+        u'sources.stagingfoldersource': {
+            'Meta': {'ordering': "('title',)", 'object_name': 'StagingFolderSource', '_ormbases': [u'sources.InteractiveSource']},
+            'delete_after_upload': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'folder_path': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            u'interactivesource_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': u"orm['sources.InteractiveSource']", 'unique': 'True', 'primary_key': 'True'}),
+            'preview_height': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
+            'preview_width': ('django.db.models.fields.IntegerField', [], {}),
+            'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'})
+        },
+        u'sources.watchfoldersource': {
+            'Meta': {'ordering': "('title',)", 'object_name': 'WatchFolderSource', '_ormbases': [u'sources.OutOfProcessSource']},
+            'delete_after_upload': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'folder_path': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'interval': ('django.db.models.fields.PositiveIntegerField', [], {}),
+            u'outofprocesssource_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': u"orm['sources.OutOfProcessSource']", 'unique': 'True', 'primary_key': 'True'}),
+            'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'})
+        },
+        u'sources.webformsource': {
+            'Meta': {'ordering': "('title',)", 'object_name': 'WebFormSource', '_ormbases': [u'sources.InteractiveSource']},
+            u'interactivesource_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': u"orm['sources.InteractiveSource']", 'unique': 'True', 'primary_key': 'True'}),
+            'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'})
+        }
+    }
+
+    complete_apps = ['sources']

mayan/apps/sources/models.py

@@ -1,8 +1,13 @@
 from __future__ import absolute_import
 
 from ast import literal_eval
+from email.Utils import collapse_rfc2231_value
+from email import message_from_string
+import json
+import imaplib
 import logging
 import os
+import poplib
 
 from django.contrib.contenttypes import generic
 from django.contrib.contenttypes.models import ContentType
@@ -15,15 +20,18 @@ from model_utils.managers import InheritanceManager
 from common.compressed_files import CompressedFile, NotACompressedFile
 from converter.api import get_available_transformations_choices
 from converter.literals import DIMENSION_SEPARATOR
-from documents.models import Document
+from djcelery.models import PeriodicTask, IntervalSchedule
+from documents.models import Document, DocumentType
 from metadata.api import save_metadata_list
 
-from .classes import StagingFile
+from .classes import Attachment, StagingFile
-from .literals import (SOURCE_CHOICES, SOURCE_CHOICES_PLURAL,
+from .literals import (DEFAULT_INTERVAL, DEFAULT_POP3_TIMEOUT,
-                       SOURCE_CHOICE_STAGING, SOURCE_CHOICE_WATCH,
+                       DEFAULT_IMAP_MAILBOX, SOURCE_CHOICES,
-                       SOURCE_CHOICE_WEB_FORM,
+                       SOURCE_CHOICES_PLURAL, SOURCE_CHOICE_STAGING,
+                       SOURCE_CHOICE_WATCH, SOURCE_CHOICE_WEB_FORM,
                        SOURCE_INTERACTIVE_UNCOMPRESS_CHOICES,
-                       SOURCE_UNCOMPRESS_CHOICES)
+                       SOURCE_UNCOMPRESS_CHOICES, SOURCE_UNCOMPRESS_CHOICE_Y,
+                       SOURCE_CHOICE_EMAIL_IMAP, SOURCE_CHOICE_EMAIL_POP3)
 from .managers import SourceTransformationManager
 
 logger = logging.getLogger(__name__)
@@ -32,6 +40,7 @@ logger = logging.getLogger(__name__)
 class Source(models.Model):
     title = models.CharField(max_length=64, verbose_name=_(u'Title'))
     enabled = models.BooleanField(default=True, verbose_name=_(u'Enabled'))
+    # TODO: remove whitelist and blacklists
     whitelist = models.TextField(blank=True, verbose_name=_(u'Whitelist'), editable=False)
     blacklist = models.TextField(blank=True, verbose_name=_(u'Blacklist'), editable=False)
 
@@ -190,8 +199,178 @@ class OutOfProcessSource(Source):
         verbose_name_plural = _(u'Out of process')
 
 
+class IntervalBaseModel(OutOfProcessSource):
+    interval = models.PositiveIntegerField(default=DEFAULT_INTERVAL, verbose_name=_('Interval'), help_text=_('Interval in seconds between document downloads from this source.'))
+    document_type = models.ForeignKey(DocumentType, null=True, blank=True, verbose_name=_('Document type'), help_text=_('Assign a document type to documents uploaded from this source.'))
+    uncompress = models.CharField(max_length=1, choices=SOURCE_UNCOMPRESS_CHOICES, verbose_name=_('Uncompress'), help_text=_('Whether to expand or not, compressed archives.'))
+
+    def _get_periodic_task_name(self, pk=None):
+        return 'check_interval_source-%i' % (pk or self.pk)
+
+    def _delete_periodic_task(self, pk=None):
+        periodic_task = PeriodicTask.objects.get(name=self._get_periodic_task_name(pk))
+
+        interval_instance = periodic_task.interval
+
+        if tuple(interval_instance.periodictask_set.values_list('id', flat=True)) == (periodic_task.pk,):
+            # Only delete the interval if nobody else is using it
+            interval_instance.delete()
+        else:
+            periodic_task.delete()
+
+    def save(self, *args, **kwargs):
+        new_source = not self.pk
+        super(IntervalBaseModel, self).save(*args, **kwargs)
+
+        if not new_source:
+            self._delete_periodic_task()
+
+        interval_instance, created = IntervalSchedule.objects.get_or_create(every=self.interval, period='seconds')
+        # Create a new interval or reuse someone else's
+        PeriodicTask.objects.create(
+            name=self._get_periodic_task_name(),
+            interval=interval_instance,
+            task='sources.tasks.task_check_interval_source',
+            queue='mailing',
+            kwargs=json.dumps({'source_id': self.pk})
+        )
+
+    def delete(self, *args, **kwargs):
+        pk = self.pk
+        super(IntervalBaseModel, self).delete(*args, **kwargs)
+        self._delete_periodic_task(pk)
+
+    class Meta:
+        verbose_name = _('Interval source')
+        verbose_name_plural = _('Interval sources')
+
+
+class EmailBaseModel(IntervalBaseModel):
+    host = models.CharField(max_length=128, verbose_name=_('Host'))
+    ssl = models.BooleanField(verbose_name=_('SSL'))
+    port = models.PositiveIntegerField(blank=True, null=True, verbose_name=_('Port'), help_text=_('Typical choices are 110 for POP3, 995 for POP3 over SSL, 143 for IMAP, 993 for IMAP over SSL.'))
+    username = models.CharField(max_length=96, verbose_name=_('Username'))
+    password = models.CharField(max_length=96, verbose_name=_('Password'))
+
+    # From: http://bookmarks.honewatson.com/2009/08/11/python-gmail-imaplib-search-subject-get-attachments/
+    # TODO: Add lock to avoid being running more than once concurrent
+    @staticmethod
+    def process_message(source, message):
+        email = message_from_string(message)
+        counter = 1
+
+        for part in email.walk():
+            disposition = part.get('Content-Disposition', 'none')
+            logger.debug('Disposition: %s' % disposition)
+
+            if disposition.startswith('attachment'):
+                raw_filename = part.get_filename()
+
+                if raw_filename:
+                    filename = collapse_rfc2231_value(raw_filename)
+                else:
+                    filename = _('attachment-%i') % counter
+                    counter += 1
+
+                logger.debug('filename: %s' % filename)
+
+                document_file = Attachment(part, name=filename)
+                source.upload_file(document_file, expand=(source.uncompress == SOURCE_UNCOMPRESS_CHOICE_Y), document_type=source.document_type)
+
+    class Meta:
+        verbose_name = _('Email source')
+        verbose_name_plural = _('Email sources')
+
+
+class POP3Email(EmailBaseModel):
+    source_type = SOURCE_CHOICE_EMAIL_POP3
+
+    timeout = models.PositiveIntegerField(default=DEFAULT_POP3_TIMEOUT, verbose_name=_('Timeout'))
+
+    def fetch_mail(self):
+        try:
+            logger.debug('Starting POP3 email fetch')
+            logger.debug('host: %s' % self.host)
+            logger.debug('ssl: %s' % self.ssl)
+
+            if self.ssl:
+                mailbox = poplib.POP3_SSL(self.host, self.port)
+            else:
+                mailbox = poplib.POP3(self.host, self.port, timeout=POP3_TIMEOUT)
+
+            mailbox.getwelcome()
+            mailbox.user(self.username)
+            mailbox.pass_(self.password)
+            messages_info = mailbox.list()
+
+            logger.debug('messages_info:')
+            logger.debug(messages_info)
+            logger.debug('messages count: %s' % len(messages_info[1]))
+
+            for message_info in messages_info[1]:
+                message_number, message_size = message_info.split()
+                logger.debug('message_number: %s' % message_number)
+                logger.debug('message_size: %s' % message_size)
+
+                complete_message = '\n'.join(mailbox.retr(message_number)[1])
+
+                EmailBaseModel.process_message(source=self, message=complete_message)
+                mailbox.dele(message_number)
+
+            mailbox.quit()
+        except Exception as exception:
+            logger.error('Unhandled exception: %s' % exception)
+            # TODO: Add user notification
+
+    class Meta:
+        verbose_name = _('POP email')
+        verbose_name_plural = _('POP email')
+
+
+class IMAPEmail(EmailBaseModel):
+    source_type = SOURCE_CHOICE_EMAIL_IMAP
+
+    mailbox = models.CharField(max_length=64, default=DEFAULT_IMAP_MAILBOX, verbose_name=_('Mailbox'), help_text=_('Mail from which to check for messages with attached documents.'))
+
+    # http://www.doughellmann.com/PyMOTW/imaplib/
+    def fetch_mail(self):
+        try:
+            logger.debug('Starting IMAP email fetch')
+            logger.debug('host: %s' % self.host)
+            logger.debug('ssl: %s' % self.ssl)
+
+            if self.ssl:
+                mailbox = imaplib.IMAP4_SSL(self.host, self.port)
+            else:
+                mailbox = imaplib.IMAP4(self.host, self.port)
+
+            mailbox.login(self.username, self.password)
+            mailbox.select(self.mailbox)
+
+            status, data = mailbox.search(None, 'NOT', 'DELETED')
+            if data:
+                messages_info = data[0].split()
+                logger.debug('messages count: %s' % len(messages_info))
+
+                for message_number in messages_info:
+                    logger.debug('message_number: %s' % message_number)
+                    status, data = mailbox.fetch(message_number, '(RFC822)')
+                    EmailBaseModel.process_message(source=self, message=data[0][1])
+                    mailbox.store(message_number, '+FLAGS', '\\Deleted')
+
+            mailbox.expunge()
+            mailbox.close()
+            mailbox.logout()
+        except Exception as exception:
+            logger.error('Unhandled exception: %s' % exception)
+            # TODO: Add user notification
+
+    class Meta:
+        verbose_name = _('IMAP email')
+        verbose_name_plural = _('IMAP email')
+
+
 class WatchFolderSource(OutOfProcessSource):
-    is_interactive = False
     source_type = SOURCE_CHOICE_WATCH
 
     folder_path = models.CharField(max_length=255, verbose_name=_(u'Folder path'), help_text=_(u'Server side filesystem path.'))

mayan/apps/sources/tasks.py

@@ -13,6 +13,13 @@ from .models import Source
 logger = logging.getLogger(__name__)
 
 
+@app.task(ignore_result=True)
+def task_check_interval_source(source_id):
+    source = Source.objects.get_subclass(pk=source_id)
+    if source.enabled:
+        source.fetch_mail()
+
+
 @app.task(ignore_result=True)
 def task_upload_document(source_id, file_path, filename=None, use_file_name=False, document_type_id=None, expand=False, metadata_dict_list=None, user_id=None, document_id=None, new_version_data=None, command_line=False, description=None):
     source = Source.objects.get_subclass(pk=source_id)

mayan/apps/sources/urls.py

@@ -5,7 +5,8 @@ from django.conf.urls import patterns, url
 from .api_views import (APIDocumentCreateView, APIStagingSourceFileView,
                         APIStagingSourceFileImageView, APIStagingSourceListView,
                         APIStagingSourceView)
-from .literals import (SOURCE_CHOICE_STAGING, SOURCE_CHOICE_WATCH,
+from .literals import (SOURCE_CHOICE_EMAIL_POP3, SOURCE_CHOICE_EMAIL_IMAP,
+                       SOURCE_CHOICE_STAGING, SOURCE_CHOICE_WATCH,
                        SOURCE_CHOICE_WEB_FORM)
 from .wizards import DocumentCreateWizard
 
@@ -23,6 +24,8 @@ urlpatterns = patterns('sources.views',
     url(r'^setup/interactive/%s/list/$' % SOURCE_CHOICE_WEB_FORM, 'setup_source_list', {'source_type': SOURCE_CHOICE_WEB_FORM}, 'setup_web_form_list'),
     url(r'^setup/interactive/%s/list/$' % SOURCE_CHOICE_STAGING, 'setup_source_list', {'source_type': SOURCE_CHOICE_STAGING}, 'setup_staging_folder_list'),
     url(r'^setup/interactive/%s/list/$' % SOURCE_CHOICE_WATCH, 'setup_source_list', {'source_type': SOURCE_CHOICE_WATCH}, 'setup_watch_folder_list'),
+    url(r'^setup/interactive/%s/list/$' % SOURCE_CHOICE_EMAIL_POP3, 'setup_source_list', {'source_type': SOURCE_CHOICE_EMAIL_POP3}, 'setup_pop3_email_list'),
+    url(r'^setup/interactive/%s/list/$' % SOURCE_CHOICE_EMAIL_IMAP, 'setup_source_list', {'source_type': SOURCE_CHOICE_EMAIL_IMAP}, 'setup_imap_email_list'),
 
     url(r'^setup/interactive/(?P<source_type>\w+)/list/$', 'setup_source_list', (), 'setup_source_list'),
     url(r'^setup/interactive/(?P<source_id>\d+)/edit/$', 'setup_source_edit', (), 'setup_source_edit'),

mayan/apps/sources/views.py

@@ -23,14 +23,16 @@ from documents.permissions import (PERMISSION_DOCUMENT_CREATE,
 from metadata.api import decode_metadata_from_url, metadata_repr_as_list
 from permissions.models import Permission
 
-from .forms import (StagingDocumentForm, StagingFolderSetupForm,
+from .forms import (POP3EmailSetupForm, IMAPEmailSetupForm, StagingDocumentForm,
-                    SourceTransformationForm, SourceTransformationForm_create,
+                    StagingFolderSetupForm, SourceTransformationForm,
-                    WatchFolderSetupForm, WebFormForm, WebFormSetupForm)
+                    SourceTransformationForm_create, WatchFolderSetupForm,
-from .literals import (SOURCE_CHOICE_STAGING, SOURCE_CHOICE_WATCH,
+                    WebFormForm, WebFormSetupForm)
+from .literals import (SOURCE_CHOICE_EMAIL_IMAP, SOURCE_CHOICE_EMAIL_POP3,
+                       SOURCE_CHOICE_STAGING, SOURCE_CHOICE_WATCH,
                        SOURCE_CHOICE_WEB_FORM, SOURCE_UNCOMPRESS_CHOICE_ASK,
                        SOURCE_UNCOMPRESS_CHOICE_Y)
-from .models import (Source, StagingFolderSource, SourceTransformation,
+from .models import (IMAPEmail, POP3Email, Source, StagingFolderSource,
-                     WatchFolderSource, WebFormSource)
+                     SourceTransformation, WatchFolderSource, WebFormSource)
 from .permissions import (PERMISSION_SOURCES_SETUP_CREATE,
                           PERMISSION_SOURCES_SETUP_DELETE,
                           PERMISSION_SOURCES_SETUP_EDIT,
@@ -38,6 +40,32 @@ from .permissions import (PERMISSION_SOURCES_SETUP_CREATE,
 from .tasks import task_upload_document
 
 
+def get_class(source_type):
+    if source_type == SOURCE_CHOICE_WEB_FORM:
+        return WebFormSource
+    elif source_type == SOURCE_CHOICE_STAGING:
+        return StagingFolderSource
+    elif source_type == SOURCE_CHOICE_WATCH:
+        return WatchFolderSource
+    elif source_type == SOURCE_CHOICE_EMAIL_POP3:
+        return POP3Email
+    elif source_type == SOURCE_CHOICE_EMAIL_IMAP:
+        return IMAPEmail
+
+
+def get_form_class(source_type):
+    if source_type == SOURCE_CHOICE_WEB_FORM:
+        return WebFormSetupForm
+    elif source_type == SOURCE_CHOICE_STAGING:
+        return StagingFolderSetupForm
+    elif source_type == SOURCE_CHOICE_WATCH:
+        return WatchFolderSetupForm
+    elif source_type == SOURCE_CHOICE_EMAIL_POP3:
+        return POP3EmailSetupForm
+    elif source_type == SOURCE_CHOICE_EMAIL_IMAP:
+        return IMAPEmailSetupForm
+
+
 def document_create_siblings(request, document_id):
     Permission.objects.check_permissions(request.user, [PERMISSION_DOCUMENT_CREATE])
 
@@ -86,10 +114,20 @@ def get_active_tab_links(document=None):
     for staging_folder in staging_folders:
         tab_links.append(get_tab_link_for_source(staging_folder, document))
 
+    pop3_emails = POP3Email.objects.filter(enabled=True)
+    for source_instance in pop3_emails:
+        tab_links.append(get_tab_link_for_source(source_instance, document))
+
+    imap_emails = IMAPEmail.objects.filter(enabled=True)
+    for source_instance in imap_emails:
+        tab_links.append(get_tab_link_for_source(source_instance, document))
+
     return {
         'tab_links': tab_links,
         SOURCE_CHOICE_WEB_FORM: web_forms,
-        SOURCE_CHOICE_STAGING: staging_folders
+        SOURCE_CHOICE_STAGING: staging_folders,
+        SOURCE_CHOICE_EMAIL_POP3: pop3_emails,
+        SOURCE_CHOICE_EMAIL_IMAP: imap_emails
     }
 
 
@@ -370,13 +408,9 @@ def staging_file_delete(request, staging_folder_pk, encoded_filename):
 def setup_source_list(request, source_type):
     Permission.objects.check_permissions(request.user, [PERMISSION_SOURCES_SETUP_VIEW])
 
-    if source_type == SOURCE_CHOICE_WEB_FORM:
+    cls = get_class(source_type)
-        cls = WebFormSource
-    elif source_type == SOURCE_CHOICE_STAGING:
-        cls = StagingFolderSource
-    elif source_type == SOURCE_CHOICE_WATCH:
-        cls = WatchFolderSource
 
+    # TODO: remove plurals
     context = {
         'object_list': cls.objects.all(),
         'title': cls.class_fullname_plural(),
@@ -393,12 +427,7 @@ def setup_source_edit(request, source_id):
     Permission.objects.check_permissions(request.user, [PERMISSION_SOURCES_SETUP_EDIT])
 
     source = get_object_or_404(Source.objects.select_subclasses(), pk=source_id)
-    if isinstance(source, WebFormSource):
+    form_class = get_form_class(source.source_type)
-        form_class = WebFormSetupForm
-    elif isinstance(source, StagingFolderSource):
-        form_class = StagingFolderSetupForm
-    elif isinstance(source, WatchFolderSource):
-        form_class = WatchFolderSetupForm
 
     next = request.POST.get('next', request.GET.get('next', request.META.get('HTTP_REFERER', reverse(settings.LOGIN_REDIRECT_URL))))
 
@@ -439,6 +468,12 @@ def setup_source_delete(request, source_id):
     elif isinstance(source, WatchFolderSource):
         form_icon = u'folder_delete.png'
         redirect_view = 'sources:setup_watch_folder_list'
+    elif isinstance(source, POP3Email):
+        form_icon = u'folder_delete.png'
+        redirect_view = 'sources:setup_pop3_email_list'
+    elif isinstance(source, IMAPEmail):
+        form_icon = u'folder_delete.png'
+        redirect_view = 'sources:setup_imap_email_list'
 
     previous = request.POST.get('previous', request.GET.get('previous', request.META.get('HTTP_REFERER', redirect_view)))
 
@@ -450,8 +485,7 @@ def setup_source_delete(request, source_id):
             messages.error(request, _(u'Error deleting source "%(source)s": %(error)s') % {
                 'source': source, 'error': exception
             })
-
+        return HttpResponseRedirect(reverse(redirect_view))
-        return HttpResponseRedirect(redirect_view)
 
     context = {
         'title': _(u'Are you sure you wish to delete the source: %s?') % source.fullname(),
@@ -471,15 +505,8 @@ def setup_source_delete(request, source_id):
 def setup_source_create(request, source_type):
     Permission.objects.check_permissions(request.user, [PERMISSION_SOURCES_SETUP_CREATE])
 
-    if source_type == SOURCE_CHOICE_WEB_FORM:
+    cls = get_class(source_type)
-        cls = WebFormSource
+    form_class = get_form_class(source_type)
-        form_class = WebFormSetupForm
-    elif source_type == SOURCE_CHOICE_STAGING:
-        cls = WebFormSource
-        form_class = StagingFolderSetupForm
-    elif source_type == SOURCE_CHOICE_WATCH:
-        cls = WebFormSource
-        form_class = WatchFolderSetupForm
 
     if request.method == 'POST':
         form = form_class(data=request.POST)

mayan/settings/base.py

@@ -49,6 +49,7 @@ INSTALLED_APPS = (
     # 3rd party
     'compressor',
     'corsheaders',
+    'djcelery',
     'filetransfers',
     'mptt',
     'rest_framework',
@@ -266,6 +267,7 @@ REST_FRAMEWORK = {
 CELERY_TIMEZONE = 'UTC'
 CELERY_ENABLE_UTC = True
 CELERY_ALWAYS_EAGER = True
+CELERYBEAT_SCHEDULER = 'djcelery.schedulers.DatabaseScheduler'
 # ------------ CORS ------------
 CORS_ORIGIN_ALLOW_ALL = True
 # ------ Django REST Swagger -----