Merge branch 'feature/email_source' into development

apps/main/__init__.py

@@ -20,9 +20,9 @@ __status__ = 'Production'
 
 __version_info__ = {
     'major': 0,
-    'minor': 12,
+    'minor': 13,
     'micro': 0,
-    'releaselevel': 'final',
+    'releaselevel': 'alpha',
     'serial': 0
 }
 

apps/sources/__init__.py

@@ -8,26 +8,31 @@ from common.utils import encapsulate
 from project_setup.api import register_setup
 from documents.permissions import (PERMISSION_DOCUMENT_NEW_VERSION, 
     PERMISSION_DOCUMENT_CREATE)
+from scheduler.api import register_interval_job
 
 from .staging import StagingFile
 from .models import (WebForm, StagingFolder, SourceTransformation,
-    WatchFolder)
+    WatchFolder, POP3Email)
 from .widgets import staging_file_thumbnail
 from .permissions import (PERMISSION_SOURCES_SETUP_VIEW,
     PERMISSION_SOURCES_SETUP_EDIT, PERMISSION_SOURCES_SETUP_DELETE,
     PERMISSION_SOURCES_SETUP_CREATE)
+from .tasks import task_fetch_pop3_emails
+from .conf.settings import POP3_EMAIL_PROCESSING_INTERVAL
 
 staging_file_preview = {'text': _(u'preview'), 'class': 'fancybox-noscaling', 'view': 'staging_file_preview', 'args': ['source.source_type', 'source.pk', 'object.id'], 'famfam': 'zoom', 'permissions': [PERMISSION_DOCUMENT_NEW_VERSION, PERMISSION_DOCUMENT_CREATE]}
 staging_file_delete = {'text': _(u'delete'), 'view': 'staging_file_delete', 'args': ['source.source_type', 'source.pk', 'object.id'], 'famfam': 'delete', 'keep_query': True, 'permissions': [PERMISSION_DOCUMENT_NEW_VERSION, PERMISSION_DOCUMENT_CREATE]}
 
-setup_sources = {'text': _(u'sources'), 'view': 'setup_web_form_list', 'famfam': 'application_form', 'icon': 'application_form.png', 'children_classes': [WebForm], 'permissions': [PERMISSION_SOURCES_SETUP_VIEW], 'children_view_regex': [r'setup_web_form', r'setup_staging_folder', r'setup_source_']}
+setup_sources = {'text': _(u'sources'), 'view': 'setup_web_form_list', 'famfam': 'application_form', 'icon': 'application_form.png', 'children_classes': [WebForm], 'permissions': [PERMISSION_SOURCES_SETUP_VIEW], 'children_view_regex': [r'setup_web_form', r'setup_staging_folder', r'setup_source_', r'setup_pop3']}
 setup_web_form_list = {'text': _(u'web forms'), 'view': 'setup_web_form_list', 'famfam': 'application_form', 'icon': 'application_form.png', 'children_classes': [WebForm], 'permissions': [PERMISSION_SOURCES_SETUP_VIEW]}
 setup_staging_folder_list = {'text': _(u'staging folders'), 'view': 'setup_staging_folder_list', 'famfam': 'folder_camera', 'children_classes': [StagingFolder], 'permissions': [PERMISSION_SOURCES_SETUP_VIEW]}
 setup_watch_folder_list = {'text': _(u'watch folders'), 'view': 'setup_watch_folder_list', 'famfam': 'folder_magnify', 'children_classes': [WatchFolder], 'permissions': [PERMISSION_SOURCES_SETUP_VIEW]}
+setup_pop3_email_list = {'text': _(u'POP3 email'), 'view': 'setup_pop3_email_list', 'famfam': 'email', 'children_classes': [POP3Email], 'permissions': [PERMISSION_SOURCES_SETUP_VIEW]}
 
 setup_source_edit = {'text': _(u'edit'), 'view': 'setup_source_edit', 'args': ['source.source_type', 'source.pk'], 'famfam': 'application_form_edit', 'permissions': [PERMISSION_SOURCES_SETUP_EDIT]}
 setup_source_delete = {'text': _(u'delete'), 'view': 'setup_source_delete', 'args': ['source.source_type', 'source.pk'], 'famfam': 'application_form_delete', 'permissions': [PERMISSION_SOURCES_SETUP_DELETE]}
 setup_source_create = {'text': _(u'add new source'), 'view': 'setup_source_create', 'args': 'source_type', 'famfam': 'application_form_add', 'permissions': [PERMISSION_SOURCES_SETUP_CREATE]}
+setup_source_log_list = {'text': _(u'logs'), 'view': 'setup_source_log_list', 'args': ['source.source_type', 'source.pk'], 'famfam': 'book', 'permissions': [PERMISSION_SOURCES_SETUP_EDIT]}
 
 setup_source_transformation_list = {'text': _(u'transformations'), 'view': 'setup_source_transformation_list', 'args': ['source.source_type', 'source.pk'], 'famfam': 'shape_move_front', 'permissions': [PERMISSION_SOURCES_SETUP_EDIT]}
 setup_source_transformation_create = {'text': _(u'add transformation'), 'view': 'setup_source_transformation_create', 'args': ['source.source_type', 'source.pk'], 'famfam': 'shape_square_add', 'permissions': [PERMISSION_SOURCES_SETUP_EDIT]}
@@ -43,18 +48,23 @@ register_links(StagingFile, [staging_file_delete])
 register_links(SourceTransformation, [setup_source_transformation_edit, setup_source_transformation_delete])
 
 #register_links(['setup_web_form_list', 'setup_staging_folder_list', 'setup_watch_folder_list', 'setup_source_create'], [setup_web_form_list, setup_staging_folder_list, setup_watch_folder_list], menu_name='form_header')
-register_links(['setup_web_form_list', 'setup_staging_folder_list', 'setup_watch_folder_list', 'setup_source_create'], [setup_web_form_list, setup_staging_folder_list], menu_name='form_header')
+register_links(['setup_pop3_email_list', 'setup_web_form_list', 'setup_staging_folder_list', 'setup_watch_folder_list', 'setup_source_create'], [setup_web_form_list, setup_staging_folder_list, setup_pop3_email_list], menu_name='form_header')
 
 #register_links(WebForm, [setup_web_form_list, setup_staging_folder_list, setup_watch_folder_list], menu_name='form_header')
-register_links(WebForm, [setup_web_form_list, setup_staging_folder_list], menu_name='form_header')
+register_links(WebForm, [setup_web_form_list, setup_staging_folder_list, setup_pop3_email_list], menu_name='form_header')
 register_links(WebForm, [setup_source_transformation_list, setup_source_edit, setup_source_delete])
 
-register_links(['setup_web_form_list', 'setup_staging_folder_list', 'setup_watch_folder_list', 'setup_source_edit', 'setup_source_delete', 'setup_source_create'], [setup_sources, setup_source_create], menu_name='sidebar')
+#register_links(['setup_source_log_list', 'setup_web_form_list', 'setup_staging_folder_list', 'setup_watch_folder_list', 'setup_source_edit', 'setup_source_delete', 'setup_source_create', 'setup_pop3_email_list'], [setup_sources, setup_source_create], menu_name='secondary_menu')
+register_links([WebForm, StagingFolder, POP3Email, 'setup_web_form_list', 'setup_staging_folder_list', 'setup_watch_folder_list', 'setup_source_create', 'setup_pop3_email_list'], [setup_source_create], menu_name='secondary_menu')
 
 #register_links(StagingFolder, [setup_web_form_list, setup_staging_folder_list, setup_watch_folder_list], menu_name='form_header')
-register_links(StagingFolder, [setup_web_form_list, setup_staging_folder_list], menu_name='form_header')
+register_links(StagingFolder, [setup_web_form_list, setup_staging_folder_list, setup_pop3_email_list], menu_name='form_header')
 register_links(StagingFolder, [setup_source_transformation_list, setup_source_edit, setup_source_delete])
 
+register_links(POP3Email, [setup_web_form_list, setup_staging_folder_list, setup_pop3_email_list], menu_name='form_header')
+register_links(POP3Email, [setup_source_transformation_list, setup_source_edit, setup_source_delete])
+register_links(POP3Email, [setup_source_log_list])
+
 register_links(WatchFolder, [setup_web_form_list, setup_staging_folder_list, setup_watch_folder_list], menu_name='form_header')
 register_links(WatchFolder, [setup_source_transformation_list, setup_source_edit, setup_source_delete])
 
@@ -72,3 +82,5 @@ register_model_list_columns(StagingFile, [
     ])
 
 register_setup(setup_sources)
+
+register_interval_job('task_fetch_pop3_emails', _(u'Connects to the POP3 email sources and fetches the attached documents.'), task_fetch_pop3_emails, seconds=POP3_EMAIL_PROCESSING_INTERVAL)

apps/sources/conf/settings.py

@@ -0,0 +1,34 @@
+"""Configuration options for the sources app"""
+import os
+
+from django.utils.translation import ugettext_lazy as _
+from django.conf import settings
+
+from smart_settings.api import Setting, SettingNamespace
+
+namespace = SettingNamespace('sources', _(u'Sources'), module='sources.conf.settings')
+
+POP3_DEFAULT_TIMEOUT = 10 # for POP3 only not POP3_SSL
+POP3_DEFAULT_EMAIL_PROCESSING_INTERVAL = 5
+DEFAULT_POP3_EMAIL_LOG_COUNT = 10 # Max log entries to store
+
+Setting(
+    namespace=namespace,
+    name='POP3_TIMEOUT',
+    global_name='SOURCES_POP3_TIMEOUT',
+    default=POP3_DEFAULT_TIMEOUT,
+)
+
+Setting(
+    namespace=namespace,
+    name='POP3_EMAIL_PROCESSING_INTERVAL',
+    global_name='SOURCES_POP3_EMAIL_PROCESSING_INTERVAL',
+    default=POP3_DEFAULT_EMAIL_PROCESSING_INTERVAL,
+)
+
+Setting(
+    namespace=namespace,
+    name='POP3_EMAIL_LOG_SIZE',
+    global_name='SOURCES_POP3_EMAIL_LOG_SIZE',
+    default=DEFAULT_POP3_EMAIL_LOG_COUNT,
+)

apps/sources/forms.py

@@ -7,7 +7,7 @@ from django.utils.translation import ugettext
 from documents.forms import DocumentForm
 
 from .models import (WebForm, StagingFolder, SourceTransformation,
-    WatchFolder)
+    WatchFolder, POP3Email)
 from .widgets import FamFamRadioSelect
 from .utils import validate_whitelist_blacklist
 
@@ -114,3 +114,8 @@ class SourceTransformationForm_create(forms.ModelForm):
     class Meta:
         model = SourceTransformation
         exclude = ('content_type', 'object_id')
+
+
+class POP3EmailSetupForm(forms.ModelForm):
+    class Meta:
+        model = POP3Email

apps/sources/literals.py

@@ -1,5 +1,9 @@
 from django.utils.translation import ugettext_lazy as _
 
+POP3_PORT = 110
+POP3_SSL_PORT = 995
+DEFAULT_POP3_INTERVAL = 15 * 60 # 15 minutes in seconds
+
 SOURCE_UNCOMPRESS_CHOICE_Y = 'y'
 SOURCE_UNCOMPRESS_CHOICE_N = 'n'
 SOURCE_UNCOMPRESS_CHOICE_ASK = 'a'
@@ -42,15 +46,18 @@ SOURCE_ICON_CHOICES = (
 SOURCE_CHOICE_WEB_FORM = 'webform'
 SOURCE_CHOICE_STAGING = 'staging'
 SOURCE_CHOICE_WATCH = 'watch'
+SOURCE_CHOICE_POP3_EMAIL = 'pop3'
 
 SOURCE_CHOICES = (
     (SOURCE_CHOICE_WEB_FORM, _(u'web form')),
     (SOURCE_CHOICE_STAGING, _(u'server staging folder')),
     (SOURCE_CHOICE_WATCH, _(u'server watch folder')),
+    (SOURCE_CHOICE_POP3_EMAIL, _(u'pop3 email')),
 )
 
 SOURCE_CHOICES_PLURAL = (
     (SOURCE_CHOICE_WEB_FORM, _(u'web forms')),
     (SOURCE_CHOICE_STAGING, _(u'server staging folders')),
     (SOURCE_CHOICE_WATCH, _(u'server watch folders')),
+    (SOURCE_CHOICE_POP3_EMAIL, _(u'pop3 emails')),
 )

apps/sources/managers.py

@@ -1,8 +1,12 @@
+from __future__ import absolute_import
+
 from ast import literal_eval
 
 from django.db import models
 from django.contrib.contenttypes.models import ContentType
 
+from .conf.settings import POP3_EMAIL_LOG_SIZE
+
 
 class SourceTransformationManager(models.Manager):
     def get_for_object(self, obj):
@@ -24,3 +28,12 @@ class SourceTransformationManager(models.Manager):
                 warnings.append(e)
 
         return transformations, warnings
+
+
+class POP3EmailLogManager(models.Manager):
+    def save_status(self, pop3_email, status):
+        new_recent = self.model(pop3_email=pop3_email, status=status)
+        new_recent.save()
+        to_delete = self.model.objects.filter(pop3_email=pop3_email).order_by('-creation_datetime')[POP3_EMAIL_LOG_SIZE:]
+        for recent_to_delete in to_delete:
+            recent_to_delete.delete()

apps/sources/migrations/0003_auto__add_pop3email.py

@@ -0,0 +1,113 @@
+# encoding: utf-8
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        
+        # Adding model 'POP3Email'
+        db.create_table('sources_pop3email', (
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('title', self.gf('django.db.models.fields.CharField')(max_length=64)),
+            ('enabled', self.gf('django.db.models.fields.BooleanField')(default=True)),
+            ('whitelist', self.gf('django.db.models.fields.TextField')(blank=True)),
+            ('blacklist', self.gf('django.db.models.fields.TextField')(blank=True)),
+            ('uncompress', self.gf('django.db.models.fields.CharField')(max_length=1)),
+            ('ssl', self.gf('django.db.models.fields.BooleanField')(default=False)),
+            ('port', self.gf('django.db.models.fields.PositiveIntegerField')(blank=True)),
+            ('host', self.gf('django.db.models.fields.CharField')(max_length=64)),
+            ('username', self.gf('django.db.models.fields.CharField')(max_length=64)),
+            ('password', self.gf('django.db.models.fields.CharField')(max_length=64)),
+            ('delete', self.gf('django.db.models.fields.BooleanField')(default=False)),
+        ))
+        db.send_create_signal('sources', ['POP3Email'])
+
+
+    def backwards(self, orm):
+        
+        # Deleting model 'POP3Email'
+        db.delete_table('sources_pop3email')
+
+
+    models = {
+        'contenttypes.contenttype': {
+            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        },
+        'sources.outofprocess': {
+            'Meta': {'ordering': "('title',)", 'object_name': 'OutOfProcess'},
+            'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+            'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'})
+        },
+        'sources.pop3email': {
+            'Meta': {'ordering': "('title',)", 'object_name': 'POP3Email'},
+            'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'delete': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'host': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'password': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+            'port': ('django.db.models.fields.PositiveIntegerField', [], {'blank': 'True'}),
+            'ssl': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+            'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}),
+            'username': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+            'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'})
+        },
+        'sources.sourcetransformation': {
+            'Meta': {'ordering': "('order',)", 'object_name': 'SourceTransformation'},
+            'arguments': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'object_id': ('django.db.models.fields.PositiveIntegerField', [], {}),
+            'order': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0', 'null': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'transformation': ('django.db.models.fields.CharField', [], {'max_length': '128'})
+        },
+        'sources.stagingfolder': {
+            'Meta': {'ordering': "('title',)", 'object_name': 'StagingFolder'},
+            'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'delete_after_upload': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'folder_path': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'icon': ('django.db.models.fields.CharField', [], {'max_length': '24', 'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'preview_height': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
+            'preview_width': ('django.db.models.fields.IntegerField', [], {}),
+            'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+            'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}),
+            'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'})
+        },
+        'sources.watchfolder': {
+            'Meta': {'ordering': "('title',)", 'object_name': 'WatchFolder'},
+            'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'delete_after_upload': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'folder_path': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'interval': ('django.db.models.fields.PositiveIntegerField', [], {}),
+            'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+            'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}),
+            'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'})
+        },
+        'sources.webform': {
+            'Meta': {'ordering': "('title',)", 'object_name': 'WebForm'},
+            'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'icon': ('django.db.models.fields.CharField', [], {'max_length': '24', 'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+            'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}),
+            'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'})
+        }
+    }
+
+    complete_apps = ['sources']

apps/sources/migrations/0004_make_port_nullable.py

@@ -0,0 +1,99 @@
+# encoding: utf-8
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        
+        # Changing field 'POP3Email.port'
+        db.alter_column('sources_pop3email', 'port', self.gf('django.db.models.fields.PositiveIntegerField')(null=True))
+
+
+    def backwards(self, orm):
+        
+        # Changing field 'POP3Email.port'
+        db.alter_column('sources_pop3email', 'port', self.gf('django.db.models.fields.PositiveIntegerField')(default=110))
+
+
+    models = {
+        'contenttypes.contenttype': {
+            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        },
+        'sources.outofprocess': {
+            'Meta': {'ordering': "('title',)", 'object_name': 'OutOfProcess'},
+            'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+            'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'})
+        },
+        'sources.pop3email': {
+            'Meta': {'ordering': "('title',)", 'object_name': 'POP3Email'},
+            'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'delete': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'host': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'password': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+            'port': ('django.db.models.fields.PositiveIntegerField', [], {'null': 'True', 'blank': 'True'}),
+            'ssl': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+            'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}),
+            'username': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+            'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'})
+        },
+        'sources.sourcetransformation': {
+            'Meta': {'ordering': "('order',)", 'object_name': 'SourceTransformation'},
+            'arguments': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'object_id': ('django.db.models.fields.PositiveIntegerField', [], {}),
+            'order': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0', 'null': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'transformation': ('django.db.models.fields.CharField', [], {'max_length': '128'})
+        },
+        'sources.stagingfolder': {
+            'Meta': {'ordering': "('title',)", 'object_name': 'StagingFolder'},
+            'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'delete_after_upload': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'folder_path': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'icon': ('django.db.models.fields.CharField', [], {'max_length': '24', 'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'preview_height': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
+            'preview_width': ('django.db.models.fields.IntegerField', [], {}),
+            'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+            'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}),
+            'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'})
+        },
+        'sources.watchfolder': {
+            'Meta': {'ordering': "('title',)", 'object_name': 'WatchFolder'},
+            'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'delete_after_upload': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'folder_path': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'interval': ('django.db.models.fields.PositiveIntegerField', [], {}),
+            'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+            'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}),
+            'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'})
+        },
+        'sources.webform': {
+            'Meta': {'ordering': "('title',)", 'object_name': 'WebForm'},
+            'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'icon': ('django.db.models.fields.CharField', [], {'max_length': '24', 'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+            'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}),
+            'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'})
+        }
+    }
+
+    complete_apps = ['sources']

apps/sources/migrations/0005_rename_delete_field.py

@@ -0,0 +1,105 @@
+# encoding: utf-8
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        
+        # Deleting field 'POP3Email.delete'
+        db.delete_column('sources_pop3email', 'delete')
+
+        # Adding field 'POP3Email.delete_messages'
+        db.add_column('sources_pop3email', 'delete_messages', self.gf('django.db.models.fields.BooleanField')(default=False), keep_default=False)
+
+
+    def backwards(self, orm):
+        
+        # Adding field 'POP3Email.delete'
+        db.add_column('sources_pop3email', 'delete', self.gf('django.db.models.fields.BooleanField')(default=False), keep_default=False)
+
+        # Deleting field 'POP3Email.delete_messages'
+        db.delete_column('sources_pop3email', 'delete_messages')
+
+
+    models = {
+        'contenttypes.contenttype': {
+            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        },
+        'sources.outofprocess': {
+            'Meta': {'ordering': "('title',)", 'object_name': 'OutOfProcess'},
+            'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+            'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'})
+        },
+        'sources.pop3email': {
+            'Meta': {'ordering': "('title',)", 'object_name': 'POP3Email'},
+            'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'delete_messages': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'host': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'password': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+            'port': ('django.db.models.fields.PositiveIntegerField', [], {'null': 'True', 'blank': 'True'}),
+            'ssl': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+            'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}),
+            'username': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+            'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'})
+        },
+        'sources.sourcetransformation': {
+            'Meta': {'ordering': "('order',)", 'object_name': 'SourceTransformation'},
+            'arguments': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'object_id': ('django.db.models.fields.PositiveIntegerField', [], {}),
+            'order': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0', 'null': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'transformation': ('django.db.models.fields.CharField', [], {'max_length': '128'})
+        },
+        'sources.stagingfolder': {
+            'Meta': {'ordering': "('title',)", 'object_name': 'StagingFolder'},
+            'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'delete_after_upload': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'folder_path': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'icon': ('django.db.models.fields.CharField', [], {'max_length': '24', 'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'preview_height': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
+            'preview_width': ('django.db.models.fields.IntegerField', [], {}),
+            'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+            'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}),
+            'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'})
+        },
+        'sources.watchfolder': {
+            'Meta': {'ordering': "('title',)", 'object_name': 'WatchFolder'},
+            'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'delete_after_upload': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'folder_path': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'interval': ('django.db.models.fields.PositiveIntegerField', [], {}),
+            'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+            'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}),
+            'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'})
+        },
+        'sources.webform': {
+            'Meta': {'ordering': "('title',)", 'object_name': 'WebForm'},
+            'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'icon': ('django.db.models.fields.CharField', [], {'max_length': '24', 'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+            'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}),
+            'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'})
+        }
+    }
+
+    complete_apps = ['sources']

apps/sources/migrations/0006_auto__del_field_pop3email_delete_messages.py

@@ -0,0 +1,98 @@
+# encoding: utf-8
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        
+        # Deleting field 'POP3Email.delete_messages'
+        db.delete_column('sources_pop3email', 'delete_messages')
+
+
+    def backwards(self, orm):
+        
+        # Adding field 'POP3Email.delete_messages'
+        db.add_column('sources_pop3email', 'delete_messages', self.gf('django.db.models.fields.BooleanField')(default=False), keep_default=False)
+
+
+    models = {
+        'contenttypes.contenttype': {
+            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        },
+        'sources.outofprocess': {
+            'Meta': {'ordering': "('title',)", 'object_name': 'OutOfProcess'},
+            'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+            'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'})
+        },
+        'sources.pop3email': {
+            'Meta': {'ordering': "('title',)", 'object_name': 'POP3Email'},
+            'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'host': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'password': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+            'port': ('django.db.models.fields.PositiveIntegerField', [], {'default': '110', 'null': 'True', 'blank': 'True'}),
+            'ssl': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+            'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}),
+            'username': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+            'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'})
+        },
+        'sources.sourcetransformation': {
+            'Meta': {'ordering': "('order',)", 'object_name': 'SourceTransformation'},
+            'arguments': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'object_id': ('django.db.models.fields.PositiveIntegerField', [], {}),
+            'order': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0', 'null': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'transformation': ('django.db.models.fields.CharField', [], {'max_length': '128'})
+        },
+        'sources.stagingfolder': {
+            'Meta': {'ordering': "('title',)", 'object_name': 'StagingFolder'},
+            'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'delete_after_upload': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'folder_path': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'icon': ('django.db.models.fields.CharField', [], {'max_length': '24', 'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'preview_height': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
+            'preview_width': ('django.db.models.fields.IntegerField', [], {}),
+            'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+            'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}),
+            'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'})
+        },
+        'sources.watchfolder': {
+            'Meta': {'ordering': "('title',)", 'object_name': 'WatchFolder'},
+            'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'delete_after_upload': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'folder_path': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'interval': ('django.db.models.fields.PositiveIntegerField', [], {}),
+            'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+            'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}),
+            'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'})
+        },
+        'sources.webform': {
+            'Meta': {'ordering': "('title',)", 'object_name': 'WebForm'},
+            'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'icon': ('django.db.models.fields.CharField', [], {'max_length': '24', 'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+            'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}),
+            'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'})
+        }
+    }
+
+    complete_apps = ['sources']

apps/sources/migrations/0007_auto__add_pop3emaillog__add_field_pop3email_interval.py

@@ -0,0 +1,118 @@
+# encoding: utf-8
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        
+        # Adding model 'POP3EmailLog'
+        db.create_table('sources_pop3emaillog', (
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('pop3_email', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['sources.POP3Email'])),
+            ('creation_datetime', self.gf('django.db.models.fields.DateTimeField')()),
+            ('status', self.gf('django.db.models.fields.TextField')()),
+        ))
+        db.send_create_signal('sources', ['POP3EmailLog'])
+
+        # Adding field 'POP3Email.interval'
+        db.add_column('sources_pop3email', 'interval', self.gf('django.db.models.fields.PositiveIntegerField')(default=900), keep_default=False)
+
+
+    def backwards(self, orm):
+        
+        # Deleting model 'POP3EmailLog'
+        db.delete_table('sources_pop3emaillog')
+
+        # Deleting field 'POP3Email.interval'
+        db.delete_column('sources_pop3email', 'interval')
+
+
+    models = {
+        'contenttypes.contenttype': {
+            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        },
+        'sources.outofprocess': {
+            'Meta': {'ordering': "('title',)", 'object_name': 'OutOfProcess'},
+            'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+            'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'})
+        },
+        'sources.pop3email': {
+            'Meta': {'ordering': "('title',)", 'object_name': 'POP3Email'},
+            'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'host': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'interval': ('django.db.models.fields.PositiveIntegerField', [], {'default': '900'}),
+            'password': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+            'port': ('django.db.models.fields.PositiveIntegerField', [], {'null': 'True', 'blank': 'True'}),
+            'ssl': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+            'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}),
+            'username': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+            'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'})
+        },
+        'sources.pop3emaillog': {
+            'Meta': {'object_name': 'POP3EmailLog'},
+            'creation_datetime': ('django.db.models.fields.DateTimeField', [], {}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'pop3_email': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['sources.POP3Email']"}),
+            'status': ('django.db.models.fields.TextField', [], {})
+        },
+        'sources.sourcetransformation': {
+            'Meta': {'ordering': "('order',)", 'object_name': 'SourceTransformation'},
+            'arguments': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'object_id': ('django.db.models.fields.PositiveIntegerField', [], {}),
+            'order': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0', 'null': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'transformation': ('django.db.models.fields.CharField', [], {'max_length': '128'})
+        },
+        'sources.stagingfolder': {
+            'Meta': {'ordering': "('title',)", 'object_name': 'StagingFolder'},
+            'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'delete_after_upload': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'folder_path': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'icon': ('django.db.models.fields.CharField', [], {'max_length': '24', 'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'preview_height': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
+            'preview_width': ('django.db.models.fields.IntegerField', [], {}),
+            'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+            'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}),
+            'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'})
+        },
+        'sources.watchfolder': {
+            'Meta': {'ordering': "('title',)", 'object_name': 'WatchFolder'},
+            'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'delete_after_upload': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'folder_path': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'interval': ('django.db.models.fields.PositiveIntegerField', [], {}),
+            'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+            'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}),
+            'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'})
+        },
+        'sources.webform': {
+            'Meta': {'ordering': "('title',)", 'object_name': 'WebForm'},
+            'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'icon': ('django.db.models.fields.CharField', [], {'max_length': '24', 'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+            'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}),
+            'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'})
+        }
+    }
+
+    complete_apps = ['sources']

apps/sources/models.py

@@ -2,6 +2,16 @@ from __future__ import absolute_import
 
 from ast import literal_eval
 import logging
+import poplib
+from email.Utils import collapse_rfc2231_value
+from email import message_from_string
+import os
+import datetime
+
+try:
+    from cStringIO import StringIO
+except ImportError:
+    from StringIO import StringIO
 
 from django.db import models
 from django.utils.translation import ugettext_lazy as _
@@ -9,6 +19,8 @@ from django.contrib.contenttypes.models import ContentType
 from django.contrib.contenttypes import generic
 from django.core.exceptions import ValidationError
 from django.db import transaction
+from django.core.files import File
+from django.core.files.base import ContentFile
 
 from converter.api import get_available_transformations_choices
 from converter.literals import DIMENSION_SEPARATOR
@@ -21,13 +33,15 @@ from metadata.api import save_metadata_list
 from scheduler.api import register_interval_job, remove_job
 from acls.utils import apply_default_acls
 
-from .managers import SourceTransformationManager
+from .managers import SourceTransformationManager, POP3EmailLogManager
 from .literals import (SOURCE_CHOICES, SOURCE_CHOICES_PLURAL,
     SOURCE_INTERACTIVE_UNCOMPRESS_CHOICES, SOURCE_CHOICE_WEB_FORM,
     SOURCE_CHOICE_STAGING, SOURCE_ICON_DISK, SOURCE_ICON_DRIVE,
     SOURCE_ICON_CHOICES, SOURCE_CHOICE_WATCH, SOURCE_UNCOMPRESS_CHOICES,
-    SOURCE_UNCOMPRESS_CHOICE_Y)
+    SOURCE_UNCOMPRESS_CHOICE_Y, POP3_PORT, POP3_SSL_PORT,
+    SOURCE_CHOICE_POP3_EMAIL, DEFAULT_POP3_INTERVAL)
 from .compressed_file import CompressedFile, NotACompressedFile
+from .conf.settings import POP3_TIMEOUT
 
 logger = logging.getLogger(__name__)
 
@@ -149,6 +163,133 @@ class InteractiveBaseModel(BaseModel):
 
     class Meta(BaseModel.Meta):
         abstract = True
+        
+        
+class PseudoFile(File):
+    def __init__(self, file, name):
+        self.name = name
+        self.file = file
+        self.file.seek(0, os.SEEK_END)
+        self.size = self.file.tell()
+        self.file.seek(0)
+    
+        
+class Attachment(File):
+    def __init__(self, part, name):
+        self.name = name
+        self.file = PseudoFile(StringIO(part.get_payload(decode=True)), name=name)
+
+
+class POP3Email(BaseModel):
+    is_interactive = False   
+    source_type = SOURCE_CHOICE_POP3_EMAIL
+    
+    host = models.CharField(max_length=64, verbose_name=_(u'host'))
+    ssl = models.BooleanField(verbose_name=_(u'SSL'))
+    port = models.PositiveIntegerField(blank=True, null=True, verbose_name=_(u'port'), help_text=_(u'Typical values are: %d and %d for SSL.  These are the defaults if no port is specified.') % (POP3_PORT, POP3_SSL_PORT))
+    username = models.CharField(max_length=64, verbose_name=_(u'username'))
+    password = models.CharField(max_length=64, verbose_name=_(u'password'))
+    uncompress = models.CharField(max_length=1, choices=SOURCE_UNCOMPRESS_CHOICES, verbose_name=_(u'uncompress'), help_text=_(u'Whether to expand or not compressed archives.'))
+    interval = models.PositiveIntegerField(default=DEFAULT_POP3_INTERVAL, verbose_name=_(u'interval'), help_text=_(u'Interval in seconds between document downloads from this account.'))
+
+    # From: http://bookmarks.honewatson.com/2009/08/11/python-gmail-imaplib-search-subject-get-attachments/
+    @staticmethod
+    def process_message(source, message, expand=False):
+        email = message_from_string(message)
+        counter = 1
+
+        for part in email.walk():
+            disposition = part.get('Content-Disposition', 'none')
+            logger.debug('Disposition: %s' % disposition)
+
+            if disposition.startswith('attachment'):
+                raw_filename = part.get_filename()
+
+                if raw_filename:
+                    filename = collapse_rfc2231_value(raw_filename)
+                else:
+                    filename = _(u'attachment-%i') % counter
+                    counter += 1
+
+                logger.debug('filename: %s' % filename)
+               
+                document_file = Attachment(part, name=filename)
+                source.upload_file(document_file, filename=filename, expand=expand)
+
+
+    def fetch_mail(self):
+        try:
+            last_check = self.pop3emaillog_set.latest().creation_datetime
+        except POP3EmailLog.DoesNotExist:
+            # Trigger email fetch when there are no previous logs
+            initial_trigger = True
+            difference = datetime.timedelta(seconds=0)
+        else:
+            difference = datetime.datetime.now() - last_check
+            initial_trigger = False
+        
+        if difference >= datetime.timedelta(seconds=self.interval) or initial_trigger:
+            try:
+                logger.debug('Starting POP3 email fetch')
+                logger.debug('host: %s' % self.host)
+                logger.debug('ssl: %s' % self.ssl)
+                if self.ssl:
+                    port = self.port or POP3_SSL_PORT
+                    logger.debug('port: %d' % port)
+                    mailbox = poplib.POP3_SSL(self.host, int(port))
+                else:
+                    port = self.port or POP3_PORT
+                    logger.debug('port: %d' % port)
+                    mailbox = poplib.POP3(self.host, int(port), timeout=POP3_TIMEOUT) 
+
+                mailbox.getwelcome()
+                mailbox.user(self.username)
+                mailbox.pass_(self.password)
+                messages_info = mailbox.list()
+                
+                logger.debug('messages_info:')
+                logger.debug(messages_info)
+                logger.debug('messages count: %s' % len(messages_info[1]))
+                
+                for message_info in messages_info[1]:
+                    message_number, message_size = message_info.split()
+                    logger.debug('message_number: %s' % message_number)
+                    logger.debug('message_size: %s' % message_size)
+                   
+                    complete_message = '\n'.join(mailbox.retr(message_number)[1])
+
+                    POP3Email.process_message(source=self, message=complete_message, expand=self.uncompress)
+                    mailbox.dele(message_number)
+                    
+                mailbox.quit()
+                POP3EmailLog.objects.save_status(pop3_email=self, status='Successful connection.')
+
+            except Exception, exc:
+                logger.error('Unhandled exception: %s' % exc)
+                POP3EmailLog.objects.save_status(pop3_email=self, status='Error: %s' % exc)
+        
+    class Meta(BaseModel.Meta):
+        verbose_name = _(u'POP email')
+        verbose_name_plural = _(u'POP email')
+
+
+class POP3EmailLog(models.Model):
+    pop3_email = models.ForeignKey(POP3Email, verbose_name=_(u'POP3 email'))
+    creation_datetime = models.DateTimeField(verbose_name=_(u'date time'))
+    status = models.TextField(verbose_name=_(u'status'))
+    
+    objects = POP3EmailLogManager()
+    
+    def save(self, *args, **kwargs):
+        if not self.pk:
+            self.creation_datetime = datetime.datetime.now()
+        return super(POP3EmailLog, self).save(*args, **kwargs)
+    
+    class Meta:
+        verbose_name = _(u'POP3 email log')
+        verbose_name_plural = _(u'POP3 emails logs')
+        get_latest_by = 'creation_datetime'
+        ordering = ('creation_datetime',)
 
 
 class StagingFolder(InteractiveBaseModel):

apps/sources/tasks.py

@@ -0,0 +1,41 @@
+from __future__ import absolute_import
+
+from datetime import timedelta, datetime
+import platform
+import logging
+
+from django.db.models import Q
+
+from job_processor.api import process_job
+from lock_manager import Lock, LockError
+
+from .models import POP3Email
+from .conf.settings import POP3_TIMEOUT
+
+logger = logging.getLogger(__name__)
+
+
+def task_fetch_single_pop3_email(pop3_email):
+    try:
+        lock_id = u'task_fetch_pop3_email-%d' % pop3_email.pk
+        logger.debug('trying to acquire lock: %s' % lock_id)
+        lock = Lock.acquire_lock(lock_id, POP3_TIMEOUT + 60) # Lock expiration = POP3 timeout + 60 seconds
+        logger.debug('acquired lock: %s' % lock_id)
+        try:
+            pop3_email.fetch_mail()
+        except Exception, exc:
+            raise
+        finally:
+            lock.release()
+    except LockError:
+        logger.error('unable to obtain lock')
+        pass
+    
+
+def task_fetch_pop3_emails():
+    logger.debug('executing')
+    for pop3_email in POP3Email.objects.filter(enabled=True):
+        try:
+            task_fetch_single_pop3_email(pop3_email)
+        except Exception, exc:
+            logger.error('Unhandled exception: %s' % exc)

apps/sources/urls.py

@@ -3,7 +3,7 @@ from __future__ import absolute_import
 from django.conf.urls.defaults import patterns, url
 
 from .literals import (SOURCE_CHOICE_WEB_FORM, SOURCE_CHOICE_STAGING,
-    SOURCE_CHOICE_WATCH)
+    SOURCE_CHOICE_WATCH, SOURCE_CHOICE_POP3_EMAIL)
 
 urlpatterns = patterns('sources.views',
     url(r'^staging_file/type/(?P<source_type>\w+)/(?P<source_id>\d+)/(?P<staging_file_id>\w+)/preview/$', 'staging_file_preview', (), 'staging_file_preview'),
@@ -21,6 +21,7 @@ urlpatterns = patterns('sources.views',
     url(r'^setup/interactive/%s/list/$' % SOURCE_CHOICE_WEB_FORM, 'setup_source_list', {'source_type': SOURCE_CHOICE_WEB_FORM}, 'setup_web_form_list'),
     url(r'^setup/interactive/%s/list/$' % SOURCE_CHOICE_STAGING, 'setup_source_list', {'source_type': SOURCE_CHOICE_STAGING}, 'setup_staging_folder_list'),
     url(r'^setup/interactive/%s/list/$' % SOURCE_CHOICE_WATCH, 'setup_source_list', {'source_type': SOURCE_CHOICE_WATCH}, 'setup_watch_folder_list'),
+    url(r'^setup/interactive/%s/list/$' % SOURCE_CHOICE_POP3_EMAIL, 'setup_source_list', {'source_type': SOURCE_CHOICE_POP3_EMAIL}, 'setup_pop3_email_list'),
 
     url(r'^setup/interactive/(?P<source_type>\w+)/list/$', 'setup_source_list', (), 'setup_source_list'),
     url(r'^setup/interactive/(?P<source_type>\w+)/(?P<source_id>\d+)/edit/$', 'setup_source_edit', (), 'setup_source_edit'),
@@ -31,4 +32,6 @@ urlpatterns = patterns('sources.views',
     url(r'^setup/interactive/(?P<source_type>\w+)/(?P<source_id>\d+)/transformation/create/$', 'setup_source_transformation_create', (), 'setup_source_transformation_create'),
     url(r'^setup/interactive/source/transformation/(?P<transformation_id>\d+)/edit/$', 'setup_source_transformation_edit', (), 'setup_source_transformation_edit'),
     url(r'^setup/interactive/source/transformation/(?P<transformation_id>\d+)/delete/$', 'setup_source_transformation_delete', (), 'setup_source_transformation_delete'),
+    
+    url(r'^setup/source/(?P<source_type>\w+)/(?P<source_pk>\d+)/log/list/$', 'setup_source_log_list', (), 'setup_source_log_list'),    
 )

apps/sources/views.py

@@ -1,5 +1,7 @@
 from __future__ import absolute_import
 
+import logging
+
 from django.http import HttpResponseRedirect
 from django.shortcuts import render_to_response, get_object_or_404
 from django.template import RequestContext
@@ -21,21 +23,23 @@ from common.utils import encapsulate
 import sendfile
 from acls.models import AccessEntry
 
-from sources.models import (WebForm, StagingFolder, SourceTransformation,
+from .models import (WebForm, StagingFolder, SourceTransformation,
-    WatchFolder)
+    WatchFolder, POP3Email, POP3EmailLog)
-from sources.literals import (SOURCE_CHOICE_WEB_FORM, SOURCE_CHOICE_STAGING,
+from .literals import (SOURCE_CHOICE_WEB_FORM, SOURCE_CHOICE_STAGING,
-    SOURCE_CHOICE_WATCH)
+    SOURCE_CHOICE_WATCH, SOURCE_CHOICE_POP3_EMAIL)
-from sources.literals import (SOURCE_UNCOMPRESS_CHOICE_Y,
+from .literals import (SOURCE_UNCOMPRESS_CHOICE_Y,
     SOURCE_UNCOMPRESS_CHOICE_ASK)
-from sources.staging import create_staging_file_class
+from .staging import create_staging_file_class
-from sources.forms import (StagingDocumentForm, WebFormForm,
+from .forms import (StagingDocumentForm, WebFormForm,
     WatchFolderSetupForm)
-from sources.forms import WebFormSetupForm, StagingFolderSetupForm
+from .forms import WebFormSetupForm, StagingFolderSetupForm, POP3EmailSetupForm
-from sources.forms import SourceTransformationForm, SourceTransformationForm_create
+from .forms import SourceTransformationForm, SourceTransformationForm_create
 from .permissions import (PERMISSION_SOURCES_SETUP_VIEW,
     PERMISSION_SOURCES_SETUP_EDIT, PERMISSION_SOURCES_SETUP_DELETE,
     PERMISSION_SOURCES_SETUP_CREATE)
 
+logger = logging.getLogger(__name__)
+
 
 def return_function(obj):
     return lambda context: context['source'].source_type == obj.source_type and context['source'].pk == obj.pk
@@ -69,11 +73,11 @@ def get_active_tab_links(document=None):
     staging_folders = StagingFolder.objects.filter(enabled=True)
     for staging_folder in staging_folders:
         tab_links.append(get_tab_link_for_source(staging_folder, document))
-
+        
     return {
         'tab_links': tab_links,
         SOURCE_CHOICE_WEB_FORM: web_forms,
-        SOURCE_CHOICE_STAGING: staging_folders
+        SOURCE_CHOICE_STAGING: staging_folders,
     }
 
 
@@ -434,6 +438,8 @@ def setup_source_list(request, source_type):
         cls = StagingFolder
     elif source_type == SOURCE_CHOICE_WATCH:
         cls = WatchFolder
+    elif source_type == SOURCE_CHOICE_POP3_EMAIL:
+        cls = POP3Email
 
     context = {
         'object_list': cls.objects.all(),
@@ -459,7 +465,10 @@ def setup_source_edit(request, source_type, source_id):
     elif source_type == SOURCE_CHOICE_WATCH:
         cls = WatchFolder
         form_class = WatchFolderSetupForm
-
+    elif source_type == SOURCE_CHOICE_POP3_EMAIL:
+        cls = POP3Email
+        form_class = POP3EmailSetupForm
+        
     source = get_object_or_404(cls, pk=source_id)
     next = request.POST.get('next', request.GET.get('next', request.META.get('HTTP_REFERER', '/')))
 
@@ -501,6 +510,10 @@ def setup_source_delete(request, source_type, source_id):
         cls = WatchFolder
         form_icon = u'folder_delete.png'
         redirect_view = 'setup_watch_folder_list'
+    elif source_type == SOURCE_CHOICE_POP3_EMAIL:
+        cls = POP3Email
+        form_icon = u'email_delete.png'
+        redirect_view = 'setup_pop3_email_list'
 
     redirect_view = reverse('setup_source_list', args=[source_type])
     previous = request.POST.get('previous', request.GET.get('previous', request.META.get('HTTP_REFERER', redirect_view)))
@@ -512,9 +525,12 @@ def setup_source_delete(request, source_type, source_id):
             source.delete()
             messages.success(request, _(u'Source "%s" deleted successfully.') % source)
         except Exception, e:
-            messages.error(request, _(u'Error deleting source "%(source)s": %(error)s') % {
+            if settings.DEBUG:
-                'source': source, 'error': e
+                raise
-            })
+            else:
+                messages.error(request, _(u'Error deleting source "%(source)s": %(error)s') % {
+                    'source': source, 'error': e
+                })
 
         return HttpResponseRedirect(redirect_view)
 
@@ -539,12 +555,19 @@ def setup_source_create(request, source_type):
     if source_type == SOURCE_CHOICE_WEB_FORM:
         cls = WebForm
         form_class = WebFormSetupForm
+        redirect_view = 'setup_web_form_list'
     elif source_type == SOURCE_CHOICE_STAGING:
         cls = StagingFolder
         form_class = StagingFolderSetupForm
+        redirect_view = 'setup_staging_folder_list'
     elif source_type == SOURCE_CHOICE_WATCH:
         cls = WatchFolder
         form_class = WatchFolderSetupForm
+        redirect_view = 'setup_watch_folder_list'
+    elif source_type == SOURCE_CHOICE_POP3_EMAIL:
+        cls = POP3Email
+        form_class = POP3EmailSetupForm
+        redirect_view = 'setup_pop3_email_list'
 
     if request.method == 'POST':
         form = form_class(data=request.POST)
@@ -552,7 +575,7 @@ def setup_source_create(request, source_type):
             try:
                 form.save()
                 messages.success(request, _(u'Source created successfully'))
-                return HttpResponseRedirect(reverse('setup_web_form_list'))
+                return HttpResponseRedirect(reverse(redirect_view))
             except Exception, e:
                 messages.error(request, _(u'Error creating source; %s') % e)
     else:
@@ -567,6 +590,40 @@ def setup_source_create(request, source_type):
     context_instance=RequestContext(request))
 
 
+def setup_source_log_list(request, source_type, source_pk):
+    Permission.objects.check_permissions(request.user, [PERMISSION_SOURCES_SETUP_EDIT])
+
+    if source_type == SOURCE_CHOICE_WEB_FORM:
+        cls = WebForm
+    elif source_type == SOURCE_CHOICE_STAGING:
+        cls = StagingFolder
+    elif source_type == SOURCE_CHOICE_WATCH:
+        cls = WatchFolder
+    elif source_type == SOURCE_CHOICE_POP3_EMAIL:
+        cls = POP3Email
+        
+    source = get_object_or_404(cls, pk=source_pk)
+
+    context = {
+        'object_list': POP3EmailLog.objects.filter(pop3_email=source).order_by('-creation_datetime'),
+        'title': _(u'logs for: %s') % source.fullname(),
+        'source': source,
+        'object_name': _(u'source'),
+        'navigation_object_name': 'source',
+        'source_type': source_type,
+        'extra_columns': [
+            {'name': _(u'Date time'), 'attribute': 'creation_datetime'},
+            {'name': _(u'Status'), 'attribute': 'status'},
+            ],
+        'hide_link': True,
+        'hide_object': True,
+        'hide_links': True,
+    }
+
+    return render_to_response('generic_list.html', context,
+        context_instance=RequestContext(request))
+
+
 def setup_source_transformation_list(request, source_type, source_id):
     Permission.objects.check_permissions(request.user, [PERMISSION_SOURCES_SETUP_EDIT])
 
@@ -576,7 +633,9 @@ def setup_source_transformation_list(request, source_type, source_id):
         cls = StagingFolder
     elif source_type == SOURCE_CHOICE_WATCH:
         cls = WatchFolder
-
+    elif source_type == SOURCE_CHOICE_POP3_EMAIL:
+        cls = POP3Email
+        
     source = get_object_or_404(cls, pk=source_id)
 
     context = {
@@ -584,6 +643,7 @@ def setup_source_transformation_list(request, source_type, source_id):
         'title': _(u'transformations for: %s') % source.fullname(),
         'source': source,
         'object_name': _(u'source'),
+        'source_type': source_type,
         'navigation_object_name': 'source',
         'list_object_variable_name': 'transformation',
         'extra_columns': [
@@ -675,7 +735,9 @@ def setup_source_transformation_create(request, source_type, source_id):
         cls = StagingFolder
     elif source_type == SOURCE_CHOICE_WATCH:
         cls = WatchFolder
-
+    elif source_type == SOURCE_CHOICE_POP3_EMAIL:
+        cls = POP3Email
+        
     source = get_object_or_404(cls, pk=source_id)
 
     redirect_view = reverse('setup_source_transformation_list', args=[source.source_type, source.pk])
@@ -697,6 +759,7 @@ def setup_source_transformation_create(request, source_type, source_id):
     return render_to_response('generic_form.html', {
         'form': form,
         'source': source,
+        'source_type': source_type,
         'object_name': _(u'source'),
         'navigation_object_name': 'source',
         'title': _(u'Create new transformation for source: %s') % source,

docs/releases/0.13.rst

@@ -0,0 +1,94 @@
+==============================
+Mayan EDMS v0.13 release notes
+==============================
+
+*2012*
+
+Welcome to Mayan EDMS v0.13!
+
+Overview
+========
+
+* POP3 email document source
+
+
+What's new in Mayan EDMS v0.13
+==============================
+
+E-mail document source
+~~~~~~~~~~~~~~~~~~~~~~
+
+
+Upgrading from a previous version
+=================================
+
+Start off by adding the new requirements::
+
+  $ pip install -r requirements/production.txt
+
+Then create the new database structures with::
+
+    $ ./manage.py syncdb
+
+Afterwards migrate existing database schema with::
+
+    $ ./manage.py migrate permissions 0001 --fake
+    $ ./manage.py migrate permissions
+
+
+When the following message appears
+
+::
+
+    The following content types are stale and need to be deleted:
+
+        permissions | permission
+
+    Any objects related to these content types by a foreign key will also
+    be deleted. Are you sure you want to delete these content types?
+    If you're unsure, answer 'no'.
+
+        Type 'yes' to continue, or 'no' to cancel:
+    
+Type ``yes`` and press **Enter**
+
+And continue migrating database schema with::
+
+    $ ./manage.py migrate documents
+    $ ./manage.py migrate document_signatures
+    $ ./manage.py migrate folders 0001 --fake
+    $ ./manage.py migrate folders
+    $ ./manage.py migrate document_indexing 0001 --fake
+    $ ./manage.py migrate document_indexing
+    $ ./manage.py migrate sources 0001 --fake
+    $ ./manage.py migrate sources
+
+Again when a similar messages appears
+::
+
+    The following content types are stale and need to be deleted:
+
+        document_indexing | indexinstance
+
+    Any objects related to these content types by a foreign key will also
+    be deleted. Are you sure you want to delete these content types?
+    If you're unsure, answer 'no'.
+
+        Type 'yes' to continue, or 'no' to cancel:
+
+Type ``yes`` and press **Enter**
+
+The upgrade procedure is now complete.
+
+
+Backward incompatible changes
+=============================
+None so far
+
+Bugs fixed
+==========
+* Metadata unicode fixes, while creating new documents.
+
+Stuff removed
+=============
+None so far

docs/releases/index.rst

@@ -11,11 +11,12 @@ all the backwards-incompatible changes and deprecated features for
 each 'final' release from the one after your current **Mayan EDMS** version,
 up to and including the new version.
 
-Latest version (0.12)
+Latest version (0.13)
 ---------------------
 .. toctree::
    :maxdepth: 1
 
+   0.13
    0.12
 
 Historic changelogs