matthias/mayan-edms
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Add IMAP document souce support, merge with existing POP3 email support, make POP3MailLog generic

Browse Source
This commit is contained in:
Roberto Rosario
2012-03-14 01:25:49 -04:00
parent a234c76326
commit 19b3553705
13 changed files with 795 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
apps/sources/__init__.py
View File

@@ -12,22 +12,23 @@ from scheduler.api import register_interval_job
from .staging import StagingFile
from .models import (WebForm, StagingFolder, SourceTransformation,
WatchFolder, POP3Email)
WatchFolder, POP3Email, IMAPEmail)
from .widgets import staging_file_thumbnail
from .permissions import (PERMISSION_SOURCES_SETUP_VIEW,
PERMISSION_SOURCES_SETUP_EDIT, PERMISSION_SOURCES_SETUP_DELETE,
PERMISSION_SOURCES_SETUP_CREATE)
from .tasks import task_fetch_pop3_emails
from .conf.settings import POP3_EMAIL_PROCESSING_INTERVAL
from .tasks import task_fetch_pop3_emails, task_fetch_imap_emails
from .conf.settings import EMAIL_PROCESSING_INTERVAL
staging_file_preview = {'text': _(u'preview'), 'class': 'fancybox-noscaling', 'view': 'staging_file_preview', 'args': ['source.source_type', 'source.pk', 'object.id'], 'famfam': 'zoom', 'permissions': [PERMISSION_DOCUMENT_NEW_VERSION, PERMISSION_DOCUMENT_CREATE]}
staging_file_delete = {'text': _(u'delete'), 'view': 'staging_file_delete', 'args': ['source.source_type', 'source.pk', 'object.id'], 'famfam': 'delete', 'keep_query': True, 'permissions': [PERMISSION_DOCUMENT_NEW_VERSION, PERMISSION_DOCUMENT_CREATE]}
setup_sources = {'text': _(u'sources'), 'view': 'setup_web_form_list', 'famfam': 'application_form', 'icon': 'application_form.png', 'children_classes': [WebForm], 'permissions': [PERMISSION_SOURCES_SETUP_VIEW], 'children_view_regex': [r'setup_web_form', r'setup_staging_folder', r'setup_source_', r'setup_pop3']}
setup_sources = {'text': _(u'sources'), 'view': 'setup_web_form_list', 'famfam': 'application_form', 'icon': 'application_form.png', 'children_classes': [WebForm], 'permissions': [PERMISSION_SOURCES_SETUP_VIEW], 'children_view_regex': [r'setup_web_form', r'setup_staging_folder', r'setup_source_', r'setup_pop3', r'setup_imap']}
setup_web_form_list = {'text': _(u'web forms'), 'view': 'setup_web_form_list', 'famfam': 'application_form', 'icon': 'application_form.png', 'children_classes': [WebForm], 'permissions': [PERMISSION_SOURCES_SETUP_VIEW]}
setup_staging_folder_list = {'text': _(u'staging folders'), 'view': 'setup_staging_folder_list', 'famfam': 'folder_camera', 'children_classes': [StagingFolder], 'permissions': [PERMISSION_SOURCES_SETUP_VIEW]}
setup_watch_folder_list = {'text': _(u'watch folders'), 'view': 'setup_watch_folder_list', 'famfam': 'folder_magnify', 'children_classes': [WatchFolder], 'permissions': [PERMISSION_SOURCES_SETUP_VIEW]}
setup_pop3_email_list = {'text': _(u'POP3 email'), 'view': 'setup_pop3_email_list', 'famfam': 'email', 'children_classes': [POP3Email], 'permissions': [PERMISSION_SOURCES_SETUP_VIEW]}
setup_imap_email_list = {'text': _(u'IMAP email'), 'view': 'setup_imap_email_list', 'famfam': 'email', 'children_classes': [IMAPEmail], 'permissions': [PERMISSION_SOURCES_SETUP_VIEW]}
setup_source_edit = {'text': _(u'edit'), 'view': 'setup_source_edit', 'args': ['source.source_type', 'source.pk'], 'famfam': 'application_form_edit', 'permissions': [PERMISSION_SOURCES_SETUP_EDIT]}
setup_source_delete = {'text': _(u'delete'), 'view': 'setup_source_delete', 'args': ['source.source_type', 'source.pk'], 'famfam': 'application_form_delete', 'permissions': [PERMISSION_SOURCES_SETUP_DELETE]}
@@ -47,25 +48,24 @@ register_links(StagingFile, [staging_file_delete])
register_links(SourceTransformation, [setup_source_transformation_edit, setup_source_transformation_delete])
#register_links(['setup_web_form_list', 'setup_staging_folder_list', 'setup_watch_folder_list', 'setup_source_create'], [setup_web_form_list, setup_staging_folder_list, setup_watch_folder_list], menu_name='form_header')
register_links(['setup_pop3_email_list', 'setup_web_form_list', 'setup_staging_folder_list', 'setup_watch_folder_list', 'setup_source_create'], [setup_web_form_list, setup_staging_folder_list, setup_pop3_email_list], menu_name='form_header')
register_links(['setup_imap_email_list', 'setup_pop3_email_list', 'setup_web_form_list', 'setup_staging_folder_list', 'setup_watch_folder_list', 'setup_source_create'], [setup_web_form_list, setup_staging_folder_list, setup_pop3_email_list, setup_imap_email_list], menu_name='form_header')
register_links([WebForm, StagingFolder, POP3Email, IMAPEmail, 'setup_web_form_list', 'setup_staging_folder_list', 'setup_watch_folder_list', 'setup_source_create', 'setup_pop3_email_list', 'setup_imap_email_list'], [setup_source_create], menu_name='secondary_menu')
#register_links(WebForm, [setup_web_form_list, setup_staging_folder_list, setup_watch_folder_list], menu_name='form_header')
register_links(WebForm, [setup_web_form_list, setup_staging_folder_list, setup_pop3_email_list], menu_name='form_header')
register_links(WebForm, [setup_web_form_list, setup_staging_folder_list, setup_pop3_email_list, setup_imap_email_list], menu_name='form_header')
register_links(WebForm, [setup_source_transformation_list, setup_source_edit, setup_source_delete])
#register_links(['setup_source_log_list', 'setup_web_form_list', 'setup_staging_folder_list', 'setup_watch_folder_list', 'setup_source_edit', 'setup_source_delete', 'setup_source_create', 'setup_pop3_email_list'], [setup_sources, setup_source_create], menu_name='secondary_menu')
register_links([WebForm, StagingFolder, POP3Email, 'setup_web_form_list', 'setup_staging_folder_list', 'setup_watch_folder_list', 'setup_source_create', 'setup_pop3_email_list'], [setup_source_create], menu_name='secondary_menu')
#register_links(StagingFolder, [setup_web_form_list, setup_staging_folder_list, setup_watch_folder_list], menu_name='form_header')
register_links(StagingFolder, [setup_web_form_list, setup_staging_folder_list, setup_pop3_email_list], menu_name='form_header')
register_links(StagingFolder, [setup_web_form_list, setup_staging_folder_list, setup_pop3_email_list, setup_imap_email_list], menu_name='form_header')
register_links(StagingFolder, [setup_source_transformation_list, setup_source_edit, setup_source_delete])
register_links(POP3Email, [setup_web_form_list, setup_staging_folder_list, setup_pop3_email_list], menu_name='form_header')
register_links(POP3Email, [setup_web_form_list, setup_staging_folder_list, setup_pop3_email_list, setup_imap_email_list], menu_name='form_header')
register_links(POP3Email, [setup_source_transformation_list, setup_source_edit, setup_source_delete])
register_links(POP3Email, [setup_source_log_list])
register_links(WatchFolder, [setup_web_form_list, setup_staging_folder_list, setup_watch_folder_list], menu_name='form_header')
register_links(IMAPEmail, [setup_web_form_list, setup_staging_folder_list, setup_pop3_email_list, setup_imap_email_list], menu_name='form_header')
register_links(IMAPEmail, [setup_source_transformation_list, setup_source_edit, setup_source_delete])
register_links(IMAPEmail, [setup_source_log_list])
register_links(WatchFolder, [setup_web_form_list, setup_staging_folder_list, setup_watch_folder_list, setup_imap_email_list], menu_name='form_header')
register_links(WatchFolder, [setup_source_transformation_list, setup_source_edit, setup_source_delete])
# Document version
@@ -83,4 +83,5 @@ register_model_list_columns(StagingFile, [
register_setup(setup_sources)
register_interval_job('task_fetch_pop3_emails', _(u'Connects to the POP3 email sources and fetches the attached documents.'), task_fetch_pop3_emails, seconds=POP3_EMAIL_PROCESSING_INTERVAL)
register_interval_job('task_fetch_pop3_emails', _(u'Connects to the POP3 email sources and fetches the attached documents.'), task_fetch_pop3_emails, seconds=EMAIL_PROCESSING_INTERVAL)
register_interval_job('task_fetch_imap_emails', _(u'Connects to the IMAP email sources and fetches the attached documents.'), task_fetch_imap_emails, seconds=EMAIL_PROCESSING_INTERVAL)

12
apps/sources/conf/settings.py
View File

@@ -9,7 +9,7 @@ from smart_settings.api import Setting, SettingNamespace
namespace = SettingNamespace('sources', _(u'Sources'), module='sources.conf.settings')
POP3_DEFAULT_TIMEOUT = 10 # for POP3 only not POP3_SSL
POP3_DEFAULT_EMAIL_PROCESSING_INTERVAL = 60
DEFAULT_EMAIL_PROCESSING_INTERVAL = 60
DEFAULT_POP3_EMAIL_LOG_COUNT = 10 # Max log entries to store
Setting(
@@ -21,14 +21,14 @@ Setting(
Setting(
namespace=namespace,
name='POP3_EMAIL_PROCESSING_INTERVAL',
global_name='SOURCES_POP3_EMAIL_PROCESSING_INTERVAL',
default=POP3_DEFAULT_EMAIL_PROCESSING_INTERVAL,
name='EMAIL_PROCESSING_INTERVAL',
global_name='SOURCES_EMAIL_PROCESSING_INTERVAL',
default=DEFAULT_EMAIL_PROCESSING_INTERVAL,
)
Setting(
namespace=namespace,
name='POP3_EMAIL_LOG_SIZE',
global_name='SOURCES_POP3_EMAIL_LOG_SIZE',
name='LOG_SIZE',
global_name='SOURCES_LOG_SIZE',
default=DEFAULT_POP3_EMAIL_LOG_COUNT,
)

15
apps/sources/forms.py
View File

@@ -7,7 +7,7 @@ from django.utils.translation import ugettext
from documents.forms import DocumentForm
from .models import (WebForm, StagingFolder, SourceTransformation,
WatchFolder, POP3Email)
WatchFolder, POP3Email, IMAPEmail)
from .widgets import FamFamRadioSelect
from .utils import validate_whitelist_blacklist
@@ -116,9 +116,18 @@ class SourceTransformationForm_create(forms.ModelForm):
exclude = ('content_type', 'object_id')
class POP3EmailSetupForm(forms.ModelForm):
class EmailSetupForm(forms.ModelForm):
class Meta:
model = POP3Email
widgets = {
'password': forms.widgets.PasswordInput,
}
class POP3EmailSetupForm(EmailSetupForm):
class Meta(EmailSetupForm.Meta):
model = POP3Email
class IMAPEmailSetupForm(EmailSetupForm):
class Meta(EmailSetupForm.Meta):
model = IMAPEmail

13
apps/sources/literals.py
View File

@@ -4,6 +4,12 @@ POP3_PORT = 110
POP3_SSL_PORT = 995
DEFAULT_POP3_INTERVAL = 15 * 60 # 15 minutes in seconds
IMAP_PORT = 143
IMAP_SSL_PORT = 993
DEFAULT_IMAP_INTERVAL = 15 * 60 # 15 minutes in seconds
IMAP_LOCK_TIMEOUT = 60
IMAP_DEFAULT_MAILBOX = 'INBOX'
SOURCE_UNCOMPRESS_CHOICE_Y = 'y'
SOURCE_UNCOMPRESS_CHOICE_N = 'n'
SOURCE_UNCOMPRESS_CHOICE_ASK = 'a'
@@ -47,17 +53,20 @@ SOURCE_CHOICE_WEB_FORM = 'webform'
SOURCE_CHOICE_STAGING = 'staging'
SOURCE_CHOICE_WATCH = 'watch'
SOURCE_CHOICE_POP3_EMAIL = 'pop3'
SOURCE_CHOICE_IMAP_EMAIL = 'imap'
SOURCE_CHOICES = (
(SOURCE_CHOICE_WEB_FORM, _(u'web form')),
(SOURCE_CHOICE_STAGING, _(u'server staging folder')),
(SOURCE_CHOICE_WATCH, _(u'server watch folder')),
(SOURCE_CHOICE_POP3_EMAIL, _(u'pop3 email')),
(SOURCE_CHOICE_POP3_EMAIL, _(u'POP3 email')),
(SOURCE_CHOICE_IMAP_EMAIL, _(u'IMAP email')),
)
SOURCE_CHOICES_PLURAL = (
(SOURCE_CHOICE_WEB_FORM, _(u'web forms')),
(SOURCE_CHOICE_STAGING, _(u'server staging folders')),
(SOURCE_CHOICE_WATCH, _(u'server watch folders')),
(SOURCE_CHOICE_POP3_EMAIL, _(u'pop3 emails')),
(SOURCE_CHOICE_POP3_EMAIL, _(u'POP3 emails')),
(SOURCE_CHOICE_IMAP_EMAIL, _(u'IMAP emails')),
)

19
apps/sources/managers.py
View File

@@ -5,7 +5,7 @@ from ast import literal_eval
from django.db import models
from django.contrib.contenttypes.models import ContentType
from .conf.settings import POP3_EMAIL_LOG_SIZE
from .conf.settings import LOG_SIZE
class SourceTransformationManager(models.Manager):
@@ -30,10 +30,19 @@ class SourceTransformationManager(models.Manager):
return transformations, warnings
class POP3EmailLogManager(models.Manager):
def save_status(self, pop3_email, status):
new_recent = self.model(pop3_email=pop3_email, status=status)
class SourceLogManager(models.Manager):
def save_status(self, source, status):
new_recent = self.model(source=source, status=status)
new_recent.save()
to_delete = self.model.objects.filter(pop3_email=pop3_email).order_by('-creation_datetime')[POP3_EMAIL_LOG_SIZE:]
content_type = ContentType.objects.get_for_model(source)
to_delete = self.model.objects.filter(content_type=content_type, object_id=source.pk).order_by('-creation_datetime')[LOG_SIZE:]
for recent_to_delete in to_delete:
recent_to_delete.delete()
def get_for_source(self, source):
content_type = ContentType.objects.get_for_model(source)
return self.model.objects.filter(content_type=content_type, object_id=source.pk).order_by('-creation_datetime')
def get_latest_for(self, source):
content_type = ContentType.objects.get_for_model(source)
return self.model.objects.filter(content_type=content_type, object_id=source.pk).latest().creation_datetime

111
apps/sources/migrations/0009_auto__del_pop3email__del_pop3emaillog.py Normal file
View File

@@ -0,0 +1,111 @@
# encoding: utf-8
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
class Migration(SchemaMigration):
def forwards(self, orm):
# Deleting model 'POP3Email'
db.delete_table('sources_pop3email')
# Deleting model 'POP3EmailLog'
db.delete_table('sources_pop3emaillog')
def backwards(self, orm):
# Adding model 'POP3Email'
db.create_table('sources_pop3email', (
('username', self.gf('django.db.models.fields.CharField')(max_length=64)),
('uncompress', self.gf('django.db.models.fields.CharField')(max_length=1)),
('whitelist', self.gf('django.db.models.fields.TextField')(blank=True)),
('blacklist', self.gf('django.db.models.fields.TextField')(blank=True)),
('ssl', self.gf('django.db.models.fields.BooleanField')(default=False)),
('host', self.gf('django.db.models.fields.CharField')(max_length=64)),
('password', self.gf('django.db.models.fields.CharField')(max_length=64)),
('port', self.gf('django.db.models.fields.PositiveIntegerField')(null=True, blank=True)),
('document_type', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['documents.DocumentType'], null=True, blank=True)),
('title', self.gf('django.db.models.fields.CharField')(max_length=64)),
('interval', self.gf('django.db.models.fields.PositiveIntegerField')(default=900)),
('enabled', self.gf('django.db.models.fields.BooleanField')(default=True)),
('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
))
db.send_create_signal('sources', ['POP3Email'])
# Adding model 'POP3EmailLog'
db.create_table('sources_pop3emaillog', (
('status', self.gf('django.db.models.fields.TextField')()),
('creation_datetime', self.gf('django.db.models.fields.DateTimeField')()),
('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
('pop3_email', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['sources.POP3Email'])),
))
db.send_create_signal('sources', ['POP3EmailLog'])
models = {
'contenttypes.contenttype': {
'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
},
'sources.outofprocess': {
'Meta': {'ordering': "('title',)", 'object_name': 'OutOfProcess'},
'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'})
},
'sources.sourcetransformation': {
'Meta': {'ordering': "('order',)", 'object_name': 'SourceTransformation'},
'arguments': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'object_id': ('django.db.models.fields.PositiveIntegerField', [], {}),
'order': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0', 'null': 'True', 'db_index': 'True', 'blank': 'True'}),
'transformation': ('django.db.models.fields.CharField', [], {'max_length': '128'})
},
'sources.stagingfolder': {
'Meta': {'ordering': "('title',)", 'object_name': 'StagingFolder'},
'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'delete_after_upload': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'folder_path': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
'icon': ('django.db.models.fields.CharField', [], {'max_length': '24', 'null': 'True', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'preview_height': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
'preview_width': ('django.db.models.fields.IntegerField', [], {}),
'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}),
'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'})
},
'sources.watchfolder': {
'Meta': {'ordering': "('title',)", 'object_name': 'WatchFolder'},
'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'delete_after_upload': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'folder_path': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'interval': ('django.db.models.fields.PositiveIntegerField', [], {}),
'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}),
'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'})
},
'sources.webform': {
'Meta': {'ordering': "('title',)", 'object_name': 'WebForm'},
'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'icon': ('django.db.models.fields.CharField', [], {'max_length': '24', 'null': 'True', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}),
'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'})
}
}
complete_apps = ['sources']

180
apps/sources/migrations/0010_auto__add_imapemail__add_sourcelog__add_pop3email.py Normal file
View File

@@ -0,0 +1,180 @@
# encoding: utf-8
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
class Migration(SchemaMigration):
def forwards(self, orm):
# Adding model 'ImapEmail'
db.create_table('sources_imapemail', (
('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
('title', self.gf('django.db.models.fields.CharField')(max_length=64)),
('enabled', self.gf('django.db.models.fields.BooleanField')(default=True)),
('whitelist', self.gf('django.db.models.fields.TextField')(blank=True)),
('blacklist', self.gf('django.db.models.fields.TextField')(blank=True)),
('interval', self.gf('django.db.models.fields.PositiveIntegerField')(default=900)),
('document_type', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['documents.DocumentType'], null=True, blank=True)),
('uncompress', self.gf('django.db.models.fields.CharField')(max_length=1)),
('host', self.gf('django.db.models.fields.CharField')(max_length=64)),
('ssl', self.gf('django.db.models.fields.BooleanField')(default=False)),
('port', self.gf('django.db.models.fields.PositiveIntegerField')(null=True, blank=True)),
('username', self.gf('django.db.models.fields.CharField')(max_length=64)),
('password', self.gf('django.db.models.fields.CharField')(max_length=64)),
('mailbox', self.gf('django.db.models.fields.CharField')(max_length=64, blank=True)),
))
db.send_create_signal('sources', ['ImapEmail'])
# Adding model 'SourceLog'
db.create_table('sources_sourcelog', (
('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
('content_type', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['contenttypes.ContentType'])),
('object_id', self.gf('django.db.models.fields.PositiveIntegerField')()),
('creation_datetime', self.gf('django.db.models.fields.DateTimeField')()),
('status', self.gf('django.db.models.fields.TextField')()),
))
db.send_create_signal('sources', ['SourceLog'])
# Adding model 'POP3Email'
db.create_table('sources_pop3email', (
('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
('title', self.gf('django.db.models.fields.CharField')(max_length=64)),
('enabled', self.gf('django.db.models.fields.BooleanField')(default=True)),
('whitelist', self.gf('django.db.models.fields.TextField')(blank=True)),
('blacklist', self.gf('django.db.models.fields.TextField')(blank=True)),
('interval', self.gf('django.db.models.fields.PositiveIntegerField')(default=900)),
('document_type', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['documents.DocumentType'], null=True, blank=True)),
('uncompress', self.gf('django.db.models.fields.CharField')(max_length=1)),
('host', self.gf('django.db.models.fields.CharField')(max_length=64)),
('ssl', self.gf('django.db.models.fields.BooleanField')(default=False)),
('port', self.gf('django.db.models.fields.PositiveIntegerField')(null=True, blank=True)),
('username', self.gf('django.db.models.fields.CharField')(max_length=64)),
('password', self.gf('django.db.models.fields.CharField')(max_length=64)),
))
db.send_create_signal('sources', ['POP3Email'])
def backwards(self, orm):
# Deleting model 'ImapEmail'
db.delete_table('sources_imapemail')
# Deleting model 'SourceLog'
db.delete_table('sources_sourcelog')
# Deleting model 'POP3Email'
db.delete_table('sources_pop3email')
models = {
'contenttypes.contenttype': {
'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
},
'documents.documenttype': {
'Meta': {'ordering': "['name']", 'object_name': 'DocumentType'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '32'})
},
'sources.imapemail': {
'Meta': {'ordering': "('title',)", 'object_name': 'ImapEmail'},
'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'document_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['documents.DocumentType']", 'null': 'True', 'blank': 'True'}),
'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'host': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'interval': ('django.db.models.fields.PositiveIntegerField', [], {'default': '900'}),
'mailbox': ('django.db.models.fields.CharField', [], {'max_length': '64', 'blank': 'True'}),
'password': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
'port': ('django.db.models.fields.PositiveIntegerField', [], {'null': 'True', 'blank': 'True'}),
'ssl': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}),
'username': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'})
},
'sources.outofprocess': {
'Meta': {'ordering': "('title',)", 'object_name': 'OutOfProcess'},
'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'})
},
'sources.pop3email': {
'Meta': {'ordering': "('title',)", 'object_name': 'POP3Email'},
'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'document_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['documents.DocumentType']", 'null': 'True', 'blank': 'True'}),
'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'host': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'interval': ('django.db.models.fields.PositiveIntegerField', [], {'default': '900'}),
'password': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
'port': ('django.db.models.fields.PositiveIntegerField', [], {'null': 'True', 'blank': 'True'}),
'ssl': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}),
'username': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'})
},
'sources.sourcelog': {
'Meta': {'ordering': "('creation_datetime',)", 'object_name': 'SourceLog'},
'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
'creation_datetime': ('django.db.models.fields.DateTimeField', [], {}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'object_id': ('django.db.models.fields.PositiveIntegerField', [], {}),
'status': ('django.db.models.fields.TextField', [], {})
},
'sources.sourcetransformation': {
'Meta': {'ordering': "('order',)", 'object_name': 'SourceTransformation'},
'arguments': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'object_id': ('django.db.models.fields.PositiveIntegerField', [], {}),
'order': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0', 'null': 'True', 'db_index': 'True', 'blank': 'True'}),
'transformation': ('django.db.models.fields.CharField', [], {'max_length': '128'})
},
'sources.stagingfolder': {
'Meta': {'ordering': "('title',)", 'object_name': 'StagingFolder'},
'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'delete_after_upload': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'folder_path': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
'icon': ('django.db.models.fields.CharField', [], {'max_length': '24', 'null': 'True', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'preview_height': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
'preview_width': ('django.db.models.fields.IntegerField', [], {}),
'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}),
'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'})
},
'sources.watchfolder': {
'Meta': {'ordering': "('title',)", 'object_name': 'WatchFolder'},
'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'delete_after_upload': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'folder_path': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'interval': ('django.db.models.fields.PositiveIntegerField', [], {}),
'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}),
'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'})
},
'sources.webform': {
'Meta': {'ordering': "('title',)", 'object_name': 'WebForm'},
'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'icon': ('django.db.models.fields.CharField', [], {'max_length': '24', 'null': 'True', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}),
'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'})
}
}
complete_apps = ['sources']

129
apps/sources/migrations/0011_auto__del_imapemail.py Normal file
View File

@@ -0,0 +1,129 @@
# encoding: utf-8
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
class Migration(SchemaMigration):
def forwards(self, orm):
# Deleting model 'ImapEmail'
db.delete_table('sources_imapemail')
def backwards(self, orm):
# Adding model 'ImapEmail'
db.create_table('sources_imapemail', (
('username', self.gf('django.db.models.fields.CharField')(max_length=64)),
('uncompress', self.gf('django.db.models.fields.CharField')(max_length=1)),
('whitelist', self.gf('django.db.models.fields.TextField')(blank=True)),
('mailbox', self.gf('django.db.models.fields.CharField')(max_length=64, blank=True)),
('blacklist', self.gf('django.db.models.fields.TextField')(blank=True)),
('ssl', self.gf('django.db.models.fields.BooleanField')(default=False)),
('host', self.gf('django.db.models.fields.CharField')(max_length=64)),
('password', self.gf('django.db.models.fields.CharField')(max_length=64)),
('port', self.gf('django.db.models.fields.PositiveIntegerField')(null=True, blank=True)),
('document_type', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['documents.DocumentType'], null=True, blank=True)),
('title', self.gf('django.db.models.fields.CharField')(max_length=64)),
('interval', self.gf('django.db.models.fields.PositiveIntegerField')(default=900)),
('enabled', self.gf('django.db.models.fields.BooleanField')(default=True)),
('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
))
db.send_create_signal('sources', ['ImapEmail'])
models = {
'contenttypes.contenttype': {
'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
},
'documents.documenttype': {
'Meta': {'ordering': "['name']", 'object_name': 'DocumentType'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '32'})
},
'sources.outofprocess': {
'Meta': {'ordering': "('title',)", 'object_name': 'OutOfProcess'},
'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'})
},
'sources.pop3email': {
'Meta': {'ordering': "('title',)", 'object_name': 'POP3Email'},
'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'document_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['documents.DocumentType']", 'null': 'True', 'blank': 'True'}),
'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'host': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'interval': ('django.db.models.fields.PositiveIntegerField', [], {'default': '900'}),
'password': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
'port': ('django.db.models.fields.PositiveIntegerField', [], {'null': 'True', 'blank': 'True'}),
'ssl': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}),
'username': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'})
},
'sources.sourcelog': {
'Meta': {'ordering': "('creation_datetime',)", 'object_name': 'SourceLog'},
'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
'creation_datetime': ('django.db.models.fields.DateTimeField', [], {}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'object_id': ('django.db.models.fields.PositiveIntegerField', [], {}),
'status': ('django.db.models.fields.TextField', [], {})
},
'sources.sourcetransformation': {
'Meta': {'ordering': "('order',)", 'object_name': 'SourceTransformation'},
'arguments': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'object_id': ('django.db.models.fields.PositiveIntegerField', [], {}),
'order': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0', 'null': 'True', 'db_index': 'True', 'blank': 'True'}),
'transformation': ('django.db.models.fields.CharField', [], {'max_length': '128'})
},
'sources.stagingfolder': {
'Meta': {'ordering': "('title',)", 'object_name': 'StagingFolder'},
'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'delete_after_upload': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'folder_path': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
'icon': ('django.db.models.fields.CharField', [], {'max_length': '24', 'null': 'True', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'preview_height': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
'preview_width': ('django.db.models.fields.IntegerField', [], {}),
'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}),
'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'})
},
'sources.watchfolder': {
'Meta': {'ordering': "('title',)", 'object_name': 'WatchFolder'},
'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'delete_after_upload': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'folder_path': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'interval': ('django.db.models.fields.PositiveIntegerField', [], {}),
'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}),
'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'})
},
'sources.webform': {
'Meta': {'ordering': "('title',)", 'object_name': 'WebForm'},
'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'icon': ('django.db.models.fields.CharField', [], {'max_length': '24', 'null': 'True', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}),
'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'})
}
}
complete_apps = ['sources']

146
apps/sources/migrations/0012_auto__add_imapemail.py Normal file
View File

@@ -0,0 +1,146 @@
# encoding: utf-8
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
class Migration(SchemaMigration):
def forwards(self, orm):
# Adding model 'IMAPEmail'
db.create_table('sources_imapemail', (
('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
('title', self.gf('django.db.models.fields.CharField')(max_length=64)),
('enabled', self.gf('django.db.models.fields.BooleanField')(default=True)),
('whitelist', self.gf('django.db.models.fields.TextField')(blank=True)),
('blacklist', self.gf('django.db.models.fields.TextField')(blank=True)),
('interval', self.gf('django.db.models.fields.PositiveIntegerField')(default=900)),
('document_type', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['documents.DocumentType'], null=True, blank=True)),
('uncompress', self.gf('django.db.models.fields.CharField')(max_length=1)),
('host', self.gf('django.db.models.fields.CharField')(max_length=64)),
('ssl', self.gf('django.db.models.fields.BooleanField')(default=False)),
('port', self.gf('django.db.models.fields.PositiveIntegerField')(null=True, blank=True)),
('username', self.gf('django.db.models.fields.CharField')(max_length=64)),
('password', self.gf('django.db.models.fields.CharField')(max_length=64)),
('mailbox', self.gf('django.db.models.fields.CharField')(max_length=64, blank=True)),
))
db.send_create_signal('sources', ['IMAPEmail'])
def backwards(self, orm):
# Deleting model 'IMAPEmail'
db.delete_table('sources_imapemail')
models = {
'contenttypes.contenttype': {
'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
},
'documents.documenttype': {
'Meta': {'ordering': "['name']", 'object_name': 'DocumentType'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '32'})
},
'sources.imapemail': {
'Meta': {'ordering': "('title',)", 'object_name': 'IMAPEmail'},
'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'document_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['documents.DocumentType']", 'null': 'True', 'blank': 'True'}),
'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'host': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'interval': ('django.db.models.fields.PositiveIntegerField', [], {'default': '900'}),
'mailbox': ('django.db.models.fields.CharField', [], {'max_length': '64', 'blank': 'True'}),
'password': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
'port': ('django.db.models.fields.PositiveIntegerField', [], {'null': 'True', 'blank': 'True'}),
'ssl': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}),
'username': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'})
},
'sources.outofprocess': {
'Meta': {'ordering': "('title',)", 'object_name': 'OutOfProcess'},
'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'})
},
'sources.pop3email': {
'Meta': {'ordering': "('title',)", 'object_name': 'POP3Email'},
'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'document_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['documents.DocumentType']", 'null': 'True', 'blank': 'True'}),
'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'host': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'interval': ('django.db.models.fields.PositiveIntegerField', [], {'default': '900'}),
'password': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
'port': ('django.db.models.fields.PositiveIntegerField', [], {'null': 'True', 'blank': 'True'}),
'ssl': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}),
'username': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'})
},
'sources.sourcelog': {
'Meta': {'ordering': "('creation_datetime',)", 'object_name': 'SourceLog'},
'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
'creation_datetime': ('django.db.models.fields.DateTimeField', [], {}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'object_id': ('django.db.models.fields.PositiveIntegerField', [], {}),
'status': ('django.db.models.fields.TextField', [], {})
},
'sources.sourcetransformation': {
'Meta': {'ordering': "('order',)", 'object_name': 'SourceTransformation'},
'arguments': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'object_id': ('django.db.models.fields.PositiveIntegerField', [], {}),
'order': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0', 'null': 'True', 'db_index': 'True', 'blank': 'True'}),
'transformation': ('django.db.models.fields.CharField', [], {'max_length': '128'})
},
'sources.stagingfolder': {
'Meta': {'ordering': "('title',)", 'object_name': 'StagingFolder'},
'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'delete_after_upload': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'folder_path': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
'icon': ('django.db.models.fields.CharField', [], {'max_length': '24', 'null': 'True', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'preview_height': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
'preview_width': ('django.db.models.fields.IntegerField', [], {}),
'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}),
'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'})
},
'sources.watchfolder': {
'Meta': {'ordering': "('title',)", 'object_name': 'WatchFolder'},
'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'delete_after_upload': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'folder_path': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'interval': ('django.db.models.fields.PositiveIntegerField', [], {}),
'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}),
'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'})
},
'sources.webform': {
'Meta': {'ordering': "('title',)", 'object_name': 'WebForm'},
'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'icon': ('django.db.models.fields.CharField', [], {'max_length': '24', 'null': 'True', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}),
'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'})
}
}
complete_apps = ['sources']

138
apps/sources/models.py
View File

@@ -3,6 +3,7 @@ from __future__ import absolute_import
from ast import literal_eval
import logging
import poplib
import imaplib
from email.Utils import collapse_rfc2231_value
from email import message_from_string
import os
@@ -33,13 +34,17 @@ from metadata.api import save_metadata_list
from scheduler.api import register_interval_job, remove_job
from acls.utils import apply_default_acls
from .managers import SourceTransformationManager, POP3EmailLogManager
from .managers import SourceTransformationManager, SourceLogManager
from .literals import (SOURCE_CHOICES, SOURCE_CHOICES_PLURAL,
SOURCE_INTERACTIVE_UNCOMPRESS_CHOICES, SOURCE_CHOICE_WEB_FORM,
SOURCE_CHOICE_STAGING, SOURCE_ICON_DISK, SOURCE_ICON_DRIVE,
SOURCE_ICON_CHOICES, SOURCE_CHOICE_WATCH, SOURCE_UNCOMPRESS_CHOICES,
SOURCE_UNCOMPRESS_CHOICE_Y, POP3_PORT, POP3_SSL_PORT,
SOURCE_CHOICE_POP3_EMAIL, DEFAULT_POP3_INTERVAL)
SOURCE_UNCOMPRESS_CHOICE_Y,
POP3_PORT, POP3_SSL_PORT,
SOURCE_CHOICE_POP3_EMAIL, DEFAULT_POP3_INTERVAL,
IMAP_PORT, IMAP_SSL_PORT,
SOURCE_CHOICE_IMAP_EMAIL, DEFAULT_IMAP_INTERVAL,
IMAP_DEFAULT_MAILBOX)
from .compressed_file import CompressedFile, NotACompressedFile
from .conf.settings import POP3_TIMEOUT
@@ -162,6 +167,27 @@ class BaseModel(models.Model):
abstract = True
class SourceLog(models.Model):
content_type = models.ForeignKey(ContentType)
object_id = models.PositiveIntegerField()
source = generic.GenericForeignKey('content_type', 'object_id')
creation_datetime = models.DateTimeField(verbose_name=_(u'date time'))
status = models.TextField(verbose_name=_(u'status'))
objects = SourceLogManager()
def save(self, *args, **kwargs):
if not self.pk:
self.creation_datetime = datetime.datetime.now()
return super(SourceLog, self).save(*args, **kwargs)
class Meta:
verbose_name = _(u'source log')
verbose_name_plural = _(u'sources logs')
get_latest_by = 'creation_datetime'
ordering = ('creation_datetime',)
class InteractiveBaseModel(BaseModel):
icon = models.CharField(blank=True, null=True, max_length=24, choices=SOURCE_ICON_CHOICES, verbose_name=_(u'icon'), help_text=_(u'An icon to visually distinguish this source.'))
@@ -189,18 +215,25 @@ class Attachment(File):
self.file = PseudoFile(StringIO(part.get_payload(decode=True)), name=name)
class POP3Email(BaseModel):
class IntervalBaseModel(BaseModel):
is_interactive = False
source_type = SOURCE_CHOICE_POP3_EMAIL
interval = models.PositiveIntegerField(default=DEFAULT_POP3_INTERVAL, verbose_name=_(u'interval'), help_text=_(u'Interval in seconds between document downloads from this source.'))
document_type = models.ForeignKey(DocumentType, null=True, blank=True, verbose_name=_(u'document type'), help_text=_(u'Assign a document type to documents uploaded from this source.'))
uncompress = models.CharField(max_length=1, choices=SOURCE_UNCOMPRESS_CHOICES, verbose_name=_(u'uncompress'), help_text=_(u'Whether to expand or not, compressed archives.'))
class Meta(BaseModel.Meta):
verbose_name = _(u'interval source')
verbose_name_plural = _(u'interval sources')
abstract = True
class EmailBaseModel(IntervalBaseModel):
host = models.CharField(max_length=64, verbose_name=_(u'host'))
ssl = models.BooleanField(verbose_name=_(u'SSL'))
port = models.PositiveIntegerField(blank=True, null=True, verbose_name=_(u'port'), help_text=_(u'Override the defaults values of %d and %d for SSL, can be left blank otherwise.') % (POP3_PORT, POP3_SSL_PORT))
username = models.CharField(max_length=64, verbose_name=_(u'username'))
password = models.CharField(max_length=64, verbose_name=_(u'password'))
uncompress = models.CharField(max_length=1, choices=SOURCE_UNCOMPRESS_CHOICES, verbose_name=_(u'uncompress'), help_text=_(u'Whether to expand or not, compressed archives.'))
interval = models.PositiveIntegerField(default=DEFAULT_POP3_INTERVAL, verbose_name=_(u'interval'), help_text=_(u'Interval in seconds between document downloads from this account.'))
document_type = models.ForeignKey(DocumentType, null=True, blank=True, verbose_name=_(u'document type'), help_text=_(u'Assign a document type to documents uploaded from this source.'))
# From: http://bookmarks.honewatson.com/2009/08/11/python-gmail-imaplib-search-subject-get-attachments/
@staticmethod
@@ -226,10 +259,19 @@ class POP3Email(BaseModel):
document_file = Attachment(part, name=filename)
source.upload_file(document_file, expand=(source.uncompress == SOURCE_UNCOMPRESS_CHOICE_Y), document_type=source.document_type)
class Meta(IntervalBaseModel.Meta):
verbose_name = _(u'email source')
verbose_name_plural = _(u'email sources')
abstract = True
class POP3Email(EmailBaseModel):
source_type = SOURCE_CHOICE_POP3_EMAIL
def fetch_mail(self):
try:
last_check = self.pop3emaillog_set.latest().creation_datetime
except POP3EmailLog.DoesNotExist:
last_check = SourceLog.objects.get_latest_for(self)
except SourceLog.DoesNotExist:
# Trigger email fetch when there are no previous logs
initial_trigger = True
difference = datetime.timedelta(seconds=0)
@@ -267,38 +309,78 @@ class POP3Email(BaseModel):
complete_message = '\n'.join(mailbox.retr(message_number)[1])
POP3Email.process_message(source=self, message=complete_message)
EmailBaseModel.process_message(source=self, message=complete_message)
mailbox.dele(message_number)
mailbox.quit()
POP3EmailLog.objects.save_status(pop3_email=self, status='Successful connection.')
SourceLog.objects.save_status(source=self, status='Successful connection.')
except Exception, exc:
logger.error('Unhandled exception: %s' % exc)
POP3EmailLog.objects.save_status(pop3_email=self, status='Error: %s' % exc)
SourceLog.objects.save_status(source=self, status='Error: %s' % exc)
class Meta(BaseModel.Meta):
class Meta(EmailBaseModel.Meta):
verbose_name = _(u'POP email')
verbose_name_plural = _(u'POP email')
class POP3EmailLog(models.Model):
pop3_email = models.ForeignKey(POP3Email, verbose_name=_(u'POP3 email'))
creation_datetime = models.DateTimeField(verbose_name=_(u'date time'))
status = models.TextField(verbose_name=_(u'status'))
class IMAPEmail(EmailBaseModel):
source_type = SOURCE_CHOICE_IMAP_EMAIL
objects = POP3EmailLogManager()
mailbox = models.CharField(max_length=64, blank=True, verbose_name=_(u'mailbox'), help_text=_(u'Mail from which to check for messages with attached documents. If none is specified, the default mailbox is %s') % IMAP_DEFAULT_MAILBOX)
def save(self, *args, **kwargs):
if not self.pk:
self.creation_datetime = datetime.datetime.now()
return super(POP3EmailLog, self).save(*args, **kwargs)
# http://www.doughellmann.com/PyMOTW/imaplib/
def fetch_mail(self):
try:
last_check = SourceLog.objects.get_latest_for(self)
except SourceLog.DoesNotExist:
# Trigger email fetch when there are no previous logs
initial_trigger = True
difference = datetime.timedelta(seconds=0)
else:
difference = datetime.datetime.now() - last_check
initial_trigger = False
class Meta:
verbose_name = _(u'POP3 email log')
verbose_name_plural = _(u'POP3 emails logs')
get_latest_by = 'creation_datetime'
ordering = ('creation_datetime',)
if difference >= datetime.timedelta(seconds=self.interval) or initial_trigger:
try:
logger.debug('Starting IMAP email fetch')
logger.debug('host: %s' % self.host)
logger.debug('ssl: %s' % self.ssl)
if self.ssl:
port = self.port or IMAP_SSL_PORT
logger.debug('port: %d' % port)
mailbox = imaplib.IMAP4_SSL(self.host, int(port))
else:
port = self.port or IMAP_PORT
logger.debug('port: %d' % port)
mailbox = imaplib.IMAP4(self.host, int(port))
mailbox.login(self.username, self.password)
mailbox.select(self.mailbox or IMAP_DEFAULT_MAILBOX)
status, data = mailbox.search(None, 'NOT', 'DELETED')
if data:
messages_info = data[0].split()
logger.debug('messages count: %s' % len(messages_info))
for message_number in messages_info:
logger.debug('message_number: %s' % message_number)
status, data = mailbox.fetch(message_number, '(RFC822)')
EmailBaseModel.process_message(source=self, message=data[0][1])
mailbox.store(message_number, '+FLAGS', '\\Deleted')
mailbox.expunge()
mailbox.close()
mailbox.logout()
SourceLog.objects.save_status(source=self, status='Successful connection.')
except Exception, exc:
logger.error('Unhandled exception: %s' % exc)
SourceLog.objects.save_status(source=self, status='Error: %s' % exc)
class Meta(EmailBaseModel.Meta):
verbose_name = _(u'IMAP email')
verbose_name_plural = _(u'IMAP email')
class StagingFolder(InteractiveBaseModel):

29
apps/sources/tasks.py
View File

@@ -9,8 +9,9 @@ from django.db.models import Q
from job_processor.api import process_job
from lock_manager import Lock, LockError
from .models import POP3Email
from .models import POP3Email, IMAPEmail
from .conf.settings import POP3_TIMEOUT
from .literals import IMAP_LOCK_TIMEOUT
logger = logging.getLogger(__name__)
@@ -39,3 +40,29 @@ def task_fetch_pop3_emails():
task_fetch_single_pop3_email(pop3_email)
except Exception, exc:
logger.error('Unhandled exception: %s' % exc)
def task_fetch_single_imap_email(imap_email):
try:
lock_id = u'task_fetch_iamp_email-%d' % imap_email.pk
logger.debug('trying to acquire lock: %s' % lock_id)
lock = Lock.acquire_lock(lock_id, IMAP_LOCK_TIMEOUT)
logger.debug('acquired lock: %s' % lock_id)
try:
imap_email.fetch_mail()
except Exception, exc:
raise
finally:
lock.release()
except LockError:
logger.error('unable to obtain lock')
pass
def task_fetch_imap_emails():
logger.debug('executing')
for imap_email in IMAPEmail.objects.filter(enabled=True):
try:
task_fetch_single_imap_email(imap_email)
except Exception, exc:
logger.error('Unhandled exception: %s' % exc)

3
apps/sources/urls.py
View File

@@ -3,7 +3,7 @@ from __future__ import absolute_import
from django.conf.urls.defaults import patterns, url
from .literals import (SOURCE_CHOICE_WEB_FORM, SOURCE_CHOICE_STAGING,
SOURCE_CHOICE_WATCH, SOURCE_CHOICE_POP3_EMAIL)
SOURCE_CHOICE_WATCH, SOURCE_CHOICE_POP3_EMAIL, SOURCE_CHOICE_IMAP_EMAIL)
urlpatterns = patterns('sources.views',
url(r'^staging_file/type/(?P<source_type>\w+)/(?P<source_id>\d+)/(?P<staging_file_id>\w+)/preview/$', 'staging_file_preview', (), 'staging_file_preview'),
@@ -22,6 +22,7 @@ urlpatterns = patterns('sources.views',
url(r'^setup/interactive/%s/list/$' % SOURCE_CHOICE_STAGING, 'setup_source_list', {'source_type': SOURCE_CHOICE_STAGING}, 'setup_staging_folder_list'),
url(r'^setup/interactive/%s/list/$' % SOURCE_CHOICE_WATCH, 'setup_source_list', {'source_type': SOURCE_CHOICE_WATCH}, 'setup_watch_folder_list'),
url(r'^setup/interactive/%s/list/$' % SOURCE_CHOICE_POP3_EMAIL, 'setup_source_list', {'source_type': SOURCE_CHOICE_POP3_EMAIL}, 'setup_pop3_email_list'),
url(r'^setup/interactive/%s/list/$' % SOURCE_CHOICE_IMAP_EMAIL, 'setup_source_list', {'source_type': SOURCE_CHOICE_IMAP_EMAIL}, 'setup_imap_email_list'),
url(r'^setup/interactive/(?P<source_type>\w+)/list/$', 'setup_source_list', (), 'setup_source_list'),
url(r'^setup/interactive/(?P<source_type>\w+)/(?P<source_id>\d+)/edit/$', 'setup_source_edit', (), 'setup_source_edit'),

28
apps/sources/views.py
View File

@@ -25,15 +25,16 @@ import sendfile
from acls.models import AccessEntry
from .models import (WebForm, StagingFolder, SourceTransformation,
WatchFolder, POP3Email, POP3EmailLog)
WatchFolder, POP3Email, SourceLog, IMAPEmail)
from .literals import (SOURCE_CHOICE_WEB_FORM, SOURCE_CHOICE_STAGING,
SOURCE_CHOICE_WATCH, SOURCE_CHOICE_POP3_EMAIL)
SOURCE_CHOICE_WATCH, SOURCE_CHOICE_POP3_EMAIL, SOURCE_CHOICE_IMAP_EMAIL)
from .literals import (SOURCE_UNCOMPRESS_CHOICE_Y,
SOURCE_UNCOMPRESS_CHOICE_ASK)
from .staging import create_staging_file_class
from .forms import (StagingDocumentForm, WebFormForm,
WatchFolderSetupForm)
from .forms import WebFormSetupForm, StagingFolderSetupForm, POP3EmailSetupForm
from .forms import (WebFormSetupForm, StagingFolderSetupForm,
POP3EmailSetupForm, IMAPEmailSetupForm)
from .forms import SourceTransformationForm, SourceTransformationForm_create
from .permissions import (PERMISSION_SOURCES_SETUP_VIEW,
PERMISSION_SOURCES_SETUP_EDIT, PERMISSION_SOURCES_SETUP_DELETE,
@@ -441,6 +442,8 @@ def setup_source_list(request, source_type):
cls = WatchFolder
elif source_type == SOURCE_CHOICE_POP3_EMAIL:
cls = POP3Email
elif source_type == SOURCE_CHOICE_IMAP_EMAIL:
cls = IMAPEmail
context = {
'object_list': cls.objects.all(),
@@ -472,6 +475,9 @@ def setup_source_edit(request, source_type, source_id):
elif source_type == SOURCE_CHOICE_POP3_EMAIL:
cls = POP3Email
form_class = POP3EmailSetupForm
elif source_type == SOURCE_CHOICE_IMAP_EMAIL:
cls = IMAPEmail
form_class = IMAPEmailSetupForm
source = get_object_or_404(cls, pk=source_id)
next = request.POST.get('next', request.GET.get('next', request.META.get('HTTP_REFERER', '/')))
@@ -518,6 +524,10 @@ def setup_source_delete(request, source_type, source_id):
cls = POP3Email
form_icon = u'email_delete.png'
redirect_view = 'setup_pop3_email_list'
elif source_type == SOURCE_CHOICE_IMAP_EMAIL:
cls = IMAPEmail
form_icon = u'email_delete.png'
redirect_view = 'setup_imap_email_list'
redirect_view = reverse('setup_source_list', args=[source_type])
previous = request.POST.get('previous', request.GET.get('previous', request.META.get('HTTP_REFERER', redirect_view)))
@@ -572,6 +582,10 @@ def setup_source_create(request, source_type):
cls = POP3Email
form_class = POP3EmailSetupForm
redirect_view = 'setup_pop3_email_list'
elif source_type == SOURCE_CHOICE_IMAP_EMAIL:
cls = IMAPEmail
form_class = IMAPEmailSetupForm
redirect_view = 'setup_imap_email_list'
if request.method == 'POST':
form = form_class(data=request.POST)
@@ -605,11 +619,13 @@ def setup_source_log_list(request, source_type, source_pk):
cls = WatchFolder
elif source_type == SOURCE_CHOICE_POP3_EMAIL:
cls = POP3Email
elif source_type == SOURCE_CHOICE_IMAP_EMAIL:
cls = IMAPEmail
source = get_object_or_404(cls, pk=source_pk)
context = {
'object_list': POP3EmailLog.objects.filter(pop3_email=source).order_by('-creation_datetime'),
'object_list': SourceLog.objects.get_for_source(source),
'title': _(u'logs for: %s') % source.fullname(),
'source': source,
'object_name': _(u'source'),
@@ -639,6 +655,8 @@ def setup_source_transformation_list(request, source_type, source_id):
cls = WatchFolder
elif source_type == SOURCE_CHOICE_POP3_EMAIL:
cls = POP3Email
elif source_type == SOURCE_CHOICE_IMAP_EMAIL:
cls = IMAPEmail
source = get_object_or_404(cls, pk=source_id)
@@ -741,6 +759,8 @@ def setup_source_transformation_create(request, source_type, source_id):
cls = WatchFolder
elif source_type == SOURCE_CHOICE_POP3_EMAIL:
cls = POP3Email
elif source_type == SOURCE_CHOICE_IMAP_EMAIL:
cls = IMAPEmail
source = get_object_or_404(cls, pk=source_id)