diff --git a/apps/sources/__init__.py b/apps/sources/__init__.py index f6c4933d4f..aad255b28d 100644 --- a/apps/sources/__init__.py +++ b/apps/sources/__init__.py @@ -12,22 +12,23 @@ from scheduler.api import register_interval_job from .staging import StagingFile from .models import (WebForm, StagingFolder, SourceTransformation, - WatchFolder, POP3Email) + WatchFolder, POP3Email, IMAPEmail) from .widgets import staging_file_thumbnail from .permissions import (PERMISSION_SOURCES_SETUP_VIEW, PERMISSION_SOURCES_SETUP_EDIT, PERMISSION_SOURCES_SETUP_DELETE, PERMISSION_SOURCES_SETUP_CREATE) -from .tasks import task_fetch_pop3_emails -from .conf.settings import POP3_EMAIL_PROCESSING_INTERVAL +from .tasks import task_fetch_pop3_emails, task_fetch_imap_emails +from .conf.settings import EMAIL_PROCESSING_INTERVAL staging_file_preview = {'text': _(u'preview'), 'class': 'fancybox-noscaling', 'view': 'staging_file_preview', 'args': ['source.source_type', 'source.pk', 'object.id'], 'famfam': 'zoom', 'permissions': [PERMISSION_DOCUMENT_NEW_VERSION, PERMISSION_DOCUMENT_CREATE]} staging_file_delete = {'text': _(u'delete'), 'view': 'staging_file_delete', 'args': ['source.source_type', 'source.pk', 'object.id'], 'famfam': 'delete', 'keep_query': True, 'permissions': [PERMISSION_DOCUMENT_NEW_VERSION, PERMISSION_DOCUMENT_CREATE]} -setup_sources = {'text': _(u'sources'), 'view': 'setup_web_form_list', 'famfam': 'application_form', 'icon': 'application_form.png', 'children_classes': [WebForm], 'permissions': [PERMISSION_SOURCES_SETUP_VIEW], 'children_view_regex': [r'setup_web_form', r'setup_staging_folder', r'setup_source_', r'setup_pop3']} +setup_sources = {'text': _(u'sources'), 'view': 'setup_web_form_list', 'famfam': 'application_form', 'icon': 'application_form.png', 'children_classes': [WebForm], 'permissions': [PERMISSION_SOURCES_SETUP_VIEW], 'children_view_regex': [r'setup_web_form', r'setup_staging_folder', r'setup_source_', r'setup_pop3', r'setup_imap']} setup_web_form_list = {'text': _(u'web forms'), 'view': 'setup_web_form_list', 'famfam': 'application_form', 'icon': 'application_form.png', 'children_classes': [WebForm], 'permissions': [PERMISSION_SOURCES_SETUP_VIEW]} setup_staging_folder_list = {'text': _(u'staging folders'), 'view': 'setup_staging_folder_list', 'famfam': 'folder_camera', 'children_classes': [StagingFolder], 'permissions': [PERMISSION_SOURCES_SETUP_VIEW]} setup_watch_folder_list = {'text': _(u'watch folders'), 'view': 'setup_watch_folder_list', 'famfam': 'folder_magnify', 'children_classes': [WatchFolder], 'permissions': [PERMISSION_SOURCES_SETUP_VIEW]} setup_pop3_email_list = {'text': _(u'POP3 email'), 'view': 'setup_pop3_email_list', 'famfam': 'email', 'children_classes': [POP3Email], 'permissions': [PERMISSION_SOURCES_SETUP_VIEW]} +setup_imap_email_list = {'text': _(u'IMAP email'), 'view': 'setup_imap_email_list', 'famfam': 'email', 'children_classes': [IMAPEmail], 'permissions': [PERMISSION_SOURCES_SETUP_VIEW]} setup_source_edit = {'text': _(u'edit'), 'view': 'setup_source_edit', 'args': ['source.source_type', 'source.pk'], 'famfam': 'application_form_edit', 'permissions': [PERMISSION_SOURCES_SETUP_EDIT]} setup_source_delete = {'text': _(u'delete'), 'view': 'setup_source_delete', 'args': ['source.source_type', 'source.pk'], 'famfam': 'application_form_delete', 'permissions': [PERMISSION_SOURCES_SETUP_DELETE]} @@ -47,25 +48,24 @@ register_links(StagingFile, [staging_file_delete]) register_links(SourceTransformation, [setup_source_transformation_edit, setup_source_transformation_delete]) -#register_links(['setup_web_form_list', 'setup_staging_folder_list', 'setup_watch_folder_list', 'setup_source_create'], [setup_web_form_list, setup_staging_folder_list, setup_watch_folder_list], menu_name='form_header') -register_links(['setup_pop3_email_list', 'setup_web_form_list', 'setup_staging_folder_list', 'setup_watch_folder_list', 'setup_source_create'], [setup_web_form_list, setup_staging_folder_list, setup_pop3_email_list], menu_name='form_header') +register_links(['setup_imap_email_list', 'setup_pop3_email_list', 'setup_web_form_list', 'setup_staging_folder_list', 'setup_watch_folder_list', 'setup_source_create'], [setup_web_form_list, setup_staging_folder_list, setup_pop3_email_list, setup_imap_email_list], menu_name='form_header') +register_links([WebForm, StagingFolder, POP3Email, IMAPEmail, 'setup_web_form_list', 'setup_staging_folder_list', 'setup_watch_folder_list', 'setup_source_create', 'setup_pop3_email_list', 'setup_imap_email_list'], [setup_source_create], menu_name='secondary_menu') -#register_links(WebForm, [setup_web_form_list, setup_staging_folder_list, setup_watch_folder_list], menu_name='form_header') -register_links(WebForm, [setup_web_form_list, setup_staging_folder_list, setup_pop3_email_list], menu_name='form_header') +register_links(WebForm, [setup_web_form_list, setup_staging_folder_list, setup_pop3_email_list, setup_imap_email_list], menu_name='form_header') register_links(WebForm, [setup_source_transformation_list, setup_source_edit, setup_source_delete]) -#register_links(['setup_source_log_list', 'setup_web_form_list', 'setup_staging_folder_list', 'setup_watch_folder_list', 'setup_source_edit', 'setup_source_delete', 'setup_source_create', 'setup_pop3_email_list'], [setup_sources, setup_source_create], menu_name='secondary_menu') -register_links([WebForm, StagingFolder, POP3Email, 'setup_web_form_list', 'setup_staging_folder_list', 'setup_watch_folder_list', 'setup_source_create', 'setup_pop3_email_list'], [setup_source_create], menu_name='secondary_menu') - -#register_links(StagingFolder, [setup_web_form_list, setup_staging_folder_list, setup_watch_folder_list], menu_name='form_header') -register_links(StagingFolder, [setup_web_form_list, setup_staging_folder_list, setup_pop3_email_list], menu_name='form_header') +register_links(StagingFolder, [setup_web_form_list, setup_staging_folder_list, setup_pop3_email_list, setup_imap_email_list], menu_name='form_header') register_links(StagingFolder, [setup_source_transformation_list, setup_source_edit, setup_source_delete]) -register_links(POP3Email, [setup_web_form_list, setup_staging_folder_list, setup_pop3_email_list], menu_name='form_header') +register_links(POP3Email, [setup_web_form_list, setup_staging_folder_list, setup_pop3_email_list, setup_imap_email_list], menu_name='form_header') register_links(POP3Email, [setup_source_transformation_list, setup_source_edit, setup_source_delete]) register_links(POP3Email, [setup_source_log_list]) -register_links(WatchFolder, [setup_web_form_list, setup_staging_folder_list, setup_watch_folder_list], menu_name='form_header') +register_links(IMAPEmail, [setup_web_form_list, setup_staging_folder_list, setup_pop3_email_list, setup_imap_email_list], menu_name='form_header') +register_links(IMAPEmail, [setup_source_transformation_list, setup_source_edit, setup_source_delete]) +register_links(IMAPEmail, [setup_source_log_list]) + +register_links(WatchFolder, [setup_web_form_list, setup_staging_folder_list, setup_watch_folder_list, setup_imap_email_list], menu_name='form_header') register_links(WatchFolder, [setup_source_transformation_list, setup_source_edit, setup_source_delete]) # Document version @@ -83,4 +83,5 @@ register_model_list_columns(StagingFile, [ register_setup(setup_sources) -register_interval_job('task_fetch_pop3_emails', _(u'Connects to the POP3 email sources and fetches the attached documents.'), task_fetch_pop3_emails, seconds=POP3_EMAIL_PROCESSING_INTERVAL) +register_interval_job('task_fetch_pop3_emails', _(u'Connects to the POP3 email sources and fetches the attached documents.'), task_fetch_pop3_emails, seconds=EMAIL_PROCESSING_INTERVAL) +register_interval_job('task_fetch_imap_emails', _(u'Connects to the IMAP email sources and fetches the attached documents.'), task_fetch_imap_emails, seconds=EMAIL_PROCESSING_INTERVAL) diff --git a/apps/sources/conf/settings.py b/apps/sources/conf/settings.py index b5e3d7a02c..25e53e75d5 100644 --- a/apps/sources/conf/settings.py +++ b/apps/sources/conf/settings.py @@ -9,7 +9,7 @@ from smart_settings.api import Setting, SettingNamespace namespace = SettingNamespace('sources', _(u'Sources'), module='sources.conf.settings') POP3_DEFAULT_TIMEOUT = 10 # for POP3 only not POP3_SSL -POP3_DEFAULT_EMAIL_PROCESSING_INTERVAL = 60 +DEFAULT_EMAIL_PROCESSING_INTERVAL = 60 DEFAULT_POP3_EMAIL_LOG_COUNT = 10 # Max log entries to store Setting( @@ -21,14 +21,14 @@ Setting( Setting( namespace=namespace, - name='POP3_EMAIL_PROCESSING_INTERVAL', - global_name='SOURCES_POP3_EMAIL_PROCESSING_INTERVAL', - default=POP3_DEFAULT_EMAIL_PROCESSING_INTERVAL, + name='EMAIL_PROCESSING_INTERVAL', + global_name='SOURCES_EMAIL_PROCESSING_INTERVAL', + default=DEFAULT_EMAIL_PROCESSING_INTERVAL, ) Setting( namespace=namespace, - name='POP3_EMAIL_LOG_SIZE', - global_name='SOURCES_POP3_EMAIL_LOG_SIZE', + name='LOG_SIZE', + global_name='SOURCES_LOG_SIZE', default=DEFAULT_POP3_EMAIL_LOG_COUNT, ) diff --git a/apps/sources/forms.py b/apps/sources/forms.py index 1ec31b9c9e..84b6f79281 100644 --- a/apps/sources/forms.py +++ b/apps/sources/forms.py @@ -7,7 +7,7 @@ from django.utils.translation import ugettext from documents.forms import DocumentForm from .models import (WebForm, StagingFolder, SourceTransformation, - WatchFolder, POP3Email) + WatchFolder, POP3Email, IMAPEmail) from .widgets import FamFamRadioSelect from .utils import validate_whitelist_blacklist @@ -116,9 +116,18 @@ class SourceTransformationForm_create(forms.ModelForm): exclude = ('content_type', 'object_id') -class POP3EmailSetupForm(forms.ModelForm): +class EmailSetupForm(forms.ModelForm): class Meta: - model = POP3Email widgets = { 'password': forms.widgets.PasswordInput, } + + +class POP3EmailSetupForm(EmailSetupForm): + class Meta(EmailSetupForm.Meta): + model = POP3Email + + +class IMAPEmailSetupForm(EmailSetupForm): + class Meta(EmailSetupForm.Meta): + model = IMAPEmail diff --git a/apps/sources/literals.py b/apps/sources/literals.py index 839e934bc6..8faeb2f4aa 100644 --- a/apps/sources/literals.py +++ b/apps/sources/literals.py @@ -4,6 +4,12 @@ POP3_PORT = 110 POP3_SSL_PORT = 995 DEFAULT_POP3_INTERVAL = 15 * 60 # 15 minutes in seconds +IMAP_PORT = 143 +IMAP_SSL_PORT = 993 +DEFAULT_IMAP_INTERVAL = 15 * 60 # 15 minutes in seconds +IMAP_LOCK_TIMEOUT = 60 +IMAP_DEFAULT_MAILBOX = 'INBOX' + SOURCE_UNCOMPRESS_CHOICE_Y = 'y' SOURCE_UNCOMPRESS_CHOICE_N = 'n' SOURCE_UNCOMPRESS_CHOICE_ASK = 'a' @@ -47,17 +53,20 @@ SOURCE_CHOICE_WEB_FORM = 'webform' SOURCE_CHOICE_STAGING = 'staging' SOURCE_CHOICE_WATCH = 'watch' SOURCE_CHOICE_POP3_EMAIL = 'pop3' +SOURCE_CHOICE_IMAP_EMAIL = 'imap' SOURCE_CHOICES = ( (SOURCE_CHOICE_WEB_FORM, _(u'web form')), (SOURCE_CHOICE_STAGING, _(u'server staging folder')), (SOURCE_CHOICE_WATCH, _(u'server watch folder')), - (SOURCE_CHOICE_POP3_EMAIL, _(u'pop3 email')), + (SOURCE_CHOICE_POP3_EMAIL, _(u'POP3 email')), + (SOURCE_CHOICE_IMAP_EMAIL, _(u'IMAP email')), ) SOURCE_CHOICES_PLURAL = ( (SOURCE_CHOICE_WEB_FORM, _(u'web forms')), (SOURCE_CHOICE_STAGING, _(u'server staging folders')), (SOURCE_CHOICE_WATCH, _(u'server watch folders')), - (SOURCE_CHOICE_POP3_EMAIL, _(u'pop3 emails')), + (SOURCE_CHOICE_POP3_EMAIL, _(u'POP3 emails')), + (SOURCE_CHOICE_IMAP_EMAIL, _(u'IMAP emails')), ) diff --git a/apps/sources/managers.py b/apps/sources/managers.py index b390befa67..750c2dc932 100644 --- a/apps/sources/managers.py +++ b/apps/sources/managers.py @@ -5,7 +5,7 @@ from ast import literal_eval from django.db import models from django.contrib.contenttypes.models import ContentType -from .conf.settings import POP3_EMAIL_LOG_SIZE +from .conf.settings import LOG_SIZE class SourceTransformationManager(models.Manager): @@ -30,10 +30,19 @@ class SourceTransformationManager(models.Manager): return transformations, warnings -class POP3EmailLogManager(models.Manager): - def save_status(self, pop3_email, status): - new_recent = self.model(pop3_email=pop3_email, status=status) +class SourceLogManager(models.Manager): + def save_status(self, source, status): + new_recent = self.model(source=source, status=status) new_recent.save() - to_delete = self.model.objects.filter(pop3_email=pop3_email).order_by('-creation_datetime')[POP3_EMAIL_LOG_SIZE:] + content_type = ContentType.objects.get_for_model(source) + to_delete = self.model.objects.filter(content_type=content_type, object_id=source.pk).order_by('-creation_datetime')[LOG_SIZE:] for recent_to_delete in to_delete: recent_to_delete.delete() + + def get_for_source(self, source): + content_type = ContentType.objects.get_for_model(source) + return self.model.objects.filter(content_type=content_type, object_id=source.pk).order_by('-creation_datetime') + + def get_latest_for(self, source): + content_type = ContentType.objects.get_for_model(source) + return self.model.objects.filter(content_type=content_type, object_id=source.pk).latest().creation_datetime diff --git a/apps/sources/migrations/0009_auto__del_pop3email__del_pop3emaillog.py b/apps/sources/migrations/0009_auto__del_pop3email__del_pop3emaillog.py new file mode 100644 index 0000000000..fafe1f0a84 --- /dev/null +++ b/apps/sources/migrations/0009_auto__del_pop3email__del_pop3emaillog.py @@ -0,0 +1,111 @@ +# encoding: utf-8 +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + +class Migration(SchemaMigration): + + def forwards(self, orm): + + # Deleting model 'POP3Email' + db.delete_table('sources_pop3email') + + # Deleting model 'POP3EmailLog' + db.delete_table('sources_pop3emaillog') + + + def backwards(self, orm): + + # Adding model 'POP3Email' + db.create_table('sources_pop3email', ( + ('username', self.gf('django.db.models.fields.CharField')(max_length=64)), + ('uncompress', self.gf('django.db.models.fields.CharField')(max_length=1)), + ('whitelist', self.gf('django.db.models.fields.TextField')(blank=True)), + ('blacklist', self.gf('django.db.models.fields.TextField')(blank=True)), + ('ssl', self.gf('django.db.models.fields.BooleanField')(default=False)), + ('host', self.gf('django.db.models.fields.CharField')(max_length=64)), + ('password', self.gf('django.db.models.fields.CharField')(max_length=64)), + ('port', self.gf('django.db.models.fields.PositiveIntegerField')(null=True, blank=True)), + ('document_type', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['documents.DocumentType'], null=True, blank=True)), + ('title', self.gf('django.db.models.fields.CharField')(max_length=64)), + ('interval', self.gf('django.db.models.fields.PositiveIntegerField')(default=900)), + ('enabled', self.gf('django.db.models.fields.BooleanField')(default=True)), + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + )) + db.send_create_signal('sources', ['POP3Email']) + + # Adding model 'POP3EmailLog' + db.create_table('sources_pop3emaillog', ( + ('status', self.gf('django.db.models.fields.TextField')()), + ('creation_datetime', self.gf('django.db.models.fields.DateTimeField')()), + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('pop3_email', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['sources.POP3Email'])), + )) + db.send_create_signal('sources', ['POP3EmailLog']) + + + models = { + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'sources.outofprocess': { + 'Meta': {'ordering': "('title',)", 'object_name': 'OutOfProcess'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + 'sources.sourcetransformation': { + 'Meta': {'ordering': "('order',)", 'object_name': 'SourceTransformation'}, + 'arguments': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'object_id': ('django.db.models.fields.PositiveIntegerField', [], {}), + 'order': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0', 'null': 'True', 'db_index': 'True', 'blank': 'True'}), + 'transformation': ('django.db.models.fields.CharField', [], {'max_length': '128'}) + }, + 'sources.stagingfolder': { + 'Meta': {'ordering': "('title',)", 'object_name': 'StagingFolder'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'delete_after_upload': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'folder_path': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'icon': ('django.db.models.fields.CharField', [], {'max_length': '24', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'preview_height': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}), + 'preview_width': ('django.db.models.fields.IntegerField', [], {}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + 'sources.watchfolder': { + 'Meta': {'ordering': "('title',)", 'object_name': 'WatchFolder'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'delete_after_upload': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'folder_path': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'interval': ('django.db.models.fields.PositiveIntegerField', [], {}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + 'sources.webform': { + 'Meta': {'ordering': "('title',)", 'object_name': 'WebForm'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'icon': ('django.db.models.fields.CharField', [], {'max_length': '24', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + } + } + + complete_apps = ['sources'] diff --git a/apps/sources/migrations/0010_auto__add_imapemail__add_sourcelog__add_pop3email.py b/apps/sources/migrations/0010_auto__add_imapemail__add_sourcelog__add_pop3email.py new file mode 100644 index 0000000000..c34e9ea7a6 --- /dev/null +++ b/apps/sources/migrations/0010_auto__add_imapemail__add_sourcelog__add_pop3email.py @@ -0,0 +1,180 @@ +# encoding: utf-8 +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + +class Migration(SchemaMigration): + + def forwards(self, orm): + + # Adding model 'ImapEmail' + db.create_table('sources_imapemail', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('title', self.gf('django.db.models.fields.CharField')(max_length=64)), + ('enabled', self.gf('django.db.models.fields.BooleanField')(default=True)), + ('whitelist', self.gf('django.db.models.fields.TextField')(blank=True)), + ('blacklist', self.gf('django.db.models.fields.TextField')(blank=True)), + ('interval', self.gf('django.db.models.fields.PositiveIntegerField')(default=900)), + ('document_type', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['documents.DocumentType'], null=True, blank=True)), + ('uncompress', self.gf('django.db.models.fields.CharField')(max_length=1)), + ('host', self.gf('django.db.models.fields.CharField')(max_length=64)), + ('ssl', self.gf('django.db.models.fields.BooleanField')(default=False)), + ('port', self.gf('django.db.models.fields.PositiveIntegerField')(null=True, blank=True)), + ('username', self.gf('django.db.models.fields.CharField')(max_length=64)), + ('password', self.gf('django.db.models.fields.CharField')(max_length=64)), + ('mailbox', self.gf('django.db.models.fields.CharField')(max_length=64, blank=True)), + )) + db.send_create_signal('sources', ['ImapEmail']) + + # Adding model 'SourceLog' + db.create_table('sources_sourcelog', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('content_type', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['contenttypes.ContentType'])), + ('object_id', self.gf('django.db.models.fields.PositiveIntegerField')()), + ('creation_datetime', self.gf('django.db.models.fields.DateTimeField')()), + ('status', self.gf('django.db.models.fields.TextField')()), + )) + db.send_create_signal('sources', ['SourceLog']) + + # Adding model 'POP3Email' + db.create_table('sources_pop3email', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('title', self.gf('django.db.models.fields.CharField')(max_length=64)), + ('enabled', self.gf('django.db.models.fields.BooleanField')(default=True)), + ('whitelist', self.gf('django.db.models.fields.TextField')(blank=True)), + ('blacklist', self.gf('django.db.models.fields.TextField')(blank=True)), + ('interval', self.gf('django.db.models.fields.PositiveIntegerField')(default=900)), + ('document_type', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['documents.DocumentType'], null=True, blank=True)), + ('uncompress', self.gf('django.db.models.fields.CharField')(max_length=1)), + ('host', self.gf('django.db.models.fields.CharField')(max_length=64)), + ('ssl', self.gf('django.db.models.fields.BooleanField')(default=False)), + ('port', self.gf('django.db.models.fields.PositiveIntegerField')(null=True, blank=True)), + ('username', self.gf('django.db.models.fields.CharField')(max_length=64)), + ('password', self.gf('django.db.models.fields.CharField')(max_length=64)), + )) + db.send_create_signal('sources', ['POP3Email']) + + + def backwards(self, orm): + + # Deleting model 'ImapEmail' + db.delete_table('sources_imapemail') + + # Deleting model 'SourceLog' + db.delete_table('sources_sourcelog') + + # Deleting model 'POP3Email' + db.delete_table('sources_pop3email') + + + models = { + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'documents.documenttype': { + 'Meta': {'ordering': "['name']", 'object_name': 'DocumentType'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '32'}) + }, + 'sources.imapemail': { + 'Meta': {'ordering': "('title',)", 'object_name': 'ImapEmail'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'document_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['documents.DocumentType']", 'null': 'True', 'blank': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'host': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'interval': ('django.db.models.fields.PositiveIntegerField', [], {'default': '900'}), + 'mailbox': ('django.db.models.fields.CharField', [], {'max_length': '64', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'port': ('django.db.models.fields.PositiveIntegerField', [], {'null': 'True', 'blank': 'True'}), + 'ssl': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'username': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + 'sources.outofprocess': { + 'Meta': {'ordering': "('title',)", 'object_name': 'OutOfProcess'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + 'sources.pop3email': { + 'Meta': {'ordering': "('title',)", 'object_name': 'POP3Email'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'document_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['documents.DocumentType']", 'null': 'True', 'blank': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'host': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'interval': ('django.db.models.fields.PositiveIntegerField', [], {'default': '900'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'port': ('django.db.models.fields.PositiveIntegerField', [], {'null': 'True', 'blank': 'True'}), + 'ssl': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'username': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + 'sources.sourcelog': { + 'Meta': {'ordering': "('creation_datetime',)", 'object_name': 'SourceLog'}, + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'creation_datetime': ('django.db.models.fields.DateTimeField', [], {}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'object_id': ('django.db.models.fields.PositiveIntegerField', [], {}), + 'status': ('django.db.models.fields.TextField', [], {}) + }, + 'sources.sourcetransformation': { + 'Meta': {'ordering': "('order',)", 'object_name': 'SourceTransformation'}, + 'arguments': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'object_id': ('django.db.models.fields.PositiveIntegerField', [], {}), + 'order': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0', 'null': 'True', 'db_index': 'True', 'blank': 'True'}), + 'transformation': ('django.db.models.fields.CharField', [], {'max_length': '128'}) + }, + 'sources.stagingfolder': { + 'Meta': {'ordering': "('title',)", 'object_name': 'StagingFolder'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'delete_after_upload': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'folder_path': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'icon': ('django.db.models.fields.CharField', [], {'max_length': '24', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'preview_height': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}), + 'preview_width': ('django.db.models.fields.IntegerField', [], {}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + 'sources.watchfolder': { + 'Meta': {'ordering': "('title',)", 'object_name': 'WatchFolder'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'delete_after_upload': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'folder_path': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'interval': ('django.db.models.fields.PositiveIntegerField', [], {}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + 'sources.webform': { + 'Meta': {'ordering': "('title',)", 'object_name': 'WebForm'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'icon': ('django.db.models.fields.CharField', [], {'max_length': '24', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + } + } + + complete_apps = ['sources'] diff --git a/apps/sources/migrations/0011_auto__del_imapemail.py b/apps/sources/migrations/0011_auto__del_imapemail.py new file mode 100644 index 0000000000..d879d93cf0 --- /dev/null +++ b/apps/sources/migrations/0011_auto__del_imapemail.py @@ -0,0 +1,129 @@ +# encoding: utf-8 +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + +class Migration(SchemaMigration): + + def forwards(self, orm): + + # Deleting model 'ImapEmail' + db.delete_table('sources_imapemail') + + + def backwards(self, orm): + + # Adding model 'ImapEmail' + db.create_table('sources_imapemail', ( + ('username', self.gf('django.db.models.fields.CharField')(max_length=64)), + ('uncompress', self.gf('django.db.models.fields.CharField')(max_length=1)), + ('whitelist', self.gf('django.db.models.fields.TextField')(blank=True)), + ('mailbox', self.gf('django.db.models.fields.CharField')(max_length=64, blank=True)), + ('blacklist', self.gf('django.db.models.fields.TextField')(blank=True)), + ('ssl', self.gf('django.db.models.fields.BooleanField')(default=False)), + ('host', self.gf('django.db.models.fields.CharField')(max_length=64)), + ('password', self.gf('django.db.models.fields.CharField')(max_length=64)), + ('port', self.gf('django.db.models.fields.PositiveIntegerField')(null=True, blank=True)), + ('document_type', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['documents.DocumentType'], null=True, blank=True)), + ('title', self.gf('django.db.models.fields.CharField')(max_length=64)), + ('interval', self.gf('django.db.models.fields.PositiveIntegerField')(default=900)), + ('enabled', self.gf('django.db.models.fields.BooleanField')(default=True)), + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + )) + db.send_create_signal('sources', ['ImapEmail']) + + + models = { + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'documents.documenttype': { + 'Meta': {'ordering': "['name']", 'object_name': 'DocumentType'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '32'}) + }, + 'sources.outofprocess': { + 'Meta': {'ordering': "('title',)", 'object_name': 'OutOfProcess'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + 'sources.pop3email': { + 'Meta': {'ordering': "('title',)", 'object_name': 'POP3Email'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'document_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['documents.DocumentType']", 'null': 'True', 'blank': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'host': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'interval': ('django.db.models.fields.PositiveIntegerField', [], {'default': '900'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'port': ('django.db.models.fields.PositiveIntegerField', [], {'null': 'True', 'blank': 'True'}), + 'ssl': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'username': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + 'sources.sourcelog': { + 'Meta': {'ordering': "('creation_datetime',)", 'object_name': 'SourceLog'}, + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'creation_datetime': ('django.db.models.fields.DateTimeField', [], {}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'object_id': ('django.db.models.fields.PositiveIntegerField', [], {}), + 'status': ('django.db.models.fields.TextField', [], {}) + }, + 'sources.sourcetransformation': { + 'Meta': {'ordering': "('order',)", 'object_name': 'SourceTransformation'}, + 'arguments': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'object_id': ('django.db.models.fields.PositiveIntegerField', [], {}), + 'order': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0', 'null': 'True', 'db_index': 'True', 'blank': 'True'}), + 'transformation': ('django.db.models.fields.CharField', [], {'max_length': '128'}) + }, + 'sources.stagingfolder': { + 'Meta': {'ordering': "('title',)", 'object_name': 'StagingFolder'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'delete_after_upload': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'folder_path': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'icon': ('django.db.models.fields.CharField', [], {'max_length': '24', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'preview_height': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}), + 'preview_width': ('django.db.models.fields.IntegerField', [], {}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + 'sources.watchfolder': { + 'Meta': {'ordering': "('title',)", 'object_name': 'WatchFolder'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'delete_after_upload': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'folder_path': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'interval': ('django.db.models.fields.PositiveIntegerField', [], {}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + 'sources.webform': { + 'Meta': {'ordering': "('title',)", 'object_name': 'WebForm'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'icon': ('django.db.models.fields.CharField', [], {'max_length': '24', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + } + } + + complete_apps = ['sources'] diff --git a/apps/sources/migrations/0012_auto__add_imapemail.py b/apps/sources/migrations/0012_auto__add_imapemail.py new file mode 100644 index 0000000000..d56b10bd3d --- /dev/null +++ b/apps/sources/migrations/0012_auto__add_imapemail.py @@ -0,0 +1,146 @@ +# encoding: utf-8 +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + +class Migration(SchemaMigration): + + def forwards(self, orm): + + # Adding model 'IMAPEmail' + db.create_table('sources_imapemail', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('title', self.gf('django.db.models.fields.CharField')(max_length=64)), + ('enabled', self.gf('django.db.models.fields.BooleanField')(default=True)), + ('whitelist', self.gf('django.db.models.fields.TextField')(blank=True)), + ('blacklist', self.gf('django.db.models.fields.TextField')(blank=True)), + ('interval', self.gf('django.db.models.fields.PositiveIntegerField')(default=900)), + ('document_type', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['documents.DocumentType'], null=True, blank=True)), + ('uncompress', self.gf('django.db.models.fields.CharField')(max_length=1)), + ('host', self.gf('django.db.models.fields.CharField')(max_length=64)), + ('ssl', self.gf('django.db.models.fields.BooleanField')(default=False)), + ('port', self.gf('django.db.models.fields.PositiveIntegerField')(null=True, blank=True)), + ('username', self.gf('django.db.models.fields.CharField')(max_length=64)), + ('password', self.gf('django.db.models.fields.CharField')(max_length=64)), + ('mailbox', self.gf('django.db.models.fields.CharField')(max_length=64, blank=True)), + )) + db.send_create_signal('sources', ['IMAPEmail']) + + + def backwards(self, orm): + + # Deleting model 'IMAPEmail' + db.delete_table('sources_imapemail') + + + models = { + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'documents.documenttype': { + 'Meta': {'ordering': "['name']", 'object_name': 'DocumentType'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '32'}) + }, + 'sources.imapemail': { + 'Meta': {'ordering': "('title',)", 'object_name': 'IMAPEmail'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'document_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['documents.DocumentType']", 'null': 'True', 'blank': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'host': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'interval': ('django.db.models.fields.PositiveIntegerField', [], {'default': '900'}), + 'mailbox': ('django.db.models.fields.CharField', [], {'max_length': '64', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'port': ('django.db.models.fields.PositiveIntegerField', [], {'null': 'True', 'blank': 'True'}), + 'ssl': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'username': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + 'sources.outofprocess': { + 'Meta': {'ordering': "('title',)", 'object_name': 'OutOfProcess'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + 'sources.pop3email': { + 'Meta': {'ordering': "('title',)", 'object_name': 'POP3Email'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'document_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['documents.DocumentType']", 'null': 'True', 'blank': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'host': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'interval': ('django.db.models.fields.PositiveIntegerField', [], {'default': '900'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'port': ('django.db.models.fields.PositiveIntegerField', [], {'null': 'True', 'blank': 'True'}), + 'ssl': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'username': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + 'sources.sourcelog': { + 'Meta': {'ordering': "('creation_datetime',)", 'object_name': 'SourceLog'}, + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'creation_datetime': ('django.db.models.fields.DateTimeField', [], {}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'object_id': ('django.db.models.fields.PositiveIntegerField', [], {}), + 'status': ('django.db.models.fields.TextField', [], {}) + }, + 'sources.sourcetransformation': { + 'Meta': {'ordering': "('order',)", 'object_name': 'SourceTransformation'}, + 'arguments': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'object_id': ('django.db.models.fields.PositiveIntegerField', [], {}), + 'order': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0', 'null': 'True', 'db_index': 'True', 'blank': 'True'}), + 'transformation': ('django.db.models.fields.CharField', [], {'max_length': '128'}) + }, + 'sources.stagingfolder': { + 'Meta': {'ordering': "('title',)", 'object_name': 'StagingFolder'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'delete_after_upload': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'folder_path': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'icon': ('django.db.models.fields.CharField', [], {'max_length': '24', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'preview_height': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}), + 'preview_width': ('django.db.models.fields.IntegerField', [], {}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + 'sources.watchfolder': { + 'Meta': {'ordering': "('title',)", 'object_name': 'WatchFolder'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'delete_after_upload': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'folder_path': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'interval': ('django.db.models.fields.PositiveIntegerField', [], {}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + 'sources.webform': { + 'Meta': {'ordering': "('title',)", 'object_name': 'WebForm'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'icon': ('django.db.models.fields.CharField', [], {'max_length': '24', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + } + } + + complete_apps = ['sources'] diff --git a/apps/sources/models.py b/apps/sources/models.py index 6cd35235a7..a43153d15d 100644 --- a/apps/sources/models.py +++ b/apps/sources/models.py @@ -3,6 +3,7 @@ from __future__ import absolute_import from ast import literal_eval import logging import poplib +import imaplib from email.Utils import collapse_rfc2231_value from email import message_from_string import os @@ -33,13 +34,17 @@ from metadata.api import save_metadata_list from scheduler.api import register_interval_job, remove_job from acls.utils import apply_default_acls -from .managers import SourceTransformationManager, POP3EmailLogManager +from .managers import SourceTransformationManager, SourceLogManager from .literals import (SOURCE_CHOICES, SOURCE_CHOICES_PLURAL, SOURCE_INTERACTIVE_UNCOMPRESS_CHOICES, SOURCE_CHOICE_WEB_FORM, SOURCE_CHOICE_STAGING, SOURCE_ICON_DISK, SOURCE_ICON_DRIVE, SOURCE_ICON_CHOICES, SOURCE_CHOICE_WATCH, SOURCE_UNCOMPRESS_CHOICES, - SOURCE_UNCOMPRESS_CHOICE_Y, POP3_PORT, POP3_SSL_PORT, - SOURCE_CHOICE_POP3_EMAIL, DEFAULT_POP3_INTERVAL) + SOURCE_UNCOMPRESS_CHOICE_Y, + POP3_PORT, POP3_SSL_PORT, + SOURCE_CHOICE_POP3_EMAIL, DEFAULT_POP3_INTERVAL, + IMAP_PORT, IMAP_SSL_PORT, + SOURCE_CHOICE_IMAP_EMAIL, DEFAULT_IMAP_INTERVAL, + IMAP_DEFAULT_MAILBOX) from .compressed_file import CompressedFile, NotACompressedFile from .conf.settings import POP3_TIMEOUT @@ -162,6 +167,27 @@ class BaseModel(models.Model): abstract = True +class SourceLog(models.Model): + content_type = models.ForeignKey(ContentType) + object_id = models.PositiveIntegerField() + source = generic.GenericForeignKey('content_type', 'object_id') + creation_datetime = models.DateTimeField(verbose_name=_(u'date time')) + status = models.TextField(verbose_name=_(u'status')) + + objects = SourceLogManager() + + def save(self, *args, **kwargs): + if not self.pk: + self.creation_datetime = datetime.datetime.now() + return super(SourceLog, self).save(*args, **kwargs) + + class Meta: + verbose_name = _(u'source log') + verbose_name_plural = _(u'sources logs') + get_latest_by = 'creation_datetime' + ordering = ('creation_datetime',) + + class InteractiveBaseModel(BaseModel): icon = models.CharField(blank=True, null=True, max_length=24, choices=SOURCE_ICON_CHOICES, verbose_name=_(u'icon'), help_text=_(u'An icon to visually distinguish this source.')) @@ -189,18 +215,25 @@ class Attachment(File): self.file = PseudoFile(StringIO(part.get_payload(decode=True)), name=name) -class POP3Email(BaseModel): +class IntervalBaseModel(BaseModel): is_interactive = False - source_type = SOURCE_CHOICE_POP3_EMAIL - + + interval = models.PositiveIntegerField(default=DEFAULT_POP3_INTERVAL, verbose_name=_(u'interval'), help_text=_(u'Interval in seconds between document downloads from this source.')) + document_type = models.ForeignKey(DocumentType, null=True, blank=True, verbose_name=_(u'document type'), help_text=_(u'Assign a document type to documents uploaded from this source.')) + uncompress = models.CharField(max_length=1, choices=SOURCE_UNCOMPRESS_CHOICES, verbose_name=_(u'uncompress'), help_text=_(u'Whether to expand or not, compressed archives.')) + + class Meta(BaseModel.Meta): + verbose_name = _(u'interval source') + verbose_name_plural = _(u'interval sources') + abstract = True + + +class EmailBaseModel(IntervalBaseModel): host = models.CharField(max_length=64, verbose_name=_(u'host')) ssl = models.BooleanField(verbose_name=_(u'SSL')) port = models.PositiveIntegerField(blank=True, null=True, verbose_name=_(u'port'), help_text=_(u'Override the defaults values of %d and %d for SSL, can be left blank otherwise.') % (POP3_PORT, POP3_SSL_PORT)) username = models.CharField(max_length=64, verbose_name=_(u'username')) password = models.CharField(max_length=64, verbose_name=_(u'password')) - uncompress = models.CharField(max_length=1, choices=SOURCE_UNCOMPRESS_CHOICES, verbose_name=_(u'uncompress'), help_text=_(u'Whether to expand or not, compressed archives.')) - interval = models.PositiveIntegerField(default=DEFAULT_POP3_INTERVAL, verbose_name=_(u'interval'), help_text=_(u'Interval in seconds between document downloads from this account.')) - document_type = models.ForeignKey(DocumentType, null=True, blank=True, verbose_name=_(u'document type'), help_text=_(u'Assign a document type to documents uploaded from this source.')) # From: http://bookmarks.honewatson.com/2009/08/11/python-gmail-imaplib-search-subject-get-attachments/ @staticmethod @@ -226,10 +259,19 @@ class POP3Email(BaseModel): document_file = Attachment(part, name=filename) source.upload_file(document_file, expand=(source.uncompress == SOURCE_UNCOMPRESS_CHOICE_Y), document_type=source.document_type) + class Meta(IntervalBaseModel.Meta): + verbose_name = _(u'email source') + verbose_name_plural = _(u'email sources') + abstract = True + + +class POP3Email(EmailBaseModel): + source_type = SOURCE_CHOICE_POP3_EMAIL + def fetch_mail(self): try: - last_check = self.pop3emaillog_set.latest().creation_datetime - except POP3EmailLog.DoesNotExist: + last_check = SourceLog.objects.get_latest_for(self) + except SourceLog.DoesNotExist: # Trigger email fetch when there are no previous logs initial_trigger = True difference = datetime.timedelta(seconds=0) @@ -267,38 +309,78 @@ class POP3Email(BaseModel): complete_message = '\n'.join(mailbox.retr(message_number)[1]) - POP3Email.process_message(source=self, message=complete_message) + EmailBaseModel.process_message(source=self, message=complete_message) mailbox.dele(message_number) mailbox.quit() - POP3EmailLog.objects.save_status(pop3_email=self, status='Successful connection.') + SourceLog.objects.save_status(source=self, status='Successful connection.') except Exception, exc: logger.error('Unhandled exception: %s' % exc) - POP3EmailLog.objects.save_status(pop3_email=self, status='Error: %s' % exc) + SourceLog.objects.save_status(source=self, status='Error: %s' % exc) - class Meta(BaseModel.Meta): + class Meta(EmailBaseModel.Meta): verbose_name = _(u'POP email') verbose_name_plural = _(u'POP email') -class POP3EmailLog(models.Model): - pop3_email = models.ForeignKey(POP3Email, verbose_name=_(u'POP3 email')) - creation_datetime = models.DateTimeField(verbose_name=_(u'date time')) - status = models.TextField(verbose_name=_(u'status')) - - objects = POP3EmailLogManager() - - def save(self, *args, **kwargs): - if not self.pk: - self.creation_datetime = datetime.datetime.now() - return super(POP3EmailLog, self).save(*args, **kwargs) - - class Meta: - verbose_name = _(u'POP3 email log') - verbose_name_plural = _(u'POP3 emails logs') - get_latest_by = 'creation_datetime' - ordering = ('creation_datetime',) +class IMAPEmail(EmailBaseModel): + source_type = SOURCE_CHOICE_IMAP_EMAIL + + mailbox = models.CharField(max_length=64, blank=True, verbose_name=_(u'mailbox'), help_text=_(u'Mail from which to check for messages with attached documents. If none is specified, the default mailbox is %s') % IMAP_DEFAULT_MAILBOX) + + # http://www.doughellmann.com/PyMOTW/imaplib/ + def fetch_mail(self): + try: + last_check = SourceLog.objects.get_latest_for(self) + except SourceLog.DoesNotExist: + # Trigger email fetch when there are no previous logs + initial_trigger = True + difference = datetime.timedelta(seconds=0) + else: + difference = datetime.datetime.now() - last_check + initial_trigger = False + + if difference >= datetime.timedelta(seconds=self.interval) or initial_trigger: + try: + logger.debug('Starting IMAP email fetch') + logger.debug('host: %s' % self.host) + logger.debug('ssl: %s' % self.ssl) + if self.ssl: + port = self.port or IMAP_SSL_PORT + logger.debug('port: %d' % port) + mailbox = imaplib.IMAP4_SSL(self.host, int(port)) + else: + port = self.port or IMAP_PORT + logger.debug('port: %d' % port) + mailbox = imaplib.IMAP4(self.host, int(port)) + + mailbox.login(self.username, self.password) + mailbox.select(self.mailbox or IMAP_DEFAULT_MAILBOX) + + status, data = mailbox.search(None, 'NOT', 'DELETED') + if data: + messages_info = data[0].split() + logger.debug('messages count: %s' % len(messages_info)) + + for message_number in messages_info: + logger.debug('message_number: %s' % message_number) + status, data = mailbox.fetch(message_number, '(RFC822)') + EmailBaseModel.process_message(source=self, message=data[0][1]) + mailbox.store(message_number, '+FLAGS', '\\Deleted') + + mailbox.expunge() + mailbox.close() + mailbox.logout() + SourceLog.objects.save_status(source=self, status='Successful connection.') + + except Exception, exc: + logger.error('Unhandled exception: %s' % exc) + SourceLog.objects.save_status(source=self, status='Error: %s' % exc) + + class Meta(EmailBaseModel.Meta): + verbose_name = _(u'IMAP email') + verbose_name_plural = _(u'IMAP email') class StagingFolder(InteractiveBaseModel): diff --git a/apps/sources/tasks.py b/apps/sources/tasks.py index c7da93e28a..19e5683671 100644 --- a/apps/sources/tasks.py +++ b/apps/sources/tasks.py @@ -9,8 +9,9 @@ from django.db.models import Q from job_processor.api import process_job from lock_manager import Lock, LockError -from .models import POP3Email +from .models import POP3Email, IMAPEmail from .conf.settings import POP3_TIMEOUT +from .literals import IMAP_LOCK_TIMEOUT logger = logging.getLogger(__name__) @@ -39,3 +40,29 @@ def task_fetch_pop3_emails(): task_fetch_single_pop3_email(pop3_email) except Exception, exc: logger.error('Unhandled exception: %s' % exc) + + +def task_fetch_single_imap_email(imap_email): + try: + lock_id = u'task_fetch_iamp_email-%d' % imap_email.pk + logger.debug('trying to acquire lock: %s' % lock_id) + lock = Lock.acquire_lock(lock_id, IMAP_LOCK_TIMEOUT) + logger.debug('acquired lock: %s' % lock_id) + try: + imap_email.fetch_mail() + except Exception, exc: + raise + finally: + lock.release() + except LockError: + logger.error('unable to obtain lock') + pass + + +def task_fetch_imap_emails(): + logger.debug('executing') + for imap_email in IMAPEmail.objects.filter(enabled=True): + try: + task_fetch_single_imap_email(imap_email) + except Exception, exc: + logger.error('Unhandled exception: %s' % exc) diff --git a/apps/sources/urls.py b/apps/sources/urls.py index 289afb10fa..0a1d075382 100644 --- a/apps/sources/urls.py +++ b/apps/sources/urls.py @@ -3,7 +3,7 @@ from __future__ import absolute_import from django.conf.urls.defaults import patterns, url from .literals import (SOURCE_CHOICE_WEB_FORM, SOURCE_CHOICE_STAGING, - SOURCE_CHOICE_WATCH, SOURCE_CHOICE_POP3_EMAIL) + SOURCE_CHOICE_WATCH, SOURCE_CHOICE_POP3_EMAIL, SOURCE_CHOICE_IMAP_EMAIL) urlpatterns = patterns('sources.views', url(r'^staging_file/type/(?P\w+)/(?P\d+)/(?P\w+)/preview/$', 'staging_file_preview', (), 'staging_file_preview'), @@ -22,6 +22,7 @@ urlpatterns = patterns('sources.views', url(r'^setup/interactive/%s/list/$' % SOURCE_CHOICE_STAGING, 'setup_source_list', {'source_type': SOURCE_CHOICE_STAGING}, 'setup_staging_folder_list'), url(r'^setup/interactive/%s/list/$' % SOURCE_CHOICE_WATCH, 'setup_source_list', {'source_type': SOURCE_CHOICE_WATCH}, 'setup_watch_folder_list'), url(r'^setup/interactive/%s/list/$' % SOURCE_CHOICE_POP3_EMAIL, 'setup_source_list', {'source_type': SOURCE_CHOICE_POP3_EMAIL}, 'setup_pop3_email_list'), + url(r'^setup/interactive/%s/list/$' % SOURCE_CHOICE_IMAP_EMAIL, 'setup_source_list', {'source_type': SOURCE_CHOICE_IMAP_EMAIL}, 'setup_imap_email_list'), url(r'^setup/interactive/(?P\w+)/list/$', 'setup_source_list', (), 'setup_source_list'), url(r'^setup/interactive/(?P\w+)/(?P\d+)/edit/$', 'setup_source_edit', (), 'setup_source_edit'), diff --git a/apps/sources/views.py b/apps/sources/views.py index ac60276848..351a6e80e3 100644 --- a/apps/sources/views.py +++ b/apps/sources/views.py @@ -25,15 +25,16 @@ import sendfile from acls.models import AccessEntry from .models import (WebForm, StagingFolder, SourceTransformation, - WatchFolder, POP3Email, POP3EmailLog) + WatchFolder, POP3Email, SourceLog, IMAPEmail) from .literals import (SOURCE_CHOICE_WEB_FORM, SOURCE_CHOICE_STAGING, - SOURCE_CHOICE_WATCH, SOURCE_CHOICE_POP3_EMAIL) + SOURCE_CHOICE_WATCH, SOURCE_CHOICE_POP3_EMAIL, SOURCE_CHOICE_IMAP_EMAIL) from .literals import (SOURCE_UNCOMPRESS_CHOICE_Y, SOURCE_UNCOMPRESS_CHOICE_ASK) from .staging import create_staging_file_class from .forms import (StagingDocumentForm, WebFormForm, WatchFolderSetupForm) -from .forms import WebFormSetupForm, StagingFolderSetupForm, POP3EmailSetupForm +from .forms import (WebFormSetupForm, StagingFolderSetupForm, + POP3EmailSetupForm, IMAPEmailSetupForm) from .forms import SourceTransformationForm, SourceTransformationForm_create from .permissions import (PERMISSION_SOURCES_SETUP_VIEW, PERMISSION_SOURCES_SETUP_EDIT, PERMISSION_SOURCES_SETUP_DELETE, @@ -441,6 +442,8 @@ def setup_source_list(request, source_type): cls = WatchFolder elif source_type == SOURCE_CHOICE_POP3_EMAIL: cls = POP3Email + elif source_type == SOURCE_CHOICE_IMAP_EMAIL: + cls = IMAPEmail context = { 'object_list': cls.objects.all(), @@ -472,6 +475,9 @@ def setup_source_edit(request, source_type, source_id): elif source_type == SOURCE_CHOICE_POP3_EMAIL: cls = POP3Email form_class = POP3EmailSetupForm + elif source_type == SOURCE_CHOICE_IMAP_EMAIL: + cls = IMAPEmail + form_class = IMAPEmailSetupForm source = get_object_or_404(cls, pk=source_id) next = request.POST.get('next', request.GET.get('next', request.META.get('HTTP_REFERER', '/'))) @@ -518,6 +524,10 @@ def setup_source_delete(request, source_type, source_id): cls = POP3Email form_icon = u'email_delete.png' redirect_view = 'setup_pop3_email_list' + elif source_type == SOURCE_CHOICE_IMAP_EMAIL: + cls = IMAPEmail + form_icon = u'email_delete.png' + redirect_view = 'setup_imap_email_list' redirect_view = reverse('setup_source_list', args=[source_type]) previous = request.POST.get('previous', request.GET.get('previous', request.META.get('HTTP_REFERER', redirect_view))) @@ -572,6 +582,10 @@ def setup_source_create(request, source_type): cls = POP3Email form_class = POP3EmailSetupForm redirect_view = 'setup_pop3_email_list' + elif source_type == SOURCE_CHOICE_IMAP_EMAIL: + cls = IMAPEmail + form_class = IMAPEmailSetupForm + redirect_view = 'setup_imap_email_list' if request.method == 'POST': form = form_class(data=request.POST) @@ -605,11 +619,13 @@ def setup_source_log_list(request, source_type, source_pk): cls = WatchFolder elif source_type == SOURCE_CHOICE_POP3_EMAIL: cls = POP3Email + elif source_type == SOURCE_CHOICE_IMAP_EMAIL: + cls = IMAPEmail source = get_object_or_404(cls, pk=source_pk) context = { - 'object_list': POP3EmailLog.objects.filter(pop3_email=source).order_by('-creation_datetime'), + 'object_list': SourceLog.objects.get_for_source(source), 'title': _(u'logs for: %s') % source.fullname(), 'source': source, 'object_name': _(u'source'), @@ -639,6 +655,8 @@ def setup_source_transformation_list(request, source_type, source_id): cls = WatchFolder elif source_type == SOURCE_CHOICE_POP3_EMAIL: cls = POP3Email + elif source_type == SOURCE_CHOICE_IMAP_EMAIL: + cls = IMAPEmail source = get_object_or_404(cls, pk=source_id) @@ -741,7 +759,9 @@ def setup_source_transformation_create(request, source_type, source_id): cls = WatchFolder elif source_type == SOURCE_CHOICE_POP3_EMAIL: cls = POP3Email - + elif source_type == SOURCE_CHOICE_IMAP_EMAIL: + cls = IMAPEmail + source = get_object_or_404(cls, pk=source_id) redirect_view = reverse('setup_source_transformation_list', args=[source.source_type, source.pk])