diff --git a/apps/main/__init__.py b/apps/main/__init__.py index 97aa7458f4..e057b7b219 100644 --- a/apps/main/__init__.py +++ b/apps/main/__init__.py @@ -20,9 +20,9 @@ __status__ = 'Production' __version_info__ = { 'major': 0, - 'minor': 12, + 'minor': 13, 'micro': 0, - 'releaselevel': 'final', + 'releaselevel': 'alpha', 'serial': 0 } diff --git a/apps/sources/__init__.py b/apps/sources/__init__.py index 4596235c3a..f6c4933d4f 100644 --- a/apps/sources/__init__.py +++ b/apps/sources/__init__.py @@ -8,26 +8,31 @@ from common.utils import encapsulate from project_setup.api import register_setup from documents.permissions import (PERMISSION_DOCUMENT_NEW_VERSION, PERMISSION_DOCUMENT_CREATE) +from scheduler.api import register_interval_job from .staging import StagingFile from .models import (WebForm, StagingFolder, SourceTransformation, - WatchFolder) + WatchFolder, POP3Email) from .widgets import staging_file_thumbnail from .permissions import (PERMISSION_SOURCES_SETUP_VIEW, PERMISSION_SOURCES_SETUP_EDIT, PERMISSION_SOURCES_SETUP_DELETE, PERMISSION_SOURCES_SETUP_CREATE) +from .tasks import task_fetch_pop3_emails +from .conf.settings import POP3_EMAIL_PROCESSING_INTERVAL staging_file_preview = {'text': _(u'preview'), 'class': 'fancybox-noscaling', 'view': 'staging_file_preview', 'args': ['source.source_type', 'source.pk', 'object.id'], 'famfam': 'zoom', 'permissions': [PERMISSION_DOCUMENT_NEW_VERSION, PERMISSION_DOCUMENT_CREATE]} staging_file_delete = {'text': _(u'delete'), 'view': 'staging_file_delete', 'args': ['source.source_type', 'source.pk', 'object.id'], 'famfam': 'delete', 'keep_query': True, 'permissions': [PERMISSION_DOCUMENT_NEW_VERSION, PERMISSION_DOCUMENT_CREATE]} -setup_sources = {'text': _(u'sources'), 'view': 'setup_web_form_list', 'famfam': 'application_form', 'icon': 'application_form.png', 'children_classes': [WebForm], 'permissions': [PERMISSION_SOURCES_SETUP_VIEW], 'children_view_regex': [r'setup_web_form', r'setup_staging_folder', r'setup_source_']} +setup_sources = {'text': _(u'sources'), 'view': 'setup_web_form_list', 'famfam': 'application_form', 'icon': 'application_form.png', 'children_classes': [WebForm], 'permissions': [PERMISSION_SOURCES_SETUP_VIEW], 'children_view_regex': [r'setup_web_form', r'setup_staging_folder', r'setup_source_', r'setup_pop3']} setup_web_form_list = {'text': _(u'web forms'), 'view': 'setup_web_form_list', 'famfam': 'application_form', 'icon': 'application_form.png', 'children_classes': [WebForm], 'permissions': [PERMISSION_SOURCES_SETUP_VIEW]} setup_staging_folder_list = {'text': _(u'staging folders'), 'view': 'setup_staging_folder_list', 'famfam': 'folder_camera', 'children_classes': [StagingFolder], 'permissions': [PERMISSION_SOURCES_SETUP_VIEW]} setup_watch_folder_list = {'text': _(u'watch folders'), 'view': 'setup_watch_folder_list', 'famfam': 'folder_magnify', 'children_classes': [WatchFolder], 'permissions': [PERMISSION_SOURCES_SETUP_VIEW]} +setup_pop3_email_list = {'text': _(u'POP3 email'), 'view': 'setup_pop3_email_list', 'famfam': 'email', 'children_classes': [POP3Email], 'permissions': [PERMISSION_SOURCES_SETUP_VIEW]} setup_source_edit = {'text': _(u'edit'), 'view': 'setup_source_edit', 'args': ['source.source_type', 'source.pk'], 'famfam': 'application_form_edit', 'permissions': [PERMISSION_SOURCES_SETUP_EDIT]} setup_source_delete = {'text': _(u'delete'), 'view': 'setup_source_delete', 'args': ['source.source_type', 'source.pk'], 'famfam': 'application_form_delete', 'permissions': [PERMISSION_SOURCES_SETUP_DELETE]} setup_source_create = {'text': _(u'add new source'), 'view': 'setup_source_create', 'args': 'source_type', 'famfam': 'application_form_add', 'permissions': [PERMISSION_SOURCES_SETUP_CREATE]} +setup_source_log_list = {'text': _(u'logs'), 'view': 'setup_source_log_list', 'args': ['source.source_type', 'source.pk'], 'famfam': 'book', 'permissions': [PERMISSION_SOURCES_SETUP_EDIT]} setup_source_transformation_list = {'text': _(u'transformations'), 'view': 'setup_source_transformation_list', 'args': ['source.source_type', 'source.pk'], 'famfam': 'shape_move_front', 'permissions': [PERMISSION_SOURCES_SETUP_EDIT]} setup_source_transformation_create = {'text': _(u'add transformation'), 'view': 'setup_source_transformation_create', 'args': ['source.source_type', 'source.pk'], 'famfam': 'shape_square_add', 'permissions': [PERMISSION_SOURCES_SETUP_EDIT]} @@ -43,18 +48,23 @@ register_links(StagingFile, [staging_file_delete]) register_links(SourceTransformation, [setup_source_transformation_edit, setup_source_transformation_delete]) #register_links(['setup_web_form_list', 'setup_staging_folder_list', 'setup_watch_folder_list', 'setup_source_create'], [setup_web_form_list, setup_staging_folder_list, setup_watch_folder_list], menu_name='form_header') -register_links(['setup_web_form_list', 'setup_staging_folder_list', 'setup_watch_folder_list', 'setup_source_create'], [setup_web_form_list, setup_staging_folder_list], menu_name='form_header') +register_links(['setup_pop3_email_list', 'setup_web_form_list', 'setup_staging_folder_list', 'setup_watch_folder_list', 'setup_source_create'], [setup_web_form_list, setup_staging_folder_list, setup_pop3_email_list], menu_name='form_header') #register_links(WebForm, [setup_web_form_list, setup_staging_folder_list, setup_watch_folder_list], menu_name='form_header') -register_links(WebForm, [setup_web_form_list, setup_staging_folder_list], menu_name='form_header') +register_links(WebForm, [setup_web_form_list, setup_staging_folder_list, setup_pop3_email_list], menu_name='form_header') register_links(WebForm, [setup_source_transformation_list, setup_source_edit, setup_source_delete]) -register_links(['setup_web_form_list', 'setup_staging_folder_list', 'setup_watch_folder_list', 'setup_source_edit', 'setup_source_delete', 'setup_source_create'], [setup_sources, setup_source_create], menu_name='sidebar') +#register_links(['setup_source_log_list', 'setup_web_form_list', 'setup_staging_folder_list', 'setup_watch_folder_list', 'setup_source_edit', 'setup_source_delete', 'setup_source_create', 'setup_pop3_email_list'], [setup_sources, setup_source_create], menu_name='secondary_menu') +register_links([WebForm, StagingFolder, POP3Email, 'setup_web_form_list', 'setup_staging_folder_list', 'setup_watch_folder_list', 'setup_source_create', 'setup_pop3_email_list'], [setup_source_create], menu_name='secondary_menu') #register_links(StagingFolder, [setup_web_form_list, setup_staging_folder_list, setup_watch_folder_list], menu_name='form_header') -register_links(StagingFolder, [setup_web_form_list, setup_staging_folder_list], menu_name='form_header') +register_links(StagingFolder, [setup_web_form_list, setup_staging_folder_list, setup_pop3_email_list], menu_name='form_header') register_links(StagingFolder, [setup_source_transformation_list, setup_source_edit, setup_source_delete]) +register_links(POP3Email, [setup_web_form_list, setup_staging_folder_list, setup_pop3_email_list], menu_name='form_header') +register_links(POP3Email, [setup_source_transformation_list, setup_source_edit, setup_source_delete]) +register_links(POP3Email, [setup_source_log_list]) + register_links(WatchFolder, [setup_web_form_list, setup_staging_folder_list, setup_watch_folder_list], menu_name='form_header') register_links(WatchFolder, [setup_source_transformation_list, setup_source_edit, setup_source_delete]) @@ -72,3 +82,5 @@ register_model_list_columns(StagingFile, [ ]) register_setup(setup_sources) + +register_interval_job('task_fetch_pop3_emails', _(u'Connects to the POP3 email sources and fetches the attached documents.'), task_fetch_pop3_emails, seconds=POP3_EMAIL_PROCESSING_INTERVAL) diff --git a/apps/sources/conf/__init__.py b/apps/sources/conf/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/apps/sources/conf/settings.py b/apps/sources/conf/settings.py new file mode 100644 index 0000000000..c854ed65d4 --- /dev/null +++ b/apps/sources/conf/settings.py @@ -0,0 +1,34 @@ +"""Configuration options for the sources app""" +import os + +from django.utils.translation import ugettext_lazy as _ +from django.conf import settings + +from smart_settings.api import Setting, SettingNamespace + +namespace = SettingNamespace('sources', _(u'Sources'), module='sources.conf.settings') + +POP3_DEFAULT_TIMEOUT = 10 # for POP3 only not POP3_SSL +POP3_DEFAULT_EMAIL_PROCESSING_INTERVAL = 5 +DEFAULT_POP3_EMAIL_LOG_COUNT = 10 # Max log entries to store + +Setting( + namespace=namespace, + name='POP3_TIMEOUT', + global_name='SOURCES_POP3_TIMEOUT', + default=POP3_DEFAULT_TIMEOUT, +) + +Setting( + namespace=namespace, + name='POP3_EMAIL_PROCESSING_INTERVAL', + global_name='SOURCES_POP3_EMAIL_PROCESSING_INTERVAL', + default=POP3_DEFAULT_EMAIL_PROCESSING_INTERVAL, +) + +Setting( + namespace=namespace, + name='POP3_EMAIL_LOG_SIZE', + global_name='SOURCES_POP3_EMAIL_LOG_SIZE', + default=DEFAULT_POP3_EMAIL_LOG_COUNT, +) diff --git a/apps/sources/forms.py b/apps/sources/forms.py index 714343530b..002c584f4c 100644 --- a/apps/sources/forms.py +++ b/apps/sources/forms.py @@ -7,7 +7,7 @@ from django.utils.translation import ugettext from documents.forms import DocumentForm from .models import (WebForm, StagingFolder, SourceTransformation, - WatchFolder) + WatchFolder, POP3Email) from .widgets import FamFamRadioSelect from .utils import validate_whitelist_blacklist @@ -114,3 +114,8 @@ class SourceTransformationForm_create(forms.ModelForm): class Meta: model = SourceTransformation exclude = ('content_type', 'object_id') + + +class POP3EmailSetupForm(forms.ModelForm): + class Meta: + model = POP3Email diff --git a/apps/sources/literals.py b/apps/sources/literals.py index a1555f4033..839e934bc6 100644 --- a/apps/sources/literals.py +++ b/apps/sources/literals.py @@ -1,5 +1,9 @@ from django.utils.translation import ugettext_lazy as _ +POP3_PORT = 110 +POP3_SSL_PORT = 995 +DEFAULT_POP3_INTERVAL = 15 * 60 # 15 minutes in seconds + SOURCE_UNCOMPRESS_CHOICE_Y = 'y' SOURCE_UNCOMPRESS_CHOICE_N = 'n' SOURCE_UNCOMPRESS_CHOICE_ASK = 'a' @@ -42,15 +46,18 @@ SOURCE_ICON_CHOICES = ( SOURCE_CHOICE_WEB_FORM = 'webform' SOURCE_CHOICE_STAGING = 'staging' SOURCE_CHOICE_WATCH = 'watch' +SOURCE_CHOICE_POP3_EMAIL = 'pop3' SOURCE_CHOICES = ( (SOURCE_CHOICE_WEB_FORM, _(u'web form')), (SOURCE_CHOICE_STAGING, _(u'server staging folder')), (SOURCE_CHOICE_WATCH, _(u'server watch folder')), + (SOURCE_CHOICE_POP3_EMAIL, _(u'pop3 email')), ) SOURCE_CHOICES_PLURAL = ( (SOURCE_CHOICE_WEB_FORM, _(u'web forms')), (SOURCE_CHOICE_STAGING, _(u'server staging folders')), (SOURCE_CHOICE_WATCH, _(u'server watch folders')), + (SOURCE_CHOICE_POP3_EMAIL, _(u'pop3 emails')), ) diff --git a/apps/sources/managers.py b/apps/sources/managers.py index 4e74ec7d4f..b390befa67 100644 --- a/apps/sources/managers.py +++ b/apps/sources/managers.py @@ -1,8 +1,12 @@ +from __future__ import absolute_import + from ast import literal_eval from django.db import models from django.contrib.contenttypes.models import ContentType +from .conf.settings import POP3_EMAIL_LOG_SIZE + class SourceTransformationManager(models.Manager): def get_for_object(self, obj): @@ -24,3 +28,12 @@ class SourceTransformationManager(models.Manager): warnings.append(e) return transformations, warnings + + +class POP3EmailLogManager(models.Manager): + def save_status(self, pop3_email, status): + new_recent = self.model(pop3_email=pop3_email, status=status) + new_recent.save() + to_delete = self.model.objects.filter(pop3_email=pop3_email).order_by('-creation_datetime')[POP3_EMAIL_LOG_SIZE:] + for recent_to_delete in to_delete: + recent_to_delete.delete() diff --git a/apps/sources/migrations/0003_auto__add_pop3email.py b/apps/sources/migrations/0003_auto__add_pop3email.py new file mode 100644 index 0000000000..251a308295 --- /dev/null +++ b/apps/sources/migrations/0003_auto__add_pop3email.py @@ -0,0 +1,113 @@ +# encoding: utf-8 +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + +class Migration(SchemaMigration): + + def forwards(self, orm): + + # Adding model 'POP3Email' + db.create_table('sources_pop3email', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('title', self.gf('django.db.models.fields.CharField')(max_length=64)), + ('enabled', self.gf('django.db.models.fields.BooleanField')(default=True)), + ('whitelist', self.gf('django.db.models.fields.TextField')(blank=True)), + ('blacklist', self.gf('django.db.models.fields.TextField')(blank=True)), + ('uncompress', self.gf('django.db.models.fields.CharField')(max_length=1)), + ('ssl', self.gf('django.db.models.fields.BooleanField')(default=False)), + ('port', self.gf('django.db.models.fields.PositiveIntegerField')(blank=True)), + ('host', self.gf('django.db.models.fields.CharField')(max_length=64)), + ('username', self.gf('django.db.models.fields.CharField')(max_length=64)), + ('password', self.gf('django.db.models.fields.CharField')(max_length=64)), + ('delete', self.gf('django.db.models.fields.BooleanField')(default=False)), + )) + db.send_create_signal('sources', ['POP3Email']) + + + def backwards(self, orm): + + # Deleting model 'POP3Email' + db.delete_table('sources_pop3email') + + + models = { + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'sources.outofprocess': { + 'Meta': {'ordering': "('title',)", 'object_name': 'OutOfProcess'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + 'sources.pop3email': { + 'Meta': {'ordering': "('title',)", 'object_name': 'POP3Email'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'delete': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'host': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'port': ('django.db.models.fields.PositiveIntegerField', [], {'blank': 'True'}), + 'ssl': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'username': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + 'sources.sourcetransformation': { + 'Meta': {'ordering': "('order',)", 'object_name': 'SourceTransformation'}, + 'arguments': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'object_id': ('django.db.models.fields.PositiveIntegerField', [], {}), + 'order': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0', 'null': 'True', 'db_index': 'True', 'blank': 'True'}), + 'transformation': ('django.db.models.fields.CharField', [], {'max_length': '128'}) + }, + 'sources.stagingfolder': { + 'Meta': {'ordering': "('title',)", 'object_name': 'StagingFolder'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'delete_after_upload': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'folder_path': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'icon': ('django.db.models.fields.CharField', [], {'max_length': '24', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'preview_height': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}), + 'preview_width': ('django.db.models.fields.IntegerField', [], {}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + 'sources.watchfolder': { + 'Meta': {'ordering': "('title',)", 'object_name': 'WatchFolder'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'delete_after_upload': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'folder_path': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'interval': ('django.db.models.fields.PositiveIntegerField', [], {}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + 'sources.webform': { + 'Meta': {'ordering': "('title',)", 'object_name': 'WebForm'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'icon': ('django.db.models.fields.CharField', [], {'max_length': '24', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + } + } + + complete_apps = ['sources'] diff --git a/apps/sources/migrations/0004_make_port_nullable.py b/apps/sources/migrations/0004_make_port_nullable.py new file mode 100644 index 0000000000..57e3c97c69 --- /dev/null +++ b/apps/sources/migrations/0004_make_port_nullable.py @@ -0,0 +1,99 @@ +# encoding: utf-8 +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + +class Migration(SchemaMigration): + + def forwards(self, orm): + + # Changing field 'POP3Email.port' + db.alter_column('sources_pop3email', 'port', self.gf('django.db.models.fields.PositiveIntegerField')(null=True)) + + + def backwards(self, orm): + + # Changing field 'POP3Email.port' + db.alter_column('sources_pop3email', 'port', self.gf('django.db.models.fields.PositiveIntegerField')(default=110)) + + + models = { + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'sources.outofprocess': { + 'Meta': {'ordering': "('title',)", 'object_name': 'OutOfProcess'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + 'sources.pop3email': { + 'Meta': {'ordering': "('title',)", 'object_name': 'POP3Email'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'delete': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'host': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'port': ('django.db.models.fields.PositiveIntegerField', [], {'null': 'True', 'blank': 'True'}), + 'ssl': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'username': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + 'sources.sourcetransformation': { + 'Meta': {'ordering': "('order',)", 'object_name': 'SourceTransformation'}, + 'arguments': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'object_id': ('django.db.models.fields.PositiveIntegerField', [], {}), + 'order': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0', 'null': 'True', 'db_index': 'True', 'blank': 'True'}), + 'transformation': ('django.db.models.fields.CharField', [], {'max_length': '128'}) + }, + 'sources.stagingfolder': { + 'Meta': {'ordering': "('title',)", 'object_name': 'StagingFolder'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'delete_after_upload': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'folder_path': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'icon': ('django.db.models.fields.CharField', [], {'max_length': '24', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'preview_height': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}), + 'preview_width': ('django.db.models.fields.IntegerField', [], {}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + 'sources.watchfolder': { + 'Meta': {'ordering': "('title',)", 'object_name': 'WatchFolder'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'delete_after_upload': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'folder_path': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'interval': ('django.db.models.fields.PositiveIntegerField', [], {}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + 'sources.webform': { + 'Meta': {'ordering': "('title',)", 'object_name': 'WebForm'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'icon': ('django.db.models.fields.CharField', [], {'max_length': '24', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + } + } + + complete_apps = ['sources'] diff --git a/apps/sources/migrations/0005_rename_delete_field.py b/apps/sources/migrations/0005_rename_delete_field.py new file mode 100644 index 0000000000..cb6b5cd896 --- /dev/null +++ b/apps/sources/migrations/0005_rename_delete_field.py @@ -0,0 +1,105 @@ +# encoding: utf-8 +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + +class Migration(SchemaMigration): + + def forwards(self, orm): + + # Deleting field 'POP3Email.delete' + db.delete_column('sources_pop3email', 'delete') + + # Adding field 'POP3Email.delete_messages' + db.add_column('sources_pop3email', 'delete_messages', self.gf('django.db.models.fields.BooleanField')(default=False), keep_default=False) + + + def backwards(self, orm): + + # Adding field 'POP3Email.delete' + db.add_column('sources_pop3email', 'delete', self.gf('django.db.models.fields.BooleanField')(default=False), keep_default=False) + + # Deleting field 'POP3Email.delete_messages' + db.delete_column('sources_pop3email', 'delete_messages') + + + models = { + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'sources.outofprocess': { + 'Meta': {'ordering': "('title',)", 'object_name': 'OutOfProcess'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + 'sources.pop3email': { + 'Meta': {'ordering': "('title',)", 'object_name': 'POP3Email'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'delete_messages': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'host': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'port': ('django.db.models.fields.PositiveIntegerField', [], {'null': 'True', 'blank': 'True'}), + 'ssl': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'username': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + 'sources.sourcetransformation': { + 'Meta': {'ordering': "('order',)", 'object_name': 'SourceTransformation'}, + 'arguments': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'object_id': ('django.db.models.fields.PositiveIntegerField', [], {}), + 'order': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0', 'null': 'True', 'db_index': 'True', 'blank': 'True'}), + 'transformation': ('django.db.models.fields.CharField', [], {'max_length': '128'}) + }, + 'sources.stagingfolder': { + 'Meta': {'ordering': "('title',)", 'object_name': 'StagingFolder'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'delete_after_upload': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'folder_path': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'icon': ('django.db.models.fields.CharField', [], {'max_length': '24', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'preview_height': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}), + 'preview_width': ('django.db.models.fields.IntegerField', [], {}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + 'sources.watchfolder': { + 'Meta': {'ordering': "('title',)", 'object_name': 'WatchFolder'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'delete_after_upload': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'folder_path': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'interval': ('django.db.models.fields.PositiveIntegerField', [], {}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + 'sources.webform': { + 'Meta': {'ordering': "('title',)", 'object_name': 'WebForm'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'icon': ('django.db.models.fields.CharField', [], {'max_length': '24', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + } + } + + complete_apps = ['sources'] diff --git a/apps/sources/migrations/0006_auto__del_field_pop3email_delete_messages.py b/apps/sources/migrations/0006_auto__del_field_pop3email_delete_messages.py new file mode 100644 index 0000000000..f63d5e77af --- /dev/null +++ b/apps/sources/migrations/0006_auto__del_field_pop3email_delete_messages.py @@ -0,0 +1,98 @@ +# encoding: utf-8 +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + +class Migration(SchemaMigration): + + def forwards(self, orm): + + # Deleting field 'POP3Email.delete_messages' + db.delete_column('sources_pop3email', 'delete_messages') + + + def backwards(self, orm): + + # Adding field 'POP3Email.delete_messages' + db.add_column('sources_pop3email', 'delete_messages', self.gf('django.db.models.fields.BooleanField')(default=False), keep_default=False) + + + models = { + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'sources.outofprocess': { + 'Meta': {'ordering': "('title',)", 'object_name': 'OutOfProcess'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + 'sources.pop3email': { + 'Meta': {'ordering': "('title',)", 'object_name': 'POP3Email'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'host': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'port': ('django.db.models.fields.PositiveIntegerField', [], {'default': '110', 'null': 'True', 'blank': 'True'}), + 'ssl': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'username': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + 'sources.sourcetransformation': { + 'Meta': {'ordering': "('order',)", 'object_name': 'SourceTransformation'}, + 'arguments': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'object_id': ('django.db.models.fields.PositiveIntegerField', [], {}), + 'order': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0', 'null': 'True', 'db_index': 'True', 'blank': 'True'}), + 'transformation': ('django.db.models.fields.CharField', [], {'max_length': '128'}) + }, + 'sources.stagingfolder': { + 'Meta': {'ordering': "('title',)", 'object_name': 'StagingFolder'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'delete_after_upload': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'folder_path': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'icon': ('django.db.models.fields.CharField', [], {'max_length': '24', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'preview_height': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}), + 'preview_width': ('django.db.models.fields.IntegerField', [], {}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + 'sources.watchfolder': { + 'Meta': {'ordering': "('title',)", 'object_name': 'WatchFolder'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'delete_after_upload': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'folder_path': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'interval': ('django.db.models.fields.PositiveIntegerField', [], {}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + 'sources.webform': { + 'Meta': {'ordering': "('title',)", 'object_name': 'WebForm'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'icon': ('django.db.models.fields.CharField', [], {'max_length': '24', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + } + } + + complete_apps = ['sources'] diff --git a/apps/sources/migrations/0007_auto__add_pop3emaillog__add_field_pop3email_interval.py b/apps/sources/migrations/0007_auto__add_pop3emaillog__add_field_pop3email_interval.py new file mode 100644 index 0000000000..b4777a0bc5 --- /dev/null +++ b/apps/sources/migrations/0007_auto__add_pop3emaillog__add_field_pop3email_interval.py @@ -0,0 +1,118 @@ +# encoding: utf-8 +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + +class Migration(SchemaMigration): + + def forwards(self, orm): + + # Adding model 'POP3EmailLog' + db.create_table('sources_pop3emaillog', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('pop3_email', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['sources.POP3Email'])), + ('creation_datetime', self.gf('django.db.models.fields.DateTimeField')()), + ('status', self.gf('django.db.models.fields.TextField')()), + )) + db.send_create_signal('sources', ['POP3EmailLog']) + + # Adding field 'POP3Email.interval' + db.add_column('sources_pop3email', 'interval', self.gf('django.db.models.fields.PositiveIntegerField')(default=900), keep_default=False) + + + def backwards(self, orm): + + # Deleting model 'POP3EmailLog' + db.delete_table('sources_pop3emaillog') + + # Deleting field 'POP3Email.interval' + db.delete_column('sources_pop3email', 'interval') + + + models = { + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'sources.outofprocess': { + 'Meta': {'ordering': "('title',)", 'object_name': 'OutOfProcess'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + 'sources.pop3email': { + 'Meta': {'ordering': "('title',)", 'object_name': 'POP3Email'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'host': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'interval': ('django.db.models.fields.PositiveIntegerField', [], {'default': '900'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'port': ('django.db.models.fields.PositiveIntegerField', [], {'null': 'True', 'blank': 'True'}), + 'ssl': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'username': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + 'sources.pop3emaillog': { + 'Meta': {'object_name': 'POP3EmailLog'}, + 'creation_datetime': ('django.db.models.fields.DateTimeField', [], {}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'pop3_email': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['sources.POP3Email']"}), + 'status': ('django.db.models.fields.TextField', [], {}) + }, + 'sources.sourcetransformation': { + 'Meta': {'ordering': "('order',)", 'object_name': 'SourceTransformation'}, + 'arguments': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'object_id': ('django.db.models.fields.PositiveIntegerField', [], {}), + 'order': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0', 'null': 'True', 'db_index': 'True', 'blank': 'True'}), + 'transformation': ('django.db.models.fields.CharField', [], {'max_length': '128'}) + }, + 'sources.stagingfolder': { + 'Meta': {'ordering': "('title',)", 'object_name': 'StagingFolder'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'delete_after_upload': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'folder_path': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'icon': ('django.db.models.fields.CharField', [], {'max_length': '24', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'preview_height': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}), + 'preview_width': ('django.db.models.fields.IntegerField', [], {}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + 'sources.watchfolder': { + 'Meta': {'ordering': "('title',)", 'object_name': 'WatchFolder'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'delete_after_upload': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'folder_path': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'interval': ('django.db.models.fields.PositiveIntegerField', [], {}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + 'sources.webform': { + 'Meta': {'ordering': "('title',)", 'object_name': 'WebForm'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'icon': ('django.db.models.fields.CharField', [], {'max_length': '24', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + } + } + + complete_apps = ['sources'] diff --git a/apps/sources/models.py b/apps/sources/models.py index 0941abd00d..60c1b6d796 100644 --- a/apps/sources/models.py +++ b/apps/sources/models.py @@ -2,6 +2,16 @@ from __future__ import absolute_import from ast import literal_eval import logging +import poplib +from email.Utils import collapse_rfc2231_value +from email import message_from_string +import os +import datetime + +try: + from cStringIO import StringIO +except ImportError: + from StringIO import StringIO from django.db import models from django.utils.translation import ugettext_lazy as _ @@ -9,6 +19,8 @@ from django.contrib.contenttypes.models import ContentType from django.contrib.contenttypes import generic from django.core.exceptions import ValidationError from django.db import transaction +from django.core.files import File +from django.core.files.base import ContentFile from converter.api import get_available_transformations_choices from converter.literals import DIMENSION_SEPARATOR @@ -21,13 +33,15 @@ from metadata.api import save_metadata_list from scheduler.api import register_interval_job, remove_job from acls.utils import apply_default_acls -from .managers import SourceTransformationManager +from .managers import SourceTransformationManager, POP3EmailLogManager from .literals import (SOURCE_CHOICES, SOURCE_CHOICES_PLURAL, SOURCE_INTERACTIVE_UNCOMPRESS_CHOICES, SOURCE_CHOICE_WEB_FORM, SOURCE_CHOICE_STAGING, SOURCE_ICON_DISK, SOURCE_ICON_DRIVE, SOURCE_ICON_CHOICES, SOURCE_CHOICE_WATCH, SOURCE_UNCOMPRESS_CHOICES, - SOURCE_UNCOMPRESS_CHOICE_Y) + SOURCE_UNCOMPRESS_CHOICE_Y, POP3_PORT, POP3_SSL_PORT, + SOURCE_CHOICE_POP3_EMAIL, DEFAULT_POP3_INTERVAL) from .compressed_file import CompressedFile, NotACompressedFile +from .conf.settings import POP3_TIMEOUT logger = logging.getLogger(__name__) @@ -149,6 +163,133 @@ class InteractiveBaseModel(BaseModel): class Meta(BaseModel.Meta): abstract = True + + +class PseudoFile(File): + def __init__(self, file, name): + self.name = name + self.file = file + self.file.seek(0, os.SEEK_END) + self.size = self.file.tell() + self.file.seek(0) + + +class Attachment(File): + def __init__(self, part, name): + self.name = name + self.file = PseudoFile(StringIO(part.get_payload(decode=True)), name=name) + + +class POP3Email(BaseModel): + is_interactive = False + source_type = SOURCE_CHOICE_POP3_EMAIL + + host = models.CharField(max_length=64, verbose_name=_(u'host')) + ssl = models.BooleanField(verbose_name=_(u'SSL')) + port = models.PositiveIntegerField(blank=True, null=True, verbose_name=_(u'port'), help_text=_(u'Typical values are: %d and %d for SSL. These are the defaults if no port is specified.') % (POP3_PORT, POP3_SSL_PORT)) + username = models.CharField(max_length=64, verbose_name=_(u'username')) + password = models.CharField(max_length=64, verbose_name=_(u'password')) + uncompress = models.CharField(max_length=1, choices=SOURCE_UNCOMPRESS_CHOICES, verbose_name=_(u'uncompress'), help_text=_(u'Whether to expand or not compressed archives.')) + interval = models.PositiveIntegerField(default=DEFAULT_POP3_INTERVAL, verbose_name=_(u'interval'), help_text=_(u'Interval in seconds between document downloads from this account.')) + + # From: http://bookmarks.honewatson.com/2009/08/11/python-gmail-imaplib-search-subject-get-attachments/ + @staticmethod + def process_message(source, message, expand=False): + email = message_from_string(message) + counter = 1 + + for part in email.walk(): + disposition = part.get('Content-Disposition', 'none') + logger.debug('Disposition: %s' % disposition) + + if disposition.startswith('attachment'): + raw_filename = part.get_filename() + + if raw_filename: + filename = collapse_rfc2231_value(raw_filename) + else: + filename = _(u'attachment-%i') % counter + counter += 1 + + logger.debug('filename: %s' % filename) + + document_file = Attachment(part, name=filename) + source.upload_file(document_file, filename=filename, expand=expand) + + + def fetch_mail(self): + try: + last_check = self.pop3emaillog_set.latest().creation_datetime + except POP3EmailLog.DoesNotExist: + # Trigger email fetch when there are no previous logs + initial_trigger = True + difference = datetime.timedelta(seconds=0) + else: + difference = datetime.datetime.now() - last_check + initial_trigger = False + + if difference >= datetime.timedelta(seconds=self.interval) or initial_trigger: + try: + logger.debug('Starting POP3 email fetch') + logger.debug('host: %s' % self.host) + logger.debug('ssl: %s' % self.ssl) + if self.ssl: + port = self.port or POP3_SSL_PORT + logger.debug('port: %d' % port) + mailbox = poplib.POP3_SSL(self.host, int(port)) + else: + port = self.port or POP3_PORT + logger.debug('port: %d' % port) + mailbox = poplib.POP3(self.host, int(port), timeout=POP3_TIMEOUT) + + mailbox.getwelcome() + mailbox.user(self.username) + mailbox.pass_(self.password) + messages_info = mailbox.list() + + logger.debug('messages_info:') + logger.debug(messages_info) + logger.debug('messages count: %s' % len(messages_info[1])) + + for message_info in messages_info[1]: + message_number, message_size = message_info.split() + logger.debug('message_number: %s' % message_number) + logger.debug('message_size: %s' % message_size) + + complete_message = '\n'.join(mailbox.retr(message_number)[1]) + + POP3Email.process_message(source=self, message=complete_message, expand=self.uncompress) + mailbox.dele(message_number) + + mailbox.quit() + POP3EmailLog.objects.save_status(pop3_email=self, status='Successful connection.') + + except Exception, exc: + logger.error('Unhandled exception: %s' % exc) + POP3EmailLog.objects.save_status(pop3_email=self, status='Error: %s' % exc) + + class Meta(BaseModel.Meta): + verbose_name = _(u'POP email') + verbose_name_plural = _(u'POP email') + + +class POP3EmailLog(models.Model): + pop3_email = models.ForeignKey(POP3Email, verbose_name=_(u'POP3 email')) + creation_datetime = models.DateTimeField(verbose_name=_(u'date time')) + status = models.TextField(verbose_name=_(u'status')) + + objects = POP3EmailLogManager() + + def save(self, *args, **kwargs): + if not self.pk: + self.creation_datetime = datetime.datetime.now() + return super(POP3EmailLog, self).save(*args, **kwargs) + + class Meta: + verbose_name = _(u'POP3 email log') + verbose_name_plural = _(u'POP3 emails logs') + get_latest_by = 'creation_datetime' + ordering = ('creation_datetime',) class StagingFolder(InteractiveBaseModel): diff --git a/apps/sources/static/images/icons/email_delete.png b/apps/sources/static/images/icons/email_delete.png new file mode 100644 index 0000000000..30a2f9d94f Binary files /dev/null and b/apps/sources/static/images/icons/email_delete.png differ diff --git a/apps/sources/tasks.py b/apps/sources/tasks.py new file mode 100644 index 0000000000..c7da93e28a --- /dev/null +++ b/apps/sources/tasks.py @@ -0,0 +1,41 @@ +from __future__ import absolute_import + +from datetime import timedelta, datetime +import platform +import logging + +from django.db.models import Q + +from job_processor.api import process_job +from lock_manager import Lock, LockError + +from .models import POP3Email +from .conf.settings import POP3_TIMEOUT + +logger = logging.getLogger(__name__) + + +def task_fetch_single_pop3_email(pop3_email): + try: + lock_id = u'task_fetch_pop3_email-%d' % pop3_email.pk + logger.debug('trying to acquire lock: %s' % lock_id) + lock = Lock.acquire_lock(lock_id, POP3_TIMEOUT + 60) # Lock expiration = POP3 timeout + 60 seconds + logger.debug('acquired lock: %s' % lock_id) + try: + pop3_email.fetch_mail() + except Exception, exc: + raise + finally: + lock.release() + except LockError: + logger.error('unable to obtain lock') + pass + + +def task_fetch_pop3_emails(): + logger.debug('executing') + for pop3_email in POP3Email.objects.filter(enabled=True): + try: + task_fetch_single_pop3_email(pop3_email) + except Exception, exc: + logger.error('Unhandled exception: %s' % exc) diff --git a/apps/sources/urls.py b/apps/sources/urls.py index 9b8f6ad59f..289afb10fa 100644 --- a/apps/sources/urls.py +++ b/apps/sources/urls.py @@ -3,7 +3,7 @@ from __future__ import absolute_import from django.conf.urls.defaults import patterns, url from .literals import (SOURCE_CHOICE_WEB_FORM, SOURCE_CHOICE_STAGING, - SOURCE_CHOICE_WATCH) + SOURCE_CHOICE_WATCH, SOURCE_CHOICE_POP3_EMAIL) urlpatterns = patterns('sources.views', url(r'^staging_file/type/(?P\w+)/(?P\d+)/(?P\w+)/preview/$', 'staging_file_preview', (), 'staging_file_preview'), @@ -21,6 +21,7 @@ urlpatterns = patterns('sources.views', url(r'^setup/interactive/%s/list/$' % SOURCE_CHOICE_WEB_FORM, 'setup_source_list', {'source_type': SOURCE_CHOICE_WEB_FORM}, 'setup_web_form_list'), url(r'^setup/interactive/%s/list/$' % SOURCE_CHOICE_STAGING, 'setup_source_list', {'source_type': SOURCE_CHOICE_STAGING}, 'setup_staging_folder_list'), url(r'^setup/interactive/%s/list/$' % SOURCE_CHOICE_WATCH, 'setup_source_list', {'source_type': SOURCE_CHOICE_WATCH}, 'setup_watch_folder_list'), + url(r'^setup/interactive/%s/list/$' % SOURCE_CHOICE_POP3_EMAIL, 'setup_source_list', {'source_type': SOURCE_CHOICE_POP3_EMAIL}, 'setup_pop3_email_list'), url(r'^setup/interactive/(?P\w+)/list/$', 'setup_source_list', (), 'setup_source_list'), url(r'^setup/interactive/(?P\w+)/(?P\d+)/edit/$', 'setup_source_edit', (), 'setup_source_edit'), @@ -31,4 +32,6 @@ urlpatterns = patterns('sources.views', url(r'^setup/interactive/(?P\w+)/(?P\d+)/transformation/create/$', 'setup_source_transformation_create', (), 'setup_source_transformation_create'), url(r'^setup/interactive/source/transformation/(?P\d+)/edit/$', 'setup_source_transformation_edit', (), 'setup_source_transformation_edit'), url(r'^setup/interactive/source/transformation/(?P\d+)/delete/$', 'setup_source_transformation_delete', (), 'setup_source_transformation_delete'), + + url(r'^setup/source/(?P\w+)/(?P\d+)/log/list/$', 'setup_source_log_list', (), 'setup_source_log_list'), ) diff --git a/apps/sources/views.py b/apps/sources/views.py index adb4df3611..8f27092b2f 100644 --- a/apps/sources/views.py +++ b/apps/sources/views.py @@ -1,5 +1,7 @@ from __future__ import absolute_import +import logging + from django.http import HttpResponseRedirect from django.shortcuts import render_to_response, get_object_or_404 from django.template import RequestContext @@ -21,21 +23,23 @@ from common.utils import encapsulate import sendfile from acls.models import AccessEntry -from sources.models import (WebForm, StagingFolder, SourceTransformation, - WatchFolder) -from sources.literals import (SOURCE_CHOICE_WEB_FORM, SOURCE_CHOICE_STAGING, - SOURCE_CHOICE_WATCH) -from sources.literals import (SOURCE_UNCOMPRESS_CHOICE_Y, +from .models import (WebForm, StagingFolder, SourceTransformation, + WatchFolder, POP3Email, POP3EmailLog) +from .literals import (SOURCE_CHOICE_WEB_FORM, SOURCE_CHOICE_STAGING, + SOURCE_CHOICE_WATCH, SOURCE_CHOICE_POP3_EMAIL) +from .literals import (SOURCE_UNCOMPRESS_CHOICE_Y, SOURCE_UNCOMPRESS_CHOICE_ASK) -from sources.staging import create_staging_file_class -from sources.forms import (StagingDocumentForm, WebFormForm, +from .staging import create_staging_file_class +from .forms import (StagingDocumentForm, WebFormForm, WatchFolderSetupForm) -from sources.forms import WebFormSetupForm, StagingFolderSetupForm -from sources.forms import SourceTransformationForm, SourceTransformationForm_create +from .forms import WebFormSetupForm, StagingFolderSetupForm, POP3EmailSetupForm +from .forms import SourceTransformationForm, SourceTransformationForm_create from .permissions import (PERMISSION_SOURCES_SETUP_VIEW, PERMISSION_SOURCES_SETUP_EDIT, PERMISSION_SOURCES_SETUP_DELETE, PERMISSION_SOURCES_SETUP_CREATE) +logger = logging.getLogger(__name__) + def return_function(obj): return lambda context: context['source'].source_type == obj.source_type and context['source'].pk == obj.pk @@ -69,11 +73,11 @@ def get_active_tab_links(document=None): staging_folders = StagingFolder.objects.filter(enabled=True) for staging_folder in staging_folders: tab_links.append(get_tab_link_for_source(staging_folder, document)) - + return { 'tab_links': tab_links, SOURCE_CHOICE_WEB_FORM: web_forms, - SOURCE_CHOICE_STAGING: staging_folders + SOURCE_CHOICE_STAGING: staging_folders, } @@ -434,6 +438,8 @@ def setup_source_list(request, source_type): cls = StagingFolder elif source_type == SOURCE_CHOICE_WATCH: cls = WatchFolder + elif source_type == SOURCE_CHOICE_POP3_EMAIL: + cls = POP3Email context = { 'object_list': cls.objects.all(), @@ -459,7 +465,10 @@ def setup_source_edit(request, source_type, source_id): elif source_type == SOURCE_CHOICE_WATCH: cls = WatchFolder form_class = WatchFolderSetupForm - + elif source_type == SOURCE_CHOICE_POP3_EMAIL: + cls = POP3Email + form_class = POP3EmailSetupForm + source = get_object_or_404(cls, pk=source_id) next = request.POST.get('next', request.GET.get('next', request.META.get('HTTP_REFERER', '/'))) @@ -501,6 +510,10 @@ def setup_source_delete(request, source_type, source_id): cls = WatchFolder form_icon = u'folder_delete.png' redirect_view = 'setup_watch_folder_list' + elif source_type == SOURCE_CHOICE_POP3_EMAIL: + cls = POP3Email + form_icon = u'email_delete.png' + redirect_view = 'setup_pop3_email_list' redirect_view = reverse('setup_source_list', args=[source_type]) previous = request.POST.get('previous', request.GET.get('previous', request.META.get('HTTP_REFERER', redirect_view))) @@ -512,9 +525,12 @@ def setup_source_delete(request, source_type, source_id): source.delete() messages.success(request, _(u'Source "%s" deleted successfully.') % source) except Exception, e: - messages.error(request, _(u'Error deleting source "%(source)s": %(error)s') % { - 'source': source, 'error': e - }) + if settings.DEBUG: + raise + else: + messages.error(request, _(u'Error deleting source "%(source)s": %(error)s') % { + 'source': source, 'error': e + }) return HttpResponseRedirect(redirect_view) @@ -539,12 +555,19 @@ def setup_source_create(request, source_type): if source_type == SOURCE_CHOICE_WEB_FORM: cls = WebForm form_class = WebFormSetupForm + redirect_view = 'setup_web_form_list' elif source_type == SOURCE_CHOICE_STAGING: cls = StagingFolder form_class = StagingFolderSetupForm + redirect_view = 'setup_staging_folder_list' elif source_type == SOURCE_CHOICE_WATCH: cls = WatchFolder form_class = WatchFolderSetupForm + redirect_view = 'setup_watch_folder_list' + elif source_type == SOURCE_CHOICE_POP3_EMAIL: + cls = POP3Email + form_class = POP3EmailSetupForm + redirect_view = 'setup_pop3_email_list' if request.method == 'POST': form = form_class(data=request.POST) @@ -552,7 +575,7 @@ def setup_source_create(request, source_type): try: form.save() messages.success(request, _(u'Source created successfully')) - return HttpResponseRedirect(reverse('setup_web_form_list')) + return HttpResponseRedirect(reverse(redirect_view)) except Exception, e: messages.error(request, _(u'Error creating source; %s') % e) else: @@ -567,6 +590,40 @@ def setup_source_create(request, source_type): context_instance=RequestContext(request)) +def setup_source_log_list(request, source_type, source_pk): + Permission.objects.check_permissions(request.user, [PERMISSION_SOURCES_SETUP_EDIT]) + + if source_type == SOURCE_CHOICE_WEB_FORM: + cls = WebForm + elif source_type == SOURCE_CHOICE_STAGING: + cls = StagingFolder + elif source_type == SOURCE_CHOICE_WATCH: + cls = WatchFolder + elif source_type == SOURCE_CHOICE_POP3_EMAIL: + cls = POP3Email + + source = get_object_or_404(cls, pk=source_pk) + + context = { + 'object_list': POP3EmailLog.objects.filter(pop3_email=source).order_by('-creation_datetime'), + 'title': _(u'logs for: %s') % source.fullname(), + 'source': source, + 'object_name': _(u'source'), + 'navigation_object_name': 'source', + 'source_type': source_type, + 'extra_columns': [ + {'name': _(u'Date time'), 'attribute': 'creation_datetime'}, + {'name': _(u'Status'), 'attribute': 'status'}, + ], + 'hide_link': True, + 'hide_object': True, + 'hide_links': True, + } + + return render_to_response('generic_list.html', context, + context_instance=RequestContext(request)) + + def setup_source_transformation_list(request, source_type, source_id): Permission.objects.check_permissions(request.user, [PERMISSION_SOURCES_SETUP_EDIT]) @@ -576,7 +633,9 @@ def setup_source_transformation_list(request, source_type, source_id): cls = StagingFolder elif source_type == SOURCE_CHOICE_WATCH: cls = WatchFolder - + elif source_type == SOURCE_CHOICE_POP3_EMAIL: + cls = POP3Email + source = get_object_or_404(cls, pk=source_id) context = { @@ -584,6 +643,7 @@ def setup_source_transformation_list(request, source_type, source_id): 'title': _(u'transformations for: %s') % source.fullname(), 'source': source, 'object_name': _(u'source'), + 'source_type': source_type, 'navigation_object_name': 'source', 'list_object_variable_name': 'transformation', 'extra_columns': [ @@ -675,7 +735,9 @@ def setup_source_transformation_create(request, source_type, source_id): cls = StagingFolder elif source_type == SOURCE_CHOICE_WATCH: cls = WatchFolder - + elif source_type == SOURCE_CHOICE_POP3_EMAIL: + cls = POP3Email + source = get_object_or_404(cls, pk=source_id) redirect_view = reverse('setup_source_transformation_list', args=[source.source_type, source.pk]) @@ -697,6 +759,7 @@ def setup_source_transformation_create(request, source_type, source_id): return render_to_response('generic_form.html', { 'form': form, 'source': source, + 'source_type': source_type, 'object_name': _(u'source'), 'navigation_object_name': 'source', 'title': _(u'Create new transformation for source: %s') % source, diff --git a/docs/releases/0.13.rst b/docs/releases/0.13.rst new file mode 100644 index 0000000000..c75631bc02 --- /dev/null +++ b/docs/releases/0.13.rst @@ -0,0 +1,94 @@ +============================== +Mayan EDMS v0.13 release notes +============================== + +*2012* + +Welcome to Mayan EDMS v0.13! + +Overview +======== + +* POP3 email document source + + +What's new in Mayan EDMS v0.13 +============================== + +E-mail document source +~~~~~~~~~~~~~~~~~~~~~~ + + +Upgrading from a previous version +================================= + +Start off by adding the new requirements:: + + $ pip install -r requirements/production.txt + +Then create the new database structures with:: + + $ ./manage.py syncdb + +Afterwards migrate existing database schema with:: + + $ ./manage.py migrate permissions 0001 --fake + $ ./manage.py migrate permissions + + +When the following message appears + +:: + + The following content types are stale and need to be deleted: + + permissions | permission + + Any objects related to these content types by a foreign key will also + be deleted. Are you sure you want to delete these content types? + If you're unsure, answer 'no'. + + Type 'yes' to continue, or 'no' to cancel: + +Type ``yes`` and press **Enter** + +And continue migrating database schema with:: + + $ ./manage.py migrate documents + $ ./manage.py migrate document_signatures + $ ./manage.py migrate folders 0001 --fake + $ ./manage.py migrate folders + $ ./manage.py migrate document_indexing 0001 --fake + $ ./manage.py migrate document_indexing + $ ./manage.py migrate sources 0001 --fake + $ ./manage.py migrate sources + +Again when a similar messages appears +:: + + The following content types are stale and need to be deleted: + + document_indexing | indexinstance + + Any objects related to these content types by a foreign key will also + be deleted. Are you sure you want to delete these content types? + If you're unsure, answer 'no'. + + Type 'yes' to continue, or 'no' to cancel: + +Type ``yes`` and press **Enter** + +The upgrade procedure is now complete. + + +Backward incompatible changes +============================= +None so far + +Bugs fixed +========== +* Metadata unicode fixes, while creating new documents. + +Stuff removed +============= +None so far diff --git a/docs/releases/index.rst b/docs/releases/index.rst index 3bbee2aa57..a51dcf09ac 100644 --- a/docs/releases/index.rst +++ b/docs/releases/index.rst @@ -11,11 +11,12 @@ all the backwards-incompatible changes and deprecated features for each 'final' release from the one after your current **Mayan EDMS** version, up to and including the new version. -Latest version (0.12) +Latest version (0.13) --------------------- .. toctree:: :maxdepth: 1 + 0.13 0.12 Historic changelogs