From 5651b3b4df07c5807a8154392b24f80028987572 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 7 Mar 2012 01:26:36 -0400 Subject: [PATCH 01/11] Initial commit for the email source support --- apps/sources/forms.py | 7 ++++++- apps/sources/literals.py | 3 +++ apps/sources/models.py | 32 +++++++++++++++++++++++++++++++- 3 files changed, 40 insertions(+), 2 deletions(-) diff --git a/apps/sources/forms.py b/apps/sources/forms.py index 714343530b..002c584f4c 100644 --- a/apps/sources/forms.py +++ b/apps/sources/forms.py @@ -7,7 +7,7 @@ from django.utils.translation import ugettext from documents.forms import DocumentForm from .models import (WebForm, StagingFolder, SourceTransformation, - WatchFolder) + WatchFolder, POP3Email) from .widgets import FamFamRadioSelect from .utils import validate_whitelist_blacklist @@ -114,3 +114,8 @@ class SourceTransformationForm_create(forms.ModelForm): class Meta: model = SourceTransformation exclude = ('content_type', 'object_id') + + +class POP3EmailSetupForm(forms.ModelForm): + class Meta: + model = POP3Email diff --git a/apps/sources/literals.py b/apps/sources/literals.py index a1555f4033..f040e1fabc 100644 --- a/apps/sources/literals.py +++ b/apps/sources/literals.py @@ -1,5 +1,8 @@ from django.utils.translation import ugettext_lazy as _ +POP3_PORT = 110 +POP3_SSL_PORT = 995 + SOURCE_UNCOMPRESS_CHOICE_Y = 'y' SOURCE_UNCOMPRESS_CHOICE_N = 'n' SOURCE_UNCOMPRESS_CHOICE_ASK = 'a' diff --git a/apps/sources/models.py b/apps/sources/models.py index 0941abd00d..498c9db419 100644 --- a/apps/sources/models.py +++ b/apps/sources/models.py @@ -26,7 +26,7 @@ from .literals import (SOURCE_CHOICES, SOURCE_CHOICES_PLURAL, SOURCE_INTERACTIVE_UNCOMPRESS_CHOICES, SOURCE_CHOICE_WEB_FORM, SOURCE_CHOICE_STAGING, SOURCE_ICON_DISK, SOURCE_ICON_DRIVE, SOURCE_ICON_CHOICES, SOURCE_CHOICE_WATCH, SOURCE_UNCOMPRESS_CHOICES, - SOURCE_UNCOMPRESS_CHOICE_Y) + SOURCE_UNCOMPRESS_CHOICE_Y, POP3_PORT, POP3_SSL_PORT) from .compressed_file import CompressedFile, NotACompressedFile logger = logging.getLogger(__name__) @@ -151,6 +151,36 @@ class InteractiveBaseModel(BaseModel): abstract = True +class POP3Email(BaseModel): + #is_interactive = False + uncompress = models.CharField(max_length=1, choices=SOURCE_UNCOMPRESS_CHOICES, verbose_name=_(u'uncompress'), help_text=_(u'Whether to expand or not compressed archives.')) + ssl = models.BooleanField(verbose_name=_(u'SSL')) + port = models.PositiveIntegerField(blank=True, verbose_name=_(u'port')) + host = models.CharField(max_length=32, verbose_name=_(u'host')) + username = models.CharField(max_length=64, verbose_name=_(u'username')) + password = models.CharField(max_length=64, verbose_name=_(u'password')) + delete = models.BooleanField(verbose_name=_(u'delete messages')) + + def fetch_mail(self): + if ssl: + port = self.port or POP3_PORT + mailbox = poplib.POP3_SSL(self.host, port) + else: + port = self.port or POP3_SSL_PORT + mailbox = poplib.POP3(self.host, port) + + mailbox.user(self.username) + mailbox.pass_(self.password) + message_count = len(mailbox.list()[1]) + for message_list in range(message_count): + for message in mailbox.retr(message_list+1)[1]: + mail = email.message_from_string(''.join(message)) + + class Meta(BaseModel.Meta): + verbose_name = _(u'POP email') + verbose_name_plural = _(u'POP email') + + class StagingFolder(InteractiveBaseModel): is_interactive = True source_type = SOURCE_CHOICE_STAGING From ef54f946a8f054b446bfc7c6c76e0f35e74120af Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Thu, 8 Mar 2012 16:36:35 -0400 Subject: [PATCH 02/11] Add POP3 email creation, edit, delete views and links --- apps/sources/__init__.py | 14 ++- apps/sources/literals.py | 3 + .../migrations/0003_auto__add_pop3email.py | 113 ++++++++++++++++++ .../migrations/0004_make_port_nullable.py | 99 +++++++++++++++ .../migrations/0005_rename_delete_field.py | 105 ++++++++++++++++ apps/sources/models.py | 22 ++-- .../static/images/icons/email_delete.png | Bin 0 -> 1525 bytes apps/sources/urls.py | 3 +- apps/sources/views.py | 63 +++++++--- 9 files changed, 390 insertions(+), 32 deletions(-) create mode 100644 apps/sources/migrations/0003_auto__add_pop3email.py create mode 100644 apps/sources/migrations/0004_make_port_nullable.py create mode 100644 apps/sources/migrations/0005_rename_delete_field.py create mode 100644 apps/sources/static/images/icons/email_delete.png diff --git a/apps/sources/__init__.py b/apps/sources/__init__.py index 4596235c3a..d03a2a1f6f 100644 --- a/apps/sources/__init__.py +++ b/apps/sources/__init__.py @@ -11,7 +11,7 @@ from documents.permissions import (PERMISSION_DOCUMENT_NEW_VERSION, from .staging import StagingFile from .models import (WebForm, StagingFolder, SourceTransformation, - WatchFolder) + WatchFolder, POP3Email) from .widgets import staging_file_thumbnail from .permissions import (PERMISSION_SOURCES_SETUP_VIEW, PERMISSION_SOURCES_SETUP_EDIT, PERMISSION_SOURCES_SETUP_DELETE, @@ -24,6 +24,7 @@ setup_sources = {'text': _(u'sources'), 'view': 'setup_web_form_list', 'famfam': setup_web_form_list = {'text': _(u'web forms'), 'view': 'setup_web_form_list', 'famfam': 'application_form', 'icon': 'application_form.png', 'children_classes': [WebForm], 'permissions': [PERMISSION_SOURCES_SETUP_VIEW]} setup_staging_folder_list = {'text': _(u'staging folders'), 'view': 'setup_staging_folder_list', 'famfam': 'folder_camera', 'children_classes': [StagingFolder], 'permissions': [PERMISSION_SOURCES_SETUP_VIEW]} setup_watch_folder_list = {'text': _(u'watch folders'), 'view': 'setup_watch_folder_list', 'famfam': 'folder_magnify', 'children_classes': [WatchFolder], 'permissions': [PERMISSION_SOURCES_SETUP_VIEW]} +setup_pop3_email_list = {'text': _(u'POP3 email'), 'view': 'setup_pop3_email_list', 'famfam': 'email', 'children_classes': [POP3Email], 'permissions': [PERMISSION_SOURCES_SETUP_VIEW]} setup_source_edit = {'text': _(u'edit'), 'view': 'setup_source_edit', 'args': ['source.source_type', 'source.pk'], 'famfam': 'application_form_edit', 'permissions': [PERMISSION_SOURCES_SETUP_EDIT]} setup_source_delete = {'text': _(u'delete'), 'view': 'setup_source_delete', 'args': ['source.source_type', 'source.pk'], 'famfam': 'application_form_delete', 'permissions': [PERMISSION_SOURCES_SETUP_DELETE]} @@ -43,18 +44,21 @@ register_links(StagingFile, [staging_file_delete]) register_links(SourceTransformation, [setup_source_transformation_edit, setup_source_transformation_delete]) #register_links(['setup_web_form_list', 'setup_staging_folder_list', 'setup_watch_folder_list', 'setup_source_create'], [setup_web_form_list, setup_staging_folder_list, setup_watch_folder_list], menu_name='form_header') -register_links(['setup_web_form_list', 'setup_staging_folder_list', 'setup_watch_folder_list', 'setup_source_create'], [setup_web_form_list, setup_staging_folder_list], menu_name='form_header') +register_links(['setup_pop3_email_list', 'setup_web_form_list', 'setup_staging_folder_list', 'setup_watch_folder_list', 'setup_source_create'], [setup_web_form_list, setup_staging_folder_list, setup_pop3_email_list], menu_name='form_header') #register_links(WebForm, [setup_web_form_list, setup_staging_folder_list, setup_watch_folder_list], menu_name='form_header') -register_links(WebForm, [setup_web_form_list, setup_staging_folder_list], menu_name='form_header') +register_links(WebForm, [setup_web_form_list, setup_staging_folder_list, setup_pop3_email_list], menu_name='form_header') register_links(WebForm, [setup_source_transformation_list, setup_source_edit, setup_source_delete]) -register_links(['setup_web_form_list', 'setup_staging_folder_list', 'setup_watch_folder_list', 'setup_source_edit', 'setup_source_delete', 'setup_source_create'], [setup_sources, setup_source_create], menu_name='sidebar') +register_links(['setup_web_form_list', 'setup_staging_folder_list', 'setup_watch_folder_list', 'setup_source_edit', 'setup_source_delete', 'setup_source_create', 'setup_pop3_email_list'], [setup_sources, setup_source_create], menu_name='sidebar') #register_links(StagingFolder, [setup_web_form_list, setup_staging_folder_list, setup_watch_folder_list], menu_name='form_header') -register_links(StagingFolder, [setup_web_form_list, setup_staging_folder_list], menu_name='form_header') +register_links(StagingFolder, [setup_web_form_list, setup_staging_folder_list, setup_pop3_email_list], menu_name='form_header') register_links(StagingFolder, [setup_source_transformation_list, setup_source_edit, setup_source_delete]) +register_links(POP3Email, [setup_web_form_list, setup_staging_folder_list, setup_pop3_email_list], menu_name='form_header') +register_links(POP3Email, [setup_source_transformation_list, setup_source_edit, setup_source_delete]) + register_links(WatchFolder, [setup_web_form_list, setup_staging_folder_list, setup_watch_folder_list], menu_name='form_header') register_links(WatchFolder, [setup_source_transformation_list, setup_source_edit, setup_source_delete]) diff --git a/apps/sources/literals.py b/apps/sources/literals.py index f040e1fabc..5a3a7dcef3 100644 --- a/apps/sources/literals.py +++ b/apps/sources/literals.py @@ -45,15 +45,18 @@ SOURCE_ICON_CHOICES = ( SOURCE_CHOICE_WEB_FORM = 'webform' SOURCE_CHOICE_STAGING = 'staging' SOURCE_CHOICE_WATCH = 'watch' +SOURCE_CHOICE_POP3_EMAIL = 'pop3' SOURCE_CHOICES = ( (SOURCE_CHOICE_WEB_FORM, _(u'web form')), (SOURCE_CHOICE_STAGING, _(u'server staging folder')), (SOURCE_CHOICE_WATCH, _(u'server watch folder')), + (SOURCE_CHOICE_POP3_EMAIL, _(u'pop3 email')), ) SOURCE_CHOICES_PLURAL = ( (SOURCE_CHOICE_WEB_FORM, _(u'web forms')), (SOURCE_CHOICE_STAGING, _(u'server staging folders')), (SOURCE_CHOICE_WATCH, _(u'server watch folders')), + (SOURCE_CHOICE_POP3_EMAIL, _(u'pop3 emails')), ) diff --git a/apps/sources/migrations/0003_auto__add_pop3email.py b/apps/sources/migrations/0003_auto__add_pop3email.py new file mode 100644 index 0000000000..251a308295 --- /dev/null +++ b/apps/sources/migrations/0003_auto__add_pop3email.py @@ -0,0 +1,113 @@ +# encoding: utf-8 +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + +class Migration(SchemaMigration): + + def forwards(self, orm): + + # Adding model 'POP3Email' + db.create_table('sources_pop3email', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('title', self.gf('django.db.models.fields.CharField')(max_length=64)), + ('enabled', self.gf('django.db.models.fields.BooleanField')(default=True)), + ('whitelist', self.gf('django.db.models.fields.TextField')(blank=True)), + ('blacklist', self.gf('django.db.models.fields.TextField')(blank=True)), + ('uncompress', self.gf('django.db.models.fields.CharField')(max_length=1)), + ('ssl', self.gf('django.db.models.fields.BooleanField')(default=False)), + ('port', self.gf('django.db.models.fields.PositiveIntegerField')(blank=True)), + ('host', self.gf('django.db.models.fields.CharField')(max_length=64)), + ('username', self.gf('django.db.models.fields.CharField')(max_length=64)), + ('password', self.gf('django.db.models.fields.CharField')(max_length=64)), + ('delete', self.gf('django.db.models.fields.BooleanField')(default=False)), + )) + db.send_create_signal('sources', ['POP3Email']) + + + def backwards(self, orm): + + # Deleting model 'POP3Email' + db.delete_table('sources_pop3email') + + + models = { + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'sources.outofprocess': { + 'Meta': {'ordering': "('title',)", 'object_name': 'OutOfProcess'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + 'sources.pop3email': { + 'Meta': {'ordering': "('title',)", 'object_name': 'POP3Email'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'delete': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'host': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'port': ('django.db.models.fields.PositiveIntegerField', [], {'blank': 'True'}), + 'ssl': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'username': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + 'sources.sourcetransformation': { + 'Meta': {'ordering': "('order',)", 'object_name': 'SourceTransformation'}, + 'arguments': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'object_id': ('django.db.models.fields.PositiveIntegerField', [], {}), + 'order': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0', 'null': 'True', 'db_index': 'True', 'blank': 'True'}), + 'transformation': ('django.db.models.fields.CharField', [], {'max_length': '128'}) + }, + 'sources.stagingfolder': { + 'Meta': {'ordering': "('title',)", 'object_name': 'StagingFolder'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'delete_after_upload': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'folder_path': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'icon': ('django.db.models.fields.CharField', [], {'max_length': '24', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'preview_height': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}), + 'preview_width': ('django.db.models.fields.IntegerField', [], {}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + 'sources.watchfolder': { + 'Meta': {'ordering': "('title',)", 'object_name': 'WatchFolder'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'delete_after_upload': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'folder_path': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'interval': ('django.db.models.fields.PositiveIntegerField', [], {}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + 'sources.webform': { + 'Meta': {'ordering': "('title',)", 'object_name': 'WebForm'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'icon': ('django.db.models.fields.CharField', [], {'max_length': '24', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + } + } + + complete_apps = ['sources'] diff --git a/apps/sources/migrations/0004_make_port_nullable.py b/apps/sources/migrations/0004_make_port_nullable.py new file mode 100644 index 0000000000..57e3c97c69 --- /dev/null +++ b/apps/sources/migrations/0004_make_port_nullable.py @@ -0,0 +1,99 @@ +# encoding: utf-8 +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + +class Migration(SchemaMigration): + + def forwards(self, orm): + + # Changing field 'POP3Email.port' + db.alter_column('sources_pop3email', 'port', self.gf('django.db.models.fields.PositiveIntegerField')(null=True)) + + + def backwards(self, orm): + + # Changing field 'POP3Email.port' + db.alter_column('sources_pop3email', 'port', self.gf('django.db.models.fields.PositiveIntegerField')(default=110)) + + + models = { + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'sources.outofprocess': { + 'Meta': {'ordering': "('title',)", 'object_name': 'OutOfProcess'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + 'sources.pop3email': { + 'Meta': {'ordering': "('title',)", 'object_name': 'POP3Email'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'delete': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'host': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'port': ('django.db.models.fields.PositiveIntegerField', [], {'null': 'True', 'blank': 'True'}), + 'ssl': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'username': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + 'sources.sourcetransformation': { + 'Meta': {'ordering': "('order',)", 'object_name': 'SourceTransformation'}, + 'arguments': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'object_id': ('django.db.models.fields.PositiveIntegerField', [], {}), + 'order': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0', 'null': 'True', 'db_index': 'True', 'blank': 'True'}), + 'transformation': ('django.db.models.fields.CharField', [], {'max_length': '128'}) + }, + 'sources.stagingfolder': { + 'Meta': {'ordering': "('title',)", 'object_name': 'StagingFolder'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'delete_after_upload': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'folder_path': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'icon': ('django.db.models.fields.CharField', [], {'max_length': '24', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'preview_height': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}), + 'preview_width': ('django.db.models.fields.IntegerField', [], {}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + 'sources.watchfolder': { + 'Meta': {'ordering': "('title',)", 'object_name': 'WatchFolder'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'delete_after_upload': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'folder_path': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'interval': ('django.db.models.fields.PositiveIntegerField', [], {}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + 'sources.webform': { + 'Meta': {'ordering': "('title',)", 'object_name': 'WebForm'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'icon': ('django.db.models.fields.CharField', [], {'max_length': '24', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + } + } + + complete_apps = ['sources'] diff --git a/apps/sources/migrations/0005_rename_delete_field.py b/apps/sources/migrations/0005_rename_delete_field.py new file mode 100644 index 0000000000..cb6b5cd896 --- /dev/null +++ b/apps/sources/migrations/0005_rename_delete_field.py @@ -0,0 +1,105 @@ +# encoding: utf-8 +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + +class Migration(SchemaMigration): + + def forwards(self, orm): + + # Deleting field 'POP3Email.delete' + db.delete_column('sources_pop3email', 'delete') + + # Adding field 'POP3Email.delete_messages' + db.add_column('sources_pop3email', 'delete_messages', self.gf('django.db.models.fields.BooleanField')(default=False), keep_default=False) + + + def backwards(self, orm): + + # Adding field 'POP3Email.delete' + db.add_column('sources_pop3email', 'delete', self.gf('django.db.models.fields.BooleanField')(default=False), keep_default=False) + + # Deleting field 'POP3Email.delete_messages' + db.delete_column('sources_pop3email', 'delete_messages') + + + models = { + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'sources.outofprocess': { + 'Meta': {'ordering': "('title',)", 'object_name': 'OutOfProcess'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + 'sources.pop3email': { + 'Meta': {'ordering': "('title',)", 'object_name': 'POP3Email'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'delete_messages': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'host': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'port': ('django.db.models.fields.PositiveIntegerField', [], {'null': 'True', 'blank': 'True'}), + 'ssl': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'username': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + 'sources.sourcetransformation': { + 'Meta': {'ordering': "('order',)", 'object_name': 'SourceTransformation'}, + 'arguments': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'object_id': ('django.db.models.fields.PositiveIntegerField', [], {}), + 'order': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0', 'null': 'True', 'db_index': 'True', 'blank': 'True'}), + 'transformation': ('django.db.models.fields.CharField', [], {'max_length': '128'}) + }, + 'sources.stagingfolder': { + 'Meta': {'ordering': "('title',)", 'object_name': 'StagingFolder'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'delete_after_upload': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'folder_path': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'icon': ('django.db.models.fields.CharField', [], {'max_length': '24', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'preview_height': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}), + 'preview_width': ('django.db.models.fields.IntegerField', [], {}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + 'sources.watchfolder': { + 'Meta': {'ordering': "('title',)", 'object_name': 'WatchFolder'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'delete_after_upload': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'folder_path': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'interval': ('django.db.models.fields.PositiveIntegerField', [], {}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + 'sources.webform': { + 'Meta': {'ordering': "('title',)", 'object_name': 'WebForm'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'icon': ('django.db.models.fields.CharField', [], {'max_length': '24', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + } + } + + complete_apps = ['sources'] diff --git a/apps/sources/models.py b/apps/sources/models.py index 498c9db419..6e8c807184 100644 --- a/apps/sources/models.py +++ b/apps/sources/models.py @@ -26,7 +26,7 @@ from .literals import (SOURCE_CHOICES, SOURCE_CHOICES_PLURAL, SOURCE_INTERACTIVE_UNCOMPRESS_CHOICES, SOURCE_CHOICE_WEB_FORM, SOURCE_CHOICE_STAGING, SOURCE_ICON_DISK, SOURCE_ICON_DRIVE, SOURCE_ICON_CHOICES, SOURCE_CHOICE_WATCH, SOURCE_UNCOMPRESS_CHOICES, - SOURCE_UNCOMPRESS_CHOICE_Y, POP3_PORT, POP3_SSL_PORT) + SOURCE_UNCOMPRESS_CHOICE_Y, POP3_PORT, POP3_SSL_PORT, SOURCE_CHOICE_POP3_EMAIL) from .compressed_file import CompressedFile, NotACompressedFile logger = logging.getLogger(__name__) @@ -152,14 +152,16 @@ class InteractiveBaseModel(BaseModel): class POP3Email(BaseModel): - #is_interactive = False - uncompress = models.CharField(max_length=1, choices=SOURCE_UNCOMPRESS_CHOICES, verbose_name=_(u'uncompress'), help_text=_(u'Whether to expand or not compressed archives.')) + is_interactive = False + source_type = SOURCE_CHOICE_POP3_EMAIL + + host = models.CharField(max_length=64, verbose_name=_(u'host')) ssl = models.BooleanField(verbose_name=_(u'SSL')) - port = models.PositiveIntegerField(blank=True, verbose_name=_(u'port')) - host = models.CharField(max_length=32, verbose_name=_(u'host')) + port = models.PositiveIntegerField(blank=True, null=True, verbose_name=_(u'port')) username = models.CharField(max_length=64, verbose_name=_(u'username')) password = models.CharField(max_length=64, verbose_name=_(u'password')) - delete = models.BooleanField(verbose_name=_(u'delete messages')) + uncompress = models.CharField(max_length=1, choices=SOURCE_UNCOMPRESS_CHOICES, verbose_name=_(u'uncompress'), help_text=_(u'Whether to expand or not compressed archives.')) + delete_messages = models.BooleanField(verbose_name=_(u'delete messages'), help_text=_(u'Delete messages after downloading their respective attached documents.')) def fetch_mail(self): if ssl: @@ -172,10 +174,14 @@ class POP3Email(BaseModel): mailbox.user(self.username) mailbox.pass_(self.password) message_count = len(mailbox.list()[1]) - for message_list in range(message_count): + logger.debug('message_count: %n' % message_count) + for message_list in range(message_count): for message in mailbox.retr(message_list+1)[1]: mail = email.message_from_string(''.join(message)) - + for part in mail.walk(): + logger.debug(part) + #f.write(part.get_payload(decode=True)) + class Meta(BaseModel.Meta): verbose_name = _(u'POP email') verbose_name_plural = _(u'POP email') diff --git a/apps/sources/static/images/icons/email_delete.png b/apps/sources/static/images/icons/email_delete.png new file mode 100644 index 0000000000000000000000000000000000000000..30a2f9d94fd25145b4d92a2625e33b631bcebb42 GIT binary patch literal 1525 zcmV5pdsQAb1stPoZsXd zKJxZjV|y+8B)MS|aH-^VTm>4xAHb_?3(?wVGEN2oj_A`Rz#hWhUQ-D6#XjQ+2-Z|d z7R@a{!`T76yrvLm`b`2t++Q?etJcl$nj#pOjIHW1g&3Sg<@wzEw6bS;T%mbXfGHG_gx@^O?jKNrpX<+L!5a2mkn8Jg3 znQoL9`LX*%Hj9oA(Vwq*<{4mVN9nL#~h*Hy;a+ zvGtj9K5(y7EO}E5jj`1ApaNPHv>_s_6L~#<9!xY7lxS1$Zw!m&W zM4&ho`218iQe6txRn9?_+H_rYIhua#!RCgarhrtG=NWTRBC(;S3@x1je0FXaTUTYH z->7lI)J$)>tRyBNtR&=*JaT^EBxkZ06||pG&FpL%wd=Z7<=C{kJhm8)DzDzC@f2X9 z<2O_{)+5sT4Z66=8ipED);)udCkpSWm%{edS*rg9`IwvUfvQby$u^Pr z^Hk^LrE(8__&)Swjd-NC4!%$JKu$|D7)Eb}F>8O7Yp)^6LOzbXH|&lRl>El@P%1(?m9megpCLnwR zN?L{?{LN!y#|~rROFQ7p&UxYBvPY|}ber6Qrz#5JaY-gHw_jxzLIiGpxf^pfJ&TF6 zEzpJrAh}$Ss3avSB}4d|#?*iN5jowA()G0{aLOOSgy5A(#sLkdk+AoD=8v}>h3~C* zaqH(35UL725=qeb1q9W(+H(b2kFP|EByj}XCIO*^cQ#(|sFcq4b3@dry+u+pvE7vd z>F-{`5y))roY5v!R|(PAWBcJghDgJfs?A;ek z;3i2gN&f@hys}gRRYyaRGcpl=?n&s}weGxaQ*;F71M6tPV63j0zywL~|6Kh75FNp> z@nwE5fv334<}*D7$fuw~Nx}7C2%}N8gKd>0+`J>LL!tWI)N6-w{eH@N2jZ{yxZ_&R z&tHSr&^i>XXInWLSO4=P;gKh@Gs-4zh8Gm&=Nlv>%6LIZ%5RxNPh)}zw0B*`)nK@3 z*YAHGAls2-5a7zyjtmTURA+ihZjT1%yQwPD($gWOP<1#QM(BqF0bIIr6_xCP~relHAoB3VhqL75One?w}ivsu!CE1N#s44gFvxe280dvIy8D z==1?y%s@9oc&ssg`1lHwTqg-wm8T~27cl-*cUZq(&NzGy2DisCtLh%^AMq5%5BvB0 bKLG{+%T&{Yo2>p500000NkvXXu0mjfLgv@N literal 0 HcmV?d00001 diff --git a/apps/sources/urls.py b/apps/sources/urls.py index 9b8f6ad59f..9498708241 100644 --- a/apps/sources/urls.py +++ b/apps/sources/urls.py @@ -3,7 +3,7 @@ from __future__ import absolute_import from django.conf.urls.defaults import patterns, url from .literals import (SOURCE_CHOICE_WEB_FORM, SOURCE_CHOICE_STAGING, - SOURCE_CHOICE_WATCH) + SOURCE_CHOICE_WATCH, SOURCE_CHOICE_POP3_EMAIL) urlpatterns = patterns('sources.views', url(r'^staging_file/type/(?P\w+)/(?P\d+)/(?P\w+)/preview/$', 'staging_file_preview', (), 'staging_file_preview'), @@ -21,6 +21,7 @@ urlpatterns = patterns('sources.views', url(r'^setup/interactive/%s/list/$' % SOURCE_CHOICE_WEB_FORM, 'setup_source_list', {'source_type': SOURCE_CHOICE_WEB_FORM}, 'setup_web_form_list'), url(r'^setup/interactive/%s/list/$' % SOURCE_CHOICE_STAGING, 'setup_source_list', {'source_type': SOURCE_CHOICE_STAGING}, 'setup_staging_folder_list'), url(r'^setup/interactive/%s/list/$' % SOURCE_CHOICE_WATCH, 'setup_source_list', {'source_type': SOURCE_CHOICE_WATCH}, 'setup_watch_folder_list'), + url(r'^setup/interactive/%s/list/$' % SOURCE_CHOICE_POP3_EMAIL, 'setup_source_list', {'source_type': SOURCE_CHOICE_POP3_EMAIL}, 'setup_pop3_email_list'), url(r'^setup/interactive/(?P\w+)/list/$', 'setup_source_list', (), 'setup_source_list'), url(r'^setup/interactive/(?P\w+)/(?P\d+)/edit/$', 'setup_source_edit', (), 'setup_source_edit'), diff --git a/apps/sources/views.py b/apps/sources/views.py index adb4df3611..c7462221e4 100644 --- a/apps/sources/views.py +++ b/apps/sources/views.py @@ -1,5 +1,7 @@ from __future__ import absolute_import +import logging + from django.http import HttpResponseRedirect from django.shortcuts import render_to_response, get_object_or_404 from django.template import RequestContext @@ -21,21 +23,23 @@ from common.utils import encapsulate import sendfile from acls.models import AccessEntry -from sources.models import (WebForm, StagingFolder, SourceTransformation, - WatchFolder) -from sources.literals import (SOURCE_CHOICE_WEB_FORM, SOURCE_CHOICE_STAGING, - SOURCE_CHOICE_WATCH) -from sources.literals import (SOURCE_UNCOMPRESS_CHOICE_Y, +from .models import (WebForm, StagingFolder, SourceTransformation, + WatchFolder, POP3Email) +from .literals import (SOURCE_CHOICE_WEB_FORM, SOURCE_CHOICE_STAGING, + SOURCE_CHOICE_WATCH, SOURCE_CHOICE_POP3_EMAIL) +from .literals import (SOURCE_UNCOMPRESS_CHOICE_Y, SOURCE_UNCOMPRESS_CHOICE_ASK) -from sources.staging import create_staging_file_class -from sources.forms import (StagingDocumentForm, WebFormForm, +from .staging import create_staging_file_class +from .forms import (StagingDocumentForm, WebFormForm, WatchFolderSetupForm) -from sources.forms import WebFormSetupForm, StagingFolderSetupForm -from sources.forms import SourceTransformationForm, SourceTransformationForm_create +from .forms import WebFormSetupForm, StagingFolderSetupForm, POP3EmailSetupForm +from .forms import SourceTransformationForm, SourceTransformationForm_create from .permissions import (PERMISSION_SOURCES_SETUP_VIEW, PERMISSION_SOURCES_SETUP_EDIT, PERMISSION_SOURCES_SETUP_DELETE, PERMISSION_SOURCES_SETUP_CREATE) +logger = logging.getLogger(__name__) + def return_function(obj): return lambda context: context['source'].source_type == obj.source_type and context['source'].pk == obj.pk @@ -69,11 +73,11 @@ def get_active_tab_links(document=None): staging_folders = StagingFolder.objects.filter(enabled=True) for staging_folder in staging_folders: tab_links.append(get_tab_link_for_source(staging_folder, document)) - + return { 'tab_links': tab_links, SOURCE_CHOICE_WEB_FORM: web_forms, - SOURCE_CHOICE_STAGING: staging_folders + SOURCE_CHOICE_STAGING: staging_folders, } @@ -434,6 +438,8 @@ def setup_source_list(request, source_type): cls = StagingFolder elif source_type == SOURCE_CHOICE_WATCH: cls = WatchFolder + elif source_type == SOURCE_CHOICE_POP3_EMAIL: + cls = POP3Email context = { 'object_list': cls.objects.all(), @@ -459,7 +465,10 @@ def setup_source_edit(request, source_type, source_id): elif source_type == SOURCE_CHOICE_WATCH: cls = WatchFolder form_class = WatchFolderSetupForm - + elif source_type == SOURCE_CHOICE_POP3_EMAIL: + cls = POP3Email + form_class = POP3EmailSetupForm + source = get_object_or_404(cls, pk=source_id) next = request.POST.get('next', request.GET.get('next', request.META.get('HTTP_REFERER', '/'))) @@ -501,6 +510,10 @@ def setup_source_delete(request, source_type, source_id): cls = WatchFolder form_icon = u'folder_delete.png' redirect_view = 'setup_watch_folder_list' + elif source_type == SOURCE_CHOICE_POP3_EMAIL: + cls = POP3Email + form_icon = u'email_delete.png' + redirect_view = 'setup_pop3_email_list' redirect_view = reverse('setup_source_list', args=[source_type]) previous = request.POST.get('previous', request.GET.get('previous', request.META.get('HTTP_REFERER', redirect_view))) @@ -512,9 +525,12 @@ def setup_source_delete(request, source_type, source_id): source.delete() messages.success(request, _(u'Source "%s" deleted successfully.') % source) except Exception, e: - messages.error(request, _(u'Error deleting source "%(source)s": %(error)s') % { - 'source': source, 'error': e - }) + if settings.DEBUG: + raise + else: + messages.error(request, _(u'Error deleting source "%(source)s": %(error)s') % { + 'source': source, 'error': e + }) return HttpResponseRedirect(redirect_view) @@ -539,12 +555,19 @@ def setup_source_create(request, source_type): if source_type == SOURCE_CHOICE_WEB_FORM: cls = WebForm form_class = WebFormSetupForm + redirect_view = 'setup_web_form_list' elif source_type == SOURCE_CHOICE_STAGING: cls = StagingFolder form_class = StagingFolderSetupForm + redirect_view = 'setup_staging_folder_list' elif source_type == SOURCE_CHOICE_WATCH: cls = WatchFolder form_class = WatchFolderSetupForm + redirect_view = 'setup_watch_folder_list' + elif source_type == SOURCE_CHOICE_POP3_EMAIL: + cls = POP3Email + form_class = POP3EmailSetupForm + redirect_view = 'setup_pop3_email_list' if request.method == 'POST': form = form_class(data=request.POST) @@ -552,7 +575,7 @@ def setup_source_create(request, source_type): try: form.save() messages.success(request, _(u'Source created successfully')) - return HttpResponseRedirect(reverse('setup_web_form_list')) + return HttpResponseRedirect(reverse(redirect_view)) except Exception, e: messages.error(request, _(u'Error creating source; %s') % e) else: @@ -576,7 +599,9 @@ def setup_source_transformation_list(request, source_type, source_id): cls = StagingFolder elif source_type == SOURCE_CHOICE_WATCH: cls = WatchFolder - + elif source_type == SOURCE_CHOICE_POP3_EMAIL: + cls = POP3Email + source = get_object_or_404(cls, pk=source_id) context = { @@ -675,7 +700,9 @@ def setup_source_transformation_create(request, source_type, source_id): cls = StagingFolder elif source_type == SOURCE_CHOICE_WATCH: cls = WatchFolder - + elif source_type == SOURCE_CHOICE_POP3_EMAIL: + cls = POP3Email + source = get_object_or_404(cls, pk=source_id) redirect_view = reverse('setup_source_transformation_list', args=[source.source_type, source.pk]) From 7f57336d8694655e92e38bc8a0fe46f0040a5723 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Fri, 9 Mar 2012 03:13:40 -0400 Subject: [PATCH 03/11] Initial pop3 email fetch task --- apps/sources/__init__.py | 4 ++ apps/sources/tasks.py | 102 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+) create mode 100644 apps/sources/tasks.py diff --git a/apps/sources/__init__.py b/apps/sources/__init__.py index d03a2a1f6f..3cd5b85b62 100644 --- a/apps/sources/__init__.py +++ b/apps/sources/__init__.py @@ -8,6 +8,7 @@ from common.utils import encapsulate from project_setup.api import register_setup from documents.permissions import (PERMISSION_DOCUMENT_NEW_VERSION, PERMISSION_DOCUMENT_CREATE) +from scheduler.api import register_interval_job from .staging import StagingFile from .models import (WebForm, StagingFolder, SourceTransformation, @@ -16,6 +17,7 @@ from .widgets import staging_file_thumbnail from .permissions import (PERMISSION_SOURCES_SETUP_VIEW, PERMISSION_SOURCES_SETUP_EDIT, PERMISSION_SOURCES_SETUP_DELETE, PERMISSION_SOURCES_SETUP_CREATE) +from .tasks import task_fetch_pop3_emails staging_file_preview = {'text': _(u'preview'), 'class': 'fancybox-noscaling', 'view': 'staging_file_preview', 'args': ['source.source_type', 'source.pk', 'object.id'], 'famfam': 'zoom', 'permissions': [PERMISSION_DOCUMENT_NEW_VERSION, PERMISSION_DOCUMENT_CREATE]} staging_file_delete = {'text': _(u'delete'), 'view': 'staging_file_delete', 'args': ['source.source_type', 'source.pk', 'object.id'], 'famfam': 'delete', 'keep_query': True, 'permissions': [PERMISSION_DOCUMENT_NEW_VERSION, PERMISSION_DOCUMENT_CREATE]} @@ -76,3 +78,5 @@ register_model_list_columns(StagingFile, [ ]) register_setup(setup_sources) + +#register_interval_job('task_fetch_pop3_emails', _(u'Connects to the POP3 email sources and fetches the attached documents.'), task_fetch_pop3_emails, seconds=5)#QUEUE_PROCESSING_INTERVAL) diff --git a/apps/sources/tasks.py b/apps/sources/tasks.py new file mode 100644 index 0000000000..be2a11ac3c --- /dev/null +++ b/apps/sources/tasks.py @@ -0,0 +1,102 @@ +from __future__ import absolute_import + +from datetime import timedelta, datetime +import platform +import logging + +from django.db.models import Q + +from job_processor.api import process_job +from lock_manager import Lock, LockError + +#from .api import do_document_ocr +#from .literals import (QUEUEDOCUMENT_STATE_PENDING, +# QUEUEDOCUMENT_STATE_PROCESSING, DOCUMENTQUEUE_STATE_ACTIVE, +# QUEUEDOCUMENT_STATE_ERROR) +from .models import POP3Email +#from .conf.settings import (NODE_CONCURRENT_EXECUTION, REPLICATION_DELAY, +# QUEUE_PROCESSING_INTERVAL) + +LOCK_EXPIRE = 30 * 1 # Lock expires in 10 minutes +# TODO: Tie LOCK_EXPIRATION with hard task timeout + +logger = logging.getLogger(__name__) + +""" +def task_process_queue_document(queue_document_id): + lock_id = u'task_proc_queue_doc-%d' % queue_document_id + try: + logger.debug('trying to acquire lock: %s' % lock_id) + lock = Lock.acquire_lock(lock_id, LOCK_EXPIRE) + logger.debug('acquired lock: %s' % lock_id) + queue_document = QueueDocument.objects.get(pk=queue_document_id) + queue_document.state = QUEUEDOCUMENT_STATE_PROCESSING + queue_document.node_name = platform.node() + queue_document.save() + try: + do_document_ocr(queue_document) + queue_document.delete() + except Exception, e: + queue_document.state = QUEUEDOCUMENT_STATE_ERROR + queue_document.result = e + queue_document.save() + + lock.release() + except LockError: + logger.debug('unable to obtain lock') + pass +""" + +def task_fetch_single_pop3_email(pop3_email): + try: + lock_id = u'task_fetch_pop3_email-%d' % pop3_email.pk + logger.debug('trying to acquire lock: %s' % lock_id) + lock = Lock.acquire_lock(lock_id, LOCK_EXPIRE) + logger.debug('acquired lock: %s' % lock_id) + #try: + pop3_email.fetch_mail() + #except Exception, exc: + #raise + #finally: + lock.release() + except LockError: + logger.error('unable to obtain lock') + pass + + +def task_fetch_pop3_emails(): + logger.debug('executing') + for pop3_email in POP3Email.objects.filter(enabled=True): + try: + task_fetch_single_pop3_email(pop3_email) + except Exception, exc: + logger.error('Unhandled exception: %s' % exc) + + """ + # TODO: reset_orphans() + q_pending = Q(state=QUEUEDOCUMENT_STATE_PENDING) + q_delayed = Q(delay=True) + q_delay_interval = Q(datetime_submitted__lt=datetime.now() - timedelta(seconds=REPLICATION_DELAY)) + for document_queue in DocumentQueue.objects.filter(state=DOCUMENTQUEUE_STATE_ACTIVE): + current_local_processing_count = QueueDocument.objects.filter( + state=QUEUEDOCUMENT_STATE_PROCESSING).filter( + node_name=platform.node()).count() + if current_local_processing_count < NODE_CONCURRENT_EXECUTION: + try: + oldest_queued_document_qs = document_queue.queuedocument_set.filter( + (q_pending & ~q_delayed) | (q_pending & q_delayed & q_delay_interval)) + + if oldest_queued_document_qs: + oldest_queued_document = oldest_queued_document_qs.order_by('datetime_submitted')[0] + process_job(task_process_queue_document, oldest_queued_document.pk) + except Exception, e: + pass + #print 'DocumentQueueWatcher exception: %s' % e + finally: + # Don't process anymore from this queryset, might be stale + break + else: + logger.debug('already processing maximun') + else: + logger.debug('nothing to process') + """ From 2998fe0611a66be611b2cd026b67d981c77c0afd Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Fri, 9 Mar 2012 03:14:10 -0400 Subject: [PATCH 04/11] Improve attachment retreive code --- apps/sources/models.py | 39 ++++++++++++++++++++++++++++++--------- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/apps/sources/models.py b/apps/sources/models.py index 6e8c807184..a2fe72bab9 100644 --- a/apps/sources/models.py +++ b/apps/sources/models.py @@ -2,6 +2,13 @@ from __future__ import absolute_import from ast import literal_eval import logging +import poplib +import email + +try: + from cStringIO import StringIO +except ImportError: + from StringIO import StringIO from django.db import models from django.utils.translation import ugettext_lazy as _ @@ -31,6 +38,7 @@ from .compressed_file import CompressedFile, NotACompressedFile logger = logging.getLogger(__name__) +POP3_TIMEOUT = 30 class BaseModel(models.Model): title = models.CharField(max_length=64, verbose_name=_(u'title')) @@ -157,30 +165,43 @@ class POP3Email(BaseModel): host = models.CharField(max_length=64, verbose_name=_(u'host')) ssl = models.BooleanField(verbose_name=_(u'SSL')) - port = models.PositiveIntegerField(blank=True, null=True, verbose_name=_(u'port')) + port = models.PositiveIntegerField(default=POP3_PORT, blank=True, null=True, verbose_name=_(u'port'), help_text=_(u'Typical values are: %d and %d for SSL. These are the defaults if no port is specified.') % (POP3_PORT, POP3_SSL_PORT)) username = models.CharField(max_length=64, verbose_name=_(u'username')) password = models.CharField(max_length=64, verbose_name=_(u'password')) uncompress = models.CharField(max_length=1, choices=SOURCE_UNCOMPRESS_CHOICES, verbose_name=_(u'uncompress'), help_text=_(u'Whether to expand or not compressed archives.')) delete_messages = models.BooleanField(verbose_name=_(u'delete messages'), help_text=_(u'Delete messages after downloading their respective attached documents.')) def fetch_mail(self): - if ssl: - port = self.port or POP3_PORT - mailbox = poplib.POP3_SSL(self.host, port) - else: + logger.debug('Starting POP3 email fetch') + logger.debug('host: %s' % self.host) + logger.debug('ssl: %s' % self.ssl) + if self.ssl: port = self.port or POP3_SSL_PORT - mailbox = poplib.POP3(self.host, port) - + logger.debug('port: %d' % port) + mailbox = poplib.POP3_SSL(self.host, port)#, POP3_TIMEOUT) + else: + port = self.port or POP3_PORT + logger.debug('port: %d' % port) + mailbox = poplib.POP3(self.host, port)#, timeout=POP3_TIMEOUT) + mailbox.user(self.username) mailbox.pass_(self.password) message_count = len(mailbox.list()[1]) - logger.debug('message_count: %n' % message_count) + logger.debug('message_count: %d' % message_count) for message_list in range(message_count): for message in mailbox.retr(message_list+1)[1]: mail = email.message_from_string(''.join(message)) for part in mail.walk(): - logger.debug(part) + #logger.debug('part: %s' % part) #f.write(part.get_payload(decode=True)) + container = StringIO() + attachment_name = part.get_filename() + container.write(part.get_payload(decode=True)) + container.seek(0) + f = file('tmp/attach','wb') + f.write(container.read()) + f.close() + class Meta(BaseModel.Meta): verbose_name = _(u'POP email') From 8650e9acb56086b6dbb2ce6ad07da39c1d181137 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Fri, 9 Mar 2012 17:03:30 -0400 Subject: [PATCH 05/11] Improve messages information decoding --- apps/sources/models.py | 64 +++++++++++++++++++++++++++++++----------- 1 file changed, 47 insertions(+), 17 deletions(-) diff --git a/apps/sources/models.py b/apps/sources/models.py index a2fe72bab9..7259e4148c 100644 --- a/apps/sources/models.py +++ b/apps/sources/models.py @@ -171,6 +171,23 @@ class POP3Email(BaseModel): uncompress = models.CharField(max_length=1, choices=SOURCE_UNCOMPRESS_CHOICES, verbose_name=_(u'uncompress'), help_text=_(u'Whether to expand or not compressed archives.')) delete_messages = models.BooleanField(verbose_name=_(u'delete messages'), help_text=_(u'Delete messages after downloading their respective attached documents.')) + @staticmethod + def process_message(message): + pass + #for part in message.walk(): + #logger.debug('part: %s' % part) + # #f.write(part.get_payload(decode=True)) + # logger.debug('payload: %s' % message.get_payload(decode=True) + # container = StringIO() + # attachment_name = part.get_filename() + # container.write(part.get_payload(decode=True)) + # container.seek(0) + # f = file('tmp/attach','wb') + # f.write(container.read()) + # f.close() + + + def fetch_mail(self): logger.debug('Starting POP3 email fetch') logger.debug('host: %s' % self.host) @@ -178,29 +195,42 @@ class POP3Email(BaseModel): if self.ssl: port = self.port or POP3_SSL_PORT logger.debug('port: %d' % port) - mailbox = poplib.POP3_SSL(self.host, port)#, POP3_TIMEOUT) + mailbox = poplib.POP3_SSL(self.host, int(port))#, POP3_TIMEOUT) else: port = self.port or POP3_PORT logger.debug('port: %d' % port) - mailbox = poplib.POP3(self.host, port)#, timeout=POP3_TIMEOUT) + mailbox = poplib.POP3(self.host, int(port))#, timeout=POP3_TIMEOUT) + mailbox.getwelcome() mailbox.user(self.username) mailbox.pass_(self.password) - message_count = len(mailbox.list()[1]) - logger.debug('message_count: %d' % message_count) - for message_list in range(message_count): - for message in mailbox.retr(message_list+1)[1]: - mail = email.message_from_string(''.join(message)) - for part in mail.walk(): - #logger.debug('part: %s' % part) - #f.write(part.get_payload(decode=True)) - container = StringIO() - attachment_name = part.get_filename() - container.write(part.get_payload(decode=True)) - container.seek(0) - f = file('tmp/attach','wb') - f.write(container.read()) - f.close() + messages_info = mailbox.list() + + logger.debug('messages_info:') + logger.debug(messages_info) + logger.debug('messages count: %s' % len(messages_info[1])) + + for message_info in messages_info[1]: + message_number, message_size = message_info.split() + logger.debug('message_number: %s' % message_number) + + #message_size = message.split(' ')[1] + logger.debug('message_size: %s' % message_size) + + complete_message = '\n'.join(mailbox.retr(message_number)[1]) + logger.debug('complete_message: %s' % complete_message) + + POP3Email.process_message(complete_message) + + #for messages in server.list()[1]: + #logger.debug('message_count: %d' % message_count) + #for message_number in range(message_count): + # #message = '\n'.join(mailbox.retr(message_number)[1]) + # message = '\n'.join(mailbox.retr(message_number+1)[1]) + # #message = email.message_from_string('\n'.join(mailbox.retr(message_list+1)[1])) + + # #for message in mailbox.retr(message_list+1)[1]: + # # mail = email.message_from_string(''.join(message)) class Meta(BaseModel.Meta): From 29c374616342505af49575e8a380123629dd73ed Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Sat, 10 Mar 2012 03:10:45 -0400 Subject: [PATCH 06/11] Fetch documents from email attachments --- apps/sources/conf/__init__.py | 0 apps/sources/conf/settings.py | 16 +++++++ apps/sources/models.py | 84 +++++++++++++++++++++-------------- 3 files changed, 66 insertions(+), 34 deletions(-) create mode 100644 apps/sources/conf/__init__.py create mode 100644 apps/sources/conf/settings.py diff --git a/apps/sources/conf/__init__.py b/apps/sources/conf/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/apps/sources/conf/settings.py b/apps/sources/conf/settings.py new file mode 100644 index 0000000000..9babb1d457 --- /dev/null +++ b/apps/sources/conf/settings.py @@ -0,0 +1,16 @@ +"""Configuration options for the sources app""" +import os + +from django.utils.translation import ugettext_lazy as _ +from django.conf import settings + +from smart_settings.api import Setting, SettingNamespace + +namespace = SettingNamespace('sources', _(u'Sources'), module='sources.conf.settings') + +Setting( + namespace=namespace, + name='POP3_TIMEOUT', + global_name='SOURCES_POP3_TIMEOUT', + default=5, +) diff --git a/apps/sources/models.py b/apps/sources/models.py index 7259e4148c..e227661a47 100644 --- a/apps/sources/models.py +++ b/apps/sources/models.py @@ -3,7 +3,9 @@ from __future__ import absolute_import from ast import literal_eval import logging import poplib -import email +from email.Utils import collapse_rfc2231_value +from email import message_from_string +import os try: from cStringIO import StringIO @@ -16,6 +18,8 @@ from django.contrib.contenttypes.models import ContentType from django.contrib.contenttypes import generic from django.core.exceptions import ValidationError from django.db import transaction +from django.core.files import File +from django.core.files.base import ContentFile from converter.api import get_available_transformations_choices from converter.literals import DIMENSION_SEPARATOR @@ -35,10 +39,10 @@ from .literals import (SOURCE_CHOICES, SOURCE_CHOICES_PLURAL, SOURCE_ICON_CHOICES, SOURCE_CHOICE_WATCH, SOURCE_UNCOMPRESS_CHOICES, SOURCE_UNCOMPRESS_CHOICE_Y, POP3_PORT, POP3_SSL_PORT, SOURCE_CHOICE_POP3_EMAIL) from .compressed_file import CompressedFile, NotACompressedFile +from .conf.settings import POP3_TIMEOUT logger = logging.getLogger(__name__) -POP3_TIMEOUT = 30 class BaseModel(models.Model): title = models.CharField(max_length=64, verbose_name=_(u'title')) @@ -157,6 +161,21 @@ class InteractiveBaseModel(BaseModel): class Meta(BaseModel.Meta): abstract = True + + +class PseudoFile(File): + def __init__(self, file, name): + self.name = name + self.file = file + self.file.seek(0, os.SEEK_END) + self.size = self.file.tell() + self.file.seek(0) + + +class Attachment(File): + def __init__(self, part, name): + self.name = name + self.file = PseudoFile(StringIO(part.get_payload(decode=True)), name=name) class POP3Email(BaseModel): @@ -171,22 +190,30 @@ class POP3Email(BaseModel): uncompress = models.CharField(max_length=1, choices=SOURCE_UNCOMPRESS_CHOICES, verbose_name=_(u'uncompress'), help_text=_(u'Whether to expand or not compressed archives.')) delete_messages = models.BooleanField(verbose_name=_(u'delete messages'), help_text=_(u'Delete messages after downloading their respective attached documents.')) + # From: http://bookmarks.honewatson.com/2009/08/11/python-gmail-imaplib-search-subject-get-attachments/ @staticmethod - def process_message(message): - pass - #for part in message.walk(): - #logger.debug('part: %s' % part) - # #f.write(part.get_payload(decode=True)) - # logger.debug('payload: %s' % message.get_payload(decode=True) - # container = StringIO() - # attachment_name = part.get_filename() - # container.write(part.get_payload(decode=True)) - # container.seek(0) - # f = file('tmp/attach','wb') - # f.write(container.read()) - # f.close() - - + def process_message(source, message): + email = message_from_string(message) + counter = 1 + + for part in email.walk(): + disposition = part.get('Content-Disposition') + logger.debug('Disposition: %s' % disposition) + + if disposition.startswith('attachment'): + raw_filename = part.get_filename() + + if raw_filename: + filename = collapse_rfc2231_value(raw_filename) + else: + filename = _(u'attachment-%i') % counter + counter += 1 + + logger.debug('filename: %s' % filename) + + document_file = Attachment(part, name=filename) + source.upload_file(document_file, filename=filename) + def fetch_mail(self): logger.debug('Starting POP3 email fetch') @@ -199,7 +226,7 @@ class POP3Email(BaseModel): else: port = self.port or POP3_PORT logger.debug('port: %d' % port) - mailbox = poplib.POP3(self.host, int(port))#, timeout=POP3_TIMEOUT) + mailbox = poplib.POP3(self.host, int(port), timeout=POP3_TIMEOUT) mailbox.getwelcome() mailbox.user(self.username) @@ -213,26 +240,15 @@ class POP3Email(BaseModel): for message_info in messages_info[1]: message_number, message_size = message_info.split() logger.debug('message_number: %s' % message_number) - - #message_size = message.split(' ')[1] logger.debug('message_size: %s' % message_size) - + complete_message = '\n'.join(mailbox.retr(message_number)[1]) - logger.debug('complete_message: %s' % complete_message) - - POP3Email.process_message(complete_message) - - #for messages in server.list()[1]: - #logger.debug('message_count: %d' % message_count) - #for message_number in range(message_count): - # #message = '\n'.join(mailbox.retr(message_number)[1]) - # message = '\n'.join(mailbox.retr(message_number+1)[1]) - # #message = email.message_from_string('\n'.join(mailbox.retr(message_list+1)[1])) - - # #for message in mailbox.retr(message_list+1)[1]: - # # mail = email.message_from_string(''.join(message)) + POP3Email.process_message(self, complete_message) + mailbox.dele(message_number) + mailbox.quit() + class Meta(BaseModel.Meta): verbose_name = _(u'POP email') verbose_name_plural = _(u'POP email') From 0e060aec46b7b1a30745b7362705c1d461dd2493 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Sat, 10 Mar 2012 03:30:21 -0400 Subject: [PATCH 07/11] Remove SSL timeout option, added POP3_EMAIL_PROCESSING_INTERVAL configuration option --- apps/sources/__init__.py | 3 +- apps/sources/conf/settings.py | 11 +++++ apps/sources/models.py | 15 +++---- apps/sources/tasks.py | 77 ++++------------------------------- 4 files changed, 29 insertions(+), 77 deletions(-) diff --git a/apps/sources/__init__.py b/apps/sources/__init__.py index 3cd5b85b62..1ac08e782a 100644 --- a/apps/sources/__init__.py +++ b/apps/sources/__init__.py @@ -18,6 +18,7 @@ from .permissions import (PERMISSION_SOURCES_SETUP_VIEW, PERMISSION_SOURCES_SETUP_EDIT, PERMISSION_SOURCES_SETUP_DELETE, PERMISSION_SOURCES_SETUP_CREATE) from .tasks import task_fetch_pop3_emails +from .conf.settings import POP3_EMAIL_PROCESSING_INTERVAL staging_file_preview = {'text': _(u'preview'), 'class': 'fancybox-noscaling', 'view': 'staging_file_preview', 'args': ['source.source_type', 'source.pk', 'object.id'], 'famfam': 'zoom', 'permissions': [PERMISSION_DOCUMENT_NEW_VERSION, PERMISSION_DOCUMENT_CREATE]} staging_file_delete = {'text': _(u'delete'), 'view': 'staging_file_delete', 'args': ['source.source_type', 'source.pk', 'object.id'], 'famfam': 'delete', 'keep_query': True, 'permissions': [PERMISSION_DOCUMENT_NEW_VERSION, PERMISSION_DOCUMENT_CREATE]} @@ -79,4 +80,4 @@ register_model_list_columns(StagingFile, [ register_setup(setup_sources) -#register_interval_job('task_fetch_pop3_emails', _(u'Connects to the POP3 email sources and fetches the attached documents.'), task_fetch_pop3_emails, seconds=5)#QUEUE_PROCESSING_INTERVAL) +register_interval_job('task_fetch_pop3_emails', _(u'Connects to the POP3 email sources and fetches the attached documents.'), task_fetch_pop3_emails, seconds=POP3_EMAIL_PROCESSING_INTERVAL) diff --git a/apps/sources/conf/settings.py b/apps/sources/conf/settings.py index 9babb1d457..c4bf07fb46 100644 --- a/apps/sources/conf/settings.py +++ b/apps/sources/conf/settings.py @@ -8,9 +8,20 @@ from smart_settings.api import Setting, SettingNamespace namespace = SettingNamespace('sources', _(u'Sources'), module='sources.conf.settings') +POP3_DEFAULT_TIMEOUT = 5 +POP3_DEFAULT_EMAIL_PROCESSING_INTERVAL = 15 * 60 + Setting( namespace=namespace, name='POP3_TIMEOUT', global_name='SOURCES_POP3_TIMEOUT', default=5, ) + +Setting( + namespace=namespace, + name='POP3_EMAIL_PROCESSING_INTERVAL', + global_name='SOURCES_POP3_EMAIL_PROCESSING_INTERVAL', + default=POP3_DEFAULT_EMAIL_PROCESSING_INTERVAL, +) + diff --git a/apps/sources/models.py b/apps/sources/models.py index e227661a47..f4254c5ebc 100644 --- a/apps/sources/models.py +++ b/apps/sources/models.py @@ -188,16 +188,16 @@ class POP3Email(BaseModel): username = models.CharField(max_length=64, verbose_name=_(u'username')) password = models.CharField(max_length=64, verbose_name=_(u'password')) uncompress = models.CharField(max_length=1, choices=SOURCE_UNCOMPRESS_CHOICES, verbose_name=_(u'uncompress'), help_text=_(u'Whether to expand or not compressed archives.')) - delete_messages = models.BooleanField(verbose_name=_(u'delete messages'), help_text=_(u'Delete messages after downloading their respective attached documents.')) + delete_messages = models.BooleanField(default=True, verbose_name=_(u'delete messages'), help_text=_(u'Delete messages after downloading their respective attached documents.')) # From: http://bookmarks.honewatson.com/2009/08/11/python-gmail-imaplib-search-subject-get-attachments/ @staticmethod - def process_message(source, message): + def process_message(source, message, expand=False): email = message_from_string(message) counter = 1 for part in email.walk(): - disposition = part.get('Content-Disposition') + disposition = part.get('Content-Disposition', 'attachment') logger.debug('Disposition: %s' % disposition) if disposition.startswith('attachment'): @@ -212,7 +212,7 @@ class POP3Email(BaseModel): logger.debug('filename: %s' % filename) document_file = Attachment(part, name=filename) - source.upload_file(document_file, filename=filename) + source.upload_file(document_file, filename=filename, expand=expand) def fetch_mail(self): @@ -222,7 +222,7 @@ class POP3Email(BaseModel): if self.ssl: port = self.port or POP3_SSL_PORT logger.debug('port: %d' % port) - mailbox = poplib.POP3_SSL(self.host, int(port))#, POP3_TIMEOUT) + mailbox = poplib.POP3_SSL(self.host, int(port)) else: port = self.port or POP3_PORT logger.debug('port: %d' % port) @@ -244,8 +244,9 @@ class POP3Email(BaseModel): complete_message = '\n'.join(mailbox.retr(message_number)[1]) - POP3Email.process_message(self, complete_message) - mailbox.dele(message_number) + POP3Email.process_message(source=self, message=complete_message, expand=self.uncompress) + if self.delete_messages: + mailbox.dele(message_number) mailbox.quit() diff --git a/apps/sources/tasks.py b/apps/sources/tasks.py index be2a11ac3c..c7da93e28a 100644 --- a/apps/sources/tasks.py +++ b/apps/sources/tasks.py @@ -9,56 +9,24 @@ from django.db.models import Q from job_processor.api import process_job from lock_manager import Lock, LockError -#from .api import do_document_ocr -#from .literals import (QUEUEDOCUMENT_STATE_PENDING, -# QUEUEDOCUMENT_STATE_PROCESSING, DOCUMENTQUEUE_STATE_ACTIVE, -# QUEUEDOCUMENT_STATE_ERROR) from .models import POP3Email -#from .conf.settings import (NODE_CONCURRENT_EXECUTION, REPLICATION_DELAY, -# QUEUE_PROCESSING_INTERVAL) - -LOCK_EXPIRE = 30 * 1 # Lock expires in 10 minutes -# TODO: Tie LOCK_EXPIRATION with hard task timeout +from .conf.settings import POP3_TIMEOUT logger = logging.getLogger(__name__) -""" -def task_process_queue_document(queue_document_id): - lock_id = u'task_proc_queue_doc-%d' % queue_document_id - try: - logger.debug('trying to acquire lock: %s' % lock_id) - lock = Lock.acquire_lock(lock_id, LOCK_EXPIRE) - logger.debug('acquired lock: %s' % lock_id) - queue_document = QueueDocument.objects.get(pk=queue_document_id) - queue_document.state = QUEUEDOCUMENT_STATE_PROCESSING - queue_document.node_name = platform.node() - queue_document.save() - try: - do_document_ocr(queue_document) - queue_document.delete() - except Exception, e: - queue_document.state = QUEUEDOCUMENT_STATE_ERROR - queue_document.result = e - queue_document.save() - - lock.release() - except LockError: - logger.debug('unable to obtain lock') - pass -""" def task_fetch_single_pop3_email(pop3_email): try: lock_id = u'task_fetch_pop3_email-%d' % pop3_email.pk logger.debug('trying to acquire lock: %s' % lock_id) - lock = Lock.acquire_lock(lock_id, LOCK_EXPIRE) + lock = Lock.acquire_lock(lock_id, POP3_TIMEOUT + 60) # Lock expiration = POP3 timeout + 60 seconds logger.debug('acquired lock: %s' % lock_id) - #try: - pop3_email.fetch_mail() - #except Exception, exc: - #raise - #finally: - lock.release() + try: + pop3_email.fetch_mail() + except Exception, exc: + raise + finally: + lock.release() except LockError: logger.error('unable to obtain lock') pass @@ -71,32 +39,3 @@ def task_fetch_pop3_emails(): task_fetch_single_pop3_email(pop3_email) except Exception, exc: logger.error('Unhandled exception: %s' % exc) - - """ - # TODO: reset_orphans() - q_pending = Q(state=QUEUEDOCUMENT_STATE_PENDING) - q_delayed = Q(delay=True) - q_delay_interval = Q(datetime_submitted__lt=datetime.now() - timedelta(seconds=REPLICATION_DELAY)) - for document_queue in DocumentQueue.objects.filter(state=DOCUMENTQUEUE_STATE_ACTIVE): - current_local_processing_count = QueueDocument.objects.filter( - state=QUEUEDOCUMENT_STATE_PROCESSING).filter( - node_name=platform.node()).count() - if current_local_processing_count < NODE_CONCURRENT_EXECUTION: - try: - oldest_queued_document_qs = document_queue.queuedocument_set.filter( - (q_pending & ~q_delayed) | (q_pending & q_delayed & q_delay_interval)) - - if oldest_queued_document_qs: - oldest_queued_document = oldest_queued_document_qs.order_by('datetime_submitted')[0] - process_job(task_process_queue_document, oldest_queued_document.pk) - except Exception, e: - pass - #print 'DocumentQueueWatcher exception: %s' % e - finally: - # Don't process anymore from this queryset, might be stale - break - else: - logger.debug('already processing maximun') - else: - logger.debug('nothing to process') - """ From b849352b3a256d27c90a1393a6771d6b7bcd65b5 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Sat, 10 Mar 2012 04:26:02 -0400 Subject: [PATCH 08/11] Delete all POP3 messages after their attachments are processed, remove delete_messages field --- apps/sources/conf/settings.py | 6 +- ...to__del_field_pop3email_delete_messages.py | 98 +++++++++++++++++++ apps/sources/models.py | 6 +- 3 files changed, 103 insertions(+), 7 deletions(-) create mode 100644 apps/sources/migrations/0006_auto__del_field_pop3email_delete_messages.py diff --git a/apps/sources/conf/settings.py b/apps/sources/conf/settings.py index c4bf07fb46..15fd5eceab 100644 --- a/apps/sources/conf/settings.py +++ b/apps/sources/conf/settings.py @@ -8,14 +8,14 @@ from smart_settings.api import Setting, SettingNamespace namespace = SettingNamespace('sources', _(u'Sources'), module='sources.conf.settings') -POP3_DEFAULT_TIMEOUT = 5 -POP3_DEFAULT_EMAIL_PROCESSING_INTERVAL = 15 * 60 +POP3_DEFAULT_TIMEOUT = 10 # POP3 only not SSL +POP3_DEFAULT_EMAIL_PROCESSING_INTERVAL = 15 * 60 # 15 minutes Setting( namespace=namespace, name='POP3_TIMEOUT', global_name='SOURCES_POP3_TIMEOUT', - default=5, + default=POP3_DEFAULT_TIMEOUT, ) Setting( diff --git a/apps/sources/migrations/0006_auto__del_field_pop3email_delete_messages.py b/apps/sources/migrations/0006_auto__del_field_pop3email_delete_messages.py new file mode 100644 index 0000000000..f63d5e77af --- /dev/null +++ b/apps/sources/migrations/0006_auto__del_field_pop3email_delete_messages.py @@ -0,0 +1,98 @@ +# encoding: utf-8 +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + +class Migration(SchemaMigration): + + def forwards(self, orm): + + # Deleting field 'POP3Email.delete_messages' + db.delete_column('sources_pop3email', 'delete_messages') + + + def backwards(self, orm): + + # Adding field 'POP3Email.delete_messages' + db.add_column('sources_pop3email', 'delete_messages', self.gf('django.db.models.fields.BooleanField')(default=False), keep_default=False) + + + models = { + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'sources.outofprocess': { + 'Meta': {'ordering': "('title',)", 'object_name': 'OutOfProcess'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + 'sources.pop3email': { + 'Meta': {'ordering': "('title',)", 'object_name': 'POP3Email'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'host': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'port': ('django.db.models.fields.PositiveIntegerField', [], {'default': '110', 'null': 'True', 'blank': 'True'}), + 'ssl': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'username': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + 'sources.sourcetransformation': { + 'Meta': {'ordering': "('order',)", 'object_name': 'SourceTransformation'}, + 'arguments': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'object_id': ('django.db.models.fields.PositiveIntegerField', [], {}), + 'order': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0', 'null': 'True', 'db_index': 'True', 'blank': 'True'}), + 'transformation': ('django.db.models.fields.CharField', [], {'max_length': '128'}) + }, + 'sources.stagingfolder': { + 'Meta': {'ordering': "('title',)", 'object_name': 'StagingFolder'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'delete_after_upload': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'folder_path': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'icon': ('django.db.models.fields.CharField', [], {'max_length': '24', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'preview_height': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}), + 'preview_width': ('django.db.models.fields.IntegerField', [], {}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + 'sources.watchfolder': { + 'Meta': {'ordering': "('title',)", 'object_name': 'WatchFolder'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'delete_after_upload': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'folder_path': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'interval': ('django.db.models.fields.PositiveIntegerField', [], {}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + 'sources.webform': { + 'Meta': {'ordering': "('title',)", 'object_name': 'WebForm'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'icon': ('django.db.models.fields.CharField', [], {'max_length': '24', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + } + } + + complete_apps = ['sources'] diff --git a/apps/sources/models.py b/apps/sources/models.py index f4254c5ebc..9648b6109b 100644 --- a/apps/sources/models.py +++ b/apps/sources/models.py @@ -188,7 +188,6 @@ class POP3Email(BaseModel): username = models.CharField(max_length=64, verbose_name=_(u'username')) password = models.CharField(max_length=64, verbose_name=_(u'password')) uncompress = models.CharField(max_length=1, choices=SOURCE_UNCOMPRESS_CHOICES, verbose_name=_(u'uncompress'), help_text=_(u'Whether to expand or not compressed archives.')) - delete_messages = models.BooleanField(default=True, verbose_name=_(u'delete messages'), help_text=_(u'Delete messages after downloading their respective attached documents.')) # From: http://bookmarks.honewatson.com/2009/08/11/python-gmail-imaplib-search-subject-get-attachments/ @staticmethod @@ -197,7 +196,7 @@ class POP3Email(BaseModel): counter = 1 for part in email.walk(): - disposition = part.get('Content-Disposition', 'attachment') + disposition = part.get('Content-Disposition', 'none') logger.debug('Disposition: %s' % disposition) if disposition.startswith('attachment'): @@ -245,8 +244,7 @@ class POP3Email(BaseModel): complete_message = '\n'.join(mailbox.retr(message_number)[1]) POP3Email.process_message(source=self, message=complete_message, expand=self.uncompress) - if self.delete_messages: - mailbox.dele(message_number) + mailbox.dele(message_number) mailbox.quit() From d1cf15b3eb8b57febe4e1802a4ae75ace35c2aaa Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Mon, 12 Mar 2012 13:36:56 -0400 Subject: [PATCH 09/11] Add per POP email source interval, add per email source log --- apps/sources/__init__.py | 7 +- apps/sources/conf/settings.py | 11 +- apps/sources/literals.py | 1 + apps/sources/managers.py | 13 ++ ...3emaillog__add_field_pop3email_interval.py | 118 ++++++++++++++++++ apps/sources/models.py | 105 +++++++++++----- apps/sources/urls.py | 2 + apps/sources/views.py | 38 +++++- 8 files changed, 257 insertions(+), 38 deletions(-) create mode 100644 apps/sources/migrations/0007_auto__add_pop3emaillog__add_field_pop3email_interval.py diff --git a/apps/sources/__init__.py b/apps/sources/__init__.py index 1ac08e782a..f6c4933d4f 100644 --- a/apps/sources/__init__.py +++ b/apps/sources/__init__.py @@ -23,7 +23,7 @@ from .conf.settings import POP3_EMAIL_PROCESSING_INTERVAL staging_file_preview = {'text': _(u'preview'), 'class': 'fancybox-noscaling', 'view': 'staging_file_preview', 'args': ['source.source_type', 'source.pk', 'object.id'], 'famfam': 'zoom', 'permissions': [PERMISSION_DOCUMENT_NEW_VERSION, PERMISSION_DOCUMENT_CREATE]} staging_file_delete = {'text': _(u'delete'), 'view': 'staging_file_delete', 'args': ['source.source_type', 'source.pk', 'object.id'], 'famfam': 'delete', 'keep_query': True, 'permissions': [PERMISSION_DOCUMENT_NEW_VERSION, PERMISSION_DOCUMENT_CREATE]} -setup_sources = {'text': _(u'sources'), 'view': 'setup_web_form_list', 'famfam': 'application_form', 'icon': 'application_form.png', 'children_classes': [WebForm], 'permissions': [PERMISSION_SOURCES_SETUP_VIEW], 'children_view_regex': [r'setup_web_form', r'setup_staging_folder', r'setup_source_']} +setup_sources = {'text': _(u'sources'), 'view': 'setup_web_form_list', 'famfam': 'application_form', 'icon': 'application_form.png', 'children_classes': [WebForm], 'permissions': [PERMISSION_SOURCES_SETUP_VIEW], 'children_view_regex': [r'setup_web_form', r'setup_staging_folder', r'setup_source_', r'setup_pop3']} setup_web_form_list = {'text': _(u'web forms'), 'view': 'setup_web_form_list', 'famfam': 'application_form', 'icon': 'application_form.png', 'children_classes': [WebForm], 'permissions': [PERMISSION_SOURCES_SETUP_VIEW]} setup_staging_folder_list = {'text': _(u'staging folders'), 'view': 'setup_staging_folder_list', 'famfam': 'folder_camera', 'children_classes': [StagingFolder], 'permissions': [PERMISSION_SOURCES_SETUP_VIEW]} setup_watch_folder_list = {'text': _(u'watch folders'), 'view': 'setup_watch_folder_list', 'famfam': 'folder_magnify', 'children_classes': [WatchFolder], 'permissions': [PERMISSION_SOURCES_SETUP_VIEW]} @@ -32,6 +32,7 @@ setup_pop3_email_list = {'text': _(u'POP3 email'), 'view': 'setup_pop3_email_lis setup_source_edit = {'text': _(u'edit'), 'view': 'setup_source_edit', 'args': ['source.source_type', 'source.pk'], 'famfam': 'application_form_edit', 'permissions': [PERMISSION_SOURCES_SETUP_EDIT]} setup_source_delete = {'text': _(u'delete'), 'view': 'setup_source_delete', 'args': ['source.source_type', 'source.pk'], 'famfam': 'application_form_delete', 'permissions': [PERMISSION_SOURCES_SETUP_DELETE]} setup_source_create = {'text': _(u'add new source'), 'view': 'setup_source_create', 'args': 'source_type', 'famfam': 'application_form_add', 'permissions': [PERMISSION_SOURCES_SETUP_CREATE]} +setup_source_log_list = {'text': _(u'logs'), 'view': 'setup_source_log_list', 'args': ['source.source_type', 'source.pk'], 'famfam': 'book', 'permissions': [PERMISSION_SOURCES_SETUP_EDIT]} setup_source_transformation_list = {'text': _(u'transformations'), 'view': 'setup_source_transformation_list', 'args': ['source.source_type', 'source.pk'], 'famfam': 'shape_move_front', 'permissions': [PERMISSION_SOURCES_SETUP_EDIT]} setup_source_transformation_create = {'text': _(u'add transformation'), 'view': 'setup_source_transformation_create', 'args': ['source.source_type', 'source.pk'], 'famfam': 'shape_square_add', 'permissions': [PERMISSION_SOURCES_SETUP_EDIT]} @@ -53,7 +54,8 @@ register_links(['setup_pop3_email_list', 'setup_web_form_list', 'setup_staging_f register_links(WebForm, [setup_web_form_list, setup_staging_folder_list, setup_pop3_email_list], menu_name='form_header') register_links(WebForm, [setup_source_transformation_list, setup_source_edit, setup_source_delete]) -register_links(['setup_web_form_list', 'setup_staging_folder_list', 'setup_watch_folder_list', 'setup_source_edit', 'setup_source_delete', 'setup_source_create', 'setup_pop3_email_list'], [setup_sources, setup_source_create], menu_name='sidebar') +#register_links(['setup_source_log_list', 'setup_web_form_list', 'setup_staging_folder_list', 'setup_watch_folder_list', 'setup_source_edit', 'setup_source_delete', 'setup_source_create', 'setup_pop3_email_list'], [setup_sources, setup_source_create], menu_name='secondary_menu') +register_links([WebForm, StagingFolder, POP3Email, 'setup_web_form_list', 'setup_staging_folder_list', 'setup_watch_folder_list', 'setup_source_create', 'setup_pop3_email_list'], [setup_source_create], menu_name='secondary_menu') #register_links(StagingFolder, [setup_web_form_list, setup_staging_folder_list, setup_watch_folder_list], menu_name='form_header') register_links(StagingFolder, [setup_web_form_list, setup_staging_folder_list, setup_pop3_email_list], menu_name='form_header') @@ -61,6 +63,7 @@ register_links(StagingFolder, [setup_source_transformation_list, setup_source_ed register_links(POP3Email, [setup_web_form_list, setup_staging_folder_list, setup_pop3_email_list], menu_name='form_header') register_links(POP3Email, [setup_source_transformation_list, setup_source_edit, setup_source_delete]) +register_links(POP3Email, [setup_source_log_list]) register_links(WatchFolder, [setup_web_form_list, setup_staging_folder_list, setup_watch_folder_list], menu_name='form_header') register_links(WatchFolder, [setup_source_transformation_list, setup_source_edit, setup_source_delete]) diff --git a/apps/sources/conf/settings.py b/apps/sources/conf/settings.py index 15fd5eceab..c854ed65d4 100644 --- a/apps/sources/conf/settings.py +++ b/apps/sources/conf/settings.py @@ -8,8 +8,9 @@ from smart_settings.api import Setting, SettingNamespace namespace = SettingNamespace('sources', _(u'Sources'), module='sources.conf.settings') -POP3_DEFAULT_TIMEOUT = 10 # POP3 only not SSL -POP3_DEFAULT_EMAIL_PROCESSING_INTERVAL = 15 * 60 # 15 minutes +POP3_DEFAULT_TIMEOUT = 10 # for POP3 only not POP3_SSL +POP3_DEFAULT_EMAIL_PROCESSING_INTERVAL = 5 +DEFAULT_POP3_EMAIL_LOG_COUNT = 10 # Max log entries to store Setting( namespace=namespace, @@ -25,3 +26,9 @@ Setting( default=POP3_DEFAULT_EMAIL_PROCESSING_INTERVAL, ) +Setting( + namespace=namespace, + name='POP3_EMAIL_LOG_SIZE', + global_name='SOURCES_POP3_EMAIL_LOG_SIZE', + default=DEFAULT_POP3_EMAIL_LOG_COUNT, +) diff --git a/apps/sources/literals.py b/apps/sources/literals.py index 5a3a7dcef3..839e934bc6 100644 --- a/apps/sources/literals.py +++ b/apps/sources/literals.py @@ -2,6 +2,7 @@ from django.utils.translation import ugettext_lazy as _ POP3_PORT = 110 POP3_SSL_PORT = 995 +DEFAULT_POP3_INTERVAL = 15 * 60 # 15 minutes in seconds SOURCE_UNCOMPRESS_CHOICE_Y = 'y' SOURCE_UNCOMPRESS_CHOICE_N = 'n' diff --git a/apps/sources/managers.py b/apps/sources/managers.py index 4e74ec7d4f..b390befa67 100644 --- a/apps/sources/managers.py +++ b/apps/sources/managers.py @@ -1,8 +1,12 @@ +from __future__ import absolute_import + from ast import literal_eval from django.db import models from django.contrib.contenttypes.models import ContentType +from .conf.settings import POP3_EMAIL_LOG_SIZE + class SourceTransformationManager(models.Manager): def get_for_object(self, obj): @@ -24,3 +28,12 @@ class SourceTransformationManager(models.Manager): warnings.append(e) return transformations, warnings + + +class POP3EmailLogManager(models.Manager): + def save_status(self, pop3_email, status): + new_recent = self.model(pop3_email=pop3_email, status=status) + new_recent.save() + to_delete = self.model.objects.filter(pop3_email=pop3_email).order_by('-creation_datetime')[POP3_EMAIL_LOG_SIZE:] + for recent_to_delete in to_delete: + recent_to_delete.delete() diff --git a/apps/sources/migrations/0007_auto__add_pop3emaillog__add_field_pop3email_interval.py b/apps/sources/migrations/0007_auto__add_pop3emaillog__add_field_pop3email_interval.py new file mode 100644 index 0000000000..b4777a0bc5 --- /dev/null +++ b/apps/sources/migrations/0007_auto__add_pop3emaillog__add_field_pop3email_interval.py @@ -0,0 +1,118 @@ +# encoding: utf-8 +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + +class Migration(SchemaMigration): + + def forwards(self, orm): + + # Adding model 'POP3EmailLog' + db.create_table('sources_pop3emaillog', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('pop3_email', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['sources.POP3Email'])), + ('creation_datetime', self.gf('django.db.models.fields.DateTimeField')()), + ('status', self.gf('django.db.models.fields.TextField')()), + )) + db.send_create_signal('sources', ['POP3EmailLog']) + + # Adding field 'POP3Email.interval' + db.add_column('sources_pop3email', 'interval', self.gf('django.db.models.fields.PositiveIntegerField')(default=900), keep_default=False) + + + def backwards(self, orm): + + # Deleting model 'POP3EmailLog' + db.delete_table('sources_pop3emaillog') + + # Deleting field 'POP3Email.interval' + db.delete_column('sources_pop3email', 'interval') + + + models = { + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'sources.outofprocess': { + 'Meta': {'ordering': "('title',)", 'object_name': 'OutOfProcess'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + 'sources.pop3email': { + 'Meta': {'ordering': "('title',)", 'object_name': 'POP3Email'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'host': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'interval': ('django.db.models.fields.PositiveIntegerField', [], {'default': '900'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'port': ('django.db.models.fields.PositiveIntegerField', [], {'null': 'True', 'blank': 'True'}), + 'ssl': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'username': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + 'sources.pop3emaillog': { + 'Meta': {'object_name': 'POP3EmailLog'}, + 'creation_datetime': ('django.db.models.fields.DateTimeField', [], {}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'pop3_email': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['sources.POP3Email']"}), + 'status': ('django.db.models.fields.TextField', [], {}) + }, + 'sources.sourcetransformation': { + 'Meta': {'ordering': "('order',)", 'object_name': 'SourceTransformation'}, + 'arguments': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'object_id': ('django.db.models.fields.PositiveIntegerField', [], {}), + 'order': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0', 'null': 'True', 'db_index': 'True', 'blank': 'True'}), + 'transformation': ('django.db.models.fields.CharField', [], {'max_length': '128'}) + }, + 'sources.stagingfolder': { + 'Meta': {'ordering': "('title',)", 'object_name': 'StagingFolder'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'delete_after_upload': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'folder_path': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'icon': ('django.db.models.fields.CharField', [], {'max_length': '24', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'preview_height': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}), + 'preview_width': ('django.db.models.fields.IntegerField', [], {}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + 'sources.watchfolder': { + 'Meta': {'ordering': "('title',)", 'object_name': 'WatchFolder'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'delete_after_upload': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'folder_path': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'interval': ('django.db.models.fields.PositiveIntegerField', [], {}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + 'sources.webform': { + 'Meta': {'ordering': "('title',)", 'object_name': 'WebForm'}, + 'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'icon': ('django.db.models.fields.CharField', [], {'max_length': '24', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + } + } + + complete_apps = ['sources'] diff --git a/apps/sources/models.py b/apps/sources/models.py index 9648b6109b..60c1b6d796 100644 --- a/apps/sources/models.py +++ b/apps/sources/models.py @@ -6,6 +6,7 @@ import poplib from email.Utils import collapse_rfc2231_value from email import message_from_string import os +import datetime try: from cStringIO import StringIO @@ -32,12 +33,13 @@ from metadata.api import save_metadata_list from scheduler.api import register_interval_job, remove_job from acls.utils import apply_default_acls -from .managers import SourceTransformationManager +from .managers import SourceTransformationManager, POP3EmailLogManager from .literals import (SOURCE_CHOICES, SOURCE_CHOICES_PLURAL, SOURCE_INTERACTIVE_UNCOMPRESS_CHOICES, SOURCE_CHOICE_WEB_FORM, SOURCE_CHOICE_STAGING, SOURCE_ICON_DISK, SOURCE_ICON_DRIVE, SOURCE_ICON_CHOICES, SOURCE_CHOICE_WATCH, SOURCE_UNCOMPRESS_CHOICES, - SOURCE_UNCOMPRESS_CHOICE_Y, POP3_PORT, POP3_SSL_PORT, SOURCE_CHOICE_POP3_EMAIL) + SOURCE_UNCOMPRESS_CHOICE_Y, POP3_PORT, POP3_SSL_PORT, + SOURCE_CHOICE_POP3_EMAIL, DEFAULT_POP3_INTERVAL) from .compressed_file import CompressedFile, NotACompressedFile from .conf.settings import POP3_TIMEOUT @@ -184,10 +186,11 @@ class POP3Email(BaseModel): host = models.CharField(max_length=64, verbose_name=_(u'host')) ssl = models.BooleanField(verbose_name=_(u'SSL')) - port = models.PositiveIntegerField(default=POP3_PORT, blank=True, null=True, verbose_name=_(u'port'), help_text=_(u'Typical values are: %d and %d for SSL. These are the defaults if no port is specified.') % (POP3_PORT, POP3_SSL_PORT)) + port = models.PositiveIntegerField(blank=True, null=True, verbose_name=_(u'port'), help_text=_(u'Typical values are: %d and %d for SSL. These are the defaults if no port is specified.') % (POP3_PORT, POP3_SSL_PORT)) username = models.CharField(max_length=64, verbose_name=_(u'username')) password = models.CharField(max_length=64, verbose_name=_(u'password')) uncompress = models.CharField(max_length=1, choices=SOURCE_UNCOMPRESS_CHOICES, verbose_name=_(u'uncompress'), help_text=_(u'Whether to expand or not compressed archives.')) + interval = models.PositiveIntegerField(default=DEFAULT_POP3_INTERVAL, verbose_name=_(u'interval'), help_text=_(u'Interval in seconds between document downloads from this account.')) # From: http://bookmarks.honewatson.com/2009/08/11/python-gmail-imaplib-search-subject-get-attachments/ @staticmethod @@ -215,43 +218,79 @@ class POP3Email(BaseModel): def fetch_mail(self): - logger.debug('Starting POP3 email fetch') - logger.debug('host: %s' % self.host) - logger.debug('ssl: %s' % self.ssl) - if self.ssl: - port = self.port or POP3_SSL_PORT - logger.debug('port: %d' % port) - mailbox = poplib.POP3_SSL(self.host, int(port)) + try: + last_check = self.pop3emaillog_set.latest().creation_datetime + except POP3EmailLog.DoesNotExist: + # Trigger email fetch when there are no previous logs + initial_trigger = True + difference = datetime.timedelta(seconds=0) else: - port = self.port or POP3_PORT - logger.debug('port: %d' % port) - mailbox = poplib.POP3(self.host, int(port), timeout=POP3_TIMEOUT) - - mailbox.getwelcome() - mailbox.user(self.username) - mailbox.pass_(self.password) - messages_info = mailbox.list() + difference = datetime.datetime.now() - last_check + initial_trigger = False - logger.debug('messages_info:') - logger.debug(messages_info) - logger.debug('messages count: %s' % len(messages_info[1])) - - for message_info in messages_info[1]: - message_number, message_size = message_info.split() - logger.debug('message_number: %s' % message_number) - logger.debug('message_size: %s' % message_size) - - complete_message = '\n'.join(mailbox.retr(message_number)[1]) + if difference >= datetime.timedelta(seconds=self.interval) or initial_trigger: + try: + logger.debug('Starting POP3 email fetch') + logger.debug('host: %s' % self.host) + logger.debug('ssl: %s' % self.ssl) + if self.ssl: + port = self.port or POP3_SSL_PORT + logger.debug('port: %d' % port) + mailbox = poplib.POP3_SSL(self.host, int(port)) + else: + port = self.port or POP3_PORT + logger.debug('port: %d' % port) + mailbox = poplib.POP3(self.host, int(port), timeout=POP3_TIMEOUT) - POP3Email.process_message(source=self, message=complete_message, expand=self.uncompress) - mailbox.dele(message_number) - - mailbox.quit() + mailbox.getwelcome() + mailbox.user(self.username) + mailbox.pass_(self.password) + messages_info = mailbox.list() + + logger.debug('messages_info:') + logger.debug(messages_info) + logger.debug('messages count: %s' % len(messages_info[1])) + + for message_info in messages_info[1]: + message_number, message_size = message_info.split() + logger.debug('message_number: %s' % message_number) + logger.debug('message_size: %s' % message_size) + + complete_message = '\n'.join(mailbox.retr(message_number)[1]) + + POP3Email.process_message(source=self, message=complete_message, expand=self.uncompress) + mailbox.dele(message_number) + + mailbox.quit() + POP3EmailLog.objects.save_status(pop3_email=self, status='Successful connection.') + + except Exception, exc: + logger.error('Unhandled exception: %s' % exc) + POP3EmailLog.objects.save_status(pop3_email=self, status='Error: %s' % exc) class Meta(BaseModel.Meta): verbose_name = _(u'POP email') verbose_name_plural = _(u'POP email') - + + +class POP3EmailLog(models.Model): + pop3_email = models.ForeignKey(POP3Email, verbose_name=_(u'POP3 email')) + creation_datetime = models.DateTimeField(verbose_name=_(u'date time')) + status = models.TextField(verbose_name=_(u'status')) + + objects = POP3EmailLogManager() + + def save(self, *args, **kwargs): + if not self.pk: + self.creation_datetime = datetime.datetime.now() + return super(POP3EmailLog, self).save(*args, **kwargs) + + class Meta: + verbose_name = _(u'POP3 email log') + verbose_name_plural = _(u'POP3 emails logs') + get_latest_by = 'creation_datetime' + ordering = ('creation_datetime',) + class StagingFolder(InteractiveBaseModel): is_interactive = True diff --git a/apps/sources/urls.py b/apps/sources/urls.py index 9498708241..289afb10fa 100644 --- a/apps/sources/urls.py +++ b/apps/sources/urls.py @@ -32,4 +32,6 @@ urlpatterns = patterns('sources.views', url(r'^setup/interactive/(?P\w+)/(?P\d+)/transformation/create/$', 'setup_source_transformation_create', (), 'setup_source_transformation_create'), url(r'^setup/interactive/source/transformation/(?P\d+)/edit/$', 'setup_source_transformation_edit', (), 'setup_source_transformation_edit'), url(r'^setup/interactive/source/transformation/(?P\d+)/delete/$', 'setup_source_transformation_delete', (), 'setup_source_transformation_delete'), + + url(r'^setup/source/(?P\w+)/(?P\d+)/log/list/$', 'setup_source_log_list', (), 'setup_source_log_list'), ) diff --git a/apps/sources/views.py b/apps/sources/views.py index c7462221e4..8f27092b2f 100644 --- a/apps/sources/views.py +++ b/apps/sources/views.py @@ -24,7 +24,7 @@ import sendfile from acls.models import AccessEntry from .models import (WebForm, StagingFolder, SourceTransformation, - WatchFolder, POP3Email) + WatchFolder, POP3Email, POP3EmailLog) from .literals import (SOURCE_CHOICE_WEB_FORM, SOURCE_CHOICE_STAGING, SOURCE_CHOICE_WATCH, SOURCE_CHOICE_POP3_EMAIL) from .literals import (SOURCE_UNCOMPRESS_CHOICE_Y, @@ -590,6 +590,40 @@ def setup_source_create(request, source_type): context_instance=RequestContext(request)) +def setup_source_log_list(request, source_type, source_pk): + Permission.objects.check_permissions(request.user, [PERMISSION_SOURCES_SETUP_EDIT]) + + if source_type == SOURCE_CHOICE_WEB_FORM: + cls = WebForm + elif source_type == SOURCE_CHOICE_STAGING: + cls = StagingFolder + elif source_type == SOURCE_CHOICE_WATCH: + cls = WatchFolder + elif source_type == SOURCE_CHOICE_POP3_EMAIL: + cls = POP3Email + + source = get_object_or_404(cls, pk=source_pk) + + context = { + 'object_list': POP3EmailLog.objects.filter(pop3_email=source).order_by('-creation_datetime'), + 'title': _(u'logs for: %s') % source.fullname(), + 'source': source, + 'object_name': _(u'source'), + 'navigation_object_name': 'source', + 'source_type': source_type, + 'extra_columns': [ + {'name': _(u'Date time'), 'attribute': 'creation_datetime'}, + {'name': _(u'Status'), 'attribute': 'status'}, + ], + 'hide_link': True, + 'hide_object': True, + 'hide_links': True, + } + + return render_to_response('generic_list.html', context, + context_instance=RequestContext(request)) + + def setup_source_transformation_list(request, source_type, source_id): Permission.objects.check_permissions(request.user, [PERMISSION_SOURCES_SETUP_EDIT]) @@ -609,6 +643,7 @@ def setup_source_transformation_list(request, source_type, source_id): 'title': _(u'transformations for: %s') % source.fullname(), 'source': source, 'object_name': _(u'source'), + 'source_type': source_type, 'navigation_object_name': 'source', 'list_object_variable_name': 'transformation', 'extra_columns': [ @@ -724,6 +759,7 @@ def setup_source_transformation_create(request, source_type, source_id): return render_to_response('generic_form.html', { 'form': form, 'source': source, + 'source_type': source_type, 'object_name': _(u'source'), 'navigation_object_name': 'source', 'title': _(u'Create new transformation for source: %s') % source, From c2a0ceb22e735314e4a942868237ffb4a5555250 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Mon, 12 Mar 2012 13:41:47 -0400 Subject: [PATCH 10/11] Initial release notes for 0.13 --- docs/releases/0.13.rst | 94 +++++++++++++++++++++++++++++++++++++++++ docs/releases/index.rst | 3 +- 2 files changed, 96 insertions(+), 1 deletion(-) create mode 100644 docs/releases/0.13.rst diff --git a/docs/releases/0.13.rst b/docs/releases/0.13.rst new file mode 100644 index 0000000000..c75631bc02 --- /dev/null +++ b/docs/releases/0.13.rst @@ -0,0 +1,94 @@ +============================== +Mayan EDMS v0.13 release notes +============================== + +*2012* + +Welcome to Mayan EDMS v0.13! + +Overview +======== + +* POP3 email document source + + +What's new in Mayan EDMS v0.13 +============================== + +E-mail document source +~~~~~~~~~~~~~~~~~~~~~~ + + +Upgrading from a previous version +================================= + +Start off by adding the new requirements:: + + $ pip install -r requirements/production.txt + +Then create the new database structures with:: + + $ ./manage.py syncdb + +Afterwards migrate existing database schema with:: + + $ ./manage.py migrate permissions 0001 --fake + $ ./manage.py migrate permissions + + +When the following message appears + +:: + + The following content types are stale and need to be deleted: + + permissions | permission + + Any objects related to these content types by a foreign key will also + be deleted. Are you sure you want to delete these content types? + If you're unsure, answer 'no'. + + Type 'yes' to continue, or 'no' to cancel: + +Type ``yes`` and press **Enter** + +And continue migrating database schema with:: + + $ ./manage.py migrate documents + $ ./manage.py migrate document_signatures + $ ./manage.py migrate folders 0001 --fake + $ ./manage.py migrate folders + $ ./manage.py migrate document_indexing 0001 --fake + $ ./manage.py migrate document_indexing + $ ./manage.py migrate sources 0001 --fake + $ ./manage.py migrate sources + +Again when a similar messages appears +:: + + The following content types are stale and need to be deleted: + + document_indexing | indexinstance + + Any objects related to these content types by a foreign key will also + be deleted. Are you sure you want to delete these content types? + If you're unsure, answer 'no'. + + Type 'yes' to continue, or 'no' to cancel: + +Type ``yes`` and press **Enter** + +The upgrade procedure is now complete. + + +Backward incompatible changes +============================= +None so far + +Bugs fixed +========== +* Metadata unicode fixes, while creating new documents. + +Stuff removed +============= +None so far diff --git a/docs/releases/index.rst b/docs/releases/index.rst index 3bbee2aa57..a51dcf09ac 100644 --- a/docs/releases/index.rst +++ b/docs/releases/index.rst @@ -11,11 +11,12 @@ all the backwards-incompatible changes and deprecated features for each 'final' release from the one after your current **Mayan EDMS** version, up to and including the new version. -Latest version (0.12) +Latest version (0.13) --------------------- .. toctree:: :maxdepth: 1 + 0.13 0.12 Historic changelogs From 70cb93961189f18fd4c46aebbb746d775a0a71fd Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Mon, 12 Mar 2012 13:42:27 -0400 Subject: [PATCH 11/11] Change version to 0.13 alpha --- apps/main/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/main/__init__.py b/apps/main/__init__.py index 97aa7458f4..e057b7b219 100644 --- a/apps/main/__init__.py +++ b/apps/main/__init__.py @@ -20,9 +20,9 @@ __status__ = 'Production' __version_info__ = { 'major': 0, - 'minor': 12, + 'minor': 13, 'micro': 0, - 'releaselevel': 'final', + 'releaselevel': 'alpha', 'serial': 0 }