matthias/mayan-edms
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge branch 'feature/email_receive' into development

Browse Source
This commit is contained in:
Roberto Rosario
2014-10-20 21:57:58 -04:00
parent f2670e27c8 2575e08499
commit b0c76eab42
11 changed files with 466 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
mayan/apps/sources/__init__.py
View File

@@ -10,6 +10,7 @@ from rest_api.classes import APIEndPoint
from .classes import StagingFile
from .links import (document_create_multiple, document_create_siblings,
setup_imap_email_list, setup_pop3_email_list,
setup_sources, setup_web_form_list, setup_source_create,
setup_source_delete, setup_source_edit,
setup_source_transformation_create,
@@ -18,8 +19,8 @@ from .links import (document_create_multiple, document_create_siblings,
setup_source_transformation_list,
setup_staging_folder_list, setup_watch_folder_list,
staging_file_delete, upload_version)
from .models import (SourceTransformation, StagingFolderSource,
WatchFolderSource, WebFormSource)
from .models import (IMAPEmail, POP3Email, SourceTransformation,
StagingFolderSource, WatchFolderSource, WebFormSource)
from .urls import api_urls
from .widgets import staging_file_thumbnail
@@ -27,17 +28,23 @@ register_links([StagingFile], [staging_file_delete])
register_links(SourceTransformation, [setup_source_transformation_edit, setup_source_transformation_delete])
register_links(['sources:setup_web_form_list', 'sources:setup_staging_folder_list', 'sources:setup_watch_folder_list', 'sources:setup_source_create'], [setup_web_form_list, setup_staging_folder_list], menu_name='form_header')
register_links(['sources:setup_imap_email_list', 'sources:setup_pop3_email_list', 'sources:setup_web_form_list', 'sources:setup_staging_folder_list', 'sources:setup_watch_folder_list', 'sources:setup_source_create'], [setup_web_form_list, setup_staging_folder_list, setup_pop3_email_list, setup_imap_email_list], menu_name='form_header')
register_links(WebFormSource, [setup_web_form_list, setup_staging_folder_list], menu_name='form_header')
register_links(WebFormSource, [setup_web_form_list, setup_staging_folder_list, setup_pop3_email_list, setup_imap_email_list], menu_name='form_header')
register_links(WebFormSource, [setup_source_transformation_list, setup_source_edit, setup_source_delete])
register_links(['sources:setup_web_form_list', 'sources:setup_staging_folder_list', 'sources:setup_watch_folder_list', 'sources:setup_source_edit', 'sources:setup_source_delete', 'sources:setup_source_create'], [setup_sources, setup_source_create], menu_name='sidebar')
register_links(['sources:setup_imap_email_list', 'sources:setup_pop3_email_list', 'sources:setup_web_form_list', 'sources:setup_staging_folder_list', 'sources:setup_watch_folder_list', 'sources:setup_source_edit', 'sources:setup_source_delete', 'sources:setup_source_create'], [setup_sources, setup_source_create], menu_name='sidebar')
register_links(StagingFolderSource, [setup_web_form_list, setup_staging_folder_list], menu_name='form_header')
register_links(StagingFolderSource, [setup_web_form_list, setup_staging_folder_list, setup_pop3_email_list, setup_imap_email_list], menu_name='form_header')
register_links(StagingFolderSource, [setup_source_transformation_list, setup_source_edit, setup_source_delete])
register_links(WatchFolderSource, [setup_web_form_list, setup_staging_folder_list, setup_watch_folder_list], menu_name='form_header')
register_links(POP3Email, [setup_web_form_list, setup_staging_folder_list, setup_pop3_email_list, setup_imap_email_list], menu_name='form_header')
register_links(POP3Email, [setup_source_transformation_list, setup_source_edit, setup_source_delete])
register_links(IMAPEmail, [setup_web_form_list, setup_staging_folder_list, setup_pop3_email_list, setup_imap_email_list], menu_name='form_header')
register_links(IMAPEmail, [setup_source_transformation_list, setup_source_edit, setup_source_delete])
register_links(WatchFolderSource, [setup_web_form_list, setup_staging_folder_list, setup_pop3_email_list, setup_imap_email_list], menu_name='form_header')
register_links(WatchFolderSource, [setup_source_transformation_list, setup_source_edit, setup_source_delete])
# Document version
@@ -45,7 +52,7 @@ register_links(['documents:document_version_list', 'documents:upload_version', '
register_links(['sources:setup_source_transformation_create', 'sources:setup_source_transformation_edit', 'sources:setup_source_transformation_delete', 'sources:setup_source_transformation_list'], [setup_source_transformation_create], menu_name='sidebar')
source_views = ['sources:setup_web_form_list', 'sources:setup_staging_folder_list', 'sources:setup_watch_folder_list', 'sources:setup_source_edit', 'sources:setup_source_delete', 'sources:setup_source_create', 'sources:setup_source_transformation_list', 'sources:setup_source_transformation_edit', 'sources:setup_source_transformation_delete', 'sources:setup_source_transformation_create']
source_views = ['sources:setup_imap_email_list', 'sources:setup_pop3_email_list', 'sources:setup_web_form_list', 'sources:setup_staging_folder_list', 'sources:setup_watch_folder_list', 'sources:setup_source_edit', 'sources:setup_source_delete', 'sources:setup_source_create', 'sources:setup_source_transformation_list', 'sources:setup_source_transformation_edit', 'sources:setup_source_transformation_delete', 'sources:setup_source_transformation_create']
register_model_list_columns(StagingFile, [
{

20
mayan/apps/sources/classes.py
View File

@@ -4,12 +4,32 @@ import base64
import os
import urllib
try:
from cStringIO import StringIO
except ImportError:
from StringIO import StringIO
from django.core.files import File
from converter.api import convert
from mimetype.api import get_mimetype
class PseudoFile(File):
def __init__(self, file, name):
self.name = name
self.file = file
self.file.seek(0, os.SEEK_END)
self.size = self.file.tell()
self.file.seek(0)
class Attachment(File):
def __init__(self, part, name):
self.name = name
self.file = PseudoFile(StringIO(part.get_payload(decode=True)), name=name)
class StagingFile(object):
"""
Simple class to extend the File class to add preview capabilities

14
mayan/apps/sources/forms.py
View File

@@ -8,8 +8,8 @@ from django.utils.translation import ugettext_lazy as _
from documents.forms import DocumentForm
from .models import (SourceTransformation, StagingFolderSource, WebFormSource,
WatchFolderSource)
from .models import (IMAPEmail, POP3Email, SourceTransformation,
StagingFolderSource, WebFormSource, WatchFolderSource)
from .utils import validate_whitelist_blacklist
logger = logging.getLogger(__name__)
@@ -85,6 +85,16 @@ class StagingFolderSetupForm(forms.ModelForm):
model = StagingFolderSource
class POP3EmailSetupForm(forms.ModelForm):
class Meta:
model = POP3Email
class IMAPEmailSetupForm(forms.ModelForm):
class Meta:
model = IMAPEmail
class WatchFolderSetupForm(forms.ModelForm):
class Meta:
model = WatchFolderSource

5
mayan/apps/sources/links.py
View File

@@ -5,7 +5,8 @@ from django.utils.translation import ugettext_lazy as _
from documents.permissions import (PERMISSION_DOCUMENT_CREATE,
PERMISSION_DOCUMENT_NEW_VERSION)
from .models import StagingFolderSource, WatchFolderSource, WebFormSource
from .models import (IMAPEmail, POP3Email, StagingFolderSource,
WatchFolderSource, WebFormSource)
from .permissions import (PERMISSION_SOURCES_SETUP_CREATE,
PERMISSION_SOURCES_SETUP_DELETE,
PERMISSION_SOURCES_SETUP_EDIT,
@@ -20,6 +21,8 @@ setup_sources = {'text': _(u'Sources'), 'view': 'sources:setup_web_form_list', '
setup_web_form_list = {'text': _(u'Web forms'), 'view': 'sources:setup_web_form_list', 'famfam': 'application_form', 'icon': 'application_form.png', 'children_classes': [WebFormSource], 'permissions': [PERMISSION_SOURCES_SETUP_VIEW]}
setup_staging_folder_list = {'text': _(u'Staging folders'), 'view': 'sources:setup_staging_folder_list', 'famfam': 'folder_camera', 'children_classes': [StagingFolderSource], 'permissions': [PERMISSION_SOURCES_SETUP_VIEW]}
setup_watch_folder_list = {'text': _(u'Watch folders'), 'view': 'sources:setup_watch_folder_list', 'famfam': 'folder_magnify', 'children_classes': [WatchFolderSource], 'permissions': [PERMISSION_SOURCES_SETUP_VIEW]}
setup_pop3_email_list = {'text': _(u'POP3 emails'), 'view': 'sources:setup_pop3_email_list', 'famfam': 'email', 'children_classes': [POP3Email], 'permissions': [PERMISSION_SOURCES_SETUP_VIEW]}
setup_imap_email_list = {'text': _(u'IMAP emails'), 'view': 'sources:setup_imap_email_list', 'famfam': 'email', 'children_classes': [IMAPEmail], 'permissions': [PERMISSION_SOURCES_SETUP_VIEW]}
setup_source_edit = {'text': _(u'Edit'), 'view': 'sources:setup_source_edit', 'args': ['source.pk'], 'famfam': 'application_form_edit', 'permissions': [PERMISSION_SOURCES_SETUP_EDIT]}
setup_source_delete = {'text': _(u'Delete'), 'view': 'sources:setup_source_delete', 'args': ['source.pk'], 'famfam': 'application_form_delete', 'permissions': [PERMISSION_SOURCES_SETUP_DELETE]}

12
mayan/apps/sources/literals.py
View File

@@ -18,15 +18,25 @@ SOURCE_INTERACTIVE_UNCOMPRESS_CHOICES = (
SOURCE_CHOICE_WEB_FORM = 'webform'
SOURCE_CHOICE_STAGING = 'staging'
SOURCE_CHOICE_WATCH = 'watch'
SOURCE_CHOICE_EMAIL_POP3 = 'pop3'
SOURCE_CHOICE_EMAIL_IMAP = 'imap'
SOURCE_CHOICES = (
(SOURCE_CHOICE_WEB_FORM, _(u'Web form')),
(SOURCE_CHOICE_STAGING, _(u'Server staging folder')),
(SOURCE_CHOICE_WATCH, _(u'Server watch folder')),
(SOURCE_CHOICE_EMAIL_POP3, _(u'POP3 email')),
(SOURCE_CHOICE_EMAIL_IMAP, _(u'IMAP email')),
)
# TODO: remove PLURALS
SOURCE_CHOICES_PLURAL = (
(SOURCE_CHOICE_WEB_FORM, _(u'Web forms')),
(SOURCE_CHOICE_STAGING, _(u'Server staging folders')),
(SOURCE_CHOICE_WATCH, _(u'Server watch folders')),
)
(SOURCE_CHOICE_EMAIL_POP3, _(u'POP3 emails')),
(SOURCE_CHOICE_EMAIL_IMAP, _(u'IMAP emails')),)
DEFAULT_INTERVAL = 60
DEFAULT_POP3_TIMEOUT = 60
DEFAULT_IMAP_MAILBOX = 'INBOX'

148
mayan/apps/sources/migrations/0005_auto__add_imapemail__add_intervalbasemodel__add_pop3email__add_emailba.py Normal file
View File

@@ -0,0 +1,148 @@
# -*- coding: utf-8 -*-
from south.utils import datetime_utils as datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
class Migration(SchemaMigration):
def forwards(self, orm):
# Adding model 'IMAPEmail'
db.create_table(u'sources_imapemail', (
(u'emailbasemodel_ptr', self.gf('django.db.models.fields.related.OneToOneField')(to=orm['sources.EmailBaseModel'], unique=True, primary_key=True)),
('mailbox', self.gf('django.db.models.fields.CharField')(default='INBOX', max_length=64)),
))
db.send_create_signal(u'sources', ['IMAPEmail'])
# Adding model 'IntervalBaseModel'
db.create_table(u'sources_intervalbasemodel', (
(u'outofprocesssource_ptr', self.gf('django.db.models.fields.related.OneToOneField')(to=orm['sources.OutOfProcessSource'], unique=True, primary_key=True)),
('interval', self.gf('django.db.models.fields.PositiveIntegerField')(default=60)),
('document_type', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['documents.DocumentType'], null=True, blank=True)),
('uncompress', self.gf('django.db.models.fields.CharField')(max_length=1)),
))
db.send_create_signal(u'sources', ['IntervalBaseModel'])
# Adding model 'POP3Email'
db.create_table(u'sources_pop3email', (
(u'emailbasemodel_ptr', self.gf('django.db.models.fields.related.OneToOneField')(to=orm['sources.EmailBaseModel'], unique=True, primary_key=True)),
('timeout', self.gf('django.db.models.fields.PositiveIntegerField')(default=60)),
))
db.send_create_signal(u'sources', ['POP3Email'])
# Adding model 'EmailBaseModel'
db.create_table(u'sources_emailbasemodel', (
(u'intervalbasemodel_ptr', self.gf('django.db.models.fields.related.OneToOneField')(to=orm['sources.IntervalBaseModel'], unique=True, primary_key=True)),
('host', self.gf('django.db.models.fields.CharField')(max_length=128)),
('ssl', self.gf('django.db.models.fields.BooleanField')()),
('port', self.gf('django.db.models.fields.PositiveIntegerField')(null=True, blank=True)),
('username', self.gf('django.db.models.fields.CharField')(max_length=96)),
('password', self.gf('django.db.models.fields.CharField')(max_length=96)),
))
db.send_create_signal(u'sources', ['EmailBaseModel'])
def backwards(self, orm):
# Deleting model 'IMAPEmail'
db.delete_table(u'sources_imapemail')
# Deleting model 'IntervalBaseModel'
db.delete_table(u'sources_intervalbasemodel')
# Deleting model 'POP3Email'
db.delete_table(u'sources_pop3email')
# Deleting model 'EmailBaseModel'
db.delete_table(u'sources_emailbasemodel')
models = {
u'contenttypes.contenttype': {
'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
},
u'documents.documenttype': {
'Meta': {'ordering': "['name']", 'object_name': 'DocumentType'},
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '32'})
},
u'sources.emailbasemodel': {
'Meta': {'ordering': "('title',)", 'object_name': 'EmailBaseModel', '_ormbases': [u'sources.IntervalBaseModel']},
'host': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
u'intervalbasemodel_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': u"orm['sources.IntervalBaseModel']", 'unique': 'True', 'primary_key': 'True'}),
'password': ('django.db.models.fields.CharField', [], {'max_length': '96'}),
'port': ('django.db.models.fields.PositiveIntegerField', [], {'null': 'True', 'blank': 'True'}),
'ssl': ('django.db.models.fields.BooleanField', [], {}),
'username': ('django.db.models.fields.CharField', [], {'max_length': '96'})
},
u'sources.imapemail': {
'Meta': {'ordering': "('title',)", 'object_name': 'IMAPEmail', '_ormbases': [u'sources.EmailBaseModel']},
u'emailbasemodel_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': u"orm['sources.EmailBaseModel']", 'unique': 'True', 'primary_key': 'True'}),
'mailbox': ('django.db.models.fields.CharField', [], {'default': "'INBOX'", 'max_length': '64'})
},
u'sources.interactivesource': {
'Meta': {'ordering': "('title',)", 'object_name': 'InteractiveSource', '_ormbases': [u'sources.Source']},
u'source_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': u"orm['sources.Source']", 'unique': 'True', 'primary_key': 'True'})
},
u'sources.intervalbasemodel': {
'Meta': {'ordering': "('title',)", 'object_name': 'IntervalBaseModel', '_ormbases': [u'sources.OutOfProcessSource']},
'document_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['documents.DocumentType']", 'null': 'True', 'blank': 'True'}),
'interval': ('django.db.models.fields.PositiveIntegerField', [], {'default': '60'}),
u'outofprocesssource_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': u"orm['sources.OutOfProcessSource']", 'unique': 'True', 'primary_key': 'True'}),
'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'})
},
u'sources.outofprocesssource': {
'Meta': {'ordering': "('title',)", 'object_name': 'OutOfProcessSource', '_ormbases': [u'sources.Source']},
u'source_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': u"orm['sources.Source']", 'unique': 'True', 'primary_key': 'True'})
},
u'sources.pop3email': {
'Meta': {'ordering': "('title',)", 'object_name': 'POP3Email', '_ormbases': [u'sources.EmailBaseModel']},
u'emailbasemodel_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': u"orm['sources.EmailBaseModel']", 'unique': 'True', 'primary_key': 'True'}),
'timeout': ('django.db.models.fields.PositiveIntegerField', [], {'default': '60'})
},
u'sources.source': {
'Meta': {'ordering': "('title',)", 'object_name': 'Source'},
'blacklist': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
'whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'})
},
u'sources.sourcetransformation': {
'Meta': {'ordering': "('order',)", 'object_name': 'SourceTransformation'},
'arguments': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'object_id': ('django.db.models.fields.PositiveIntegerField', [], {}),
'order': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0', 'null': 'True', 'db_index': 'True', 'blank': 'True'}),
'transformation': ('django.db.models.fields.CharField', [], {'max_length': '128'})
},
u'sources.stagingfoldersource': {
'Meta': {'ordering': "('title',)", 'object_name': 'StagingFolderSource', '_ormbases': [u'sources.InteractiveSource']},
'delete_after_upload': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'folder_path': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
u'interactivesource_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': u"orm['sources.InteractiveSource']", 'unique': 'True', 'primary_key': 'True'}),
'preview_height': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
'preview_width': ('django.db.models.fields.IntegerField', [], {}),
'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'})
},
u'sources.watchfoldersource': {
'Meta': {'ordering': "('title',)", 'object_name': 'WatchFolderSource', '_ormbases': [u'sources.OutOfProcessSource']},
'delete_after_upload': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'folder_path': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
'interval': ('django.db.models.fields.PositiveIntegerField', [], {}),
u'outofprocesssource_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': u"orm['sources.OutOfProcessSource']", 'unique': 'True', 'primary_key': 'True'}),
'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'})
},
u'sources.webformsource': {
'Meta': {'ordering': "('title',)", 'object_name': 'WebFormSource', '_ormbases': [u'sources.InteractiveSource']},
u'interactivesource_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': u"orm['sources.InteractiveSource']", 'unique': 'True', 'primary_key': 'True'}),
'uncompress': ('django.db.models.fields.CharField', [], {'max_length': '1'})
}
}
complete_apps = ['sources']

193
mayan/apps/sources/models.py
View File

@@ -1,8 +1,13 @@
from __future__ import absolute_import
from ast import literal_eval
from email.Utils import collapse_rfc2231_value
from email import message_from_string
import json
import imaplib
import logging
import os
import poplib
from django.contrib.contenttypes import generic
from django.contrib.contenttypes.models import ContentType
@@ -15,15 +20,18 @@ from model_utils.managers import InheritanceManager
from common.compressed_files import CompressedFile, NotACompressedFile
from converter.api import get_available_transformations_choices
from converter.literals import DIMENSION_SEPARATOR
from documents.models import Document
from djcelery.models import PeriodicTask, IntervalSchedule
from documents.models import Document, DocumentType
from metadata.api import save_metadata_list
from .classes import StagingFile
from .literals import (SOURCE_CHOICES, SOURCE_CHOICES_PLURAL,
SOURCE_CHOICE_STAGING, SOURCE_CHOICE_WATCH,
SOURCE_CHOICE_WEB_FORM,
from .classes import Attachment, StagingFile
from .literals import (DEFAULT_INTERVAL, DEFAULT_POP3_TIMEOUT,
DEFAULT_IMAP_MAILBOX, SOURCE_CHOICES,
SOURCE_CHOICES_PLURAL, SOURCE_CHOICE_STAGING,
SOURCE_CHOICE_WATCH, SOURCE_CHOICE_WEB_FORM,
SOURCE_INTERACTIVE_UNCOMPRESS_CHOICES,
SOURCE_UNCOMPRESS_CHOICES)
SOURCE_UNCOMPRESS_CHOICES, SOURCE_UNCOMPRESS_CHOICE_Y,
SOURCE_CHOICE_EMAIL_IMAP, SOURCE_CHOICE_EMAIL_POP3)
from .managers import SourceTransformationManager
logger = logging.getLogger(__name__)
@@ -32,6 +40,7 @@ logger = logging.getLogger(__name__)
class Source(models.Model):
title = models.CharField(max_length=64, verbose_name=_(u'Title'))
enabled = models.BooleanField(default=True, verbose_name=_(u'Enabled'))
# TODO: remove whitelist and blacklists
whitelist = models.TextField(blank=True, verbose_name=_(u'Whitelist'), editable=False)
blacklist = models.TextField(blank=True, verbose_name=_(u'Blacklist'), editable=False)
@@ -190,8 +199,178 @@ class OutOfProcessSource(Source):
verbose_name_plural = _(u'Out of process')
class IntervalBaseModel(OutOfProcessSource):
interval = models.PositiveIntegerField(default=DEFAULT_INTERVAL, verbose_name=_('Interval'), help_text=_('Interval in seconds between document downloads from this source.'))
document_type = models.ForeignKey(DocumentType, null=True, blank=True, verbose_name=_('Document type'), help_text=_('Assign a document type to documents uploaded from this source.'))
uncompress = models.CharField(max_length=1, choices=SOURCE_UNCOMPRESS_CHOICES, verbose_name=_('Uncompress'), help_text=_('Whether to expand or not, compressed archives.'))
def _get_periodic_task_name(self, pk=None):
return 'check_interval_source-%i' % (pk or self.pk)
def _delete_periodic_task(self, pk=None):
periodic_task = PeriodicTask.objects.get(name=self._get_periodic_task_name(pk))
interval_instance = periodic_task.interval
if tuple(interval_instance.periodictask_set.values_list('id', flat=True)) == (periodic_task.pk,):
# Only delete the interval if nobody else is using it
interval_instance.delete()
else:
periodic_task.delete()
def save(self, *args, **kwargs):
new_source = not self.pk
super(IntervalBaseModel, self).save(*args, **kwargs)
if not new_source:
self._delete_periodic_task()
interval_instance, created = IntervalSchedule.objects.get_or_create(every=self.interval, period='seconds')
# Create a new interval or reuse someone else's
PeriodicTask.objects.create(
name=self._get_periodic_task_name(),
interval=interval_instance,
task='sources.tasks.task_check_interval_source',
queue='mailing',
kwargs=json.dumps({'source_id': self.pk})
)
def delete(self, *args, **kwargs):
pk = self.pk
super(IntervalBaseModel, self).delete(*args, **kwargs)
self._delete_periodic_task(pk)
class Meta:
verbose_name = _('Interval source')
verbose_name_plural = _('Interval sources')
class EmailBaseModel(IntervalBaseModel):
host = models.CharField(max_length=128, verbose_name=_('Host'))
ssl = models.BooleanField(verbose_name=_('SSL'))
port = models.PositiveIntegerField(blank=True, null=True, verbose_name=_('Port'), help_text=_('Typical choices are 110 for POP3, 995 for POP3 over SSL, 143 for IMAP, 993 for IMAP over SSL.'))
username = models.CharField(max_length=96, verbose_name=_('Username'))
password = models.CharField(max_length=96, verbose_name=_('Password'))
# From: http://bookmarks.honewatson.com/2009/08/11/python-gmail-imaplib-search-subject-get-attachments/
# TODO: Add lock to avoid being running more than once concurrent
@staticmethod
def process_message(source, message):
email = message_from_string(message)
counter = 1
for part in email.walk():
disposition = part.get('Content-Disposition', 'none')
logger.debug('Disposition: %s' % disposition)
if disposition.startswith('attachment'):
raw_filename = part.get_filename()
if raw_filename:
filename = collapse_rfc2231_value(raw_filename)
else:
filename = _('attachment-%i') % counter
counter += 1
logger.debug('filename: %s' % filename)
document_file = Attachment(part, name=filename)
source.upload_file(document_file, expand=(source.uncompress == SOURCE_UNCOMPRESS_CHOICE_Y), document_type=source.document_type)
class Meta:
verbose_name = _('Email source')
verbose_name_plural = _('Email sources')
class POP3Email(EmailBaseModel):
source_type = SOURCE_CHOICE_EMAIL_POP3
timeout = models.PositiveIntegerField(default=DEFAULT_POP3_TIMEOUT, verbose_name=_('Timeout'))
def fetch_mail(self):
try:
logger.debug('Starting POP3 email fetch')
logger.debug('host: %s' % self.host)
logger.debug('ssl: %s' % self.ssl)
if self.ssl:
mailbox = poplib.POP3_SSL(self.host, self.port)
else:
mailbox = poplib.POP3(self.host, self.port, timeout=POP3_TIMEOUT)
mailbox.getwelcome()
mailbox.user(self.username)
mailbox.pass_(self.password)
messages_info = mailbox.list()
logger.debug('messages_info:')
logger.debug(messages_info)
logger.debug('messages count: %s' % len(messages_info[1]))
for message_info in messages_info[1]:
message_number, message_size = message_info.split()
logger.debug('message_number: %s' % message_number)
logger.debug('message_size: %s' % message_size)
complete_message = '\n'.join(mailbox.retr(message_number)[1])
EmailBaseModel.process_message(source=self, message=complete_message)
mailbox.dele(message_number)
mailbox.quit()
except Exception as exception:
logger.error('Unhandled exception: %s' % exception)
# TODO: Add user notification
class Meta:
verbose_name = _('POP email')
verbose_name_plural = _('POP email')
class IMAPEmail(EmailBaseModel):
source_type = SOURCE_CHOICE_EMAIL_IMAP
mailbox = models.CharField(max_length=64, default=DEFAULT_IMAP_MAILBOX, verbose_name=_('Mailbox'), help_text=_('Mail from which to check for messages with attached documents.'))
# http://www.doughellmann.com/PyMOTW/imaplib/
def fetch_mail(self):
try:
logger.debug('Starting IMAP email fetch')
logger.debug('host: %s' % self.host)
logger.debug('ssl: %s' % self.ssl)
if self.ssl:
mailbox = imaplib.IMAP4_SSL(self.host, self.port)
else:
mailbox = imaplib.IMAP4(self.host, self.port)
mailbox.login(self.username, self.password)
mailbox.select(self.mailbox)
status, data = mailbox.search(None, 'NOT', 'DELETED')
if data:
messages_info = data[0].split()
logger.debug('messages count: %s' % len(messages_info))
for message_number in messages_info:
logger.debug('message_number: %s' % message_number)
status, data = mailbox.fetch(message_number, '(RFC822)')
EmailBaseModel.process_message(source=self, message=data[0][1])
mailbox.store(message_number, '+FLAGS', '\\Deleted')
mailbox.expunge()
mailbox.close()
mailbox.logout()
except Exception as exception:
logger.error('Unhandled exception: %s' % exception)
# TODO: Add user notification
class Meta:
verbose_name = _('IMAP email')
verbose_name_plural = _('IMAP email')
class WatchFolderSource(OutOfProcessSource):
is_interactive = False
source_type = SOURCE_CHOICE_WATCH
folder_path = models.CharField(max_length=255, verbose_name=_(u'Folder path'), help_text=_(u'Server side filesystem path.'))

7
mayan/apps/sources/tasks.py
View File

@@ -13,6 +13,13 @@ from .models import Source
logger = logging.getLogger(__name__)
@app.task(ignore_result=True)
def task_check_interval_source(source_id):
source = Source.objects.get_subclass(pk=source_id)
if source.enabled:
source.fetch_mail()
@app.task(ignore_result=True)
def task_upload_document(source_id, file_path, filename=None, use_file_name=False, document_type_id=None, expand=False, metadata_dict_list=None, user_id=None, document_id=None, new_version_data=None, command_line=False, description=None):
source = Source.objects.get_subclass(pk=source_id)

5
mayan/apps/sources/urls.py
View File

@@ -5,7 +5,8 @@ from django.conf.urls import patterns, url
from .api_views import (APIDocumentCreateView, APIStagingSourceFileView,
APIStagingSourceFileImageView, APIStagingSourceListView,
APIStagingSourceView)
from .literals import (SOURCE_CHOICE_STAGING, SOURCE_CHOICE_WATCH,
from .literals import (SOURCE_CHOICE_EMAIL_POP3, SOURCE_CHOICE_EMAIL_IMAP,
SOURCE_CHOICE_STAGING, SOURCE_CHOICE_WATCH,
SOURCE_CHOICE_WEB_FORM)
from .wizards import DocumentCreateWizard
@@ -23,6 +24,8 @@ urlpatterns = patterns('sources.views',
url(r'^setup/interactive/%s/list/$' % SOURCE_CHOICE_WEB_FORM, 'setup_source_list', {'source_type': SOURCE_CHOICE_WEB_FORM}, 'setup_web_form_list'),
url(r'^setup/interactive/%s/list/$' % SOURCE_CHOICE_STAGING, 'setup_source_list', {'source_type': SOURCE_CHOICE_STAGING}, 'setup_staging_folder_list'),
url(r'^setup/interactive/%s/list/$' % SOURCE_CHOICE_WATCH, 'setup_source_list', {'source_type': SOURCE_CHOICE_WATCH}, 'setup_watch_folder_list'),
url(r'^setup/interactive/%s/list/$' % SOURCE_CHOICE_EMAIL_POP3, 'setup_source_list', {'source_type': SOURCE_CHOICE_EMAIL_POP3}, 'setup_pop3_email_list'),
url(r'^setup/interactive/%s/list/$' % SOURCE_CHOICE_EMAIL_IMAP, 'setup_source_list', {'source_type': SOURCE_CHOICE_EMAIL_IMAP}, 'setup_imap_email_list'),
url(r'^setup/interactive/(?P<source_type>\w+)/list/$', 'setup_source_list', (), 'setup_source_list'),
url(r'^setup/interactive/(?P<source_id>\d+)/edit/$', 'setup_source_edit', (), 'setup_source_edit'),

87
mayan/apps/sources/views.py
View File

@@ -23,14 +23,16 @@ from documents.permissions import (PERMISSION_DOCUMENT_CREATE,
from metadata.api import decode_metadata_from_url, metadata_repr_as_list
from permissions.models import Permission
from .forms import (StagingDocumentForm, StagingFolderSetupForm,
SourceTransformationForm, SourceTransformationForm_create,
WatchFolderSetupForm, WebFormForm, WebFormSetupForm)
from .literals import (SOURCE_CHOICE_STAGING, SOURCE_CHOICE_WATCH,
from .forms import (POP3EmailSetupForm, IMAPEmailSetupForm, StagingDocumentForm,
StagingFolderSetupForm, SourceTransformationForm,
SourceTransformationForm_create, WatchFolderSetupForm,
WebFormForm, WebFormSetupForm)
from .literals import (SOURCE_CHOICE_EMAIL_IMAP, SOURCE_CHOICE_EMAIL_POP3,
SOURCE_CHOICE_STAGING, SOURCE_CHOICE_WATCH,
SOURCE_CHOICE_WEB_FORM, SOURCE_UNCOMPRESS_CHOICE_ASK,
SOURCE_UNCOMPRESS_CHOICE_Y)
from .models import (Source, StagingFolderSource, SourceTransformation,
WatchFolderSource, WebFormSource)
from .models import (IMAPEmail, POP3Email, Source, StagingFolderSource,
SourceTransformation, WatchFolderSource, WebFormSource)
from .permissions import (PERMISSION_SOURCES_SETUP_CREATE,
PERMISSION_SOURCES_SETUP_DELETE,
PERMISSION_SOURCES_SETUP_EDIT,
@@ -38,6 +40,32 @@ from .permissions import (PERMISSION_SOURCES_SETUP_CREATE,
from .tasks import task_upload_document
def get_class(source_type):
if source_type == SOURCE_CHOICE_WEB_FORM:
return WebFormSource
elif source_type == SOURCE_CHOICE_STAGING:
return StagingFolderSource
elif source_type == SOURCE_CHOICE_WATCH:
return WatchFolderSource
elif source_type == SOURCE_CHOICE_EMAIL_POP3:
return POP3Email
elif source_type == SOURCE_CHOICE_EMAIL_IMAP:
return IMAPEmail
def get_form_class(source_type):
if source_type == SOURCE_CHOICE_WEB_FORM:
return WebFormSetupForm
elif source_type == SOURCE_CHOICE_STAGING:
return StagingFolderSetupForm
elif source_type == SOURCE_CHOICE_WATCH:
return WatchFolderSetupForm
elif source_type == SOURCE_CHOICE_EMAIL_POP3:
return POP3EmailSetupForm
elif source_type == SOURCE_CHOICE_EMAIL_IMAP:
return IMAPEmailSetupForm
def document_create_siblings(request, document_id):
Permission.objects.check_permissions(request.user, [PERMISSION_DOCUMENT_CREATE])
@@ -86,10 +114,20 @@ def get_active_tab_links(document=None):
for staging_folder in staging_folders:
tab_links.append(get_tab_link_for_source(staging_folder, document))
pop3_emails = POP3Email.objects.filter(enabled=True)
for source_instance in pop3_emails:
tab_links.append(get_tab_link_for_source(source_instance, document))
imap_emails = IMAPEmail.objects.filter(enabled=True)
for source_instance in imap_emails:
tab_links.append(get_tab_link_for_source(source_instance, document))
return {
'tab_links': tab_links,
SOURCE_CHOICE_WEB_FORM: web_forms,
SOURCE_CHOICE_STAGING: staging_folders
SOURCE_CHOICE_STAGING: staging_folders,
SOURCE_CHOICE_EMAIL_POP3: pop3_emails,
SOURCE_CHOICE_EMAIL_IMAP: imap_emails
}
@@ -370,13 +408,9 @@ def staging_file_delete(request, staging_folder_pk, encoded_filename):
def setup_source_list(request, source_type):
Permission.objects.check_permissions(request.user, [PERMISSION_SOURCES_SETUP_VIEW])
if source_type == SOURCE_CHOICE_WEB_FORM:
cls = WebFormSource
elif source_type == SOURCE_CHOICE_STAGING:
cls = StagingFolderSource
elif source_type == SOURCE_CHOICE_WATCH:
cls = WatchFolderSource
cls = get_class(source_type)
# TODO: remove plurals
context = {
'object_list': cls.objects.all(),
'title': cls.class_fullname_plural(),
@@ -393,12 +427,7 @@ def setup_source_edit(request, source_id):
Permission.objects.check_permissions(request.user, [PERMISSION_SOURCES_SETUP_EDIT])
source = get_object_or_404(Source.objects.select_subclasses(), pk=source_id)
if isinstance(source, WebFormSource):
form_class = WebFormSetupForm
elif isinstance(source, StagingFolderSource):
form_class = StagingFolderSetupForm
elif isinstance(source, WatchFolderSource):
form_class = WatchFolderSetupForm
form_class = get_form_class(source.source_type)
next = request.POST.get('next', request.GET.get('next', request.META.get('HTTP_REFERER', reverse(settings.LOGIN_REDIRECT_URL))))
@@ -439,6 +468,12 @@ def setup_source_delete(request, source_id):
elif isinstance(source, WatchFolderSource):
form_icon = u'folder_delete.png'
redirect_view = 'sources:setup_watch_folder_list'
elif isinstance(source, POP3Email):
form_icon = u'folder_delete.png'
redirect_view = 'sources:setup_pop3_email_list'
elif isinstance(source, IMAPEmail):
form_icon = u'folder_delete.png'
redirect_view = 'sources:setup_imap_email_list'
previous = request.POST.get('previous', request.GET.get('previous', request.META.get('HTTP_REFERER', redirect_view)))
@@ -450,8 +485,7 @@ def setup_source_delete(request, source_id):
messages.error(request, _(u'Error deleting source "%(source)s": %(error)s') % {
'source': source, 'error': exception
})
return HttpResponseRedirect(redirect_view)
return HttpResponseRedirect(reverse(redirect_view))
context = {
'title': _(u'Are you sure you wish to delete the source: %s?') % source.fullname(),
@@ -471,15 +505,8 @@ def setup_source_delete(request, source_id):
def setup_source_create(request, source_type):
Permission.objects.check_permissions(request.user, [PERMISSION_SOURCES_SETUP_CREATE])
if source_type == SOURCE_CHOICE_WEB_FORM:
cls = WebFormSource
form_class = WebFormSetupForm
elif source_type == SOURCE_CHOICE_STAGING:
cls = WebFormSource
form_class = StagingFolderSetupForm
elif source_type == SOURCE_CHOICE_WATCH:
cls = WebFormSource
form_class = WatchFolderSetupForm
cls = get_class(source_type)
form_class = get_form_class(source_type)
if request.method == 'POST':
form = form_class(data=request.POST)

2
mayan/settings/base.py
View File

@@ -49,6 +49,7 @@ INSTALLED_APPS = (
# 3rd party
'compressor',
'corsheaders',
'djcelery',
'filetransfers',
'mptt',
'rest_framework',
@@ -266,6 +267,7 @@ REST_FRAMEWORK = {
CELERY_TIMEZONE = 'UTC'
CELERY_ENABLE_UTC = True
CELERY_ALWAYS_EAGER = True
CELERYBEAT_SCHEDULER = 'djcelery.schedulers.DatabaseScheduler'
# ------------ CORS ------------
CORS_ORIGIN_ALLOW_ALL = True
# ------ Django REST Swagger -----