matthias/mayan-edms
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Add per POP email source interval, add per email source log

Browse Source
This commit is contained in:
Roberto Rosario
2012-03-12 13:36:56 -04:00
parent b849352b3a
commit d1cf15b3eb
8 changed files with 257 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

105
apps/sources/models.py
View File

@@ -6,6 +6,7 @@ import poplib
from email.Utils import collapse_rfc2231_value
from email import message_from_string
import os
import datetime
try:
from cStringIO import StringIO
@@ -32,12 +33,13 @@ from metadata.api import save_metadata_list
from scheduler.api import register_interval_job, remove_job
from acls.utils import apply_default_acls
from .managers import SourceTransformationManager
from .managers import SourceTransformationManager, POP3EmailLogManager
from .literals import (SOURCE_CHOICES, SOURCE_CHOICES_PLURAL,
SOURCE_INTERACTIVE_UNCOMPRESS_CHOICES, SOURCE_CHOICE_WEB_FORM,
SOURCE_CHOICE_STAGING, SOURCE_ICON_DISK, SOURCE_ICON_DRIVE,
SOURCE_ICON_CHOICES, SOURCE_CHOICE_WATCH, SOURCE_UNCOMPRESS_CHOICES,
SOURCE_UNCOMPRESS_CHOICE_Y, POP3_PORT, POP3_SSL_PORT, SOURCE_CHOICE_POP3_EMAIL)
SOURCE_UNCOMPRESS_CHOICE_Y, POP3_PORT, POP3_SSL_PORT,
SOURCE_CHOICE_POP3_EMAIL, DEFAULT_POP3_INTERVAL)
from .compressed_file import CompressedFile, NotACompressedFile
from .conf.settings import POP3_TIMEOUT
@@ -184,10 +186,11 @@ class POP3Email(BaseModel):
host = models.CharField(max_length=64, verbose_name=_(u'host'))
ssl = models.BooleanField(verbose_name=_(u'SSL'))
port = models.PositiveIntegerField(default=POP3_PORT, blank=True, null=True, verbose_name=_(u'port'), help_text=_(u'Typical values are: %d and %d for SSL. These are the defaults if no port is specified.') % (POP3_PORT, POP3_SSL_PORT))
port = models.PositiveIntegerField(blank=True, null=True, verbose_name=_(u'port'), help_text=_(u'Typical values are: %d and %d for SSL. These are the defaults if no port is specified.') % (POP3_PORT, POP3_SSL_PORT))
username = models.CharField(max_length=64, verbose_name=_(u'username'))
password = models.CharField(max_length=64, verbose_name=_(u'password'))
uncompress = models.CharField(max_length=1, choices=SOURCE_UNCOMPRESS_CHOICES, verbose_name=_(u'uncompress'), help_text=_(u'Whether to expand or not compressed archives.'))
interval = models.PositiveIntegerField(default=DEFAULT_POP3_INTERVAL, verbose_name=_(u'interval'), help_text=_(u'Interval in seconds between document downloads from this account.'))
# From: http://bookmarks.honewatson.com/2009/08/11/python-gmail-imaplib-search-subject-get-attachments/
@staticmethod
@@ -215,43 +218,79 @@ class POP3Email(BaseModel):
def fetch_mail(self):
logger.debug('Starting POP3 email fetch')
logger.debug('host: %s' % self.host)
logger.debug('ssl: %s' % self.ssl)
if self.ssl:
port = self.port or POP3_SSL_PORT
logger.debug('port: %d' % port)
mailbox = poplib.POP3_SSL(self.host, int(port))
try:
last_check = self.pop3emaillog_set.latest().creation_datetime
except POP3EmailLog.DoesNotExist:
# Trigger email fetch when there are no previous logs
initial_trigger = True
difference = datetime.timedelta(seconds=0)
else:
port = self.port or POP3_PORT
logger.debug('port: %d' % port)
mailbox = poplib.POP3(self.host, int(port), timeout=POP3_TIMEOUT)
mailbox.getwelcome()
mailbox.user(self.username)
mailbox.pass_(self.password)
messages_info = mailbox.list()
difference = datetime.datetime.now() - last_check
initial_trigger = False
logger.debug('messages_info:')
logger.debug(messages_info)
logger.debug('messages count: %s' % len(messages_info[1]))
for message_info in messages_info[1]:
message_number, message_size = message_info.split()
logger.debug('message_number: %s' % message_number)
logger.debug('message_size: %s' % message_size)
complete_message = '\n'.join(mailbox.retr(message_number)[1])
if difference >= datetime.timedelta(seconds=self.interval) or initial_trigger:
try:
logger.debug('Starting POP3 email fetch')
logger.debug('host: %s' % self.host)
logger.debug('ssl: %s' % self.ssl)
if self.ssl:
port = self.port or POP3_SSL_PORT
logger.debug('port: %d' % port)
mailbox = poplib.POP3_SSL(self.host, int(port))
else:
port = self.port or POP3_PORT
logger.debug('port: %d' % port)
mailbox = poplib.POP3(self.host, int(port), timeout=POP3_TIMEOUT)
POP3Email.process_message(source=self, message=complete_message, expand=self.uncompress)
mailbox.dele(message_number)
mailbox.quit()
mailbox.getwelcome()
mailbox.user(self.username)
mailbox.pass_(self.password)
messages_info = mailbox.list()
logger.debug('messages_info:')
logger.debug(messages_info)
logger.debug('messages count: %s' % len(messages_info[1]))
for message_info in messages_info[1]:
message_number, message_size = message_info.split()
logger.debug('message_number: %s' % message_number)
logger.debug('message_size: %s' % message_size)
complete_message = '\n'.join(mailbox.retr(message_number)[1])
POP3Email.process_message(source=self, message=complete_message, expand=self.uncompress)
mailbox.dele(message_number)
mailbox.quit()
POP3EmailLog.objects.save_status(pop3_email=self, status='Successful connection.')
except Exception, exc:
logger.error('Unhandled exception: %s' % exc)
POP3EmailLog.objects.save_status(pop3_email=self, status='Error: %s' % exc)
class Meta(BaseModel.Meta):
verbose_name = _(u'POP email')
verbose_name_plural = _(u'POP email')
class POP3EmailLog(models.Model):
pop3_email = models.ForeignKey(POP3Email, verbose_name=_(u'POP3 email'))
creation_datetime = models.DateTimeField(verbose_name=_(u'date time'))
status = models.TextField(verbose_name=_(u'status'))
objects = POP3EmailLogManager()
def save(self, *args, **kwargs):
if not self.pk:
self.creation_datetime = datetime.datetime.now()
return super(POP3EmailLog, self).save(*args, **kwargs)
class Meta:
verbose_name = _(u'POP3 email log')
verbose_name_plural = _(u'POP3 emails logs')
get_latest_by = 'creation_datetime'
ordering = ('creation_datetime',)
class StagingFolder(InteractiveBaseModel):
is_interactive = True