matthias/mayan-edms
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Add error logging to document source models

Browse Source
This commit is contained in:
Roberto Rosario
2015-06-09 20:22:58 -04:00
parent adf88b39bf
commit dea882d023
7 changed files with 134 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
mayan/apps/sources/apps.py
View File

@@ -20,7 +20,8 @@ from .links import (
link_setup_source_create_pop3_email,
link_setup_source_create_watch_folder, link_setup_source_create_webform,
link_setup_source_create_staging_folder, link_setup_source_delete,
link_setup_source_edit, link_staging_file_delete, link_upload_version
link_setup_source_edit, link_setup_source_logs, link_staging_file_delete,
link_upload_version
)
from .models import Source
from .widgets import staging_file_thumbnail
@@ -36,7 +37,7 @@ class SourcesApp(apps.AppConfig):
menu_front_page.bind_links(links=[link_document_create_multiple])
menu_object.bind_links(links=[link_document_create_siblings], sources=[Document])
menu_object.bind_links(links=[link_setup_source_edit, link_setup_source_delete, link_transformation_list], sources=[Source])
menu_object.bind_links(links=[link_setup_source_edit, link_setup_source_delete, link_transformation_list, link_setup_source_logs], sources=[Source])
menu_object.bind_links(links=[link_staging_file_delete], sources=[StagingFile])
menu_secondary.bind_links(links=[link_setup_sources, link_setup_source_create_webform, link_setup_source_create_staging_folder, link_setup_source_create_pop3_email, link_setup_source_create_imap_email, link_setup_source_create_watch_folder], sources=[Source, 'sources:setup_source_list', 'sources:setup_source_create'])
menu_setup.bind_links(links=[link_setup_sources])

2
mayan/apps/sources/links.py
View File

@@ -31,3 +31,5 @@ link_setup_source_edit = Link(text=_('Edit'), view='sources:setup_source_edit',
link_source_list = Link(permissions=[PERMISSION_SOURCES_SETUP_VIEW], text=_('Document sources'), view='sources:setup_web_form_list')
link_staging_file_delete = Link(keep_query=True, permissions=[PERMISSION_DOCUMENT_NEW_VERSION, PERMISSION_DOCUMENT_CREATE], tags='dangerous', text=_('Delete'), view='sources:staging_file_delete', args=['source.pk', 'object.encoded_filename'])
link_upload_version = Link(permissions=[PERMISSION_DOCUMENT_NEW_VERSION], text=_('Upload new version'), view='sources:upload_version', args='object.pk')
link_setup_source_logs = Link(text=_('Logs'), view='sources:setup_source_logs', args=['resolved_object.pk'], permissions=[PERMISSION_SOURCES_SETUP_VIEW])

30
mayan/apps/sources/migrations/0003_sourcelog.py Normal file
View File

@@ -0,0 +1,30 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models, migrations
class Migration(migrations.Migration):
dependencies = [
('sources', '0002_auto_20150608_1902'),
]
operations = [
migrations.CreateModel(
name='SourceLog',
fields=[
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
('datetime', models.DateTimeField(auto_now_add=True, verbose_name='Date time')),
('message', models.TextField(verbose_name='Message', editable=False, blank=True)),
('source', models.ForeignKey(related_name='logs', verbose_name='Source', to='sources.Source')),
],
options={
'ordering': ['-datetime'],
'get_latest_by': 'datetime',
'verbose_name': 'Log entry',
'verbose_name_plural': 'Log entries',
},
bases=(models.Model,),
),
]

108
mayan/apps/sources/models.py
View File

@@ -116,6 +116,7 @@ class StagingFolderSource(InteractiveSource):
for entry in sorted([os.path.normcase(f) for f in os.listdir(self.folder_path) if os.path.isfile(os.path.join(self.folder_path, f))]):
yield self.get_file(filename=entry)
except OSError as exception:
logger.error('Unable get list of staging files from source: %s; %s', self, exception)
raise Exception(_('Unable get list of staging files: %s') % exception)
def get_upload_file_object(self, form_data):
@@ -127,6 +128,7 @@ class StagingFolderSource(InteractiveSource):
try:
upload_file_object.extra_data.delete()
except Exception as exception:
logger.error('Error deleting staging file: %s; %s', upload_file_object, exception)
raise Exception(_('Error deleting staging file; %s') % exception)
class Meta:
@@ -248,39 +250,35 @@ class POP3Email(EmailBaseModel):
timeout = models.PositiveIntegerField(default=DEFAULT_POP3_TIMEOUT, verbose_name=_('Timeout'))
def check_source(self):
try:
logger.debug('Starting POP3 email fetch')
logger.debug('host: %s', self.host)
logger.debug('ssl: %s', self.ssl)
logger.debug('Starting POP3 email fetch')
logger.debug('host: %s', self.host)
logger.debug('ssl: %s', self.ssl)
if self.ssl:
mailbox = poplib.POP3_SSL(self.host, self.port)
else:
mailbox = poplib.POP3(self.host, self.port, timeout=self.timeout)
if self.ssl:
mailbox = poplib.POP3_SSL(self.host, self.port)
else:
mailbox = poplib.POP3(self.host, self.port, timeout=self.timeout)
mailbox.getwelcome()
mailbox.user(self.username)
mailbox.pass_(self.password)
messages_info = mailbox.list()
mailbox.getwelcome()
mailbox.user(self.username)
mailbox.pass_(self.password)
messages_info = mailbox.list()
logger.debug('messages_info:')
logger.debug(messages_info)
logger.debug('messages count: %s', len(messages_info[1]))
logger.debug('messages_info:')
logger.debug(messages_info)
logger.debug('messages count: %s', len(messages_info[1]))
for message_info in messages_info[1]:
message_number, message_size = message_info.split()
logger.debug('message_number: %s', message_number)
logger.debug('message_size: %s', message_size)
for message_info in messages_info[1]:
message_number, message_size = message_info.split()
logger.debug('message_number: %s', message_number)
logger.debug('message_size: %s', message_size)
complete_message = '\n'.join(mailbox.retr(message_number)[1])
complete_message = '\n'.join(mailbox.retr(message_number)[1])
EmailBaseModel.process_message(source=self, message=complete_message)
mailbox.dele(message_number)
EmailBaseModel.process_message(source=self, message=complete_message)
mailbox.dele(message_number)
mailbox.quit()
except Exception as exception:
logger.error('Unhandled exception: %s', exception)
# TODO: Add user notification
mailbox.quit()
class Meta:
verbose_name = _('POP email')
@@ -294,36 +292,32 @@ class IMAPEmail(EmailBaseModel):
# http://www.doughellmann.com/PyMOTW/imaplib/
def check_source(self):
try:
logger.debug('Starting IMAP email fetch')
logger.debug('host: %s', self.host)
logger.debug('ssl: %s', self.ssl)
logger.debug('Starting IMAP email fetch')
logger.debug('host: %s', self.host)
logger.debug('ssl: %s', self.ssl)
if self.ssl:
mailbox = imaplib.IMAP4_SSL(self.host, self.port)
else:
mailbox = imaplib.IMAP4(self.host, self.port)
if self.ssl:
mailbox = imaplib.IMAP4_SSL(self.host, self.port)
else:
mailbox = imaplib.IMAP4(self.host, self.port)
mailbox.login(self.username, self.password)
mailbox.select(self.mailbox)
mailbox.login(self.username, self.password)
mailbox.select(self.mailbox)
status, data = mailbox.search(None, 'NOT', 'DELETED')
if data:
messages_info = data[0].split()
logger.debug('messages count: %s', len(messages_info))
status, data = mailbox.search(None, 'NOT', 'DELETED')
if data:
messages_info = data[0].split()
logger.debug('messages count: %s', len(messages_info))
for message_number in messages_info:
logger.debug('message_number: %s', message_number)
status, data = mailbox.fetch(message_number, '(RFC822)')
EmailBaseModel.process_message(source=self, message=data[0][1])
mailbox.store(message_number, '+FLAGS', '\\Deleted')
for message_number in messages_info:
logger.debug('message_number: %s', message_number)
status, data = mailbox.fetch(message_number, '(RFC822)')
EmailBaseModel.process_message(source=self, message=data[0][1])
mailbox.store(message_number, '+FLAGS', '\\Deleted')
mailbox.expunge()
mailbox.close()
mailbox.logout()
except Exception as exception:
logger.error('Unhandled exception: %s', exception)
# TODO: Add user notification
mailbox.expunge()
mailbox.close()
mailbox.logout()
class Meta:
verbose_name = _('IMAP email')
@@ -347,3 +341,15 @@ class WatchFolderSource(IntervalBaseModel):
class Meta:
verbose_name = _('Watch folder')
verbose_name_plural = _('Watch folders')
class SourceLog(models.Model):
source = models.ForeignKey(Source, related_name='logs', verbose_name=_('Source'))
datetime = models.DateTimeField(auto_now_add=True, editable=False, verbose_name=_('Date time'))
message = models.TextField(blank=True, editable=False, verbose_name=_('Message'))
class Meta:
verbose_name = _('Log entry')
verbose_name_plural = _('Log entries')
get_latest_by = 'datetime'
ordering = ['-datetime']

8
mayan/apps/sources/tasks.py
View File

@@ -17,7 +17,13 @@ logger = logging.getLogger(__name__)
def task_check_interval_source(source_id):
source = Source.objects.get_subclass(pk=source_id)
if source.enabled:
source.check_source()
try:
source.check_source()
except Exception as exception:
logger.error('Error processing source: %s; %s', source, exception)
self.logs.create(message=_('Error processing source: %s') % exception)
else:
self.logs.all().delete()
@app.task(ignore_result=True)

5
mayan/apps/sources/urls.py
View File

@@ -6,7 +6,9 @@ from .api_views import (
APIStagingSourceFileView, APIStagingSourceFileImageView,
APIStagingSourceListView, APIStagingSourceView
)
from .views import UploadInteractiveVersionView, UploadInteractiveView
from .views import (
SourceLogListView, UploadInteractiveVersionView, UploadInteractiveView
)
from .wizards import DocumentCreateWizard
urlpatterns = patterns(
@@ -23,6 +25,7 @@ urlpatterns = patterns(
url(r'^setup/list/$', 'setup_source_list', (), 'setup_source_list'),
url(r'^setup/(?P<source_id>\d+)/edit/$', 'setup_source_edit', (), 'setup_source_edit'),
url(r'^setup/(?P<pk>\d+)/logs/$', SourceLogListView.as_view(), name='setup_source_logs'),
url(r'^setup/(?P<source_id>\d+)/delete/$', 'setup_source_delete', (), 'setup_source_delete'),
url(r'^setup/(?P<source_type>\w+)/create/$', 'setup_source_create', (), 'setup_source_create'),

32
mayan/apps/sources/views.py
View File

@@ -14,7 +14,7 @@ from acls.models import AccessEntry
from common import menu_facet
from common.models import SharedUploadedFile
from common.utils import encapsulate
from common.views import MultiFormView
from common.views import MultiFormView, ParentChildListView
from documents.models import DocumentType, Document
from documents.permissions import (
PERMISSION_DOCUMENT_CREATE, PERMISSION_DOCUMENT_NEW_VERSION
@@ -42,6 +42,36 @@ from .tasks import task_source_upload_document
from .utils import get_class, get_form_class, get_upload_form_class
class SourceLogListView(ParentChildListView):
object_permission = PERMISSION_SOURCES_SETUP_VIEW
parent_queryset = Source.objects.select_subclasses()
def get_queryset(self):
return self.get_object().logs.all()
def get_context_data(self, **kwargs):
context = super(SourceLogListView, self).get_context_data(**kwargs)
context.update(
{
'title': _('Log entries for source: %s') % self.get_object(),
'hide_object': True,
'extra_columns': [
{
'name': _('Date time'),
'attribute': encapsulate(lambda entry: entry.datetime)
},
{
'name': _('Message'),
'attribute': encapsulate(lambda entry: entry.message)
},
]
}
)
return context
def document_create_siblings(request, document_id):
Permission.objects.check_permissions(request.user, [PERMISSION_DOCUMENT_CREATE])