From dea882d023759484e06cbbaaab35154296603583 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 9 Jun 2015 20:22:58 -0400 Subject: [PATCH] Add error logging to document source models --- mayan/apps/sources/apps.py | 5 +- mayan/apps/sources/links.py | 2 + .../apps/sources/migrations/0003_sourcelog.py | 30 +++++ mayan/apps/sources/models.py | 108 +++++++++--------- mayan/apps/sources/tasks.py | 8 +- mayan/apps/sources/urls.py | 5 +- mayan/apps/sources/views.py | 32 +++++- 7 files changed, 134 insertions(+), 56 deletions(-) create mode 100644 mayan/apps/sources/migrations/0003_sourcelog.py diff --git a/mayan/apps/sources/apps.py b/mayan/apps/sources/apps.py index 67b943e6a2..301edd1326 100644 --- a/mayan/apps/sources/apps.py +++ b/mayan/apps/sources/apps.py @@ -20,7 +20,8 @@ from .links import ( link_setup_source_create_pop3_email, link_setup_source_create_watch_folder, link_setup_source_create_webform, link_setup_source_create_staging_folder, link_setup_source_delete, - link_setup_source_edit, link_staging_file_delete, link_upload_version + link_setup_source_edit, link_setup_source_logs, link_staging_file_delete, + link_upload_version ) from .models import Source from .widgets import staging_file_thumbnail @@ -36,7 +37,7 @@ class SourcesApp(apps.AppConfig): menu_front_page.bind_links(links=[link_document_create_multiple]) menu_object.bind_links(links=[link_document_create_siblings], sources=[Document]) - menu_object.bind_links(links=[link_setup_source_edit, link_setup_source_delete, link_transformation_list], sources=[Source]) + menu_object.bind_links(links=[link_setup_source_edit, link_setup_source_delete, link_transformation_list, link_setup_source_logs], sources=[Source]) menu_object.bind_links(links=[link_staging_file_delete], sources=[StagingFile]) menu_secondary.bind_links(links=[link_setup_sources, link_setup_source_create_webform, link_setup_source_create_staging_folder, link_setup_source_create_pop3_email, link_setup_source_create_imap_email, link_setup_source_create_watch_folder], sources=[Source, 'sources:setup_source_list', 'sources:setup_source_create']) menu_setup.bind_links(links=[link_setup_sources]) diff --git a/mayan/apps/sources/links.py b/mayan/apps/sources/links.py index 40ba430de7..ad8ec0be47 100644 --- a/mayan/apps/sources/links.py +++ b/mayan/apps/sources/links.py @@ -31,3 +31,5 @@ link_setup_source_edit = Link(text=_('Edit'), view='sources:setup_source_edit', link_source_list = Link(permissions=[PERMISSION_SOURCES_SETUP_VIEW], text=_('Document sources'), view='sources:setup_web_form_list') link_staging_file_delete = Link(keep_query=True, permissions=[PERMISSION_DOCUMENT_NEW_VERSION, PERMISSION_DOCUMENT_CREATE], tags='dangerous', text=_('Delete'), view='sources:staging_file_delete', args=['source.pk', 'object.encoded_filename']) link_upload_version = Link(permissions=[PERMISSION_DOCUMENT_NEW_VERSION], text=_('Upload new version'), view='sources:upload_version', args='object.pk') +link_setup_source_logs = Link(text=_('Logs'), view='sources:setup_source_logs', args=['resolved_object.pk'], permissions=[PERMISSION_SOURCES_SETUP_VIEW]) + diff --git a/mayan/apps/sources/migrations/0003_sourcelog.py b/mayan/apps/sources/migrations/0003_sourcelog.py new file mode 100644 index 0000000000..555aa4fc20 --- /dev/null +++ b/mayan/apps/sources/migrations/0003_sourcelog.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('sources', '0002_auto_20150608_1902'), + ] + + operations = [ + migrations.CreateModel( + name='SourceLog', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('datetime', models.DateTimeField(auto_now_add=True, verbose_name='Date time')), + ('message', models.TextField(verbose_name='Message', editable=False, blank=True)), + ('source', models.ForeignKey(related_name='logs', verbose_name='Source', to='sources.Source')), + ], + options={ + 'ordering': ['-datetime'], + 'get_latest_by': 'datetime', + 'verbose_name': 'Log entry', + 'verbose_name_plural': 'Log entries', + }, + bases=(models.Model,), + ), + ] diff --git a/mayan/apps/sources/models.py b/mayan/apps/sources/models.py index 7a6136e307..51f991e727 100644 --- a/mayan/apps/sources/models.py +++ b/mayan/apps/sources/models.py @@ -116,6 +116,7 @@ class StagingFolderSource(InteractiveSource): for entry in sorted([os.path.normcase(f) for f in os.listdir(self.folder_path) if os.path.isfile(os.path.join(self.folder_path, f))]): yield self.get_file(filename=entry) except OSError as exception: + logger.error('Unable get list of staging files from source: %s; %s', self, exception) raise Exception(_('Unable get list of staging files: %s') % exception) def get_upload_file_object(self, form_data): @@ -127,6 +128,7 @@ class StagingFolderSource(InteractiveSource): try: upload_file_object.extra_data.delete() except Exception as exception: + logger.error('Error deleting staging file: %s; %s', upload_file_object, exception) raise Exception(_('Error deleting staging file; %s') % exception) class Meta: @@ -248,39 +250,35 @@ class POP3Email(EmailBaseModel): timeout = models.PositiveIntegerField(default=DEFAULT_POP3_TIMEOUT, verbose_name=_('Timeout')) def check_source(self): - try: - logger.debug('Starting POP3 email fetch') - logger.debug('host: %s', self.host) - logger.debug('ssl: %s', self.ssl) + logger.debug('Starting POP3 email fetch') + logger.debug('host: %s', self.host) + logger.debug('ssl: %s', self.ssl) - if self.ssl: - mailbox = poplib.POP3_SSL(self.host, self.port) - else: - mailbox = poplib.POP3(self.host, self.port, timeout=self.timeout) + if self.ssl: + mailbox = poplib.POP3_SSL(self.host, self.port) + else: + mailbox = poplib.POP3(self.host, self.port, timeout=self.timeout) - mailbox.getwelcome() - mailbox.user(self.username) - mailbox.pass_(self.password) - messages_info = mailbox.list() + mailbox.getwelcome() + mailbox.user(self.username) + mailbox.pass_(self.password) + messages_info = mailbox.list() - logger.debug('messages_info:') - logger.debug(messages_info) - logger.debug('messages count: %s', len(messages_info[1])) + logger.debug('messages_info:') + logger.debug(messages_info) + logger.debug('messages count: %s', len(messages_info[1])) - for message_info in messages_info[1]: - message_number, message_size = message_info.split() - logger.debug('message_number: %s', message_number) - logger.debug('message_size: %s', message_size) + for message_info in messages_info[1]: + message_number, message_size = message_info.split() + logger.debug('message_number: %s', message_number) + logger.debug('message_size: %s', message_size) - complete_message = '\n'.join(mailbox.retr(message_number)[1]) + complete_message = '\n'.join(mailbox.retr(message_number)[1]) - EmailBaseModel.process_message(source=self, message=complete_message) - mailbox.dele(message_number) + EmailBaseModel.process_message(source=self, message=complete_message) + mailbox.dele(message_number) - mailbox.quit() - except Exception as exception: - logger.error('Unhandled exception: %s', exception) - # TODO: Add user notification + mailbox.quit() class Meta: verbose_name = _('POP email') @@ -294,36 +292,32 @@ class IMAPEmail(EmailBaseModel): # http://www.doughellmann.com/PyMOTW/imaplib/ def check_source(self): - try: - logger.debug('Starting IMAP email fetch') - logger.debug('host: %s', self.host) - logger.debug('ssl: %s', self.ssl) + logger.debug('Starting IMAP email fetch') + logger.debug('host: %s', self.host) + logger.debug('ssl: %s', self.ssl) - if self.ssl: - mailbox = imaplib.IMAP4_SSL(self.host, self.port) - else: - mailbox = imaplib.IMAP4(self.host, self.port) + if self.ssl: + mailbox = imaplib.IMAP4_SSL(self.host, self.port) + else: + mailbox = imaplib.IMAP4(self.host, self.port) - mailbox.login(self.username, self.password) - mailbox.select(self.mailbox) + mailbox.login(self.username, self.password) + mailbox.select(self.mailbox) - status, data = mailbox.search(None, 'NOT', 'DELETED') - if data: - messages_info = data[0].split() - logger.debug('messages count: %s', len(messages_info)) + status, data = mailbox.search(None, 'NOT', 'DELETED') + if data: + messages_info = data[0].split() + logger.debug('messages count: %s', len(messages_info)) - for message_number in messages_info: - logger.debug('message_number: %s', message_number) - status, data = mailbox.fetch(message_number, '(RFC822)') - EmailBaseModel.process_message(source=self, message=data[0][1]) - mailbox.store(message_number, '+FLAGS', '\\Deleted') + for message_number in messages_info: + logger.debug('message_number: %s', message_number) + status, data = mailbox.fetch(message_number, '(RFC822)') + EmailBaseModel.process_message(source=self, message=data[0][1]) + mailbox.store(message_number, '+FLAGS', '\\Deleted') - mailbox.expunge() - mailbox.close() - mailbox.logout() - except Exception as exception: - logger.error('Unhandled exception: %s', exception) - # TODO: Add user notification + mailbox.expunge() + mailbox.close() + mailbox.logout() class Meta: verbose_name = _('IMAP email') @@ -347,3 +341,15 @@ class WatchFolderSource(IntervalBaseModel): class Meta: verbose_name = _('Watch folder') verbose_name_plural = _('Watch folders') + + +class SourceLog(models.Model): + source = models.ForeignKey(Source, related_name='logs', verbose_name=_('Source')) + datetime = models.DateTimeField(auto_now_add=True, editable=False, verbose_name=_('Date time')) + message = models.TextField(blank=True, editable=False, verbose_name=_('Message')) + + class Meta: + verbose_name = _('Log entry') + verbose_name_plural = _('Log entries') + get_latest_by = 'datetime' + ordering = ['-datetime'] diff --git a/mayan/apps/sources/tasks.py b/mayan/apps/sources/tasks.py index 87557df418..200533b6db 100644 --- a/mayan/apps/sources/tasks.py +++ b/mayan/apps/sources/tasks.py @@ -17,7 +17,13 @@ logger = logging.getLogger(__name__) def task_check_interval_source(source_id): source = Source.objects.get_subclass(pk=source_id) if source.enabled: - source.check_source() + try: + source.check_source() + except Exception as exception: + logger.error('Error processing source: %s; %s', source, exception) + self.logs.create(message=_('Error processing source: %s') % exception) + else: + self.logs.all().delete() @app.task(ignore_result=True) diff --git a/mayan/apps/sources/urls.py b/mayan/apps/sources/urls.py index bdfb1604e6..5aa87b7428 100644 --- a/mayan/apps/sources/urls.py +++ b/mayan/apps/sources/urls.py @@ -6,7 +6,9 @@ from .api_views import ( APIStagingSourceFileView, APIStagingSourceFileImageView, APIStagingSourceListView, APIStagingSourceView ) -from .views import UploadInteractiveVersionView, UploadInteractiveView +from .views import ( + SourceLogListView, UploadInteractiveVersionView, UploadInteractiveView +) from .wizards import DocumentCreateWizard urlpatterns = patterns( @@ -23,6 +25,7 @@ urlpatterns = patterns( url(r'^setup/list/$', 'setup_source_list', (), 'setup_source_list'), url(r'^setup/(?P\d+)/edit/$', 'setup_source_edit', (), 'setup_source_edit'), + url(r'^setup/(?P\d+)/logs/$', SourceLogListView.as_view(), name='setup_source_logs'), url(r'^setup/(?P\d+)/delete/$', 'setup_source_delete', (), 'setup_source_delete'), url(r'^setup/(?P\w+)/create/$', 'setup_source_create', (), 'setup_source_create'), diff --git a/mayan/apps/sources/views.py b/mayan/apps/sources/views.py index d203064701..aad9d8d849 100644 --- a/mayan/apps/sources/views.py +++ b/mayan/apps/sources/views.py @@ -14,7 +14,7 @@ from acls.models import AccessEntry from common import menu_facet from common.models import SharedUploadedFile from common.utils import encapsulate -from common.views import MultiFormView +from common.views import MultiFormView, ParentChildListView from documents.models import DocumentType, Document from documents.permissions import ( PERMISSION_DOCUMENT_CREATE, PERMISSION_DOCUMENT_NEW_VERSION @@ -42,6 +42,36 @@ from .tasks import task_source_upload_document from .utils import get_class, get_form_class, get_upload_form_class +class SourceLogListView(ParentChildListView): + object_permission = PERMISSION_SOURCES_SETUP_VIEW + parent_queryset = Source.objects.select_subclasses() + + def get_queryset(self): + return self.get_object().logs.all() + + def get_context_data(self, **kwargs): + context = super(SourceLogListView, self).get_context_data(**kwargs) + + context.update( + { + 'title': _('Log entries for source: %s') % self.get_object(), + 'hide_object': True, + 'extra_columns': [ + { + 'name': _('Date time'), + 'attribute': encapsulate(lambda entry: entry.datetime) + }, + { + 'name': _('Message'), + 'attribute': encapsulate(lambda entry: entry.message) + }, + ] + } + ) + + return context + + def document_create_siblings(request, document_id): Permission.objects.check_permissions(request.user, [PERMISSION_DOCUMENT_CREATE])