Split sources models into separate modules
Signed-off-by: Roberto Rosario <roberto.rosario.gonzalez@gmail.com>
This commit is contained in:
Roberto Rosario
252
mayan/apps/sources/models/base.py
Normal file
252
mayan/apps/sources/models/base.py
Normal file
@@ -0,0 +1,252 @@
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import json
|
||||
import logging
|
||||
|
||||
from django.db import models, transaction
|
||||
from django.utils.encoding import force_text, python_2_unicode_compatible
|
||||
from django.utils.translation import ugettext_lazy as _
|
||||
|
||||
from model_utils.managers import InheritanceManager
|
||||
|
||||
from common.compressed_files import Archive
|
||||
from common.exceptions import NoMIMETypeMatch
|
||||
from converter.models import Transformation
|
||||
from djcelery.models import PeriodicTask, IntervalSchedule
|
||||
from documents.models import Document, DocumentType
|
||||
from documents.settings import setting_language
|
||||
|
||||
from ..literals import (
|
||||
DEFAULT_INTERVAL, SOURCE_CHOICES, SOURCE_UNCOMPRESS_CHOICES
|
||||
)
|
||||
from ..wizards import WizardStep
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@python_2_unicode_compatible
|
||||
class Source(models.Model):
|
||||
label = models.CharField(
|
||||
db_index=True, max_length=64, unique=True, verbose_name=_('Label')
|
||||
)
|
||||
enabled = models.BooleanField(default=True, verbose_name=_('Enabled'))
|
||||
|
||||
objects = InheritanceManager()
|
||||
|
||||
class Meta:
|
||||
ordering = ('label',)
|
||||
verbose_name = _('Source')
|
||||
verbose_name_plural = _('Sources')
|
||||
|
||||
def __str__(self):
|
||||
return '%s' % self.label
|
||||
|
||||
@classmethod
|
||||
def class_fullname(cls):
|
||||
return force_text(dict(SOURCE_CHOICES).get(cls.source_type))
|
||||
|
||||
def clean_up_upload_file(self, upload_file_object):
|
||||
pass
|
||||
# TODO: Should raise NotImplementedError?
|
||||
|
||||
def fullname(self):
|
||||
return ' '.join([self.class_fullname(), '"%s"' % self.label])
|
||||
|
||||
def handle_upload(self, file_object, description=None, document_type=None, expand=False, label=None, language=None, user=None):
|
||||
"""
|
||||
Handle an upload request from a file object which may be an individual
|
||||
document or a compressed file containing multiple documents.
|
||||
"""
|
||||
documents = []
|
||||
if not document_type:
|
||||
document_type = self.document_type
|
||||
|
||||
kwargs = {
|
||||
'description': description, 'document_type': document_type,
|
||||
'label': label, 'language': language, 'user': user
|
||||
}
|
||||
|
||||
if expand:
|
||||
try:
|
||||
compressed_file = Archive.open(file_object=file_object)
|
||||
for compressed_file_child in compressed_file.members():
|
||||
with compressed_file.open_member(filename=compressed_file_child) as file_object:
|
||||
kwargs.update(
|
||||
{'label': force_text(compressed_file_child)}
|
||||
)
|
||||
documents.append(
|
||||
self.upload_document(
|
||||
file_object=file_object, **kwargs
|
||||
)
|
||||
)
|
||||
except NoMIMETypeMatch:
|
||||
logging.debug('Exception: NoMIMETypeMatch')
|
||||
documents.append(
|
||||
self.upload_document(file_object=file_object, **kwargs)
|
||||
)
|
||||
else:
|
||||
documents.append(
|
||||
self.upload_document(file_object=file_object, **kwargs)
|
||||
)
|
||||
|
||||
# Return a list of newly created documents. Used by the email source
|
||||
# to assign the from and subject metadata values.
|
||||
return documents
|
||||
|
||||
def get_upload_file_object(self, form_data):
|
||||
pass
|
||||
# TODO: Should raise NotImplementedError?
|
||||
|
||||
def upload_document(self, file_object, document_type, description=None, label=None, language=None, querystring=None, user=None):
|
||||
"""
|
||||
Upload an individual document
|
||||
"""
|
||||
try:
|
||||
with transaction.atomic():
|
||||
document = Document(
|
||||
description=description or '', document_type=document_type,
|
||||
label=label or file_object.name,
|
||||
language=language or setting_language.value
|
||||
)
|
||||
document.save(_user=user)
|
||||
except Exception as exception:
|
||||
logger.critical(
|
||||
'Unexpected exception while trying to create new document '
|
||||
'"%s" from source "%s"; %s',
|
||||
label or file_object.name, self, exception
|
||||
)
|
||||
raise
|
||||
else:
|
||||
try:
|
||||
document_version = document.new_version(
|
||||
file_object=file_object, _user=user,
|
||||
)
|
||||
|
||||
if user:
|
||||
document.add_as_recent_document_for_user(user)
|
||||
|
||||
Transformation.objects.copy(
|
||||
source=self, targets=document_version.pages.all()
|
||||
)
|
||||
|
||||
except Exception as exception:
|
||||
logger.critical(
|
||||
'Unexpected exception while trying to create version for '
|
||||
'new document "%s" from source "%s"; %s',
|
||||
label or file_object.name, self, exception, exc_info=True
|
||||
)
|
||||
document.delete(to_trash=False)
|
||||
raise
|
||||
else:
|
||||
WizardStep.post_upload_process(
|
||||
document=document, querystring=querystring
|
||||
)
|
||||
return document
|
||||
|
||||
|
||||
class InteractiveSource(Source):
|
||||
objects = InheritanceManager()
|
||||
|
||||
class Meta:
|
||||
verbose_name = _('Interactive source')
|
||||
verbose_name_plural = _('Interactive sources')
|
||||
|
||||
|
||||
class OutOfProcessSource(Source):
|
||||
is_interactive = False
|
||||
|
||||
objects = models.Manager()
|
||||
|
||||
class Meta:
|
||||
verbose_name = _('Out of process')
|
||||
verbose_name_plural = _('Out of process')
|
||||
|
||||
|
||||
class IntervalBaseModel(OutOfProcessSource):
|
||||
interval = models.PositiveIntegerField(
|
||||
default=DEFAULT_INTERVAL,
|
||||
help_text=_('Interval in seconds between checks for new documents.'),
|
||||
verbose_name=_('Interval')
|
||||
)
|
||||
document_type = models.ForeignKey(
|
||||
DocumentType,
|
||||
help_text=_(
|
||||
'Assign a document type to documents uploaded from this source.'
|
||||
), on_delete=models.CASCADE,
|
||||
verbose_name=_('Document type')
|
||||
)
|
||||
uncompress = models.CharField(
|
||||
choices=SOURCE_UNCOMPRESS_CHOICES,
|
||||
help_text=_('Whether to expand or not, compressed archives.'),
|
||||
max_length=1, verbose_name=_('Uncompress')
|
||||
)
|
||||
|
||||
objects = models.Manager()
|
||||
|
||||
class Meta:
|
||||
verbose_name = _('Interval source')
|
||||
verbose_name_plural = _('Interval sources')
|
||||
|
||||
def _delete_periodic_task(self, pk=None):
|
||||
try:
|
||||
periodic_task = PeriodicTask.objects.get(
|
||||
name=self._get_periodic_task_name(pk)
|
||||
)
|
||||
|
||||
interval_instance = periodic_task.interval
|
||||
|
||||
if tuple(interval_instance.periodictask_set.values_list('id', flat=True)) == (periodic_task.pk,):
|
||||
# Only delete the interval if nobody else is using it
|
||||
interval_instance.delete()
|
||||
else:
|
||||
periodic_task.delete()
|
||||
except PeriodicTask.DoesNotExist:
|
||||
logger.warning(
|
||||
'Tried to delete non existant periodic task "%s"',
|
||||
self._get_periodic_task_name(pk)
|
||||
)
|
||||
|
||||
def _get_periodic_task_name(self, pk=None):
|
||||
return 'check_interval_source-%i' % (pk or self.pk)
|
||||
|
||||
def delete(self, *args, **kwargs):
|
||||
pk = self.pk
|
||||
super(IntervalBaseModel, self).delete(*args, **kwargs)
|
||||
self._delete_periodic_task(pk)
|
||||
|
||||
def save(self, *args, **kwargs):
|
||||
new_source = not self.pk
|
||||
super(IntervalBaseModel, self).save(*args, **kwargs)
|
||||
|
||||
if not new_source:
|
||||
self._delete_periodic_task()
|
||||
|
||||
interval_instance, created = IntervalSchedule.objects.get_or_create(
|
||||
every=self.interval, period='seconds'
|
||||
)
|
||||
# Create a new interval or reuse someone else's
|
||||
PeriodicTask.objects.create(
|
||||
name=self._get_periodic_task_name(),
|
||||
interval=interval_instance,
|
||||
task='sources.tasks.task_check_interval_source',
|
||||
kwargs=json.dumps({'source_id': self.pk})
|
||||
)
|
||||
|
||||
|
||||
class SourceLog(models.Model):
|
||||
source = models.ForeignKey(
|
||||
on_delete=models.CASCADE, related_name='logs', to=Source,
|
||||
verbose_name=_('Source')
|
||||
)
|
||||
datetime = models.DateTimeField(
|
||||
auto_now_add=True, editable=False, verbose_name=_('Date time')
|
||||
)
|
||||
message = models.TextField(
|
||||
blank=True, editable=False, verbose_name=_('Message')
|
||||
)
|
||||
|
||||
class Meta:
|
||||
get_latest_by = 'datetime'
|
||||
ordering = ('-datetime',)
|
||||
verbose_name = _('Log entry')
|
||||
verbose_name_plural = _('Log entries')
|
||||
Reference in New Issue
Block a user