matthias/mayan-edms
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Refactor job processing app to do actual job queue and job subprocess launching, remove queue mananger app, update ocr app to use new job processing app

Browse Source
This commit is contained in:
Roberto Rosario
2012-07-30 07:36:02 -04:00
parent 20856b1589
commit 486f983d4b
27 changed files with 409 additions and 531 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
apps/ocr/__init__.py
View File

@@ -17,11 +17,11 @@ from project_tools.api import register_tool
from acls.api import class_permissions
from scheduler.api import register_interval_job
from statistics.api import register_statistics
from queue_manager.models import Queue
from job_processor.models import JobQueue, JobType
from .conf.settings import (AUTOMATIC_OCR, QUEUE_PROCESSING_INTERVAL)
from .models import OCRProcessingSingleton
from .tasks import task_process_document_queues
from .api import do_document_ocr
from .permissions import PERMISSION_OCR_DOCUMENT
from .exceptions import AlreadyQueued
from . import models as ocr_models
@@ -29,6 +29,7 @@ from .statistics import get_statistics
from .literals import OCR_QUEUE_NAME
logger = logging.getLogger(__name__)
ocr_job_queue = None
from .links import (submit_document, re_queue_multiple_document,
queue_document_multiple_delete, ocr_disable,
@@ -37,21 +38,17 @@ from .links import (submit_document, re_queue_multiple_document,
bind_links([Document], [submit_document])
bind_links([OCRProcessingSingleton], [ocr_disable, ocr_enable])
#bind_links([QueueTransformation], [setup_queue_transformation_edit, setup_queue_transformation_delete])
#register_multi_item_links(['queue_document_list'], [re_queue_multiple_document, queue_document_multiple_delete])
#bind_links(['setup_queue_transformation_create', 'setup_queue_transformation_edit', 'setup_queue_transformation_delete', 'document_queue_disable', 'document_queue_enable', 'queue_document_list', 'setup_queue_transformation_list'], [queue_document_list], menu_name='secondary_menu')
#bind_links(['setup_queue_transformation_edit', 'setup_queue_transformation_delete', 'setup_queue_transformation_list', 'setup_queue_transformation_create'], [setup_queue_transformation_create], menu_name='sidebar')
register_maintenance_links([all_document_ocr_cleanup], namespace='ocr', title=_(u'OCR'))
#register_multi_item_links(['folder_view', 'search', 'results', 'index_instance_node_view', 'document_find_duplicates', 'document_type_document_list', 'document_group_view', 'document_list', 'document_list_recent'], [submit_document_multiple])
register_multi_item_links(['folder_view', 'search', 'results', 'index_instance_node_view', 'document_find_duplicates', 'document_type_document_list', 'document_group_view', 'document_list', 'document_list_recent'], [submit_document_multiple])
@transaction.commit_on_success
def create_ocr_queue():
def create_ocr_job_queue():
global ocr_job_queue
try:
queue, created = Queue.objects.get_or_create(name=OCR_QUEUE_NAME, defaults={'label': _('OCR'), 'unique_names': True})
ocr_job_queue, created = JobQueue.objects.get_or_create(name=OCR_QUEUE_NAME, defaults={'label': _('OCR'), 'unique_jobs': True})
except DatabaseError:
transaction.rollback()
@@ -76,12 +73,6 @@ def document_post_save(sender, instance, **kwargs):
# logger.debug('got call_queue signal: %s' % kwargs)
# task_process_document_queues()
#@receiver(post_syncdb, dispatch_uid='create_ocr_queue_on_syncdb', sender=ocr_models)
#def create_ocr_queue_on_syncdb(sender, **kwargs):
#register_interval_job('task_process_document_queues', _(u'Checks the OCR queue for pending documents.'), task_process_document_queues, seconds=QUEUE_PROCESSING_INTERVAL)
register_tool(ocr_tool_link)
class_permissions(Document, [
@@ -89,4 +80,5 @@ class_permissions(Document, [
])
#register_statistics(get_statistics)
create_ocr_queue()
create_ocr_job_queue()
ocr_job_type = JobType('ocr', _(u'OCR'), do_document_ocr)

22
apps/ocr/admin.py
View File

@@ -1,22 +0,0 @@
"""
from __future__ import absolute_import
from django.contrib import admin
from .models import DocumentQueue, QueueDocument
class QueueDocumentInline(admin.StackedInline):
model = QueueDocument
extra = 1
classes = ('collapse-open',)
allow_add = True
class DocumentQueueAdmin(admin.ModelAdmin):
inlines = [QueueDocumentInline]
list_display = ('name', 'label', 'state')
admin.site.register(DocumentQueue, DocumentQueueAdmin)
"""

10
apps/ocr/api.py
View File

@@ -12,7 +12,7 @@ from django.utils.importlib import import_module
from common.conf.settings import TEMPORARY_DIRECTORY
from converter.api import convert
from documents.models import DocumentPage
from documents.models import DocumentPage, DocumentVersion
from .conf.settings import (TESSERACT_PATH, TESSERACT_LANGUAGE, UNPAPER_PATH)
from .exceptions import TesseractError, UnpaperError
@@ -81,25 +81,25 @@ def run_tesseract(input_filename, lang=None):
return text
def do_document_ocr(queue_document):
def do_document_ocr(document_version_pk):
"""
Try first to extract text from document pages using the registered
parser, if the parser fails or if there is no parser registered for
the document mimetype do a visual OCR by calling tesseract
"""
for document_page in queue_document.document_version.pages.all():
document_version = DocumentVersion.objects.get(pk=document_version_pk)
for document_page in document_version.pages.all():
try:
# Try to extract text by means of a parser
parse_document_page(document_page)
except (ParserError, ParserUnknownFile):
# Fall back to doing visual OCR
ocr_transformations, warnings = queue_document.get_transformation_list()
document_filepath = document_page.document.get_image_cache_name(page=document_page.page_number, version=document_page.document_version.pk)
unpaper_output_filename = u'%s_unpaper_out_page_%s%s%s' % (document_page.document.uuid, document_page.page_number, os.extsep, UNPAPER_FILE_FORMAT)
unpaper_output_filepath = os.path.join(TEMPORARY_DIRECTORY, unpaper_output_filename)
unpaper_input = convert(document_filepath, file_format=UNPAPER_FILE_FORMAT, transformations=ocr_transformations)
unpaper_input = convert(document_filepath, file_format=UNPAPER_FILE_FORMAT)
execute_unpaper(input_filepath=unpaper_input, output_filepath=unpaper_output_filepath)
#from PIL import Image, ImageOps

23
apps/ocr/forms.py
View File

@@ -1,23 +0,0 @@
"""
from __future__ import absolute_import
from django import forms
from .models import QueueTransformation
class QueueTransformationForm(forms.ModelForm):
class Meta:
model = QueueTransformation
def __init__(self, *args, **kwargs):
super(QueueTransformationForm, self).__init__(*args, **kwargs)
self.fields['content_type'].widget = forms.HiddenInput()
self.fields['object_id'].widget = forms.HiddenInput()
class QueueTransformationForm_create(forms.ModelForm):
class Meta:
model = QueueTransformation
exclude = ('content_type', 'object_id')
"""

5
apps/ocr/links.py
View File

@@ -30,8 +30,3 @@ queue_document_multiple_delete = Link(text=_(u'delete'), view='queue_document_mu
all_document_ocr_cleanup = Link(text=_(u'clean up pages content'), view='all_document_ocr_cleanup', sprite='text_strikethrough', permissions=[PERMISSION_OCR_CLEAN_ALL_PAGES], description=_(u'Runs a language filter to remove common OCR mistakes from document pages content.'))
ocr_tool_link = Link(text=_(u'OCR'), view='ocr_log', sprite='hourglass', icon='text.png', permissions=[PERMISSION_OCR_DOCUMENT]) # children_view_regex=[r'queue_', r'document_queue'])
#setup_queue_transformation_list = Link(text=_(u'transformations'), view='setup_queue_transformation_list', args='queue.pk', sprite='shape_move_front')
#setup_queue_transformation_create = Link(text=_(u'add transformation'), view='setup_queue_transformation_create', args='queue.pk', sprite='shape_square_add')
#setup_queue_transformation_edit = Link(text=_(u'edit'), view='setup_queue_transformation_edit', args='transformation.pk', sprite='shape_square_edit')
#setup_queue_transformation_delete = Link(text=_(u'delete'), view='setup_queue_transformation_delete', args='transformation.pk', sprite='shape_square_delete')

11
apps/ocr/literals.py
View File

@@ -9,17 +9,6 @@ OCR_STATE_CHOICES = (
(OCR_STATE_ENABLED, _(u'enabled')),
)
#QUEUEDOCUMENT_STATE_PENDING = 'p'
#QUEUEDOCUMENT_STATE_PROCESSING = 'i'
#QUEUEDOCUMENT_STATE_ERROR = 'e'
#QUEUEDOCUMENT_STATE_CHOICES = (
# (QUEUEDOCUMENT_STATE_PENDING, _(u'pending')),
# (QUEUEDOCUMENT_STATE_PROCESSING, _(u'processing')),
# (QUEUEDOCUMENT_STATE_ERROR, _(u'error')),
#)
DEFAULT_OCR_FILE_FORMAT = u'tiff'
DEFAULT_OCR_FILE_EXTENSION = u'tif'
UNPAPER_FILE_FORMAT = u'ppm'

143
apps/ocr/models.py
View File

@@ -53,146 +53,3 @@ class OCRProcessingSingleton(Singleton):
class Meta:
verbose_name = verbose_name_plural = _(u'OCR processing properties')
"""
class OCRLog(models.Model):
#queue = models.ForeignKey(Queue, verbose_name=_(u'queue'))
document_version = models.ForeignKey(DocumentVersion, verbose_name=_(u'document version'))
datetime = models.DateTimeField(verbose_name=_(u'date time'), default=lambda: datetime.datetime.now(), db_index=True)
delay = models.BooleanField(verbose_name=_(u'delay OCR'), default=False)
#state = models.CharField(max_length=4,
# choices=QUEUEDOCUMENT_STATE_CHOICES,
# default=QUEUEDOCUMENT_STATE_PENDING,
# verbose_name=_(u'state'))
result = models.TextField(blank=True, null=True, verbose_name=_(u'result'))
#node_name = models.CharField(max_length=32, verbose_name=_(u'node name'), blank=True, null=True)
class Meta:
ordering = ('datetime',)
verbose_name = _(u'OCR log entry')
verbose_name_plural = _(u'OCR log entries')
#def get_transformation_list(self):
# return QueueTransformation.transformations.get_for_object_as_list(self)
def requeue(self):
pass
#if self.state == QUEUEDOCUMENT_STATE_PROCESSING:
# raise ReQueueError
#else:
# self.datetime_submitted = datetime.now()
# self.state = QUEUEDOCUMENT_STATE_PENDING
# self.delay = False
# self.result = None
# self.node_name = None
# self.save()
def __unicode__(self):
try:
return unicode(self.document)
except ObjectDoesNotExist:
return ugettext(u'Missing document.')
"""
#class DocumentQueue(models.Model):
# name = models.CharField(max_length=64, unique=True, verbose_name=_(u'name'))
# label = models.CharField(max_length=64, verbose_name=_(u'label'))
# state = models.CharField(max_length=4,
# choices=DOCUMENTQUEUE_STATE_CHOICES,
# default=DOCUMENTQUEUE_STATE_ACTIVE,
# verbose_name=_(u'state'))
#
# objects = DocumentQueueManager()#
#
# class Meta:
# verbose_name = _(u'document queue')
# verbose_name_plural = _(u'document queues')#
#
# def __unicode__(self):
# return self.label
"""
class QueueDocument(models.Model):
document_queue = models.ForeignKey(DocumentQueue, verbose_name=_(u'document queue'))
document = models.ForeignKey(Document, verbose_name=_(u'document'))
datetime_submitted = models.DateTimeField(verbose_name=_(u'date time submitted'), auto_now_add=True, db_index=True)
delay = models.BooleanField(verbose_name=_(u'delay ocr'), default=False)
state = models.CharField(max_length=4,
choices=QUEUEDOCUMENT_STATE_CHOICES,
default=QUEUEDOCUMENT_STATE_PENDING,
verbose_name=_(u'state'))
result = models.TextField(blank=True, null=True, verbose_name=_(u'result'))
node_name = models.CharField(max_length=32, verbose_name=_(u'node name'), blank=True, null=True)
class Meta:
ordering = ('datetime_submitted',)
verbose_name = _(u'queue document')
verbose_name_plural = _(u'queue documents')
def get_transformation_list(self):
return QueueTransformation.transformations.get_for_object_as_list(self)
def requeue(self):
if self.state == QUEUEDOCUMENT_STATE_PROCESSING:
raise ReQueueError
else:
self.datetime_submitted = datetime.now()
self.state = QUEUEDOCUMENT_STATE_PENDING
self.delay = False
self.result = None
self.node_name = None
self.save()
def __unicode__(self):
try:
return unicode(self.document)
except ObjectDoesNotExist:
return ugettext(u'Missing document.')
class ArgumentsValidator(object):
message = _(u'Enter a valid value.')
code = 'invalid'
def __init__(self, message=None, code=None):
if message is not None:
self.message = message
if code is not None:
self.code = code
def __call__(self, value):
'''
Validates that the input evaluates correctly.
'''
value = value.strip()
try:
literal_eval(value)
except (ValueError, SyntaxError):
raise ValidationError(self.message, code=self.code)
class QueueTransformation(models.Model):
'''
Model that stores the transformation and transformation arguments
for a given document queue
'''
content_type = models.ForeignKey(ContentType)
object_id = models.PositiveIntegerField()
content_object = generic.GenericForeignKey('content_type', 'object_id')
order = models.PositiveIntegerField(default=0, blank=True, null=True, verbose_name=_(u'order'), db_index=True)
transformation = models.CharField(choices=get_available_transformations_choices(), max_length=128, verbose_name=_(u'transformation'))
arguments = models.TextField(blank=True, null=True, verbose_name=_(u'arguments'), help_text=_(u'Use dictionaries to indentify arguments, example: %s') % u'{\'degrees\':90}', validators=[ArgumentsValidator()])
objects = models.Manager()
transformations = SourceTransformationManager()
def __unicode__(self):
return self.get_transformation_display()
class Meta:
ordering = ('order',)
verbose_name = _(u'document queue transformation')
verbose_name_plural = _(u'document queue transformations')
"""

75
apps/ocr/tasks.py
View File

@@ -1,75 +0,0 @@
from __future__ import absolute_import
from datetime import timedelta, datetime
import platform
import logging
from django.db.models import Q
from job_processor.api import process_job
from lock_manager import Lock, LockError
from .api import do_document_ocr
#from .literals import (QUEUEDOCUMENT_STATE_PENDING,
# QUEUEDOCUMENT_STATE_PROCESSING, DOCUMENTQUEUE_STATE_ACTIVE,
# QUEUEDOCUMENT_STATE_ERROR)
#from .models import QueueDocument, DocumentQueue
from .conf.settings import NODE_CONCURRENT_EXECUTION, REPLICATION_DELAY
LOCK_EXPIRE = 60 * 10 # Lock expires in 10 minutes
# TODO: Tie LOCK_EXPIRATION with hard task timeout
logger = logging.getLogger(__name__)
def task_process_queue_document(queue_document_id):
lock_id = u'task_proc_queue_doc-%d' % queue_document_id
try:
logger.debug('trying to acquire lock: %s' % lock_id)
lock = Lock.acquire_lock(lock_id, LOCK_EXPIRE)
logger.debug('acquired lock: %s' % lock_id)
queue_document = QueueDocument.objects.get(pk=queue_document_id)
queue_document.state = QUEUEDOCUMENT_STATE_PROCESSING
queue_document.node_name = platform.node()
queue_document.save()
try:
do_document_ocr(queue_document)
queue_document.delete()
except Exception, e:
queue_document.state = QUEUEDOCUMENT_STATE_ERROR
queue_document.result = e
queue_document.save()
lock.release()
except LockError:
logger.debug('unable to obtain lock')
pass
def task_process_document_queues():
logger.debug('executed')
# TODO: reset_orphans()
q_pending = Q(state=QUEUEDOCUMENT_STATE_PENDING)
q_delayed = Q(delay=True)
q_delay_interval = Q(datetime_submitted__lt=datetime.now() - timedelta(seconds=REPLICATION_DELAY))
for document_queue in DocumentQueue.objects.filter(state=DOCUMENTQUEUE_STATE_ACTIVE):
current_local_processing_count = QueueDocument.objects.filter(
state=QUEUEDOCUMENT_STATE_PROCESSING).filter(
node_name=platform.node()).count()
if current_local_processing_count < NODE_CONCURRENT_EXECUTION:
try:
oldest_queued_document_qs = document_queue.queuedocument_set.filter(
(q_pending & ~q_delayed) | (q_pending & q_delayed & q_delay_interval))
if oldest_queued_document_qs:
oldest_queued_document = oldest_queued_document_qs.order_by('datetime_submitted')[0]
process_job(task_process_queue_document, oldest_queued_document.pk)
except Exception, e:
logger.error('unhandled exception: %s' % e)
finally:
# Don't process anymore from this queryset, might be stale
break
else:
logger.debug('already processing maximun')
else:
logger.debug('nothing to process')

17
apps/ocr/views.py
View File

@@ -14,17 +14,16 @@ from documents.models import Document
from documents.widgets import document_link, document_thumbnail
from common.utils import encapsulate
from acls.models import AccessEntry
from job_processor.exceptions import JobQueuePushError
from .permissions import (PERMISSION_OCR_DOCUMENT,
PERMISSION_OCR_DOCUMENT_DELETE, PERMISSION_OCR_QUEUE_ENABLE_DISABLE,
PERMISSION_OCR_CLEAN_ALL_PAGES, PERMISSION_OCR_QUEUE_EDIT)
from .models import OCRProcessingSingleton
#from .literals import (QUEUEDOCUMENT_STATE_PROCESSING,
# DOCUMENTQUEUE_STATE_ACTIVE, DOCUMENTQUEUE_STATE_STOPPED)
from .exceptions import (AlreadyQueued, ReQueueError, OCRProcessingAlreadyDisabled,
OCRProcessingAlreadyEnabled)
from .api import clean_pages
#from .forms import QueueTransformationForm, QueueTransformationForm_create
from . import ocr_job_queue, ocr_job_type
def ocr_log(request):
@@ -195,15 +194,15 @@ def submit_document(request, document_id):
def submit_document_to_queue(request, document, post_submit_redirect=None):
'''
"""
This view is meant to be reusable
'''
"""
try:
document_queue = DocumentQueue.objects.queue_document(document)
messages.success(request, _(u'Document: %(document)s was added to the OCR queue: %(queue)s.') % {
'document': document, 'queue': document_queue.label})
except AlreadyQueued:
ocr_job_queue.push(ocr_job_type, document_version_pk=document.latest_version.pk)
messages.success(request, _(u'Document: %(document)s was added to the OCR queue sucessfully.') % {
'document': document})
except JobQueuePushError:
messages.warning(request, _(u'Document: %(document)s is already queued.') % {
'document': document})
except Exception, e: