diff --git a/README.md b/README.md index e1e21d1eae..662d33ab2a 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,8 @@ Python: * django-mptt - Utilities for implementing a modified pre-order traversal tree in django * python-magic - A python wrapper for libmagic * django-taggit - Simple tagging for django +* slate - The simplest way to extract text from PDFs in Python + Execute pip install -r requirements/production.txt to install the python/django dependencies automatically. diff --git a/apps/converter/api.py b/apps/converter/api.py index f5d5203bae..665a980c27 100644 --- a/apps/converter/api.py +++ b/apps/converter/api.py @@ -5,8 +5,6 @@ import hashlib from common import TEMPORARY_DIRECTORY from documents.utils import document_save_to_temp_dir -from converter.conf.settings import UNPAPER_PATH -from converter.conf.settings import OCR_OPTIONS from converter.conf.settings import UNOCONV_PATH from converter.exceptions import UnpaperError, OfficeConversionError from converter.literals import DEFAULT_PAGE_NUMBER, \ @@ -36,21 +34,6 @@ def cleanup(filename): pass -def execute_unpaper(input_filepath, output_filepath): - """ - Executes the program unpaper using subprocess's Popen - """ - command = [] - command.append(UNPAPER_PATH) - command.append(u'--overwrite') - command.append(input_filepath) - command.append(output_filepath) - proc = subprocess.Popen(command, close_fds=True, stderr=subprocess.PIPE) - return_code = proc.wait() - if return_code != 0: - raise UnpaperError(proc.stderr.readline()) - - def execute_unoconv(input_filepath, arguments=''): """ Executes the program unoconv using subprocess's Popen @@ -164,38 +147,6 @@ def get_document_dimensions(document, *args, **kwargs): return [0, 0] -def convert_document_for_ocr(document, page=DEFAULT_PAGE_NUMBER, file_format=DEFAULT_OCR_FILE_FORMAT): - #Extract document file - input_filepath = document_save_to_temp_dir(document, document.uuid) - - #Convert for OCR - temp_filename, separator = os.path.splitext(os.path.basename(input_filepath)) - temp_path = os.path.join(TEMPORARY_DIRECTORY, temp_filename) - transformation_output_file = u'%s_trans%s%s%s' % (temp_path, page, os.extsep, file_format) - unpaper_input_file = u'%s_unpaper_in%s%spnm' % (temp_path, page, os.extsep) - unpaper_output_file = u'%s_unpaper_out%s%spnm' % (temp_path, page, os.extsep) - convert_output_file = u'%s_ocr%s%s%s' % (temp_path, page, os.extsep, file_format) - - try: - document_page = document.documentpage_set.get(page_number=page) - transformation_string, warnings = document_page.get_transformation_string() - - #Apply default transformations - backend.convert_file(input_filepath=input_filepath, page=page, quality=QUALITY_HIGH, arguments=transformation_string, output_filepath=transformation_output_file) - #Do OCR operations - backend.convert_file(input_filepath=transformation_output_file, arguments=OCR_OPTIONS, output_filepath=unpaper_input_file) - # Process by unpaper - execute_unpaper(input_filepath=unpaper_input_file, output_filepath=unpaper_output_file) - # Convert to tif - backend.convert_file(input_filepath=unpaper_output_file, output_filepath=convert_output_file) - finally: - cleanup(transformation_output_file) - cleanup(unpaper_input_file) - cleanup(unpaper_output_file) - - return convert_output_file - - def get_available_transformations_choices(): result = [] for transformation in backend.get_available_transformations(): diff --git a/apps/converter/backends/graphicsmagick/base.py b/apps/converter/backends/graphicsmagick/base.py index 4d3910391b..54ebbaaa95 100644 --- a/apps/converter/backends/graphicsmagick/base.py +++ b/apps/converter/backends/graphicsmagick/base.py @@ -108,7 +108,7 @@ class ConverterClass(ConverterBase): def get_available_transformations(self): return [ TRANSFORMATION_RESIZE, TRANSFORMATION_ROTATE, \ - TRANSFORMATION_DENSITY, TRANSFORMATION_ZOOM + TRANSFORMATION_ZOOM ] def get_page_count(self, input_filepath): diff --git a/apps/converter/backends/imagemagick/base.py b/apps/converter/backends/imagemagick/base.py index c9977fb3b4..4f924316ed 100644 --- a/apps/converter/backends/imagemagick/base.py +++ b/apps/converter/backends/imagemagick/base.py @@ -106,7 +106,7 @@ class ConverterClass(ConverterBase): def get_available_transformations(self): return [ TRANSFORMATION_RESIZE, TRANSFORMATION_ROTATE, \ - TRANSFORMATION_DENSITY, TRANSFORMATION_ZOOM + TRANSFORMATION_ZOOM ] diff --git a/apps/converter/backends/python/base.py b/apps/converter/backends/python/base.py index 616e997d3f..25448346ff 100644 --- a/apps/converter/backends/python/base.py +++ b/apps/converter/backends/python/base.py @@ -1,3 +1,4 @@ +import slate from PIL import Image from django.utils.translation import ugettext_lazy as _ @@ -9,12 +10,28 @@ from converter.literals import TRANSFORMATION_RESIZE, \ TRANSFORMATION_ROTATE, TRANSFORMATION_ZOOM from converter.literals import QUALITY_DEFAULT, DEFAULT_PAGE_NUMBER, \ DEFAULT_FILE_FORMAT +from converter.utils import get_mimetype + class ConverterClass(ConverterBase): def get_page_count(self, input_filepath): page_count = 1 - im = Image.open(input_filepath) - + + mimetype, encoding = get_mimetype(input_filepath) + if mimetype == 'application/pdf': + # If file is a PDF open it with slate to determine the page + # count + with open(input_filepath) as fd: + pages = slate.PDF(fd) + return len(pages) + + try: + im = Image.open(input_filepath) + except IOError: #cannot identify image file + # Return a page count of 1, to atleast allow the document + # to be created + return 1 + try: while 1: im.seek(im.tell()+1) diff --git a/apps/converter/conf/settings.py b/apps/converter/conf/settings.py index fcaa1ec9b0..95aee33b92 100644 --- a/apps/converter/conf/settings.py +++ b/apps/converter/conf/settings.py @@ -9,12 +9,11 @@ register_settings( settings=[ {'name': u'IM_CONVERT_PATH', 'global_name': u'CONVERTER_IM_CONVERT_PATH', 'default': u'/usr/bin/convert', 'description': _(u'File path to imagemagick\'s convert program.'), 'exists': True}, {'name': u'IM_IDENTIFY_PATH', 'global_name': u'CONVERTER_IM_IDENTIFY_PATH', 'default': u'/usr/bin/identify', 'description': _(u'File path to imagemagick\'s identify program.'), 'exists': True}, - {'name': u'UNPAPER_PATH', 'global_name': u'CONVERTER_UNPAPER_PATH', 'default': u'/usr/bin/unpaper', 'description': _(u'File path to unpaper program.'), 'exists': True}, {'name': u'GM_PATH', 'global_name': u'CONVERTER_GM_PATH', 'default': u'/usr/bin/gm', 'description': _(u'File path to graphicsmagick\'s program.'), 'exists': True}, {'name': u'GM_SETTINGS', 'global_name': u'CONVERTER_GM_SETTINGS', 'default': u''}, {'name': u'GRAPHICS_BACKEND', 'global_name': u'CONVERTER_GRAPHICS_BACKEND', 'default': u'converter.backends.python', 'description': _(u'Graphics conversion backend to use. Options are: converter.backends.imagemagick, converter.backends.graphicsmagick and converter.backends.python.')}, {'name': u'UNOCONV_PATH', 'global_name': u'CONVERTER_UNOCONV_PATH', 'default': u'/usr/bin/unoconv', 'exists': True}, - {'name': u'OCR_OPTIONS', 'global_name': u'CONVERTER_OCR_OPTIONS', 'default': u'-colorspace Gray -depth 8 -resample 200x200'}, + #{'name': u'OCR_OPTIONS', 'global_name': u'CONVERTER_OCR_OPTIONS', 'default': u'-colorspace Gray -depth 8 -resample 200x200'}, {'name': u'DEFAULT_OPTIONS', 'global_name': u'CONVERTER_DEFAULT_OPTIONS', 'default': u''}, {'name': u'LOW_QUALITY_OPTIONS', 'global_name': u'CONVERTER_LOW_QUALITY_OPTIONS', 'default': u''}, {'name': u'HIGH_QUALITY_OPTIONS', 'global_name': u'CONVERTER_HIGH_QUALITY_OPTIONS', 'default': u'-density 400'}, diff --git a/apps/converter/utils.py b/apps/converter/utils.py index 5fc106a940..4653b6dc9d 100644 --- a/apps/converter/utils.py +++ b/apps/converter/utils.py @@ -3,7 +3,15 @@ import os from django.core.exceptions import ImproperlyConfigured from django.utils.importlib import import_module - +try: + from python_magic import magic + USE_PYTHON_MAGIC = True +except: + import mimetypes + mimetypes.init() + USE_PYTHON_MAGIC = False + + #http://stackoverflow.com/questions/123198/how-do-i-copy-a-file-in-python def copyfile(source, dest, buffer_size=1024 * 1024): """ @@ -72,3 +80,32 @@ def load_backend(): raise ImproperlyConfigured(error_msg) else: raise # If there's some other error, this must be an error in Mayan itself. + + +def get_mimetype(filepath): + """ + Determine a file's mimetype by calling the system's libmagic + library via python-magic or fallback to use python's mimetypes + library + """ + file_mimetype = u'' + file_mime_encoding = u'' + + if USE_PYTHON_MAGIC: + if os.path.exists(filepath): + try: + source = open(filepath, 'r') + mime = magic.Magic(mime=True) + file_mimetype = mime.from_buffer(source.read()) + source.seek(0) + mime_encoding = magic.Magic(mime_encoding=True) + file_mime_encoding = mime_encoding.from_buffer(source.read()) + finally: + if source: + source.close() + else: + path, filename = os.path.split(filepath) + file_mimetype, file_mime_encoding = mimetypes.guess_type(filename) + + return file_mimetype, file_mime_encoding + diff --git a/apps/documents/models.py b/apps/documents/models.py index c058bdb694..b3eadb08e7 100644 --- a/apps/documents/models.py +++ b/apps/documents/models.py @@ -257,6 +257,9 @@ class DocumentPage(models.Model): verbose_name = _(u'document page') verbose_name_plural = _(u'document pages') + def get_transformation_list(self): + return DocumentPageTransformation.objects.get_for_document_page_as_list(self) + @models.permalink def get_absolute_url(self): return ('document_page_view', [self.pk]) diff --git a/apps/documents/views.py b/apps/documents/views.py index c727407b88..ee9be82b3e 100644 --- a/apps/documents/views.py +++ b/apps/documents/views.py @@ -305,7 +305,7 @@ def get_document_image(request, document_id, size=PREVIEW_SIZE, quality=QUALITY_ rotation = int(request.GET.get('rotation', DEFAULT_ROTATION)) % 360 document_page = get_object_or_404(document.documentpage_set, page_number=page) - transformations, warnings = DocumentPageTransformation.objects.get_for_document_page_as_list(document_page) + transformations, warnings = document_page.get_transformation_list() if warnings and (request.user.is_staff or request.user.is_superuser): for warning in warnings: diff --git a/apps/ocr/__init__.py b/apps/ocr/__init__.py index 55aa3d5b60..00efe7b276 100644 --- a/apps/ocr/__init__.py +++ b/apps/ocr/__init__.py @@ -9,7 +9,7 @@ from documents.models import Document from main.api import register_tool from ocr.conf.settings import AUTOMATIC_OCR -from ocr.models import DocumentQueue +from ocr.models import DocumentQueue, QueueTransformation #Permissions PERMISSION_OCR_DOCUMENT = {'namespace': 'ocr', 'name': 'ocr_document', 'label': _(u'Submit document for OCR')} @@ -30,20 +30,27 @@ re_queue_multiple_document = {'text': _('re-queue'), 'view': 're_queue_multiple_ queue_document_delete = {'text': _(u'delete'), 'view': 'queue_document_delete', 'args': 'object.id', 'famfam': 'hourglass_delete', 'permissions': [PERMISSION_OCR_DOCUMENT_DELETE]} queue_document_multiple_delete = {'text': _(u'delete'), 'view': 'queue_document_multiple_delete', 'famfam': 'hourglass_delete', 'permissions': [PERMISSION_OCR_DOCUMENT_DELETE]} -document_queue_disable = {'text': _(u'stop queue'), 'view': 'document_queue_disable', 'args': 'object.id', 'famfam': 'control_stop_blue', 'permissions': [PERMISSION_OCR_QUEUE_ENABLE_DISABLE]} -document_queue_enable = {'text': _(u'activate queue'), 'view': 'document_queue_enable', 'args': 'object.id', 'famfam': 'control_play_blue', 'permissions': [PERMISSION_OCR_QUEUE_ENABLE_DISABLE]} +document_queue_disable = {'text': _(u'stop queue'), 'view': 'document_queue_disable', 'args': 'queue.id', 'famfam': 'control_stop_blue', 'permissions': [PERMISSION_OCR_QUEUE_ENABLE_DISABLE]} +document_queue_enable = {'text': _(u'activate queue'), 'view': 'document_queue_enable', 'args': 'queue.id', 'famfam': 'control_play_blue', 'permissions': [PERMISSION_OCR_QUEUE_ENABLE_DISABLE]} all_document_ocr_cleanup = {'text': _(u'clean up pages content'), 'view': 'all_document_ocr_cleanup', 'famfam': 'text_strikethrough', 'permissions': [PERMISSION_OCR_CLEAN_ALL_PAGES], 'description': _(u'Runs a language filter to remove common OCR mistakes from document pages content.')} queue_document_list = {'text': _(u'queue document list'), 'view': 'queue_document_list', 'famfam': 'hourglass', 'permissions': [PERMISSION_OCR_DOCUMENT]} node_active_list = {'text': _(u'active tasks'), 'view': 'node_active_list', 'famfam': 'server_chart', 'permissions': [PERMISSION_OCR_DOCUMENT]} +setup_queue_transformation_list = {'text': _(u'transformations'), 'view': 'setup_queue_transformation_list', 'args': 'queue.pk', 'famfam': 'shape_move_front'} +setup_queue_transformation_create = {'text': _(u'add transformation'), 'view': 'setup_queue_transformation_create', 'args': 'queue.pk', 'famfam': 'shape_square_add'} +setup_queue_transformation_edit = {'text': _(u'edit'), 'view': 'setup_queue_transformation_edit', 'args': 'transformation.pk', 'famfam': 'shape_square_edit'} +setup_queue_transformation_delete = {'text': _(u'delete'), 'view': 'setup_queue_transformation_delete', 'args': 'transformation.pk', 'famfam': 'shape_square_delete'} + register_links(Document, [submit_document]) -register_links(DocumentQueue, [document_queue_disable, document_queue_enable]) +register_links(DocumentQueue, [document_queue_disable, document_queue_enable, setup_queue_transformation_list]) +register_links(QueueTransformation, [setup_queue_transformation_edit, setup_queue_transformation_delete]) register_multi_item_links(['queue_document_list'], [re_queue_multiple_document, queue_document_multiple_delete]) -register_links(['queue_document_list', 'node_active_list'], [queue_document_list, node_active_list], menu_name='secondary_menu') +register_links(['setup_queue_transformation_create', 'setup_queue_transformation_edit', 'setup_queue_transformation_delete', 'document_queue_disable', 'document_queue_enable', 'queue_document_list', 'node_active_list', 'setup_queue_transformation_list'], [queue_document_list, node_active_list], menu_name='secondary_menu') +register_links(['setup_queue_transformation_edit', 'setup_queue_transformation_delete', 'setup_queue_transformation_list', 'setup_queue_transformation_create'], [setup_queue_transformation_create], menu_name='sidebar') register_tool(all_document_ocr_cleanup, namespace='ocr', title=_(u'OCR')) diff --git a/apps/ocr/api.py b/apps/ocr/api.py index bc9f775f76..ec89a669c9 100644 --- a/apps/ocr/api.py +++ b/apps/ocr/api.py @@ -9,13 +9,15 @@ import sys from django.utils.translation import ugettext as _ from django.utils.importlib import import_module -from converter.api import convert_document_for_ocr +from converter.api import convert from documents.models import DocumentPage from ocr.conf.settings import TESSERACT_PATH from ocr.conf.settings import TESSERACT_LANGUAGE -from ocr.conf.settings import PDFTOTEXT_PATH -from ocr.exceptions import TesseractError, PdftotextError +from ocr.exceptions import TesseractError +from ocr.conf.settings import UNPAPER_PATH +from ocr.parsers import parse_document_page +from ocr.parsers.exceptions import ParserError, ParserUnknownFile def get_language_backend(): @@ -30,7 +32,7 @@ def get_language_backend(): return None return module -backend = get_language_backend() +language_backend = get_language_backend() def cleanup(filename): @@ -58,63 +60,38 @@ def run_tesseract(input_filename, output_filename_base, lang=None): raise TesseractError(error_text) -def run_pdftotext(input_filename, output_filename, page_number=None): - """ - Execute the command line binary of pdftotext - """ - command = [unicode(PDFTOTEXT_PATH)] - if page_number: - command.extend([u'-nopgbrk', u'-f', unicode(page_number), u'-l', unicode(page_number)]) - command.extend([unicode(input_filename), unicode(output_filename)]) - proc = subprocess.Popen(command, close_fds=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE) - return_code = proc.wait() - if return_code != 0: - error_text = proc.stderr.read() - raise PdftotextError(error_text) - - def do_document_ocr(document): """ - Do OCR on all the pages of the given document object, first - trying to extract text from PDF using pdftotext then by calling - tesseract + first try to extract text from document pages using the registered + parser if the parser fails or if there is no parser registered for + the document mimetype do a visual OCR by calling tesseract """ - for page_index, document_page in enumerate(document.documentpage_set.all()): - desc, filepath = tempfile.mkstemp() - imagefile = None - source = u'' + for document_page in document.documentpage_set.all(): try: - if document.file_mimetype == u'application/pdf': - pdf_filename = os.extsep.join([filepath, u'pdf']) - document.save_to_file(pdf_filename) - run_pdftotext(pdf_filename, filepath, document_page.page_number) - cleanup(pdf_filename) - if os.stat(filepath).st_size == 0: - #PDF page had no text, run tesseract on the page - imagefile = convert_document_for_ocr(document, page=page_index) - run_tesseract(imagefile, filepath, TESSERACT_LANGUAGE) - ocr_output = os.extsep.join([filepath, u'txt']) - source = _(u'Text from OCR') - else: - ocr_output = filepath - source = _(u'Text extracted from PDF') - else: - imagefile = convert_document_for_ocr(document, page=page_index) - run_tesseract(imagefile, filepath, TESSERACT_LANGUAGE) - ocr_output = os.extsep.join([filepath, u'txt']) - source = _(u'Text from OCR') - f = codecs.open(ocr_output, 'r', 'utf-8') - document_page = document.documentpage_set.get(page_number=page_index + 1) - document_page.content = ocr_cleanup(f.read().strip()) - document_page.page_label = source - document_page.save() - f.close() - cleanup(ocr_output) - finally: - os.close(desc) - cleanup(filepath) - if imagefile: - cleanup(imagefile) + # Try to extract text by means of a parser + parse_document_page(document_page) + except (ParserError, ParserUnknownFile): + # Fall back to doing visual OCR + pass + #desc, filepath = tempfile.mkstemp() + #imagefile = None + #source = u'' + #imagefile = convert_document_for_ocr(document, page=document_page.page_number) + #run_tesseract(imagefile, filepath, TESSERACT_LANGUAGE) + #ocr_output = os.extsep.join([filepath, u'txt']) + #source = _(u'Text from OCR') + #f = codecs.open(ocr_output, 'r', 'utf-8') + #document_page.content = ocr_cleanup(f.read().strip()) + #document_page.page_label = source + #document_page.save() + #f.close() + #cleanup(ocr_output) + #finally: + # pass + #os.close(desc) + #cleanup(filepath) + #if imagefile: + # cleanup(imagefile) def ocr_cleanup(text): @@ -127,8 +104,8 @@ def ocr_cleanup(text): for line in text.splitlines(): line = line.strip() for word in line.split(): - if backend: - result = backend.check_word(word) + if language_backend: + result = language_backend.check_word(word) else: result = word if result: @@ -147,3 +124,53 @@ def clean_pages(): if page.content: page.content = ocr_cleanup(page.content) page.save() + + +def execute_unpaper(input_filepath, output_filepath): + """ + Executes the program unpaper using subprocess's Popen + """ + command = [] + command.append(UNPAPER_PATH) + command.append(u'--overwrite') + command.append(input_filepath) + command.append(output_filepath) + proc = subprocess.Popen(command, close_fds=True, stderr=subprocess.PIPE) + return_code = proc.wait() + if return_code != 0: + raise UnpaperError(proc.stderr.readline()) + +''' +def convert_document_for_ocr(document, page=DEFAULT_PAGE_NUMBER, file_format=DEFAULT_OCR_FILE_FORMAT): + #Extract document file + input_filepath = document_save_to_temp_dir(document, document.uuid) + + #Convert for OCR + temp_filename, separator = os.path.splitext(os.path.basename(input_filepath)) + temp_path = os.path.join(TEMPORARY_DIRECTORY, temp_filename) + transformation_output_file = u'%s_trans%s%s%s' % (temp_path, page, os.extsep, file_format) + unpaper_input_file = u'%s_unpaper_in%s%spnm' % (temp_path, page, os.extsep) + unpaper_output_file = u'%s_unpaper_out%s%spnm' % (temp_path, page, os.extsep) + convert_output_file = u'%s_ocr%s%s%s' % (temp_path, page, os.extsep, file_format) + + try: + document_page = document.documentpage_set.get(page_number=page) + transformations, warnings = document_page.get_transformation_list() + + #Apply default transformations + backend.convert_file(input_filepath=input_filepath, page=page, quality=QUALITY_HIGH, transformations=transformations, output_filepath=transformation_output_file) + #Do OCR operations + backend.convert_file(input_filepath=transformation_output_file, arguments=OCR_OPTIONS, output_filepath=unpaper_input_file) + # Process by unpaper + execute_unpaper(input_filepath=unpaper_input_file, output_filepath=unpaper_output_file) + # Convert to tif + backend.convert_file(input_filepath=unpaper_output_file, output_filepath=convert_output_file) + finally: + cleanup(transformation_output_file) + cleanup(unpaper_input_file) + cleanup(unpaper_output_file) + + return convert_output_file +''' + + diff --git a/apps/ocr/conf/settings.py b/apps/ocr/conf/settings.py index e9024b7152..52785f46ac 100644 --- a/apps/ocr/conf/settings.py +++ b/apps/ocr/conf/settings.py @@ -13,8 +13,9 @@ register_settings( {'name': u'REPLICATION_DELAY', 'global_name': u'OCR_REPLICATION_DELAY', 'default': 10, 'description': _(u'Amount of seconds to delay OCR of documents to allow for the node\'s storage replication overhead.')}, {'name': u'NODE_CONCURRENT_EXECUTION', 'global_name': u'OCR_NODE_CONCURRENT_EXECUTION', 'default': 1, 'description': _(u'Maximum amount of concurrent document OCRs a node can perform.')}, {'name': u'AUTOMATIC_OCR', 'global_name': u'OCR_AUTOMATIC_OCR', 'default': False, 'description': _(u'Automatically queue newly created documents for OCR.')}, - {'name': u'PDFTOTEXT_PATH', 'global_name': u'OCR_PDFTOTEXT_PATH', 'default': u'/usr/bin/pdftotext', 'exists': True}, {'name': u'QUEUE_PROCESSING_INTERVAL', 'global_name': u'OCR_QUEUE_PROCESSING_INTERVAL', 'default': 10}, - {'name': u'CACHE_URI', 'global_name': u'OCR_CACHE_URI', 'default': None, 'description': _(u'URI in the form: "memcached://127.0.0.1:11211/" to specify a cache backend to use for locking. Multiple hosts can be specified separated by a semicolon.')} + {'name': u'CACHE_URI', 'global_name': u'OCR_CACHE_URI', 'default': None, 'description': _(u'URI in the form: "memcached://127.0.0.1:11211/" to specify a cache backend to use for locking. Multiple hosts can be specified separated by a semicolon.')}, + {'name': u'UNPAPER_PATH', 'global_name': u'OCR_UNPAPER_PATH', 'default': u'/usr/bin/unpaper', 'description': _(u'File path to unpaper program.'), 'exists': True}, + {'name': u'PARSERS_PDFTOTEXT_PATH', 'global_name': u'OCR_PARSERS_PDFTOTEXT_PATH', 'default': u'/usr/bin/pdftotext', 'exists': True}, ] ) diff --git a/apps/ocr/exceptions.py b/apps/ocr/exceptions.py index 4bfa8f725a..b1ec8c3fe3 100644 --- a/apps/ocr/exceptions.py +++ b/apps/ocr/exceptions.py @@ -4,7 +4,3 @@ class AlreadyQueued(Exception): class TesseractError(Exception): pass - - -class PdftotextError(Exception): - pass diff --git a/apps/ocr/forms.py b/apps/ocr/forms.py new file mode 100644 index 0000000000..c88e7a8351 --- /dev/null +++ b/apps/ocr/forms.py @@ -0,0 +1,21 @@ +from django import forms +from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import ugettext + +from ocr.models import QueueTransformation + + +class QueueTransformationForm(forms.ModelForm): + class Meta: + model = QueueTransformation + + def __init__(self, *args, **kwargs): + super(QueueTransformationForm, self).__init__(*args, **kwargs) + self.fields['content_type'].widget = forms.HiddenInput() + self.fields['object_id'].widget = forms.HiddenInput() + + +class QueueTransformationForm_create(forms.ModelForm): + class Meta: + model = QueueTransformation + exclude = ('content_type', 'object_id') diff --git a/apps/ocr/manager.py b/apps/ocr/manager.py deleted file mode 100644 index a1fdb80b8c..0000000000 --- a/apps/ocr/manager.py +++ /dev/null @@ -1,18 +0,0 @@ -from django.db import models - -from ocr.exceptions import AlreadyQueued - - -class DocumentQueueManager(models.Manager): - """ - Module manager class to handle adding documents to an OCR document - queue - """ - def queue_document(self, document, queue_name='default'): - document_queue = self.model.objects.get(name=queue_name) - if document_queue.queuedocument_set.filter(document=document): - raise AlreadyQueued - - document_queue.queuedocument_set.create(document=document, delay=True) - - return document_queue diff --git a/apps/ocr/managers.py b/apps/ocr/managers.py new file mode 100644 index 0000000000..a3ed9621ad --- /dev/null +++ b/apps/ocr/managers.py @@ -0,0 +1,41 @@ +from django.db import models +from django.contrib.contenttypes.models import ContentType + +from ocr.exceptions import AlreadyQueued + + +class DocumentQueueManager(models.Manager): + """ + Module manager class to handle adding documents to an OCR document + queue + """ + def queue_document(self, document, queue_name='default'): + document_queue = self.model.objects.get(name=queue_name) + if document_queue.queuedocument_set.filter(document=document): + raise AlreadyQueued + + document_queue.queuedocument_set.create(document=document, delay=True) + + return document_queue + + +class QueueTransformationManager(models.Manager): + def get_for_object(self, obj): + ct = ContentType.objects.get_for_model(obj) + return self.model.objects.filter(content_type=ct).filter(object_id=obj.pk) + + def get_for_object_as_list(self, obj): + warnings = [] + transformations = [] + for transformation in self.get_for_object(obj).values('transformation', 'arguments'): + try: + transformations.append( + { + 'transformation': transformation['transformation'], + 'arguments': eval(transformation['arguments'], {}) + } + ) + except Exception, e: + warnings.append(e) + + return transformations, warnings diff --git a/apps/ocr/models.py b/apps/ocr/models.py index f9567e2b0a..bfcbf74aec 100644 --- a/apps/ocr/models.py +++ b/apps/ocr/models.py @@ -2,13 +2,16 @@ from django.db import models from django.utils.translation import ugettext_lazy as _ from django.utils.translation import ugettext from django.core.exceptions import ObjectDoesNotExist +from django.contrib.contenttypes.models import ContentType +from django.contrib.contenttypes import generic from documents.models import Document +from converter.api import get_available_transformations_choices from ocr.literals import DOCUMENTQUEUE_STATE_STOPPED, \ DOCUMENTQUEUE_STATE_CHOICES, QUEUEDOCUMENT_STATE_PENDING, \ QUEUEDOCUMENT_STATE_CHOICES -from ocr.manager import DocumentQueueManager +from ocr.managers import DocumentQueueManager, QueueTransformationManager class DocumentQueue(models.Model): @@ -51,3 +54,26 @@ class QueueDocument(models.Model): return unicode(self.document) except ObjectDoesNotExist: return ugettext(u'Missing document.') + + +class QueueTransformation(models.Model): + """ + Model that stores the transformation and transformation arguments + for a given document queue + """ + content_type = models.ForeignKey(ContentType) + object_id = models.PositiveIntegerField() + content_object = generic.GenericForeignKey('content_type', 'object_id') + order = models.PositiveIntegerField(default=0, blank=True, null=True, verbose_name=_(u'order'), db_index=True) + transformation = models.CharField(choices=get_available_transformations_choices(), max_length=128, verbose_name=_(u'transformation')) + arguments = models.TextField(blank=True, null=True, verbose_name=_(u'arguments'), help_text=_(u'Use dictionaries to indentify arguments, example: %s') % u'{\'degrees\':90}') + + objects = QueueTransformationManager() + + def __unicode__(self): + return self.get_transformation_display() + + class Meta: + ordering = ('order',) + verbose_name = _(u'document queue transformation') + verbose_name_plural = _(u'document queue transformations') diff --git a/apps/ocr/parsers/__init__.py b/apps/ocr/parsers/__init__.py new file mode 100644 index 0000000000..815e868747 --- /dev/null +++ b/apps/ocr/parsers/__init__.py @@ -0,0 +1,40 @@ +import codecs +import os +import subprocess +import tempfile +import sys + +import slate + +from django.utils.translation import ugettext as _ + +from ocr.parsers.exceptions import ParserError, ParserUnknownFile + +mimetype_registry = {} + + +def register_parser(mimetype, function): + mimetype_registry[mimetype] = {'function': function} + + +def pdf_parser(document_page): + fd = document_page.document.open() + pdf_pages = slate.PDF(fd) + fd.close() + + if pdf_pages[document_page.page_number - 1] == '\x0c': + raise ParserError + + document_page.content = pdf_pages[document_page.page_number - 1] + document_page.page_label = _(u'Text extracted from PDF') + document_page.save() + + +def parse_document_page(document_page): + try: + mimetype_registry[document_page.document.file_mimetype]['function'](document_page) + except KeyError: + raise ParserUnknownFile + + +register_parser('application/pdf', pdf_parser) diff --git a/apps/ocr/parsers/exceptions.py b/apps/ocr/parsers/exceptions.py new file mode 100644 index 0000000000..e06875f222 --- /dev/null +++ b/apps/ocr/parsers/exceptions.py @@ -0,0 +1,10 @@ +class ParserError(Exception): + """ + Raised when a text parser fails to understand a file it been passed + or the resulting parsed text is invalid + """ + pass + + +class ParserUnknownFile(Exception): + pass diff --git a/apps/ocr/urls.py b/apps/ocr/urls.py index 6bddd3d7fe..cb090cf065 100644 --- a/apps/ocr/urls.py +++ b/apps/ocr/urls.py @@ -1,16 +1,22 @@ from django.conf.urls.defaults import patterns, url urlpatterns = patterns('ocr.views', - url(r'^(?P\d+)/submit/$', 'submit_document', (), 'submit_document'), - url(r'^ocr/queue/document/list/$', 'queue_document_list', (), 'queue_document_list'), - url(r'^ocr/queue/document/(?P\d+)/delete/$', 'queue_document_delete', (), 'queue_document_delete'), - url(r'^ocr/queue/document/multiple/delete/$', 'queue_document_multiple_delete', (), 'queue_document_multiple_delete'), - url(r'^ocr/queue/document/(?P\d+)/re-queue/$', 're_queue_document', (), 're_queue_document'), - url(r'^ocr/queue/document/multiple/re-queue/$', 're_queue_multiple_document', (), 're_queue_multiple_document'), + url(r'^document/(?P\d+)/submit/$', 'submit_document', (), 'submit_document'), + url(r'^queue/document/list/$', 'queue_document_list', (), 'queue_document_list'), + url(r'^queue/document/(?P\d+)/delete/$', 'queue_document_delete', (), 'queue_document_delete'), + url(r'^queue/document/multiple/delete/$', 'queue_document_multiple_delete', (), 'queue_document_multiple_delete'), + url(r'^queue/document/(?P\d+)/re-queue/$', 're_queue_document', (), 're_queue_document'), + url(r'^queue/document/multiple/re-queue/$', 're_queue_multiple_document', (), 're_queue_multiple_document'), - url(r'^ocr/queue/(?P\d+)/enable/$', 'document_queue_enable', (), 'document_queue_enable'), - url(r'^ocr/queue/(?P\d+)/disable/$', 'document_queue_disable', (), 'document_queue_disable'), + url(r'^queue/(?P\d+)/enable/$', 'document_queue_enable', (), 'document_queue_enable'), + url(r'^queue/(?P\d+)/disable/$', 'document_queue_disable', (), 'document_queue_disable'), - url(r'^ocr/document/all/clean_up/$', 'all_document_ocr_cleanup', (), 'all_document_ocr_cleanup'), - url(r'^ocr/node/active/list/$', 'node_active_list', (), 'node_active_list'), + url(r'^document/all/clean_up/$', 'all_document_ocr_cleanup', (), 'all_document_ocr_cleanup'), + url(r'^node/active/list/$', 'node_active_list', (), 'node_active_list'), + + url(r'^queue/(?P\d+)/transformation/list/$', 'setup_queue_transformation_list', (), 'setup_queue_transformation_list'), + url(r'^queue/(?P\w+)/transformation/create/$', 'setup_queue_transformation_create', (), 'setup_queue_transformation_create'), + url(r'^queue/transformation/(?P\w+)/edit/$', 'setup_queue_transformation_edit', (), 'setup_queue_transformation_edit'), + url(r'^queue/transformation/(?P\w+)/delete/$', 'setup_queue_transformation_delete', (), 'setup_queue_transformation_delete'), + ) diff --git a/apps/ocr/views.py b/apps/ocr/views.py index 1dbff3e278..6ee6a96393 100644 --- a/apps/ocr/views.py +++ b/apps/ocr/views.py @@ -7,6 +7,7 @@ from django.template import RequestContext from django.contrib import messages from django.views.generic.list_detail import object_list from django.utils.translation import ugettext_lazy as _ +from django.core.urlresolvers import reverse from celery.task.control import inspect from permissions.api import check_permissions @@ -16,12 +17,13 @@ from documents.widgets import document_link, document_thumbnail from ocr import PERMISSION_OCR_DOCUMENT, PERMISSION_OCR_DOCUMENT_DELETE, \ PERMISSION_OCR_QUEUE_ENABLE_DISABLE, PERMISSION_OCR_CLEAN_ALL_PAGES -from ocr.models import DocumentQueue, QueueDocument +from ocr.models import DocumentQueue, QueueDocument, QueueTransformation from ocr.literals import QUEUEDOCUMENT_STATE_PENDING, \ QUEUEDOCUMENT_STATE_PROCESSING, DOCUMENTQUEUE_STATE_STOPPED, \ DOCUMENTQUEUE_STATE_ACTIVE from ocr.exceptions import AlreadyQueued from ocr.api import clean_pages +from ocr.forms import QueueTransformationForm, QueueTransformationForm_create def queue_document_list(request, queue_name='default'): @@ -36,8 +38,10 @@ def queue_document_list(request, queue_name='default'): extra_context={ 'title': _(u'documents in queue: %s') % document_queue, 'hide_object': True, - 'object': document_queue, + 'queue': document_queue, 'object_name': _(u'document queue'), + 'navigation_object_name': 'queue', + 'list_object_variable_name': 'queue_document', 'extra_columns': [ {'name': 'document', 'attribute': lambda x: document_link(x.document) if hasattr(x, 'document') else _(u'Missing document.')}, {'name': _(u'thumbnail'), 'attribute': lambda x: document_thumbnail(x.document)}, @@ -210,7 +214,8 @@ def document_queue_disable(request, document_queue_id): return HttpResponseRedirect(next) return render_to_response('generic_confirm.html', { - 'object': document_queue, + 'queue': document_queue, + 'navigation_object_name': 'queue', 'title': _(u'Are you sure you wish to disable document queue: %s') % document_queue, 'next': next, 'previous': previous, @@ -236,7 +241,8 @@ def document_queue_enable(request, document_queue_id): return HttpResponseRedirect(next) return render_to_response('generic_confirm.html', { - 'object': document_queue, + 'queue': document_queue, + 'navigation_object_name': 'queue', 'title': _(u'Are you sure you wish to activate document queue: %s') % document_queue, 'next': next, 'previous': previous, @@ -315,3 +321,141 @@ def node_active_list(request): {'name': _(u'related object'), 'attribute': lambda x: display_link(x['related_object']) if x['related_object'] else u''} ], }, context_instance=RequestContext(request)) + + +def setup_queue_transformation_list(request, document_queue_id): + #check_permissions(request.user, [PERMISSION_SOURCES_SETUP_EDIT]) + + document_queue = get_object_or_404(DocumentQueue, pk=document_queue_id) + + context = { + 'object_list': QueueTransformation.objects.get_for_object(document_queue), + 'title': _(u'transformations for: %s') % document_queue, + #'object_name': _(u'document queue'), + #'object': document_queue, + 'queue': document_queue, + 'object_name': _(u'document queue'), + 'navigation_object_name': 'queue', + 'list_object_variable_name': 'transformation', + 'extra_columns': [ + {'name': _(u'order'), 'attribute': 'order'}, + {'name': _(u'transformation'), 'attribute': lambda x: x.get_transformation_display()}, + {'name': _(u'arguments'), 'attribute': 'arguments'} + ], + 'hide_link': True, + 'hide_object': True, + } + + return render_to_response('generic_list.html', context, + context_instance=RequestContext(request)) + + +def setup_queue_transformation_edit(request, transformation_id): + #check_permissions(request.user, [PERMISSION_SOURCES_SETUP_EDIT]) + + transformation = get_object_or_404(QueueTransformation, pk=transformation_id) + redirect_view = reverse('setup_queue_transformation_list', args=[transformation.content_object.pk]) + next = request.POST.get('next', request.GET.get('next', request.META.get('HTTP_REFERER', redirect_view))) + + if request.method == 'POST': + form = QueueTransformationForm(instance=transformation, data=request.POST) + if form.is_valid(): + try: + # Test the validity of the argument field + eval(form.cleaned_data['arguments'], {}) + except: + messages.error(request, _(u'Queue transformation argument error.')) + else: + try: + form.save() + messages.success(request, _(u'Queue transformation edited successfully')) + return HttpResponseRedirect(next) + except Exception, e: + messages.error(request, _(u'Error editing queue transformation; %s') % e) + else: + form = QueueTransformationForm(instance=transformation) + + return render_to_response('generic_form.html', { + 'title': _(u'Edit transformation: %s') % transformation, + 'form': form, + 'queue': transformation.content_object, + 'transformation': transformation, + 'navigation_object_list': [ + {'object': 'queue', 'name': _(u'document queue')}, + {'object': 'transformation', 'name': _(u'transformation')} + ], + 'next': next, + }, + context_instance=RequestContext(request)) + + +def setup_queue_transformation_delete(request, transformation_id): + #check_permissions(request.user, [PERMISSION_SOURCES_SETUP_EDIT]) + + transformation = get_object_or_404(QueueTransformation, pk=transformation_id) + redirect_view = reverse('setup_queue_transformation_list', args=[transformation.content_object.pk]) + previous = request.POST.get('previous', request.GET.get('previous', request.META.get('HTTP_REFERER', redirect_view))) + + if request.method == 'POST': + try: + transformation.delete() + messages.success(request, _(u'Queue transformation deleted successfully.')) + except Exception, e: + messages.error(request, _(u'Error deleting queue transformation; %(error)s') % { + 'error': e} + ) + return HttpResponseRedirect(redirect_view) + + return render_to_response('generic_confirm.html', { + 'delete_view': True, + 'transformation': transformation, + 'queue': transformation.content_object, + 'navigation_object_list': [ + {'object': 'queue', 'name': _(u'document queue')}, + {'object': 'transformation', 'name': _(u'transformation')} + ], + 'title': _(u'Are you sure you wish to delete queue transformation "%(transformation)s"') % { + 'transformation': transformation.get_transformation_display(), + }, + 'previous': previous, + 'form_icon': u'shape_square_delete.png', + }, + context_instance=RequestContext(request)) + + +def setup_queue_transformation_create(request, document_queue_id): + #check_permissions(request.user, [PERMISSION_SOURCES_SETUP_EDIT]) + + document_queue = get_object_or_404(DocumentQueue, pk=document_queue_id) + + redirect_view = reverse('setup_queue_transformation_list', args=[document_queue.pk]) + previous = request.POST.get('previous', request.GET.get('previous', request.META.get('HTTP_REFERER', redirect_view))) + + if request.method == 'POST': + form = QueueTransformationForm_create(request.POST) + if form.is_valid(): + try: + # Test the validity of the argument field + eval(form.cleaned_data['arguments'], {}) + except: + messages.error(request, _(u'Queue transformation argument error.')) + else: + try: + queue_tranformation = form.save(commit=False) + queue_tranformation.content_object = document_queue + queue_tranformation.save() + messages.success(request, _(u'Queue transformation created successfully')) + return HttpResponseRedirect(redirect_view) + except Exception, e: + messages.error(request, _(u'Error creating queue transformation; %s') % e) + else: + form = QueueTransformationForm_create() + + return render_to_response('generic_form.html', { + 'form': form, + 'queue': document_queue, + 'object_name': _(u'document queue'), + 'navigation_object_name': 'queue', + 'title': _(u'Create new transformation for queue: %s') % document_queue, + }, context_instance=RequestContext(request)) + diff --git a/apps/sources/managers.py b/apps/sources/managers.py index f45e06e340..e27a6468a9 100644 --- a/apps/sources/managers.py +++ b/apps/sources/managers.py @@ -21,4 +21,4 @@ class SourceTransformationManager(models.Manager): except Exception, e: warnings.append(e) - return transformations, warnings + return transformations, warnings diff --git a/apps/sources/urls.py b/apps/sources/urls.py index 354ec22b0a..5d6a015f92 100644 --- a/apps/sources/urls.py +++ b/apps/sources/urls.py @@ -16,12 +16,12 @@ urlpatterns = patterns('sources.views', url(r'^setup/interactive/staging_folder/list/$', 'setup_source_list', {'source_type': SOURCE_CHOICE_STAGING}, 'setup_staging_folder_list'), url(r'^setup/interactive/(?P\w+)/list/$', 'setup_source_list', (), 'setup_source_list'), - url(r'^setup/interactive/(?P\w+)/(?P\w+)/edit/$', 'setup_source_edit', (), 'setup_source_edit'), - url(r'^setup/interactive/(?P\w+)/(?P\w+)/delete/$', 'setup_source_delete', (), 'setup_source_delete'), + url(r'^setup/interactive/(?P\w+)/(?P\d+)/edit/$', 'setup_source_edit', (), 'setup_source_edit'), + url(r'^setup/interactive/(?P\w+)/(?P\d+)/delete/$', 'setup_source_delete', (), 'setup_source_delete'), url(r'^setup/interactive/(?P\w+)/create/$', 'setup_source_create', (), 'setup_source_create'), - url(r'^setup/interactive/(?P\w+)/(?P\w+)/transformation/list/$', 'setup_source_transformation_list', (), 'setup_source_transformation_list'), - url(r'^setup/interactive/(?P\w+)/(?P\w+)/transformation/create/$', 'setup_source_transformation_create', (), 'setup_source_transformation_create'), - url(r'^setup/interactive/source/transformation/(?P\w+)/edit/$', 'setup_source_transformation_edit', (), 'setup_source_transformation_edit'), - url(r'^setup/interactive/source/transformation/(?P\w+)/delete/$', 'setup_source_transformation_delete', (), 'setup_source_transformation_delete'), + url(r'^setup/interactive/(?P\w+)/(?P\d+)/transformation/list/$', 'setup_source_transformation_list', (), 'setup_source_transformation_list'), + url(r'^setup/interactive/(?P\w+)/(?P\d+)/transformation/create/$', 'setup_source_transformation_create', (), 'setup_source_transformation_create'), + url(r'^setup/interactive/source/transformation/(?P\d+)/edit/$', 'setup_source_transformation_edit', (), 'setup_source_transformation_edit'), + url(r'^setup/interactive/source/transformation/(?P\d+)/delete/$', 'setup_source_transformation_delete', (), 'setup_source_transformation_delete'), ) diff --git a/apps/sources/views.py b/apps/sources/views.py index 05766d78a9..8732d66913 100644 --- a/apps/sources/views.py +++ b/apps/sources/views.py @@ -562,9 +562,9 @@ def setup_source_transformation_delete(request, transformation_id): if request.method == 'POST': try: source_transformation.delete() - messages.success(request, _(u'Transformation deleted successfully.')) + messages.success(request, _(u'Source transformation deleted successfully.')) except Exception, e: - messages.error(request, _(u'Error deleting transformation; %(error)s') % { + messages.error(request, _(u'Error deleting source transformation; %(error)s') % { 'error': e} ) return HttpResponseRedirect(redirect_view) @@ -577,7 +577,7 @@ def setup_source_transformation_delete(request, transformation_id): {'object': 'source', 'name': _(u'source')}, {'object': 'transformation', 'name': _(u'transformation')} ], - 'title': _(u'Are you sure you wish to delete transformation "%(transformation)s"') % { + 'title': _(u'Are you sure you wish to delete source transformation "%(transformation)s"') % { 'transformation': source_transformation.get_transformation_display(), }, 'previous': previous, diff --git a/requirements/development.txt b/requirements/development.txt index 00def8c63a..3acf630b4e 100644 --- a/requirements/development.txt +++ b/requirements/development.txt @@ -9,3 +9,5 @@ django-celery==2.2.2 django-sentry==1.6.0 django-taggit==0.9.3 -e git://github.com/django-mptt/django-mptt.git@0af02a95877041b2fd6d458bd95413dc1666c321#egg=django-mptt +slate==0.3 +PIL==1.1.7 diff --git a/requirements/production.txt b/requirements/production.txt index 1f1d3a0881..02219abaee 100644 --- a/requirements/production.txt +++ b/requirements/production.txt @@ -6,3 +6,5 @@ django-celery==2.2.2 django-sentry==1.6.0 django-taggit==0.9.3 -e git://github.com/django-mptt/django-mptt.git@0af02a95877041b2fd6d458bd95413dc1666c321#egg=django-mptt +slate==0.3 +PIL==1.1.7