From eaaaa5b6456ead9d4ae3864947bf9cd687cdb5e7 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Fri, 15 Apr 2011 23:59:52 -0400 Subject: [PATCH] Added support for the command line program pdftotext from the poppler-utils packages to extract text from PDF documents without doing OCR --- apps/main/views.py | 1 + apps/ocr/api.py | 51 +++++++++++++++++++++++++++++---------- apps/ocr/conf/settings.py | 1 + apps/ocr/exceptions.py | 8 ++++++ settings.py | 1 + 5 files changed, 49 insertions(+), 13 deletions(-) diff --git a/apps/main/views.py b/apps/main/views.py index d073be4040..03f7f50b34 100644 --- a/apps/main/views.py +++ b/apps/main/views.py @@ -94,6 +94,7 @@ def check_settings(request): {'name': 'OCR_TESSERACT_LANGUAGE', 'value': ocr_settings.TESSERACT_LANGUAGE}, {'name': 'OCR_NODE_CONCURRENT_EXECUTION', 'value': ocr_settings.NODE_CONCURRENT_EXECUTION}, {'name': 'OCR_REPLICATION_DELAY', 'value': ocr_settings.REPLICATION_DELAY}, + {'name': 'OCR_PDFTOTEXT_PATH', 'value': ocr_settings.PDFTOTEXT_PATH, 'exists': True}, # Search {'name': 'SEARCH_LIMIT', 'value': search_settings.LIMIT}, diff --git a/apps/ocr/api.py b/apps/ocr/api.py index baaa83497a..0ec18988ad 100644 --- a/apps/ocr/api.py +++ b/apps/ocr/api.py @@ -14,6 +14,8 @@ from documents.models import DocumentPage from ocr.conf.settings import TESSERACT_PATH from ocr.conf.settings import TESSERACT_LANGUAGE +from ocr.conf.settings import PDFTOTEXT_PATH +from exceptions import TesseractError, PdftotextError def get_language_backend(): @@ -27,10 +29,6 @@ def get_language_backend(): backend = get_language_backend() -class TesseractError(Exception): - pass - - def cleanup(filename): ''' tries to remove the given filename. Ignores non-existent files ''' try: @@ -51,27 +49,54 @@ def run_tesseract(input_filename, output_filename_base, lang=None): raise TesseractError(error_text) +def run_pdftotext(input_filename, output_filename, page_number=None): + command = [unicode(PDFTOTEXT_PATH)] + if page_number: + command.extend(['-nopgbrk', '-f', unicode(page_number), '-l', unicode(page_number)]) + command.extend([unicode(input_filename), unicode(output_filename)]) + proc = subprocess.Popen(command, close_fds=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE) + return_code = proc.wait() + if return_code != 0: + error_text = proc.stderr.read() + raise PdftotextError(error_text) + + def do_document_ocr(document): for page_index, document_page in enumerate(document.documentpage_set.all()): - imagefile = convert_document_for_ocr(document, page=page_index) desc, filepath = tempfile.mkstemp() + imagefile = None + source = u'' try: - run_tesseract(imagefile, filepath, TESSERACT_LANGUAGE) - ocr_output = os.extsep.join([filepath, 'txt']) + if document.file_mimetype == u'application/pdf': + pdf_filename = os.extsep.join([filepath, u'pdf']) + document.save_to_file(pdf_filename) + run_pdftotext(pdf_filename, filepath, document_page.page_number) + cleanup(pdf_filename) + if os.stat(filepath).st_size == 0: + #PDF page had no text, run tesseract on the page + imagefile = convert_document_for_ocr(document, page=page_index) + run_tesseract(imagefile, filepath, TESSERACT_LANGUAGE) + ocr_output = os.extsep.join([filepath, 'txt']) + source = _(u'Text from OCR') + else: + ocr_output = filepath + source = _(u'Text extracted from PDF') + else: + imagefile = convert_document_for_ocr(document, page=page_index) + run_tesseract(imagefile, filepath, TESSERACT_LANGUAGE) + ocr_output = os.extsep.join([filepath, 'txt']) + source = _(u'Text from OCR') f = codecs.open(ocr_output, 'r', 'utf-8') document_page = document.documentpage_set.get(page_number=page_index + 1) document_page.content = ocr_cleanup(f.read().strip()) - document_page.page_label = _(u'Text from OCR') + document_page.page_label = source document_page.save() f.close() cleanup(ocr_output) - except TesseractError, e: - cleanup(filepath) - cleanup(imagefile) - raise TesseractError(e) finally: cleanup(filepath) - cleanup(imagefile) + if imagefile: + cleanup(imagefile) def ocr_cleanup(text): diff --git a/apps/ocr/conf/settings.py b/apps/ocr/conf/settings.py index 21134bf050..4ef84ea31d 100644 --- a/apps/ocr/conf/settings.py +++ b/apps/ocr/conf/settings.py @@ -6,3 +6,4 @@ TESSERACT_LANGUAGE = getattr(settings, 'OCR_TESSERACT_LANGUAGE', u'eng') REPLICATION_DELAY = getattr(settings, 'OCR_REPLICATION_DELAY', 10) # In seconds NODE_CONCURRENT_EXECUTION = getattr(settings, 'OCR_NODE_CONCURRENT_EXECUTION', 1) AUTOMATIC_OCR = getattr(settings, 'OCR_AUTOMATIC_OCR', False) +PDFTOTEXT_PATH = getattr(settings, 'OCR_PDFTOTEXT_PATH', u'/usr/bin/pdftotext') diff --git a/apps/ocr/exceptions.py b/apps/ocr/exceptions.py index 388e83c978..cc16c93a19 100644 --- a/apps/ocr/exceptions.py +++ b/apps/ocr/exceptions.py @@ -1,2 +1,10 @@ class AlreadyQueued(Exception): pass + + +class TesseractError(Exception): + pass + + +class PdftotextError(Exception): + pass diff --git a/settings.py b/settings.py index 78f41f3e28..202943c564 100644 --- a/settings.py +++ b/settings.py @@ -224,6 +224,7 @@ TEMPLATE_CONTEXT_PROCESSORS = ( #OCR_TESSERACT_LANGUAGE = u'eng' #OCR_REPLICATION_DELAY = 10 #OCR_AUTOMATIC_OCR = False +#OCR_PDFTOTEXT_PATH = u'/usr/bin/pdftotext' # Permissions #ROLES_DEFAULT_ROLES = []