Removed pdftotext from the requirements, move unpaper calling to the OCR app
This commit is contained in:
Roberto Rosario
142
apps/ocr/api.py
142
apps/ocr/api.py
@@ -9,13 +9,15 @@ import sys
|
||||
from django.utils.translation import ugettext as _
|
||||
from django.utils.importlib import import_module
|
||||
|
||||
from converter.api import convert_document_for_ocr
|
||||
from converter.api import convert
|
||||
from documents.models import DocumentPage
|
||||
|
||||
from ocr.conf.settings import TESSERACT_PATH
|
||||
from ocr.conf.settings import TESSERACT_LANGUAGE
|
||||
from ocr.conf.settings import PDFTOTEXT_PATH
|
||||
from ocr.exceptions import TesseractError, PdftotextError
|
||||
from ocr.exceptions import TesseractError
|
||||
from ocr.conf.settings import UNPAPER_PATH
|
||||
from ocr.parsers import parse_document_page
|
||||
from ocr.parsers.exceptions import ParserError, ParserUnknownFile
|
||||
|
||||
|
||||
def get_language_backend():
|
||||
@@ -30,7 +32,7 @@ def get_language_backend():
|
||||
return None
|
||||
return module
|
||||
|
||||
backend = get_language_backend()
|
||||
language_backend = get_language_backend()
|
||||
|
||||
|
||||
def cleanup(filename):
|
||||
@@ -58,62 +60,38 @@ def run_tesseract(input_filename, output_filename_base, lang=None):
|
||||
raise TesseractError(error_text)
|
||||
|
||||
|
||||
def run_pdftotext(input_filename, output_filename, page_number=None):
|
||||
"""
|
||||
Execute the command line binary of pdftotext
|
||||
"""
|
||||
command = [unicode(PDFTOTEXT_PATH)]
|
||||
if page_number:
|
||||
command.extend([u'-nopgbrk', u'-f', unicode(page_number), u'-l', unicode(page_number)])
|
||||
command.extend([unicode(input_filename), unicode(output_filename)])
|
||||
proc = subprocess.Popen(command, close_fds=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
|
||||
return_code = proc.wait()
|
||||
if return_code != 0:
|
||||
error_text = proc.stderr.read()
|
||||
raise PdftotextError(error_text)
|
||||
|
||||
|
||||
def do_document_ocr(document):
|
||||
"""
|
||||
Do OCR on all the pages of the given document object, first
|
||||
trying to extract text from PDF using pdftotext then by calling
|
||||
tesseract
|
||||
first try to extract text from document pages using the registered
|
||||
parser if the parser fails or if there is no parser registered for
|
||||
the document mimetype do a visual OCR by calling tesseract
|
||||
"""
|
||||
for document_page in document.documentpage_set.all():
|
||||
desc, filepath = tempfile.mkstemp()
|
||||
imagefile = None
|
||||
source = u''
|
||||
try:
|
||||
if document.file_mimetype == u'application/pdf':
|
||||
pdf_filename = os.extsep.join([filepath, u'pdf'])
|
||||
document.save_to_file(pdf_filename)
|
||||
run_pdftotext(pdf_filename, filepath, document_page.page_number)
|
||||
cleanup(pdf_filename)
|
||||
if os.stat(filepath).st_size == 0:
|
||||
#PDF page had no text, run tesseract on the page
|
||||
imagefile = convert_document_for_ocr(document, page=document_page.page_number)
|
||||
run_tesseract(imagefile, filepath, TESSERACT_LANGUAGE)
|
||||
ocr_output = os.extsep.join([filepath, u'txt'])
|
||||
source = _(u'Text from OCR')
|
||||
else:
|
||||
ocr_output = filepath
|
||||
source = _(u'Text extracted from PDF')
|
||||
else:
|
||||
imagefile = convert_document_for_ocr(document, page=document_page.page_number)
|
||||
run_tesseract(imagefile, filepath, TESSERACT_LANGUAGE)
|
||||
ocr_output = os.extsep.join([filepath, u'txt'])
|
||||
source = _(u'Text from OCR')
|
||||
f = codecs.open(ocr_output, 'r', 'utf-8')
|
||||
document_page.content = ocr_cleanup(f.read().strip())
|
||||
document_page.page_label = source
|
||||
document_page.save()
|
||||
f.close()
|
||||
cleanup(ocr_output)
|
||||
finally:
|
||||
os.close(desc)
|
||||
cleanup(filepath)
|
||||
if imagefile:
|
||||
cleanup(imagefile)
|
||||
# Try to extract text by means of a parser
|
||||
parse_document_page(document_page)
|
||||
except (ParserError, ParserUnknownFile):
|
||||
# Fall back to doing visual OCR
|
||||
pass
|
||||
#desc, filepath = tempfile.mkstemp()
|
||||
#imagefile = None
|
||||
#source = u''
|
||||
#imagefile = convert_document_for_ocr(document, page=document_page.page_number)
|
||||
#run_tesseract(imagefile, filepath, TESSERACT_LANGUAGE)
|
||||
#ocr_output = os.extsep.join([filepath, u'txt'])
|
||||
#source = _(u'Text from OCR')
|
||||
#f = codecs.open(ocr_output, 'r', 'utf-8')
|
||||
#document_page.content = ocr_cleanup(f.read().strip())
|
||||
#document_page.page_label = source
|
||||
#document_page.save()
|
||||
#f.close()
|
||||
#cleanup(ocr_output)
|
||||
#finally:
|
||||
# pass
|
||||
#os.close(desc)
|
||||
#cleanup(filepath)
|
||||
#if imagefile:
|
||||
# cleanup(imagefile)
|
||||
|
||||
|
||||
def ocr_cleanup(text):
|
||||
@@ -126,8 +104,8 @@ def ocr_cleanup(text):
|
||||
for line in text.splitlines():
|
||||
line = line.strip()
|
||||
for word in line.split():
|
||||
if backend:
|
||||
result = backend.check_word(word)
|
||||
if language_backend:
|
||||
result = language_backend.check_word(word)
|
||||
else:
|
||||
result = word
|
||||
if result:
|
||||
@@ -146,3 +124,53 @@ def clean_pages():
|
||||
if page.content:
|
||||
page.content = ocr_cleanup(page.content)
|
||||
page.save()
|
||||
|
||||
|
||||
def execute_unpaper(input_filepath, output_filepath):
|
||||
"""
|
||||
Executes the program unpaper using subprocess's Popen
|
||||
"""
|
||||
command = []
|
||||
command.append(UNPAPER_PATH)
|
||||
command.append(u'--overwrite')
|
||||
command.append(input_filepath)
|
||||
command.append(output_filepath)
|
||||
proc = subprocess.Popen(command, close_fds=True, stderr=subprocess.PIPE)
|
||||
return_code = proc.wait()
|
||||
if return_code != 0:
|
||||
raise UnpaperError(proc.stderr.readline())
|
||||
|
||||
'''
|
||||
def convert_document_for_ocr(document, page=DEFAULT_PAGE_NUMBER, file_format=DEFAULT_OCR_FILE_FORMAT):
|
||||
#Extract document file
|
||||
input_filepath = document_save_to_temp_dir(document, document.uuid)
|
||||
|
||||
#Convert for OCR
|
||||
temp_filename, separator = os.path.splitext(os.path.basename(input_filepath))
|
||||
temp_path = os.path.join(TEMPORARY_DIRECTORY, temp_filename)
|
||||
transformation_output_file = u'%s_trans%s%s%s' % (temp_path, page, os.extsep, file_format)
|
||||
unpaper_input_file = u'%s_unpaper_in%s%spnm' % (temp_path, page, os.extsep)
|
||||
unpaper_output_file = u'%s_unpaper_out%s%spnm' % (temp_path, page, os.extsep)
|
||||
convert_output_file = u'%s_ocr%s%s%s' % (temp_path, page, os.extsep, file_format)
|
||||
|
||||
try:
|
||||
document_page = document.documentpage_set.get(page_number=page)
|
||||
transformations, warnings = document_page.get_transformation_list()
|
||||
|
||||
#Apply default transformations
|
||||
backend.convert_file(input_filepath=input_filepath, page=page, quality=QUALITY_HIGH, transformations=transformations, output_filepath=transformation_output_file)
|
||||
#Do OCR operations
|
||||
backend.convert_file(input_filepath=transformation_output_file, arguments=OCR_OPTIONS, output_filepath=unpaper_input_file)
|
||||
# Process by unpaper
|
||||
execute_unpaper(input_filepath=unpaper_input_file, output_filepath=unpaper_output_file)
|
||||
# Convert to tif
|
||||
backend.convert_file(input_filepath=unpaper_output_file, output_filepath=convert_output_file)
|
||||
finally:
|
||||
cleanup(transformation_output_file)
|
||||
cleanup(unpaper_input_file)
|
||||
cleanup(unpaper_output_file)
|
||||
|
||||
return convert_output_file
|
||||
'''
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user