<s> from __future__ import unicode_literals <EOL> from io import BytesIO <EOL> import logging <EOL> import os <EOL> from pdfminer . pdfinterp import PDFResourceManager , PDFPageInterpreter <EOL> from pdfminer . pdfpage import PDFPage <EOL> from pdfminer . converter import TextConverter <EOL> from pdfminer . layout import LAParams <EOL> import subprocess <EOL> import tempfile <EOL> from django . utils . translation import ugettext_lazy as _ <EOL> from common . settings import setting_temporary_directory <EOL> from common . utils import copyfile <EOL> from . exceptions import ParserError , NoMIMETypeMatch <EOL> from . models import DocumentPageContent <EOL> from . settings import setting_pdftotext_path <EOL> logger = logging . getLogger ( __name__ ) <EOL> class Parser ( object ) : <EOL> """<STR_LIT>""" <EOL> _registry = { } <EOL> @ classmethod <EOL> def register ( cls , mimetypes , parser_classes ) : <EOL> for mimetype in mimetypes : <EOL> for parser_class in parser_classes : <EOL> cls . _registry . setdefault ( <EOL> mimetype , [ ] <EOL> ) . append ( parser_class ) <EOL> @ classmethod <EOL> def parse_document_version ( cls , document_version ) : <EOL> try : <EOL> for parser_class in cls . _registry [ document_version . mimetype ] : <EOL> try : <EOL> parser = parser_class ( ) <EOL> parser . process_document_version ( document_version ) <EOL> except ParserError : <EOL> pass <EOL> else : <EOL> return <EOL> raise NoMIMETypeMatch ( '<STR_LIT>' ) <EOL> except KeyError : <EOL> raise NoMIMETypeMatch <EOL> @ classmethod <EOL> def parse_document_page ( cls , document_page ) : <EOL> try : <EOL> for parser_class in cls . _registry [ document_page . document_version . mimetype ] : <EOL> try : <EOL> parser = parser_class ( ) <EOL> parser . process_document_page ( document_page ) <EOL> except ParserError : <EOL> pass <EOL> else : <EOL> return <EOL> raise NoMIMETypeMatch ( '<STR_LIT>' ) <EOL> except KeyError : <EOL> raise NoMIMETypeMatch <EOL> def process_document_version ( self , document_version ) : <EOL> logger . info ( <EOL> '<STR_LIT>' , document_version <EOL> ) <EOL> logger . debug ( '<STR_LIT>' , document_version . pk ) <EOL> for document_page in document_version . pages . all ( ) : <EOL> self . process_document_page ( document_page = document_page ) <EOL> def process_document_page ( self , document_page ) : <EOL> logger . info ( <EOL> '<STR_LIT>' , <EOL> document_page . page_number , document_page . document_version <EOL> ) <EOL> file_object = document_page . document_version . get_intermidiate_file ( ) <EOL> try : <EOL> document_page_content , created = DocumentPageContent . objects . get_or_create ( <EOL> document_page = document_page <EOL> ) <EOL> document_page_content . content = self . execute ( <EOL> file_object = file_object , page_number = document_page . page_number <EOL> ) <EOL> document_page_content . save ( ) <EOL> except Exception as exception : <EOL> error_message = _ ( '<STR_LIT>' ) % exception <EOL> logger . error ( error_message ) <EOL> raise ParserError ( error_message ) <EOL> finally : <EOL> file_object . close ( ) <EOL> logger . info ( <EOL> '<STR_LIT>' , <EOL> document_page . page_number , document_page . document_version <EOL> ) <EOL> def execute ( self , file_object , page_number ) : <EOL> raise NotImplementedError ( <EOL> '<STR_LIT>' % <EOL> self . __class__ . __name__ <EOL> ) <EOL> class PopplerParser ( Parser ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self ) : <EOL> self . pdftotext_path = setting_pdftotext_path . value <EOL> if not os . path . exists ( self . pdftotext_path ) : <EOL> error_message = _ ( <EOL> '<STR_LIT>' <EOL> ) % self . pdftotext_path <EOL> logger . error ( error_message ) <EOL> raise ParserError ( error_message ) <EOL> logger . debug ( '<STR_LIT>' , self . pdftotext_path ) <EOL> def execute ( self , file_object , page_number ) : <EOL> logger . debug ( '<STR_LIT>' , page_number ) <EOL> destination_descriptor , temp_filepath = tempfile . mkstemp ( <EOL> dir = setting_temporary_directory . value <EOL> ) <EOL> copyfile ( file_object , temp_filepath ) <EOL> command = [ ] <EOL> command . append ( self . pdftotext_path ) <EOL> command . append ( '<STR_LIT>' ) <EOL> command . append ( str ( page_number ) ) <EOL> command . append ( '<STR_LIT>' ) <EOL> command . append ( str ( page_number ) ) <EOL> command . append ( temp_filepath ) <EOL> command . append ( '<STR_LIT:->' ) <EOL> proc = subprocess . Popen ( <EOL> command , close_fds = True , stderr = subprocess . PIPE , <EOL> stdout = subprocess . PIPE <EOL> ) <EOL> return_code = proc . wait ( ) <EOL> if return_code != <NUM_LIT:0> : <EOL> logger . error ( proc . stderr . readline ( ) ) <EOL> raise ParserError <EOL> output = proc . stdout . read ( ) <EOL> if output == b'<STR_LIT>' : <EOL> logger . debug ( '<STR_LIT>' ) <EOL> return '<STR_LIT>' <EOL> if output [ - <NUM_LIT:3> : ] == b'<STR_LIT>' : <EOL> return output [ : - <NUM_LIT:3> ] <EOL> return output <EOL> class PDFMinerParser ( Parser ) : <EOL> """<STR_LIT>""" <EOL> def execute ( self , file_object , page_number ) : <EOL> logger . debug ( '<STR_LIT>' , page_number ) <EOL> with BytesIO ( ) as string_buffer : <EOL> rsrcmgr = PDFResourceManager ( ) <EOL> device = TextConverter ( <EOL> rsrcmgr , outfp = string_buffer , laparams = LAParams ( ) <EOL> ) <EOL> interpreter = PDFPageInterpreter ( rsrcmgr , device ) <EOL> page = PDFPage . get_pages ( <EOL> file_object , maxpages = <NUM_LIT:1> , pagenos = ( page_number - <NUM_LIT:1> , ) <EOL> ) <EOL> interpreter . process_page ( page . next ( ) ) <EOL> device . close ( ) <EOL> logger . debug ( '<STR_LIT>' , page_number ) <EOL> return string_buffer . getvalue ( ) <EOL> Parser . register ( <EOL> mimetypes = ( '<STR_LIT>' , ) , <EOL> parser_classes = ( PopplerParser , PDFMinerParser ) <EOL> ) </s>
