<s> import logging <EOL> from aleph import process <EOL> from aleph . core import db , get_archive <EOL> from aleph . ext import get_ingestors <EOL> from aleph . model import Document <EOL> from aleph . analyze import analyze_document <EOL> log = logging . getLogger ( __name__ ) <EOL> class Ingestor ( object ) : <EOL> DOCUMENT_TYPE = Document . TYPE_OTHER <EOL> MIME_TYPES = [ ] <EOL> EXTENSIONS = [ ] <EOL> BASE_SCORE = <NUM_LIT:5> <EOL> def __init__ ( self , source_id ) : <EOL> self . source_id = source_id <EOL> def ingest ( self , meta , local_path ) : <EOL> raise NotImplemented ( ) <EOL> def create_document ( self , meta , type = None ) : <EOL> if meta . content_hash : <EOL> q = Document . all ( ) <EOL> if meta . foreign_id : <EOL> q = q . filter ( Document . foreign_id == meta . foreign_id ) <EOL> else : <EOL> q = q . filter ( Document . content_hash == meta . content_hash ) <EOL> q = q . filter ( Document . source_id == self . source_id ) <EOL> document = q . first ( ) <EOL> if document is None : <EOL> document = Document ( ) <EOL> document . source_id = self . source_id <EOL> document . meta = meta <EOL> document . type = type or self . DOCUMENT_TYPE <EOL> db . session . add ( document ) <EOL> db . session . flush ( ) <EOL> return document <EOL> def emit ( self , document ) : <EOL> db . session . commit ( ) <EOL> log . debug ( "<STR_LIT>" , document ) <EOL> analyze_document ( document . id ) <EOL> def log_error ( self , meta , error_type = None , error_message = None , <EOL> error_details = None ) : <EOL> process . log ( process . INGEST , component = type ( self ) . __name__ , <EOL> source_id = self . source_id , meta = meta , <EOL> error_type = error_type , error_message = error_message , <EOL> error_details = error_details ) <EOL> def log_exception ( self , meta , exception ) : <EOL> process . exception ( process . INGEST , component = type ( self ) . __name__ , <EOL> source_id = self . source_id , meta = meta , <EOL> exception = exception ) <EOL> @ classmethod <EOL> def match ( cls , meta , local_path ) : <EOL> score = - <NUM_LIT:1> <EOL> if meta . mime_type in cls . MIME_TYPES : <EOL> score += cls . BASE_SCORE <EOL> if meta . extension in cls . EXTENSIONS : <EOL> score += cls . BASE_SCORE <EOL> return score <EOL> @ classmethod <EOL> def auction_file ( cls , meta , local_path ) : <EOL> best_score , best_cls = <NUM_LIT:0> , None <EOL> for cls in get_ingestors ( ) . values ( ) : <EOL> score = cls . match ( meta , local_path ) <EOL> if score > best_score : <EOL> best_score = score <EOL> best_cls = cls <EOL> return best_cls <EOL> @ classmethod <EOL> def dispatch ( cls , source_id , meta ) : <EOL> local_path = get_archive ( ) . load_file ( meta ) <EOL> best_cls = cls . auction_file ( meta , local_path ) <EOL> if best_cls is None : <EOL> message = "<STR_LIT>" % meta . file_name <EOL> process . log ( process . INGEST , component = cls . __name__ , meta = meta , <EOL> source_id = source_id , error_type = '<STR_LIT>' , <EOL> error_message = message ) <EOL> return <EOL> log . debug ( "<STR_LIT>" , meta . file_name , best_cls . __name__ ) <EOL> try : <EOL> best_cls ( source_id ) . ingest ( meta , local_path ) <EOL> except Exception as ex : <EOL> log . exception ( ex ) <EOL> process . exception ( process . INGEST , component = best_cls . __name__ , <EOL> exception = ex , meta = meta , source_id = source_id ) <EOL> finally : <EOL> get_archive ( ) . cleanup_file ( meta ) </s>
