<s> """<STR_LIT>""" <EOL> from pyes import ES <EOL> import hashlib <EOL> from scrapy . utils . project import get_project_settings <EOL> from scrapy import log <EOL> class ElasticSearchPipeline ( object ) : <EOL> def __init__ ( self ) : <EOL> self . settings = get_project_settings ( ) <EOL> basic_auth = { '<STR_LIT:username>' : self . settings [ '<STR_LIT>' ] , '<STR_LIT:password>' : self . settings [ '<STR_LIT>' ] } <EOL> if self . settings [ '<STR_LIT>' ] : <EOL> uri = "<STR_LIT>" % ( self . settings [ '<STR_LIT>' ] , self . settings [ '<STR_LIT>' ] ) <EOL> else : <EOL> uri = "<STR_LIT:%s>" % ( self . settings [ '<STR_LIT>' ] ) <EOL> self . es = ES ( [ uri ] , basic_auth = basic_auth ) <EOL> def process_item ( self , item , spider ) : <EOL> if self . __get_uniq_key ( ) is None : <EOL> log . msg ( "<STR_LIT>" ) <EOL> self . es . index ( dict ( item ) , self . settings [ '<STR_LIT>' ] , self . settings [ '<STR_LIT>' ] , <EOL> id = item [ '<STR_LIT:id>' ] , op_type = '<STR_LIT>' , ) <EOL> else : <EOL> self . es . index ( dict ( item ) , self . settings [ '<STR_LIT>' ] , self . settings [ '<STR_LIT>' ] , <EOL> self . _get_item_key ( item ) ) <EOL> log . msg ( "<STR_LIT>" % <EOL> ( self . settings [ '<STR_LIT>' ] ) , <EOL> level = log . DEBUG , spider = spider ) <EOL> return item <EOL> def _get_item_key ( self , item ) : <EOL> uniq = self . __get_uniq_key ( ) <EOL> if isinstance ( uniq , list ) : <EOL> values = [ item [ key ] for key in uniq ] <EOL> value = '<STR_LIT>' . join ( values ) <EOL> else : <EOL> value = uniq <EOL> return hashlib . sha1 ( value ) . hexdigest ( ) <EOL> def __get_uniq_key ( self ) : <EOL> if not self . settings [ '<STR_LIT>' ] or self . settings [ '<STR_LIT>' ] == "<STR_LIT>" : <EOL> return None <EOL> return self . settings [ '<STR_LIT>' ] </s>
