<s> import time <EOL> import elasticsearch . helpers <EOL> from elasticsearch import Elasticsearch <EOL> from pyspider . database . base . projectdb import ProjectDB as BaseProjectDB <EOL> class ProjectDB ( BaseProjectDB ) : <EOL> __type__ = '<STR_LIT>' <EOL> def __init__ ( self , hosts , index = '<STR_LIT>' ) : <EOL> self . index = index <EOL> self . es = Elasticsearch ( hosts = hosts ) <EOL> self . es . indices . create ( index = self . index , ignore = <NUM_LIT> ) <EOL> if not self . es . indices . get_mapping ( index = self . index , doc_type = self . __type__ ) : <EOL> self . es . indices . put_mapping ( index = self . index , doc_type = self . __type__ , body = { <EOL> "<STR_LIT>" : { "<STR_LIT>" : False } , <EOL> "<STR_LIT>" : { <EOL> "<STR_LIT>" : { "<STR_LIT:type>" : "<STR_LIT>" } <EOL> } <EOL> } ) <EOL> def insert ( self , name , obj = { } ) : <EOL> obj = dict ( obj ) <EOL> obj [ '<STR_LIT:name>' ] = name <EOL> obj [ '<STR_LIT>' ] = time . time ( ) <EOL> obj . setdefault ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> obj . setdefault ( '<STR_LIT:status>' , '<STR_LIT>' ) <EOL> obj . setdefault ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> obj . setdefault ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> obj . setdefault ( '<STR_LIT>' , <NUM_LIT:0> ) <EOL> obj . setdefault ( '<STR_LIT>' , <NUM_LIT:0> ) <EOL> return self . es . index ( index = self . index , doc_type = self . __type__ , body = obj , id = name , <EOL> refresh = True ) <EOL> def update ( self , name , obj = { } , ** kwargs ) : <EOL> obj = dict ( obj ) <EOL> obj . update ( kwargs ) <EOL> obj [ '<STR_LIT>' ] = time . time ( ) <EOL> return self . es . update ( index = self . index , doc_type = self . __type__ , <EOL> body = { '<STR_LIT>' : obj } , id = name , refresh = True , ignore = <NUM_LIT> ) <EOL> def get_all ( self , fields = None ) : <EOL> for record in elasticsearch . helpers . scan ( self . es , index = self . index , doc_type = self . __type__ , <EOL> query = { '<STR_LIT>' : { "<STR_LIT>" : { } } } , <EOL> _source_include = fields or [ ] ) : <EOL> yield record [ '<STR_LIT>' ] <EOL> def get ( self , name , fields = None ) : <EOL> ret = self . es . get ( index = self . index , doc_type = self . __type__ , id = name , <EOL> _source_include = fields or [ ] , ignore = <NUM_LIT> ) <EOL> return ret . get ( '<STR_LIT>' , None ) <EOL> def check_update ( self , timestamp , fields = None ) : <EOL> for record in elasticsearch . helpers . scan ( self . es , index = self . index , doc_type = self . __type__ , <EOL> query = { '<STR_LIT>' : { "<STR_LIT>" : { <EOL> "<STR_LIT>" : { "<STR_LIT>" : timestamp } <EOL> } } } , _source_include = fields or [ ] ) : <EOL> yield record [ '<STR_LIT>' ] <EOL> def drop ( self , name ) : <EOL> return self . es . delete ( index = self . index , doc_type = self . __type__ , id = name , refresh = True ) </s>
