<s> from __future__ import absolute_import <EOL> from __future__ import with_statement <EOL> import copy <EOL> import threading <EOL> try : <EOL> import simplejson as json <EOL> except ImportError : <EOL> import json <EOL> from types import GeneratorType <EOL> from . exceptions import BulkOperationException <EOL> __author__ = '<STR_LIT>' <EOL> class DotDict ( dict ) : <EOL> def __getattr__ ( self , attr ) : <EOL> if attr . startswith ( '<STR_LIT>' ) : <EOL> raise AttributeError <EOL> return self . get ( attr , None ) <EOL> __setattr__ = dict . __setitem__ <EOL> __delattr__ = dict . __delitem__ <EOL> def __deepcopy__ ( self , memo ) : <EOL> return DotDict ( [ ( copy . deepcopy ( k , memo ) , copy . deepcopy ( v , memo ) ) for k , v in self . items ( ) ] ) <EOL> class ElasticSearchModel ( DotDict ) : <EOL> def __init__ ( self , * args , ** kwargs ) : <EOL> from pyes import ES <EOL> self . _meta = DotDict ( ) <EOL> self . __initialised = True <EOL> if len ( args ) == <NUM_LIT:2> and isinstance ( args [ <NUM_LIT:0> ] , ES ) : <EOL> item = args [ <NUM_LIT:1> ] <EOL> self . update ( item . pop ( "<STR_LIT>" , DotDict ( ) ) ) <EOL> self . update ( item . pop ( "<STR_LIT>" , { } ) ) <EOL> self . _meta = DotDict ( [ ( k . lstrip ( "<STR_LIT:_>" ) , v ) for k , v in item . items ( ) ] ) <EOL> self . _meta . parent = self . pop ( "<STR_LIT>" , None ) <EOL> self . _meta . connection = args [ <NUM_LIT:0> ] <EOL> else : <EOL> self . update ( dict ( * args , ** kwargs ) ) <EOL> def __setattr__ ( self , key , value ) : <EOL> if '<STR_LIT>' not in list ( self . __dict__ . keys ( ) ) : <EOL> return dict . __setattr__ ( self , key , value ) <EOL> elif key in list ( self . __dict__ . keys ( ) ) : <EOL> dict . __setattr__ ( self , key , value ) <EOL> else : <EOL> self . __setitem__ ( key , value ) <EOL> def get_meta ( self ) : <EOL> return self . _meta <EOL> def delete ( self , bulk = False ) : <EOL> """<STR_LIT>""" <EOL> meta = self . _meta <EOL> conn = meta [ '<STR_LIT>' ] <EOL> conn . delete ( meta . index , meta . type , meta . id , bulk = bulk ) <EOL> def save ( self , bulk = False , id = None , parent = None , routing = None , force = False ) : <EOL> """<STR_LIT>""" <EOL> meta = self . _meta <EOL> conn = meta [ '<STR_LIT>' ] <EOL> id = id or meta . get ( "<STR_LIT:id>" , None ) <EOL> parent = parent or meta . get ( '<STR_LIT>' , None ) <EOL> routing = routing or meta . get ( '<STR_LIT>' , None ) <EOL> qargs = None <EOL> if routing : <EOL> qargs = { '<STR_LIT>' : routing } <EOL> version = meta . get ( '<STR_LIT:version>' , None ) <EOL> if force : <EOL> version = None <EOL> res = conn . index ( self , <EOL> meta . index , meta . type , id , parent = parent , bulk = bulk , <EOL> version = version , force_insert = force , <EOL> querystring_args = qargs ) <EOL> if not bulk : <EOL> self . _meta . id = res . _id <EOL> self . _meta . version = res . _version <EOL> return res . _id <EOL> return id <EOL> def reload ( self ) : <EOL> meta = self . _meta <EOL> conn = meta [ '<STR_LIT>' ] <EOL> res = conn . get ( meta . index , meta . type , meta [ "<STR_LIT:id>" ] ) <EOL> self . update ( res ) <EOL> def get_id ( self ) : <EOL> """<STR_LIT>""" <EOL> _id = self . _meta . get ( "<STR_LIT:id>" , None ) <EOL> if _id is None : <EOL> _id = self . save ( ) <EOL> return _id <EOL> def get_bulk ( self , create = False ) : <EOL> """<STR_LIT>""" <EOL> result = [ ] <EOL> op_type = "<STR_LIT:index>" <EOL> if create : <EOL> op_type = "<STR_LIT>" <EOL> meta = self . _meta <EOL> cmd = { op_type : { "<STR_LIT>" : meta . index , "<STR_LIT>" : meta . type } } <EOL> if meta . parent : <EOL> cmd [ op_type ] [ '<STR_LIT>' ] = meta . parent <EOL> if meta . version : <EOL> cmd [ op_type ] [ '<STR_LIT>' ] = meta . version <EOL> if meta . id : <EOL> cmd [ op_type ] [ '<STR_LIT>' ] = meta . id <EOL> result . append ( json . dumps ( cmd , cls = self . _meta . connection . encoder ) ) <EOL> result . append ( "<STR_LIT:\n>" ) <EOL> result . append ( json . dumps ( self , cls = self . _meta . connection . encoder ) ) <EOL> result . append ( "<STR_LIT:\n>" ) <EOL> return '<STR_LIT>' . join ( result ) <EOL> class BaseBulker ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , conn , bulk_size = <NUM_LIT> , raise_on_bulk_item_failure = False ) : <EOL> self . conn = conn <EOL> self . _bulk_size = bulk_size <EOL> self . bulk_lock = threading . RLock ( ) <EOL> with self . bulk_lock : <EOL> self . bulk_data = [ ] <EOL> self . raise_on_bulk_item_failure = raise_on_bulk_item_failure <EOL> def __bool__ ( self ) : <EOL> return self . __nonzero__ ( ) <EOL> def get_bulk_size ( self ) : <EOL> """<STR_LIT>""" <EOL> return self . _bulk_size <EOL> def set_bulk_size ( self , bulk_size ) : <EOL> """<STR_LIT>""" <EOL> self . _bulk_size = bulk_size <EOL> self . flush_bulk ( ) <EOL> bulk_size = property ( get_bulk_size , set_bulk_size ) <EOL> def add ( self , content ) : <EOL> raise NotImplementedError <EOL> def flush_bulk ( self , forced = False ) : <EOL> raise NotImplementedError <EOL> class ListBulker ( BaseBulker ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , conn , bulk_size = <NUM_LIT> , raise_on_bulk_item_failure = False ) : <EOL> super ( ListBulker , self ) . __init__ ( conn = conn , bulk_size = bulk_size , <EOL> raise_on_bulk_item_failure = raise_on_bulk_item_failure ) <EOL> with self . bulk_lock : <EOL> self . bulk_data = [ ] <EOL> def __nonzero__ ( self ) : <EOL> return not not self . bulk_data <EOL> def add ( self , content ) : <EOL> with self . bulk_lock : <EOL> self . bulk_data . append ( content ) <EOL> def flush_bulk ( self , forced = False ) : <EOL> with self . bulk_lock : <EOL> if forced or len ( self . bulk_data ) >= self . bulk_size : <EOL> batch = self . bulk_data <EOL> self . bulk_data = [ ] <EOL> else : <EOL> return None <EOL> if len ( batch ) > <NUM_LIT:0> : <EOL> bulk_result = self . conn . _send_request ( "<STR_LIT:POST>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT:\n>" . join ( batch ) + "<STR_LIT:\n>" ) <EOL> if self . raise_on_bulk_item_failure : <EOL> _raise_exception_if_bulk_item_failed ( bulk_result ) <EOL> return bulk_result <EOL> def _is_bulk_item_ok ( item ) : <EOL> operations = ( "<STR_LIT>" , "<STR_LIT:index>" , "<STR_LIT>" , "<STR_LIT>" ) <EOL> for op in operations : <EOL> try : <EOL> if '<STR_LIT>' in item [ op ] : <EOL> return True <EOL> elif '<STR_LIT:status>' in item [ op ] : <EOL> return item [ op ] [ '<STR_LIT:status>' ] in [ <NUM_LIT:200> , <NUM_LIT> ] <EOL> else : <EOL> return False <EOL> except KeyError : <EOL> continue <EOL> return False <EOL> def _raise_exception_if_bulk_item_failed ( bulk_result ) : <EOL> errors = [ item for item in bulk_result [ "<STR_LIT>" ] if not _is_bulk_item_ok ( item ) ] <EOL> if len ( errors ) > <NUM_LIT:0> : <EOL> raise BulkOperationException ( errors , bulk_result ) <EOL> return None <EOL> class SortedDict ( dict ) : <EOL> """<STR_LIT>""" <EOL> def __new__ ( cls , * args , ** kwargs ) : <EOL> instance = super ( SortedDict , cls ) . __new__ ( cls , * args , ** kwargs ) <EOL> instance . keyOrder = [ ] <EOL> return instance <EOL> def __init__ ( self , data = None ) : <EOL> if data is None : <EOL> data = { } <EOL> elif isinstance ( data , GeneratorType ) : <EOL> data = list ( data ) <EOL> super ( SortedDict , self ) . __init__ ( data ) <EOL> if isinstance ( data , dict ) : <EOL> self . keyOrder = data . keys ( ) <EOL> else : <EOL> self . keyOrder = [ ] <EOL> seen = set ( ) <EOL> for key , value in data : <EOL> if key not in seen : <EOL> self . keyOrder . append ( key ) <EOL> seen . add ( key ) <EOL> def __deepcopy__ ( self , memo ) : <EOL> return self . __class__ ( [ ( key , copy . deepcopy ( value , memo ) ) <EOL> for key , value in self . items ( ) ] ) <EOL> def __setitem__ ( self , key , value ) : <EOL> if key not in self : <EOL> self . keyOrder . append ( key ) <EOL> super ( SortedDict , self ) . __setitem__ ( key , value ) <EOL> def __delitem__ ( self , key ) : <EOL> super ( SortedDict , self ) . __delitem__ ( key ) <EOL> self . keyOrder . remove ( key ) <EOL> def __iter__ ( self ) : <EOL> return iter ( self . keyOrder ) <EOL> def pop ( self , k , * args ) : <EOL> result = super ( SortedDict , self ) . pop ( k , * args ) <EOL> try : <EOL> self . keyOrder . remove ( k ) <EOL> except ValueError : <EOL> pass <EOL> return result <EOL> def popitem ( self ) : <EOL> result = super ( SortedDict , self ) . popitem ( ) <EOL> self . keyOrder . remove ( result [ <NUM_LIT:0> ] ) <EOL> return result <EOL> def items ( self ) : <EOL> return zip ( self . keyOrder , self . values ( ) ) <EOL> def items ( self ) : <EOL> for key in self . keyOrder : <EOL> yield key , self [ key ] <EOL> def keys ( self ) : <EOL> return self . keyOrder [ : ] <EOL> def iterkeys ( self ) : <EOL> return iter ( self . keyOrder ) <EOL> def values ( self ) : <EOL> return map ( self . __getitem__ , self . keyOrder ) <EOL> def itervalues ( self ) : <EOL> for key in self . keyOrder : <EOL> yield self [ key ] <EOL> def update ( self , dict_ ) : <EOL> for k , v in dict_ . items ( ) : <EOL> self [ k ] = v <EOL> def setdefault ( self , key , default ) : <EOL> if key not in self : <EOL> self . keyOrder . append ( key ) <EOL> return super ( SortedDict , self ) . setdefault ( key , default ) <EOL> def value_for_index ( self , index ) : <EOL> """<STR_LIT>""" <EOL> return self [ self . keyOrder [ index ] ] <EOL> def insert ( self , index , key , value ) : <EOL> """<STR_LIT>""" <EOL> if key in self . keyOrder : <EOL> n = self . keyOrder . index ( key ) <EOL> del self . keyOrder [ n ] <EOL> if n < index : <EOL> index -= <NUM_LIT:1> <EOL> self . keyOrder . insert ( index , key ) <EOL> super ( SortedDict , self ) . __setitem__ ( key , value ) <EOL> def copy ( self ) : <EOL> """<STR_LIT>""" <EOL> obj = self . __class__ ( self ) <EOL> obj . keyOrder = self . keyOrder [ : ] <EOL> return obj <EOL> def __repr__ ( self ) : <EOL> """<STR_LIT>""" <EOL> return '<STR_LIT>' % '<STR_LIT:U+002CU+0020>' . join ( [ '<STR_LIT>' % ( k , v ) for k , v in self . items ( ) ] ) <EOL> def clear ( self ) : <EOL> super ( SortedDict , self ) . clear ( ) <EOL> self . keyOrder = [ ] </s>
