<s> """<STR_LIT>""" <EOL> import datetime <EOL> try : <EOL> from google . appengine . ext import ndb <EOL> except ImportError : <EOL> ndb = None <EOL> from google . appengine . ext import db <EOL> from google . appengine . ext . mapreduce import errors <EOL> from google . appengine . ext . mapreduce import util <EOL> __all__ = [ <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" ] <EOL> def should_shard_by_property_range ( filters ) : <EOL> """<STR_LIT>""" <EOL> if not filters : <EOL> return False <EOL> for f in filters : <EOL> if f [ <NUM_LIT:1> ] != "<STR_LIT:=>" : <EOL> return True <EOL> return False <EOL> class PropertyRange ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , <EOL> filters , <EOL> model_class_path ) : <EOL> """<STR_LIT>""" <EOL> self . filters = filters <EOL> self . model_class_path = model_class_path <EOL> self . model_class = util . for_name ( self . model_class_path ) <EOL> self . prop , self . start , self . end = self . _get_range_from_filters ( <EOL> self . filters , self . model_class ) <EOL> @ classmethod <EOL> def _get_range_from_filters ( cls , filters , model_class ) : <EOL> """<STR_LIT>""" <EOL> if not filters : <EOL> return None , None , None <EOL> range_property = None <EOL> start_val = None <EOL> end_val = None <EOL> start_filter = None <EOL> end_filter = None <EOL> for f in filters : <EOL> prop , op , val = f <EOL> if op in [ "<STR_LIT:>>" , "<STR_LIT>" , "<STR_LIT:<>" , "<STR_LIT>" ] : <EOL> if range_property and range_property != prop : <EOL> raise errors . BadReaderParamsError ( <EOL> "<STR_LIT>" ) <EOL> range_property = prop <EOL> if val is None : <EOL> raise errors . BadReaderParamsError ( <EOL> "<STR_LIT>" , f ) <EOL> if op in [ "<STR_LIT:>>" , "<STR_LIT>" ] : <EOL> if start_val is not None : <EOL> raise errors . BadReaderParamsError ( <EOL> "<STR_LIT>" , op ) <EOL> start_val = val <EOL> start_filter = f <EOL> else : <EOL> if end_val is not None : <EOL> raise errors . BadReaderParamsError ( <EOL> "<STR_LIT>" , op ) <EOL> end_val = val <EOL> end_filter = f <EOL> elif op != "<STR_LIT:=>" : <EOL> raise errors . BadReaderParamsError ( <EOL> "<STR_LIT>" , op ) <EOL> if not range_property : <EOL> return None , None , None <EOL> if start_val is None or end_val is None : <EOL> raise errors . BadReaderParamsError ( <EOL> "<STR_LIT>" , <EOL> range_property ) <EOL> if issubclass ( model_class , db . Model ) : <EOL> property_obj = model_class . properties ( ) [ range_property ] <EOL> else : <EOL> property_obj = ( <EOL> model_class . _properties [ <EOL> range_property ] ) <EOL> supported_properties = ( <EOL> _DISCRETE_PROPERTY_SPLIT_FUNCTIONS . keys ( ) + <EOL> _CONTINUOUS_PROPERTY_SPLIT_FUNCTIONS . keys ( ) ) <EOL> if not isinstance ( property_obj , tuple ( supported_properties ) ) : <EOL> raise errors . BadReaderParamsError ( <EOL> "<STR_LIT>" , range_property ) <EOL> if not start_val < end_val : <EOL> raise errors . BadReaderParamsError ( <EOL> "<STR_LIT>" , <EOL> start_val , end_val ) <EOL> return property_obj , start_filter , end_filter <EOL> def split ( self , n ) : <EOL> """<STR_LIT>""" <EOL> new_range_filters = [ ] <EOL> name = self . start [ <NUM_LIT:0> ] <EOL> prop_cls = self . prop . __class__ <EOL> if prop_cls in _DISCRETE_PROPERTY_SPLIT_FUNCTIONS : <EOL> splitpoints = _DISCRETE_PROPERTY_SPLIT_FUNCTIONS [ prop_cls ] ( <EOL> self . start [ <NUM_LIT:2> ] , self . end [ <NUM_LIT:2> ] , n , <EOL> self . start [ <NUM_LIT:1> ] == "<STR_LIT>" , self . end [ <NUM_LIT:1> ] == "<STR_LIT>" ) <EOL> start_filter = ( name , "<STR_LIT>" , splitpoints [ <NUM_LIT:0> ] ) <EOL> for p in splitpoints [ <NUM_LIT:1> : ] : <EOL> end_filter = ( name , "<STR_LIT:<>" , p ) <EOL> new_range_filters . append ( [ start_filter , end_filter ] ) <EOL> start_filter = ( name , "<STR_LIT>" , p ) <EOL> else : <EOL> splitpoints = _CONTINUOUS_PROPERTY_SPLIT_FUNCTIONS [ prop_cls ] ( <EOL> self . start [ <NUM_LIT:2> ] , self . end [ <NUM_LIT:2> ] , n ) <EOL> start_filter = self . start <EOL> for p in splitpoints : <EOL> end_filter = ( name , "<STR_LIT:<>" , p ) <EOL> new_range_filters . append ( [ start_filter , end_filter ] ) <EOL> start_filter = ( name , "<STR_LIT>" , p ) <EOL> new_range_filters . append ( [ start_filter , self . end ] ) <EOL> for f in new_range_filters : <EOL> f . extend ( self . _equality_filters ) <EOL> return [ self . __class__ ( f , self . model_class_path ) for f in new_range_filters ] <EOL> def make_query ( self , ns ) : <EOL> """<STR_LIT>""" <EOL> if issubclass ( self . model_class , db . Model ) : <EOL> query = db . Query ( self . model_class , namespace = ns ) <EOL> for f in self . filters : <EOL> query . filter ( "<STR_LIT>" % ( f [ <NUM_LIT:0> ] , f [ <NUM_LIT:1> ] ) , f [ <NUM_LIT:2> ] ) <EOL> else : <EOL> query = self . model_class . query ( namespace = ns ) <EOL> for f in self . filters : <EOL> query = query . filter ( ndb . FilterNode ( * f ) ) <EOL> return query <EOL> @ property <EOL> def _equality_filters ( self ) : <EOL> return [ f for f in self . filters if f [ <NUM_LIT:1> ] == "<STR_LIT:=>" ] <EOL> def to_json ( self ) : <EOL> return { "<STR_LIT>" : self . filters , <EOL> "<STR_LIT>" : self . model_class_path } <EOL> @ classmethod <EOL> def from_json ( cls , json ) : <EOL> return cls ( json [ "<STR_LIT>" ] , json [ "<STR_LIT>" ] ) <EOL> def _split_datetime_property ( start , end , n , include_start , include_end ) : <EOL> if not include_start : <EOL> start += datetime . timedelta ( microseconds = <NUM_LIT:1> ) <EOL> if include_end : <EOL> end += datetime . timedelta ( microseconds = <NUM_LIT:1> ) <EOL> delta = end - start <EOL> stride = delta // n <EOL> if stride <= datetime . timedelta ( ) : <EOL> raise ValueError ( "<STR_LIT>" , start , end ) <EOL> splitpoints = [ start ] <EOL> previous = start <EOL> for _ in range ( n - <NUM_LIT:1> ) : <EOL> point = previous + stride <EOL> if point == previous or point > end : <EOL> continue <EOL> previous = point <EOL> splitpoints . append ( point ) <EOL> if end not in splitpoints : <EOL> splitpoints . append ( end ) <EOL> return splitpoints <EOL> def _split_float_property ( start , end , n ) : <EOL> delta = float ( end - start ) <EOL> stride = delta / n <EOL> if stride <= <NUM_LIT:0> : <EOL> raise ValueError ( "<STR_LIT>" , start , end ) <EOL> splitpoints = [ ] <EOL> for i in range ( <NUM_LIT:1> , n ) : <EOL> splitpoints . append ( start + i * stride ) <EOL> return splitpoints <EOL> def _split_integer_property ( start , end , n , include_start , include_end ) : <EOL> if not include_start : <EOL> start += <NUM_LIT:1> <EOL> if include_end : <EOL> end += <NUM_LIT:1> <EOL> delta = float ( end - start ) <EOL> stride = delta / n <EOL> if stride <= <NUM_LIT:0> : <EOL> raise ValueError ( "<STR_LIT>" , start , end ) <EOL> splitpoints = [ start ] <EOL> previous = start <EOL> for i in range ( <NUM_LIT:1> , n ) : <EOL> point = start + int ( round ( i * stride ) ) <EOL> if point == previous or point > end : <EOL> continue <EOL> previous = point <EOL> splitpoints . append ( point ) <EOL> if end not in splitpoints : <EOL> splitpoints . append ( end ) <EOL> return splitpoints <EOL> def _split_string_property ( start , end , n , include_start , include_end ) : <EOL> try : <EOL> start = start . encode ( "<STR_LIT:ascii>" ) <EOL> end = end . encode ( "<STR_LIT:ascii>" ) <EOL> except UnicodeEncodeError , e : <EOL> raise ValueError ( "<STR_LIT>" , e ) <EOL> return _split_byte_string_property ( start , end , n , include_start , include_end ) <EOL> _ALPHABET = "<STR_LIT>" . join ( chr ( i ) for i in range ( <NUM_LIT> ) ) <EOL> _STRING_LENGTH = <NUM_LIT:4> <EOL> def _split_byte_string_property ( start , end , n , include_start , include_end ) : <EOL> i = <NUM_LIT:0> <EOL> for i , ( s , e ) in enumerate ( zip ( start , end ) ) : <EOL> if s != e : <EOL> break <EOL> common_prefix = start [ : i ] <EOL> start_suffix = start [ i + _STRING_LENGTH : ] <EOL> end_suffix = end [ i + _STRING_LENGTH : ] <EOL> start = start [ i : i + _STRING_LENGTH ] <EOL> end = end [ i : i + _STRING_LENGTH ] <EOL> weights = _get_weights ( _STRING_LENGTH ) <EOL> start_ord = _str_to_ord ( start , weights ) <EOL> if not include_start : <EOL> start_ord += <NUM_LIT:1> <EOL> end_ord = _str_to_ord ( end , weights ) <EOL> if include_end : <EOL> end_ord += <NUM_LIT:1> <EOL> stride = ( end_ord - start_ord ) / float ( n ) <EOL> if stride <= <NUM_LIT:0> : <EOL> raise ValueError ( "<STR_LIT>" , start , end ) <EOL> splitpoints = [ _ord_to_str ( start_ord , weights ) ] <EOL> previous = start_ord <EOL> for i in range ( <NUM_LIT:1> , n ) : <EOL> point = start_ord + int ( round ( stride * i ) ) <EOL> if point == previous or point > end_ord : <EOL> continue <EOL> previous = point <EOL> splitpoints . append ( _ord_to_str ( point , weights ) ) <EOL> end_str = _ord_to_str ( end_ord , weights ) <EOL> if end_str not in splitpoints : <EOL> splitpoints . append ( end_str ) <EOL> splitpoints [ <NUM_LIT:0> ] += start_suffix <EOL> splitpoints [ - <NUM_LIT:1> ] += end_suffix <EOL> return [ common_prefix + point for point in splitpoints ] <EOL> def _get_weights ( max_length ) : <EOL> """<STR_LIT>""" <EOL> weights = [ <NUM_LIT:1> ] <EOL> for i in range ( <NUM_LIT:1> , max_length ) : <EOL> weights . append ( weights [ i - <NUM_LIT:1> ] * len ( _ALPHABET ) + <NUM_LIT:1> ) <EOL> weights . reverse ( ) <EOL> return weights <EOL> def _str_to_ord ( content , weights ) : <EOL> """<STR_LIT>""" <EOL> ordinal = <NUM_LIT:0> <EOL> for i , c in enumerate ( content ) : <EOL> ordinal += weights [ i ] * _ALPHABET . index ( c ) + <NUM_LIT:1> <EOL> return ordinal <EOL> def _ord_to_str ( ordinal , weights ) : <EOL> """<STR_LIT>""" <EOL> chars = [ ] <EOL> for weight in weights : <EOL> if ordinal == <NUM_LIT:0> : <EOL> return "<STR_LIT>" . join ( chars ) <EOL> ordinal -= <NUM_LIT:1> <EOL> index , ordinal = divmod ( ordinal , weight ) <EOL> chars . append ( _ALPHABET [ index ] ) <EOL> return "<STR_LIT>" . join ( chars ) <EOL> _DISCRETE_PROPERTY_SPLIT_FUNCTIONS = { <EOL> db . DateTimeProperty : _split_datetime_property , <EOL> db . IntegerProperty : _split_integer_property , <EOL> db . StringProperty : _split_string_property , <EOL> db . ByteStringProperty : _split_byte_string_property , <EOL> } <EOL> if ndb : <EOL> _DISCRETE_PROPERTY_SPLIT_FUNCTIONS . update ( { <EOL> ndb . DateTimeProperty : _split_datetime_property , <EOL> ndb . IntegerProperty : _split_integer_property , <EOL> ndb . StringProperty : _split_string_property , <EOL> ndb . BlobProperty : _split_byte_string_property <EOL> } ) <EOL> _CONTINUOUS_PROPERTY_SPLIT_FUNCTIONS = { <EOL> db . FloatProperty : _split_float_property , <EOL> } <EOL> if ndb : <EOL> _CONTINUOUS_PROPERTY_SPLIT_FUNCTIONS . update ( { <EOL> ndb . FloatProperty : _split_float_property , <EOL> } ) </s>
