<s> from __future__ import unicode_literals , division , absolute_import <EOL> import time <EOL> import logging <EOL> from collections import deque <EOL> try : <EOL> from UserDict import DictMixin <EOL> except ImportError : <EOL> from collections import Mapping as DictMixin <EOL> import six <EOL> from six import iteritems <EOL> from six . moves import cPickle <EOL> class BaseCounter ( object ) : <EOL> def __init__ ( self ) : <EOL> raise NotImplementedError <EOL> def event ( self , value = <NUM_LIT:1> ) : <EOL> """<STR_LIT>""" <EOL> raise NotImplementedError <EOL> def value ( self , value ) : <EOL> """<STR_LIT>""" <EOL> raise NotImplementedError <EOL> @ property <EOL> def avg ( self ) : <EOL> """<STR_LIT>""" <EOL> raise NotImplementedError <EOL> @ property <EOL> def sum ( self ) : <EOL> """<STR_LIT>""" <EOL> raise NotImplementedError <EOL> def empty ( self ) : <EOL> """<STR_LIT>""" <EOL> raise NotImplementedError <EOL> class TotalCounter ( BaseCounter ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self ) : <EOL> self . cnt = <NUM_LIT:0> <EOL> def event ( self , value = <NUM_LIT:1> ) : <EOL> self . cnt += value <EOL> def value ( self , value ) : <EOL> self . cnt = value <EOL> @ property <EOL> def avg ( self ) : <EOL> return self . cnt <EOL> @ property <EOL> def sum ( self ) : <EOL> return self . cnt <EOL> def empty ( self ) : <EOL> return self . cnt == <NUM_LIT:0> <EOL> class AverageWindowCounter ( BaseCounter ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , window_size = <NUM_LIT> ) : <EOL> self . window_size = window_size <EOL> self . values = deque ( maxlen = window_size ) <EOL> def event ( self , value = <NUM_LIT:1> ) : <EOL> self . values . append ( value ) <EOL> value = event <EOL> @ property <EOL> def avg ( self ) : <EOL> return self . sum / len ( self . values ) <EOL> @ property <EOL> def sum ( self ) : <EOL> return sum ( self . values ) <EOL> def empty ( self ) : <EOL> if not self . values : <EOL> return True <EOL> class TimebaseAverageEventCounter ( BaseCounter ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , window_size = <NUM_LIT:30> , window_interval = <NUM_LIT:10> ) : <EOL> self . max_window_size = window_size <EOL> self . window_size = <NUM_LIT:0> <EOL> self . window_interval = window_interval <EOL> self . values = deque ( maxlen = window_size ) <EOL> self . events = deque ( maxlen = window_size ) <EOL> self . times = deque ( maxlen = window_size ) <EOL> self . cache_value = <NUM_LIT:0> <EOL> self . cache_event = <NUM_LIT:0> <EOL> self . cache_start = None <EOL> self . _first_data_time = None <EOL> def event ( self , value = <NUM_LIT:1> ) : <EOL> now = time . time ( ) <EOL> if self . _first_data_time is None : <EOL> self . _first_data_time = now <EOL> if self . cache_start is None : <EOL> self . cache_value = value <EOL> self . cache_event = <NUM_LIT:1> <EOL> self . cache_start = now <EOL> elif now - self . cache_start > self . window_interval : <EOL> self . values . append ( self . cache_value ) <EOL> self . events . append ( self . cache_event ) <EOL> self . times . append ( self . cache_start ) <EOL> self . on_append ( self . cache_value , self . cache_start ) <EOL> self . cache_value = value <EOL> self . cache_event = <NUM_LIT:1> <EOL> self . cache_start = now <EOL> else : <EOL> self . cache_value += value <EOL> self . cache_event += <NUM_LIT:1> <EOL> return self <EOL> def value ( self , value ) : <EOL> self . cache_value = value <EOL> def _trim_window ( self ) : <EOL> now = time . time ( ) <EOL> if self . cache_start and now - self . cache_start > self . window_interval : <EOL> self . values . append ( self . cache_value ) <EOL> self . events . append ( self . cache_event ) <EOL> self . times . append ( self . cache_start ) <EOL> self . on_append ( self . cache_value , self . cache_start ) <EOL> self . cache_value = <NUM_LIT:0> <EOL> self . cache_start = None <EOL> if self . window_size != self . max_window_size and self . _first_data_time is not None : <EOL> time_passed = now - self . _first_data_time <EOL> self . window_size = min ( self . max_window_size , time_passed / self . window_interval ) <EOL> window_limit = now - self . window_size * self . window_interval <EOL> while self . times and self . times [ <NUM_LIT:0> ] < window_limit : <EOL> self . times . popleft ( ) <EOL> self . events . popleft ( ) <EOL> self . values . popleft ( ) <EOL> @ property <EOL> def avg ( self ) : <EOL> events = ( sum ( self . events ) + self . cache_event ) <EOL> if not events : <EOL> return <NUM_LIT:0> <EOL> return float ( self . sum ) / events <EOL> @ property <EOL> def sum ( self ) : <EOL> self . _trim_window ( ) <EOL> return sum ( self . values ) + self . cache_value <EOL> def empty ( self ) : <EOL> self . _trim_window ( ) <EOL> if not self . values and not self . cache_start : <EOL> return True <EOL> def on_append ( self , value , time ) : <EOL> pass <EOL> class TimebaseAverageWindowCounter ( BaseCounter ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , window_size = <NUM_LIT:30> , window_interval = <NUM_LIT:10> ) : <EOL> self . max_window_size = window_size <EOL> self . window_size = <NUM_LIT:0> <EOL> self . window_interval = window_interval <EOL> self . values = deque ( maxlen = window_size ) <EOL> self . times = deque ( maxlen = window_size ) <EOL> self . cache_value = <NUM_LIT:0> <EOL> self . cache_start = None <EOL> self . _first_data_time = None <EOL> def event ( self , value = <NUM_LIT:1> ) : <EOL> now = time . time ( ) <EOL> if self . _first_data_time is None : <EOL> self . _first_data_time = now <EOL> if self . cache_start is None : <EOL> self . cache_value = value <EOL> self . cache_start = now <EOL> elif now - self . cache_start > self . window_interval : <EOL> self . values . append ( self . cache_value ) <EOL> self . times . append ( self . cache_start ) <EOL> self . on_append ( self . cache_value , self . cache_start ) <EOL> self . cache_value = value <EOL> self . cache_start = now <EOL> else : <EOL> self . cache_value += value <EOL> return self <EOL> def value ( self , value ) : <EOL> self . cache_value = value <EOL> def _trim_window ( self ) : <EOL> now = time . time ( ) <EOL> if self . cache_start and now - self . cache_start > self . window_interval : <EOL> self . values . append ( self . cache_value ) <EOL> self . times . append ( self . cache_start ) <EOL> self . on_append ( self . cache_value , self . cache_start ) <EOL> self . cache_value = <NUM_LIT:0> <EOL> self . cache_start = None <EOL> if self . window_size != self . max_window_size and self . _first_data_time is not None : <EOL> time_passed = now - self . _first_data_time <EOL> self . window_size = min ( self . max_window_size , time_passed / self . window_interval ) <EOL> window_limit = now - self . window_size * self . window_interval <EOL> while self . times and self . times [ <NUM_LIT:0> ] < window_limit : <EOL> self . times . popleft ( ) <EOL> self . values . popleft ( ) <EOL> @ property <EOL> def avg ( self ) : <EOL> sum = float ( self . sum ) <EOL> if not self . window_size : <EOL> return <NUM_LIT:0> <EOL> return sum / self . window_size / self . window_interval <EOL> @ property <EOL> def sum ( self ) : <EOL> self . _trim_window ( ) <EOL> return sum ( self . values ) + self . cache_value <EOL> def empty ( self ) : <EOL> self . _trim_window ( ) <EOL> if not self . values and not self . cache_start : <EOL> return True <EOL> def on_append ( self , value , time ) : <EOL> pass <EOL> class CounterValue ( DictMixin ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , manager , keys ) : <EOL> self . manager = manager <EOL> self . _keys = keys <EOL> def __getitem__ ( self , key ) : <EOL> if key == '<STR_LIT>' : <EOL> key = self . _keys <EOL> return self . manager . counters [ key ] <EOL> else : <EOL> key = self . _keys + ( key , ) <EOL> available_keys = [ ] <EOL> for _key in self . manager . counters . keys ( ) : <EOL> if _key [ : len ( key ) ] == key : <EOL> available_keys . append ( _key ) <EOL> if len ( available_keys ) == <NUM_LIT:0> : <EOL> raise KeyError <EOL> elif len ( available_keys ) == <NUM_LIT:1> : <EOL> if available_keys [ <NUM_LIT:0> ] == key : <EOL> return self . manager . counters [ key ] <EOL> else : <EOL> return CounterValue ( self . manager , key ) <EOL> else : <EOL> return CounterValue ( self . manager , key ) <EOL> def __len__ ( self ) : <EOL> return len ( self . keys ( ) ) <EOL> def __iter__ ( self ) : <EOL> return iter ( self . keys ( ) ) <EOL> def __contains__ ( self , key ) : <EOL> return key in self . keys ( ) <EOL> def keys ( self ) : <EOL> result = set ( ) <EOL> for key in self . manager . counters . keys ( ) : <EOL> if key [ : len ( self . _keys ) ] == self . _keys : <EOL> key = key [ len ( self . _keys ) : ] <EOL> result . add ( key [ <NUM_LIT:0> ] if key else '<STR_LIT>' ) <EOL> return result <EOL> def to_dict ( self , get_value = None ) : <EOL> """<STR_LIT>""" <EOL> result = { } <EOL> for key , value in iteritems ( self ) : <EOL> if isinstance ( value , BaseCounter ) : <EOL> if get_value is not None : <EOL> value = getattr ( value , get_value ) <EOL> result [ key ] = value <EOL> else : <EOL> result [ key ] = value . to_dict ( get_value ) <EOL> return result <EOL> class CounterManager ( DictMixin ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , cls = TimebaseAverageWindowCounter ) : <EOL> """<STR_LIT>""" <EOL> self . cls = cls <EOL> self . counters = { } <EOL> def event ( self , key , value = <NUM_LIT:1> ) : <EOL> """<STR_LIT>""" <EOL> if isinstance ( key , six . string_types ) : <EOL> key = ( key , ) <EOL> assert isinstance ( key , tuple ) , "<STR_LIT>" <EOL> if key not in self . counters : <EOL> self . counters [ key ] = self . cls ( ) <EOL> self . counters [ key ] . event ( value ) <EOL> return self <EOL> def value ( self , key , value = <NUM_LIT:1> ) : <EOL> """<STR_LIT>""" <EOL> if isinstance ( key , six . string_types ) : <EOL> key = ( key , ) <EOL> assert isinstance ( key , tuple ) , "<STR_LIT>" <EOL> if key not in self . counters : <EOL> self . counters [ key ] = self . cls ( ) <EOL> self . counters [ key ] . value ( value ) <EOL> return self <EOL> def trim ( self ) : <EOL> """<STR_LIT>""" <EOL> for key , value in list ( iteritems ( self . counters ) ) : <EOL> if value . empty ( ) : <EOL> del self . counters [ key ] <EOL> def __getitem__ ( self , key ) : <EOL> key = ( key , ) <EOL> available_keys = [ ] <EOL> for _key in self . counters . keys ( ) : <EOL> if _key [ : len ( key ) ] == key : <EOL> available_keys . append ( _key ) <EOL> if len ( available_keys ) == <NUM_LIT:0> : <EOL> raise KeyError <EOL> elif len ( available_keys ) == <NUM_LIT:1> : <EOL> if available_keys [ <NUM_LIT:0> ] == key : <EOL> return self . counters [ key ] <EOL> else : <EOL> return CounterValue ( self , key ) <EOL> else : <EOL> return CounterValue ( self , key ) <EOL> def __delitem__ ( self , key ) : <EOL> key = ( key , ) <EOL> available_keys = [ ] <EOL> for _key in self . counters . keys ( ) : <EOL> if _key [ : len ( key ) ] == key : <EOL> available_keys . append ( _key ) <EOL> for _key in available_keys : <EOL> del self . counters [ _key ] <EOL> def __iter__ ( self ) : <EOL> return iter ( self . keys ( ) ) <EOL> def __len__ ( self ) : <EOL> return len ( self . keys ( ) ) <EOL> def keys ( self ) : <EOL> result = set ( ) <EOL> for key in self . counters . keys ( ) : <EOL> result . add ( key [ <NUM_LIT:0> ] if key else ( ) ) <EOL> return result <EOL> def to_dict ( self , get_value = None ) : <EOL> """<STR_LIT>""" <EOL> self . trim ( ) <EOL> result = { } <EOL> for key , value in iteritems ( self ) : <EOL> if isinstance ( value , BaseCounter ) : <EOL> if get_value is not None : <EOL> value = getattr ( value , get_value ) <EOL> result [ key ] = value <EOL> else : <EOL> result [ key ] = value . to_dict ( get_value ) <EOL> return result <EOL> def dump ( self , filename ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> with open ( filename , '<STR_LIT:wb>' ) as fp : <EOL> cPickle . dump ( self . counters , fp ) <EOL> except : <EOL> logging . error ( "<STR_LIT>" % filename ) <EOL> return False <EOL> return True <EOL> def load ( self , filename ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> with open ( filename ) as fp : <EOL> self . counters = cPickle . load ( fp ) <EOL> except : <EOL> logging . debug ( "<STR_LIT>" % filename ) <EOL> return False <EOL> return True </s>
