<s> import sys <EOL> import traceback <EOL> import copy <EOL> import listservers <EOL> import buckets <EOL> import info <EOL> import util_cli as util <EOL> import mc_bin_client <EOL> import stats_buffer <EOL> class StatsCollector : <EOL> def __init__ ( self , log ) : <EOL> self . log = log <EOL> def seg ( self , k , v ) : <EOL> ka = k . split ( '<STR_LIT:_>' ) <EOL> k = '<STR_LIT:_>' . join ( ka [ <NUM_LIT:0> : - <NUM_LIT:1> ] ) <EOL> kstart , kend = [ int ( x ) for x in ka [ - <NUM_LIT:1> ] . split ( '<STR_LIT:U+002C>' ) ] <EOL> return ( ( k , kstart , kend ) , int ( v ) ) <EOL> def retrieve_node_stats ( self , nodeInfo , nodeStats ) : <EOL> nodeStats [ '<STR_LIT>' ] = nodeInfo [ '<STR_LIT>' ] [ '<STR_LIT>' ] <EOL> nodeStats [ '<STR_LIT>' ] = nodeInfo [ '<STR_LIT>' ] [ '<STR_LIT>' ] <EOL> nodeStats [ '<STR_LIT>' ] = nodeInfo [ '<STR_LIT>' ] <EOL> nodeStats [ '<STR_LIT>' ] = nodeInfo [ '<STR_LIT>' ] <EOL> nodeStats [ '<STR_LIT>' ] = nodeInfo [ '<STR_LIT>' ] <EOL> nodeStats [ '<STR_LIT:version>' ] = nodeInfo [ '<STR_LIT:version>' ] <EOL> nodeStats [ '<STR_LIT>' ] = { } <EOL> nodeStats [ '<STR_LIT>' ] [ '<STR_LIT>' ] = nodeInfo [ '<STR_LIT>' ] <EOL> nodeStats [ '<STR_LIT>' ] [ '<STR_LIT>' ] = nodeInfo [ '<STR_LIT>' ] <EOL> nodeStats [ '<STR_LIT>' ] [ '<STR_LIT>' ] = nodeInfo [ '<STR_LIT>' ] <EOL> nodeStats [ '<STR_LIT>' ] [ '<STR_LIT>' ] = nodeInfo [ '<STR_LIT>' ] <EOL> nodeStats [ '<STR_LIT>' ] [ '<STR_LIT>' ] = nodeInfo [ '<STR_LIT>' ] <EOL> nodeStats [ '<STR_LIT>' ] = { } <EOL> if nodeInfo [ '<STR_LIT>' ] is not None : <EOL> hdd = nodeInfo [ '<STR_LIT>' ] [ '<STR_LIT>' ] <EOL> if hdd is not None : <EOL> nodeStats [ '<STR_LIT>' ] [ '<STR_LIT>' ] = { } <EOL> nodeStats [ '<STR_LIT>' ] [ '<STR_LIT>' ] [ '<STR_LIT>' ] = hdd [ '<STR_LIT>' ] <EOL> nodeStats [ '<STR_LIT>' ] [ '<STR_LIT>' ] [ '<STR_LIT>' ] = hdd [ '<STR_LIT>' ] <EOL> nodeStats [ '<STR_LIT>' ] [ '<STR_LIT>' ] [ '<STR_LIT>' ] = hdd [ '<STR_LIT>' ] <EOL> nodeStats [ '<STR_LIT>' ] [ '<STR_LIT>' ] [ '<STR_LIT>' ] = hdd [ '<STR_LIT>' ] <EOL> nodeStats [ '<STR_LIT>' ] [ '<STR_LIT>' ] [ '<STR_LIT>' ] = hdd [ '<STR_LIT>' ] <EOL> ram = nodeInfo [ '<STR_LIT>' ] [ '<STR_LIT>' ] <EOL> if ram is not None : <EOL> nodeStats [ '<STR_LIT>' ] [ '<STR_LIT>' ] = { } <EOL> nodeStats [ '<STR_LIT>' ] [ '<STR_LIT>' ] [ '<STR_LIT>' ] = ram [ '<STR_LIT>' ] <EOL> nodeStats [ '<STR_LIT>' ] [ '<STR_LIT>' ] [ '<STR_LIT>' ] = ram [ '<STR_LIT>' ] <EOL> nodeStats [ '<STR_LIT>' ] [ '<STR_LIT>' ] [ '<STR_LIT>' ] = ram [ '<STR_LIT>' ] <EOL> nodeStats [ '<STR_LIT>' ] [ '<STR_LIT>' ] [ '<STR_LIT>' ] = ram [ '<STR_LIT>' ] <EOL> if ram . has_key ( '<STR_LIT>' ) : <EOL> nodeStats [ '<STR_LIT>' ] [ '<STR_LIT>' ] [ '<STR_LIT>' ] = ram [ '<STR_LIT>' ] <EOL> else : <EOL> nodeStats [ '<STR_LIT>' ] [ '<STR_LIT>' ] [ '<STR_LIT>' ] = <NUM_LIT:0> <EOL> nodeStats [ '<STR_LIT>' ] = { } <EOL> nodeStats [ '<STR_LIT>' ] [ '<STR_LIT>' ] = nodeInfo [ '<STR_LIT>' ] [ '<STR_LIT>' ] <EOL> nodeStats [ '<STR_LIT>' ] [ '<STR_LIT>' ] = nodeInfo [ '<STR_LIT>' ] [ '<STR_LIT>' ] <EOL> nodeStats [ '<STR_LIT>' ] [ '<STR_LIT>' ] = nodeInfo [ '<STR_LIT>' ] [ '<STR_LIT>' ] <EOL> curr_items = <NUM_LIT:0> <EOL> curr_items_tot = <NUM_LIT:0> <EOL> vb_rep_curr_items = <NUM_LIT:0> <EOL> if nodeInfo [ '<STR_LIT>' ] is not None : <EOL> if nodeInfo [ '<STR_LIT>' ] . has_key ( '<STR_LIT>' ) : <EOL> curr_items = nodeInfo [ '<STR_LIT>' ] [ '<STR_LIT>' ] <EOL> else : <EOL> curr_items = <NUM_LIT:0> <EOL> if nodeInfo [ '<STR_LIT>' ] . has_key ( '<STR_LIT>' ) : <EOL> curr_items_tot = nodeInfo [ '<STR_LIT>' ] [ '<STR_LIT>' ] <EOL> else : <EOL> curr_items_tot = <NUM_LIT:0> <EOL> if nodeInfo [ '<STR_LIT>' ] . has_key ( '<STR_LIT>' ) : <EOL> vb_rep_curr_items = nodeInfo [ '<STR_LIT>' ] [ '<STR_LIT>' ] <EOL> else : <EOL> vb_rep_curr_items = <NUM_LIT:0> <EOL> nodeStats [ '<STR_LIT>' ] [ '<STR_LIT>' ] = curr_items <EOL> nodeStats [ '<STR_LIT>' ] [ '<STR_LIT>' ] = curr_items_tot <EOL> nodeStats [ '<STR_LIT>' ] [ '<STR_LIT>' ] = vb_rep_curr_items <EOL> def get_hostlist ( self , server , port , user , password , opts ) : <EOL> try : <EOL> opts . append ( ( "<STR_LIT>" , "<STR_LIT>" ) ) <EOL> nodes = listservers . ListServers ( ) . runCmd ( '<STR_LIT>' , server , port , user , password , opts ) <EOL> for node in nodes : <EOL> ( node_server , node_port ) = util . hostport ( node [ '<STR_LIT>' ] ) <EOL> node_stats = { "<STR_LIT:host>" : node_server , <EOL> "<STR_LIT:port>" : node_port , <EOL> "<STR_LIT:status>" : node [ '<STR_LIT:status>' ] , <EOL> "<STR_LIT>" : server } <EOL> stats_buffer . nodes [ node [ '<STR_LIT>' ] ] = node_stats <EOL> if node [ '<STR_LIT:status>' ] == '<STR_LIT>' : <EOL> node_info = info . Info ( ) . runCmd ( '<STR_LIT>' , node_server , node_port , user , password , opts ) <EOL> self . retrieve_node_stats ( node_info , node_stats ) <EOL> else : <EOL> self . log . error ( "<STR_LIT>" % ( node_server , node [ '<STR_LIT:status>' ] ) ) <EOL> return nodes <EOL> except Exception , err : <EOL> traceback . print_exc ( ) <EOL> sys . exit ( <NUM_LIT:1> ) <EOL> def get_bucketlist ( self , server , port , user , password , opts ) : <EOL> try : <EOL> bucketlist = buckets . Buckets ( ) . runCmd ( '<STR_LIT>' , server , port , user , password , opts ) <EOL> for bucket in bucketlist : <EOL> bucket_name = bucket [ '<STR_LIT:name>' ] <EOL> self . log . info ( "<STR_LIT>" % bucket_name ) <EOL> bucketinfo = { } <EOL> bucketinfo [ '<STR_LIT:name>' ] = bucket_name <EOL> bucketinfo [ '<STR_LIT>' ] = bucket [ '<STR_LIT>' ] <EOL> bucketinfo [ '<STR_LIT>' ] = bucket [ '<STR_LIT>' ] <EOL> bucketinfo [ '<STR_LIT>' ] = bucket [ '<STR_LIT>' ] <EOL> bucketinfo [ '<STR_LIT>' ] = bucket [ '<STR_LIT>' ] <EOL> bucketinfo [ '<STR_LIT>' ] = bucket [ '<STR_LIT>' ] [ '<STR_LIT>' ] <EOL> bucketinfo [ '<STR_LIT>' ] = server <EOL> bucketStats = bucket [ '<STR_LIT>' ] <EOL> bucketinfo [ '<STR_LIT>' ] = { } <EOL> bucketinfo [ '<STR_LIT>' ] [ '<STR_LIT>' ] = bucketStats [ '<STR_LIT>' ] <EOL> bucketinfo [ '<STR_LIT>' ] [ '<STR_LIT>' ] = bucketStats [ '<STR_LIT>' ] <EOL> bucketinfo [ '<STR_LIT>' ] [ '<STR_LIT>' ] = bucketStats [ '<STR_LIT>' ] <EOL> bucketinfo [ '<STR_LIT>' ] [ '<STR_LIT>' ] = bucketStats [ '<STR_LIT>' ] <EOL> bucketinfo [ '<STR_LIT>' ] [ '<STR_LIT>' ] = bucketStats [ '<STR_LIT>' ] <EOL> bucketinfo [ '<STR_LIT>' ] [ '<STR_LIT>' ] = bucketStats [ '<STR_LIT>' ] <EOL> stats_buffer . bucket_info [ bucket_name ] = bucketinfo <EOL> c = buckets . BucketStats ( bucket_name ) <EOL> json = c . runCmd ( '<STR_LIT>' , server , port , user , password , opts ) <EOL> stats_buffer . buckets_summary [ bucket_name ] = json <EOL> return bucketlist <EOL> except Exception , err : <EOL> traceback . print_exc ( ) <EOL> sys . exit ( <NUM_LIT:1> ) <EOL> def get_mc_stats_per_node ( self , mc , stats ) : <EOL> cmd_list = [ "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" ] <EOL> try : <EOL> for cmd in cmd_list : <EOL> node_stats = mc . stats ( cmd ) <EOL> if node_stats : <EOL> if cmd == "<STR_LIT>" : <EOL> vals = sorted ( [ self . seg ( * kv ) for kv in node_stats . items ( ) ] ) <EOL> dd = { } <EOL> totals = { } <EOL> longest = <NUM_LIT:0> <EOL> for s in vals : <EOL> avg = ( s [ <NUM_LIT:0> ] [ <NUM_LIT:1> ] + s [ <NUM_LIT:0> ] [ <NUM_LIT:2> ] ) / <NUM_LIT:2> <EOL> k = s [ <NUM_LIT:0> ] [ <NUM_LIT:0> ] <EOL> l = dd . get ( k , [ ] ) <EOL> l . append ( ( avg , s [ <NUM_LIT:1> ] ) ) <EOL> dd [ k ] = l <EOL> totals [ k ] = totals . get ( k , <NUM_LIT:0> ) + s [ <NUM_LIT:1> ] <EOL> for k in sorted ( dd ) : <EOL> ccount = <NUM_LIT:0> <EOL> for lbl , v in dd [ k ] : <EOL> ccount += v * lbl <EOL> stats [ k ] = ccount / totals [ k ] <EOL> else : <EOL> for key , val in node_stats . items ( ) : <EOL> stats [ key ] = val <EOL> except Exception , err : <EOL> traceback . print_exc ( ) <EOL> def get_mc_stats ( self , server , bucketlist , nodes ) : <EOL> for bucket in bucketlist : <EOL> bucket_name = bucket [ '<STR_LIT:name>' ] <EOL> stats_buffer . node_stats [ bucket_name ] = { } <EOL> for node in nodes : <EOL> ( node_server , node_port ) = util . hostport ( node [ '<STR_LIT>' ] ) <EOL> self . log . info ( "<STR_LIT>" % ( node_server , node [ '<STR_LIT>' ] [ '<STR_LIT>' ] ) ) <EOL> stats = { } <EOL> mc = mc_bin_client . MemcachedClient ( node_server , node [ '<STR_LIT>' ] [ '<STR_LIT>' ] ) <EOL> if bucket [ "<STR_LIT:name>" ] != "<STR_LIT>" : <EOL> mc . sasl_auth_plain ( bucket_name . encode ( "<STR_LIT:utf8>" ) , bucket [ "<STR_LIT>" ] . encode ( "<STR_LIT:utf8>" ) ) <EOL> self . get_mc_stats_per_node ( mc , stats ) <EOL> stats_buffer . node_stats [ bucket_name ] [ node [ '<STR_LIT>' ] ] = stats <EOL> def get_ns_stats ( self , bucketlist , server , port , user , password , opts ) : <EOL> for bucket in bucketlist : <EOL> bucket_name = bucket [ '<STR_LIT:name>' ] <EOL> stats_buffer . buckets [ bucket_name ] = copy . deepcopy ( stats_buffer . stats ) <EOL> cmd = '<STR_LIT>' <EOL> for scale , stat_set in stats_buffer . buckets [ bucket_name ] . iteritems ( ) : <EOL> for stat in stat_set . iterkeys ( ) : <EOL> sys . stderr . write ( '<STR_LIT:.>' ) <EOL> self . log . debug ( "<STR_LIT>" % stat ) <EOL> c = buckets . BucketNodeStats ( bucket_name , stat , scale ) <EOL> json = c . runCmd ( '<STR_LIT>' , server , port , user , password , opts ) <EOL> stats_buffer . buckets [ bucket_name ] [ scale ] [ stat ] = json <EOL> sys . stderr . write ( '<STR_LIT:\n>' ) <EOL> def collect_data ( self , cluster , user , password , opts ) : <EOL> server , port = util . hostport ( cluster ) <EOL> nodes = self . get_hostlist ( server , port , user , password , opts ) <EOL> self . log . debug ( util . pretty_print ( stats_buffer . nodes ) ) <EOL> bucketlist = self . get_bucketlist ( server , port , user , password , opts ) <EOL> self . log . debug ( util . pretty_print ( stats_buffer . bucket_info ) ) <EOL> self . get_mc_stats ( server , bucketlist , nodes ) <EOL> self . log . debug ( util . pretty_print ( stats_buffer . node_stats ) ) <EOL> self . get_ns_stats ( bucketlist , server , port , user , password , opts ) <EOL> self . log . debug ( util . pretty_print ( stats_buffer . buckets ) ) </s>
