<s> """<STR_LIT>""" <EOL> from collections import OrderedDict <EOL> import numpy as np <EOL> from scipy import stats <EOL> from scipy import special as spc <EOL> SAMPLE_SIZE = <NUM_LIT> <EOL> def _split ( variations , control = None ) : <EOL> if control is None : <EOL> control = list ( variations . keys ( ) ) [ <NUM_LIT:0> ] <EOL> others = OrderedDict ( ( label , variation ) for label , variation <EOL> in variations . items ( ) if label != control ) <EOL> control = variations [ control ] <EOL> return control , others <EOL> def expected_posterior ( variations ) : <EOL> """<STR_LIT>""" <EOL> values = OrderedDict ( ) <EOL> for label , variation in variations . items ( ) : <EOL> values [ label ] = variation . posterior . mean ( ) <EOL> return values <EOL> def posterior_credible_interval ( variations , level = <NUM_LIT> ) : <EOL> """<STR_LIT>""" <EOL> values = OrderedDict ( ) <EOL> for label , variation in variations . items ( ) : <EOL> left_percentile = <NUM_LIT:0.5> * ( <NUM_LIT:100> - level ) / <NUM_LIT:100> <EOL> left = variation . posterior . ppf ( left_percentile ) <EOL> median = variation . posterior . ppf ( <NUM_LIT:0.5> ) <EOL> right = variation . posterior . ppf ( ( level + left_percentile ) / <NUM_LIT:100> ) <EOL> values [ label ] = min ( left , right ) , median , max ( left , right ) <EOL> return values <EOL> def expected_lift ( variations , control = None ) : <EOL> """<STR_LIT>""" <EOL> values = OrderedDict ( ) <EOL> control , others = _split ( variations , control ) <EOL> a = control . posterior <EOL> m = a . mean ( ) <EOL> for label , variation in others . items ( ) : <EOL> b = variation . posterior <EOL> lift = ( b . mean ( ) - m ) / m <EOL> values [ label ] = lift <EOL> return values <EOL> def lift_credible_interval ( variations , control = None , level = <NUM_LIT> , <EOL> sample_size = SAMPLE_SIZE ) : <EOL> """<STR_LIT>""" <EOL> values = OrderedDict ( ) <EOL> a , others = _split ( variations , control ) <EOL> for label , b in others . items ( ) : <EOL> a_samples = a . posterior . rvs ( size = sample_size ) <EOL> b_samples = b . posterior . rvs ( size = sample_size ) <EOL> lift_samples = ( b_samples - a_samples ) / a_samples <EOL> left_percentile = <NUM_LIT:0.5> * ( <NUM_LIT:100> - level ) <EOL> right = np . percentile ( lift_samples , level + left_percentile ) <EOL> median = np . percentile ( lift_samples , <NUM_LIT:50> ) <EOL> left = np . percentile ( lift_samples , left_percentile ) <EOL> values [ label ] = min ( left , right ) , median , max ( left , right ) <EOL> return values <EOL> def dominance ( variations , control = None , sample_size = SAMPLE_SIZE ) : <EOL> """<STR_LIT>""" <EOL> values = OrderedDict ( ) <EOL> a , others = _split ( variations , control ) <EOL> def is_integer ( x ) : <EOL> try : <EOL> return x . is_integer ( ) <EOL> except : <EOL> return int ( x ) == x <EOL> for label , b in others . items ( ) : <EOL> if is_integer ( a . prior_alpha ) and is_integer ( a . prior_beta ) and is_integer ( b . prior_alpha ) and is_integer ( b . prior_beta ) : <EOL> total = <NUM_LIT:0> <EOL> for i in range ( b . alpha - b . prior_alpha ) : <EOL> total += np . exp ( spc . betaln ( a . alpha + i , b . beta + a . beta ) - <EOL> np . log ( b . beta + i ) - spc . betaln ( b . prior_alpha + i , <EOL> b . beta ) - spc . betaln ( a . alpha , a . beta ) ) <EOL> values [ label ] = total <EOL> else : <EOL> a_samples = a . posterior . rvs ( sample_size ) <EOL> b_samples = b . posterior . rvs ( sample_size ) <EOL> values [ label ] = np . mean ( b_samples > a_samples ) <EOL> return values <EOL> def empirical_lift ( variations , control = None ) : <EOL> """<STR_LIT>""" <EOL> values = OrderedDict ( ) <EOL> a , others = _split ( variations , control ) <EOL> for label , b in others . items ( ) : <EOL> total_a = a . alpha - a . prior_alpha + a . beta - a . prior_beta <EOL> total_b = b . alpha - b . prior_alpha + b . beta - b . prior_beta <EOL> p_a = float ( a . alpha - a . prior_alpha ) / total_a <EOL> p_b = float ( b . alpha - b . prior_alpha ) / total_b <EOL> lift = ( p_b - p_a ) / p_a <EOL> values [ label ] = lift <EOL> return values <EOL> def ztest_dominance ( variations , control = None ) : <EOL> """<STR_LIT>""" <EOL> values = OrderedDict ( ) <EOL> a , others = _split ( variations , control ) <EOL> for label , b in others . items ( ) : <EOL> total_a = a . alpha - a . prior_alpha + a . beta - a . prior_beta <EOL> total_b = b . alpha - b . prior_alpha + b . beta - b . prior_beta <EOL> p_a = float ( a . alpha - a . prior_alpha ) / total_a <EOL> p_b = float ( b . alpha - b . prior_alpha ) / total_b <EOL> sse_a = p_a * ( <NUM_LIT:1> - p_a ) / total_a <EOL> sse_b = p_b * ( <NUM_LIT:1> - p_b ) / total_b <EOL> zscore = ( p_b - p_a ) / np . sqrt ( sse_a + sse_b ) <EOL> values [ label ] = stats . norm ( ) . cdf ( zscore ) <EOL> return values <EOL> statistic_funcs = { <EOL> '<STR_LIT>' : expected_posterior , <EOL> '<STR_LIT>' : posterior_credible_interval , <EOL> '<STR_LIT>' : expected_lift , <EOL> '<STR_LIT>' : lift_credible_interval , <EOL> '<STR_LIT>' : empirical_lift , <EOL> '<STR_LIT>' : dominance , <EOL> '<STR_LIT>' : ztest_dominance , <EOL> } </s>
