<s> import crosscat . utils . data_utils as du <EOL> import crosscat . utils . sample_utils as su <EOL> import crosscat . tests . component_model_extensions . ContinuousComponentModel as ccmext <EOL> import crosscat . tests . component_model_extensions . MultinomialComponentModel as mcmext <EOL> import crosscat . tests . component_model_extensions . CyclicComponentModel as cycmext <EOL> import random <EOL> import numpy <EOL> import math <EOL> import six <EOL> default_data_parameters = dict ( <EOL> multinomial = dict ( weights = [ <NUM_LIT:1.0> / <NUM_LIT> ] * <NUM_LIT:5> ) , <EOL> continuous = dict ( mu = <NUM_LIT:0.0> , rho = <NUM_LIT:1.0> ) , <EOL> cyclic = dict ( mu = math . pi , kappa = <NUM_LIT> ) <EOL> ) <EOL> get_data_generator = dict ( <EOL> multinomial = mcmext . p_MultinomialComponentModel . generate_data_from_parameters , <EOL> continuous = ccmext . p_ContinuousComponentModel . generate_data_from_parameters , <EOL> cyclic = cycmext . p_CyclicComponentModel . generate_data_from_parameters <EOL> ) <EOL> NaN = float ( '<STR_LIT>' ) <EOL> has_key = lambda dictionary , key : key in dictionary . keys ( ) <EOL> def p_draw ( M ) : <EOL> r = random . random ( ) <EOL> for i in range ( len ( M ) ) : <EOL> if r < M [ i ] : <EOL> return i <EOL> def add_missing_data_to_column ( X , col , proportion ) : <EOL> """<STR_LIT>""" <EOL> assert proportion >= <NUM_LIT:0> and proportion <= <NUM_LIT:1> <EOL> for row in range ( X . shape [ <NUM_LIT:0> ] ) : <EOL> if random . random ( ) < proportion : <EOL> X [ row , col ] = NaN <EOL> return X <EOL> def generate_separated_multinomial_weights ( A , C ) : <EOL> """<STR_LIT>""" <EOL> if not isinstance ( A , list ) : <EOL> raise TypeError ( "<STR_LIT>" ) <EOL> if not math . fabs ( <NUM_LIT:1> - sum ( A ) ) < <NUM_LIT> : <EOL> raise ValueError ( "<STR_LIT>" ) <EOL> if C > <NUM_LIT:1.0> or C < <NUM_LIT:0> : <EOL> raise ValueError ( "<STR_LIT>" ) <EOL> if C == <NUM_LIT:0.0> : <EOL> return A <EOL> idx = [ i [ <NUM_LIT:0> ] for i in sorted ( enumerate ( A ) , key = lambda x : x [ <NUM_LIT:1> ] ) ] <EOL> A_sum = [ A [ i ] for i in idx ] <EOL> A = numpy . array ( A ) <EOL> A_sum = numpy . cumsum ( numpy . array ( A_sum ) ) <EOL> B = numpy . copy ( A ) <EOL> t = numpy . nonzero ( A_sum >= <NUM_LIT> ) [ <NUM_LIT:0> ] [ <NUM_LIT:0> ] ; <EOL> err_up = idx [ : t ] <EOL> err_dn = idx [ t : ] <EOL> upper_bounds = <NUM_LIT:1.0> - B ; <EOL> upper_bounds [ err_dn ] = <NUM_LIT:0> <EOL> lower_bounds = numpy . copy ( B ) ; <EOL> lower_bounds [ err_up ] = <NUM_LIT:0> <EOL> for _ in range ( int ( C * <NUM_LIT> ) ) : <EOL> ups = numpy . nonzero ( upper_bounds >= <NUM_LIT> ) [ <NUM_LIT:0> ] <EOL> move_up = ups [ random . randrange ( len ( ups ) ) ] <EOL> B [ move_up ] += <NUM_LIT> <EOL> upper_bounds [ move_up ] -= <NUM_LIT> <EOL> dns = numpy . nonzero ( lower_bounds >= <NUM_LIT> ) [ <NUM_LIT:0> ] <EOL> if len ( dns ) == <NUM_LIT:0> : <EOL> maxdex = numpy . argmin ( lower_bounds ) <EOL> B [ maxdex ] = <NUM_LIT:0> <EOL> B /= numpy . sum ( B ) <EOL> break <EOL> move_down = dns [ random . randrange ( len ( dns ) ) ] <EOL> B [ move_down ] -= <NUM_LIT> <EOL> lower_bounds [ move_down ] -= <NUM_LIT> <EOL> assert math . fabs ( <NUM_LIT:1> - numpy . sum ( B ) ) < <NUM_LIT> <EOL> return B . tolist ( ) <EOL> def generate_separated_model_parameters ( cctype , C , num_clusters , get_next_seed , distargs = None ) : <EOL> """<STR_LIT>""" <EOL> if cctype == '<STR_LIT>' : <EOL> A = <NUM_LIT> <EOL> B = <NUM_LIT> <EOL> d_in_simga = lambda c : A * ( c ** <NUM_LIT> ) + B * c <EOL> rho_to_sigma = lambda rho : ( <NUM_LIT:1.0> / rho ) ** <NUM_LIT> <EOL> N = <NUM_LIT:100> <EOL> model = ccmext . p_ContinuousComponentModel . from_parameters ( N , gen_seed = get_next_seed ( ) ) <EOL> params = model . sample_parameters_given_hyper ( gen_seed = get_next_seed ( ) ) <EOL> model_params = [ params ] <EOL> for i in range ( <NUM_LIT:0> , num_clusters - <NUM_LIT:1> ) : <EOL> params = model . sample_parameters_given_hyper ( gen_seed = get_next_seed ( ) ) <EOL> last_mean = model_params [ i ] [ '<STR_LIT>' ] <EOL> std1 = rho_to_sigma ( model_params [ i ] [ '<STR_LIT>' ] ) <EOL> std2 = rho_to_sigma ( params [ '<STR_LIT>' ] ) <EOL> sumstd = std1 + std2 <EOL> push = d_in_simga ( C ) * sumstd <EOL> params [ '<STR_LIT>' ] = model_params [ i ] [ '<STR_LIT>' ] + push <EOL> model_params . append ( params ) <EOL> assert len ( model_params ) == num_clusters <EOL> random . shuffle ( model_params ) <EOL> assert len ( model_params ) == num_clusters <EOL> return model_params <EOL> elif cctype == '<STR_LIT>' : <EOL> if not isinstance ( distargs , dict ) : <EOL> raise TypeError ( "<STR_LIT>" ) <EOL> try : <EOL> K = distargs [ '<STR_LIT>' ] <EOL> except KeyError : <EOL> raise KeyError ( "<STR_LIT>" ) <EOL> weights = numpy . array ( [ <NUM_LIT:1.0> / float ( K ) ] * K ) <EOL> weights = weights . tolist ( ) <EOL> model_params = [ dict ( weights = weights ) ] <EOL> for i in range ( <NUM_LIT:0> , num_clusters - <NUM_LIT:1> ) : <EOL> weights = generate_separated_multinomial_weights ( weights , C ) <EOL> model_params . append ( dict ( weights = weights ) ) <EOL> assert len ( model_params ) == num_clusters <EOL> random . shuffle ( model_params ) <EOL> assert len ( model_params ) == num_clusters <EOL> return model_params <EOL> elif cctype == '<STR_LIT>' : <EOL> sep = ( <NUM_LIT> * math . pi / num_clusters ) <EOL> mus = [ c * sep for c in range ( num_clusters ) ] <EOL> std = sep / ( <NUM_LIT> * C ** <NUM_LIT> ) <EOL> k = <NUM_LIT:1> / ( std * std ) <EOL> model_params = [ ] <EOL> for c in range ( num_clusters ) : <EOL> model_params . append ( dict ( mu = mus [ c ] , kappa = k ) ) <EOL> return model_params <EOL> else : <EOL> raise ValueError ( "<STR_LIT>" % cctype ) <EOL> def gen_data ( cctypes , n_rows , cols_to_views , cluster_weights , separation , seed = <NUM_LIT:0> , distargs = None , return_structure = False ) : <EOL> """<STR_LIT>""" <EOL> if not isinstance ( n_rows , int ) : <EOL> raise TypeError ( "<STR_LIT>" ) <EOL> if not isinstance ( cctypes , list ) : <EOL> raise TypeError ( "<STR_LIT>" ) <EOL> n_cols_cctypes = len ( cctypes ) <EOL> for cctype in cctypes : <EOL> if not isinstance ( cctype , str ) : <EOL> raise TypeError ( "<STR_LIT>" ) <EOL> if cctype not in [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] : <EOL> raise ValueError ( "<STR_LIT>" % cctype ) <EOL> if not isinstance ( cols_to_views , list ) : <EOL> raise TypeError ( "<STR_LIT>" ) <EOL> if len ( cols_to_views ) != n_cols_cctypes : <EOL> raise ValueError ( "<STR_LIT>" ) <EOL> if min ( cols_to_views ) != <NUM_LIT:0> : <EOL> raise ValueError ( "<STR_LIT>" ) <EOL> n_views_cols_to_views = max ( cols_to_views ) + <NUM_LIT:1> <EOL> set_ctv = set ( cols_to_views ) <EOL> if len ( set_ctv ) != n_views_cols_to_views : <EOL> raise ValueError ( "<STR_LIT>" ) <EOL> if not isinstance ( cluster_weights , list ) : <EOL> raise TypeError ( "<STR_LIT>" ) <EOL> if n_views_cols_to_views != len ( cluster_weights ) : <EOL> raise ValueError ( "<STR_LIT>" ) <EOL> for W in cluster_weights : <EOL> if not isinstance ( W , list ) : <EOL> raise TypeError ( "<STR_LIT>" ) <EOL> if math . fabs ( sum ( W ) - <NUM_LIT:1.0> ) > <NUM_LIT> : <EOL> raise ValueError ( "<STR_LIT>" ) <EOL> if not isinstance ( separation , list ) : <EOL> raise TypeError ( "<STR_LIT>" ) <EOL> if len ( separation ) != n_views_cols_to_views : <EOL> raise ValueError ( "<STR_LIT>" ) <EOL> for c in separation : <EOL> if not isinstance ( c , float ) or c > <NUM_LIT:1.0> or c < <NUM_LIT:0.0> : <EOL> raise ValueError ( "<STR_LIT>" ) <EOL> num_views = len ( separation ) <EOL> n_cols = len ( cols_to_views ) <EOL> if distargs is None : <EOL> distargs = [ None for i in range ( n_cols ) ] <EOL> if not isinstance ( distargs , list ) : <EOL> raise TypeError ( "<STR_LIT>" ) <EOL> if len ( distargs ) != n_cols : <EOL> raise ValueError ( "<STR_LIT>" ) <EOL> for i in range ( n_cols ) : <EOL> if cctypes [ i ] == '<STR_LIT>' or cctypes [ i ] == '<STR_LIT>' : <EOL> if distargs [ i ] is not None : <EOL> raise ValueError ( "<STR_LIT>" ) <EOL> elif cctypes [ i ] == '<STR_LIT>' : <EOL> if not isinstance ( distargs [ i ] , dict ) : <EOL> raise TypeError ( "<STR_LIT>" ) <EOL> if len ( distargs [ i ] . keys ( ) ) != <NUM_LIT:1> : <EOL> raise KeyError ( "<STR_LIT>" ) <EOL> if '<STR_LIT>' not in distargs [ i ] . keys ( ) : <EOL> raise KeyError ( "<STR_LIT>" ) <EOL> else : <EOL> raise ValueError ( "<STR_LIT>" % cctypes [ i ] ) <EOL> random . seed ( seed ) <EOL> numpy . random . seed ( seed ) <EOL> rows_to_clusters = [ ] <EOL> for W in cluster_weights : <EOL> cW = list ( W ) <EOL> for i in range ( <NUM_LIT:1> , len ( cW ) ) : <EOL> cW [ i ] += cW [ i - <NUM_LIT:1> ] <EOL> K = len ( cW ) <EOL> rows_to_clusters_view = list ( range ( K ) ) <EOL> for r in range ( K , n_rows ) : <EOL> rows_to_clusters_view . append ( p_draw ( cW ) ) <EOL> random . shuffle ( rows_to_clusters_view ) <EOL> assert len ( rows_to_clusters_view ) == n_rows <EOL> rows_to_clusters . append ( rows_to_clusters_view ) <EOL> get_next_seed = lambda : random . randrange ( <NUM_LIT> ) <EOL> data_table = numpy . zeros ( ( n_rows , n_cols ) ) <EOL> component_params = [ ] <EOL> for col in range ( n_cols ) : <EOL> view = cols_to_views [ col ] <EOL> num_clusters = len ( cluster_weights [ view ] ) <EOL> cctype = cctypes [ col ] <EOL> C = separation [ view ] <EOL> component_parameters = generate_separated_model_parameters ( cctype , C , <EOL> num_clusters , get_next_seed , distargs = distargs [ col ] ) <EOL> component_params . append ( component_parameters ) <EOL> gen = get_data_generator [ cctype ] <EOL> for row in range ( n_rows ) : <EOL> cluster = rows_to_clusters [ view ] [ row ] <EOL> params = component_parameters [ cluster ] <EOL> x = gen ( params , <NUM_LIT:1> , gen_seed = get_next_seed ( ) ) [ <NUM_LIT:0> ] <EOL> data_table [ row , col ] = x <EOL> T = data_table . tolist ( ) <EOL> M_c = du . gen_M_c_from_T ( T , cctypes = cctypes ) <EOL> if return_structure : <EOL> structure = dict ( ) <EOL> structure [ '<STR_LIT>' ] = component_params <EOL> structure [ '<STR_LIT>' ] = cols_to_views <EOL> structure [ '<STR_LIT>' ] = rows_to_clusters <EOL> structure [ '<STR_LIT>' ] = cluster_weights <EOL> return T , M_c , structure <EOL> else : <EOL> return T , M_c <EOL> def predictive_columns ( M_c , X_L , X_D , columns_list , optional_settings = False , seed = <NUM_LIT:0> ) : <EOL> """<STR_LIT>""" <EOL> supported_arguments = [ '<STR_LIT>' ] <EOL> num_rows = len ( X_D [ <NUM_LIT:0> ] ) <EOL> num_cols = len ( M_c [ '<STR_LIT>' ] ) <EOL> if not isinstance ( columns_list , list ) : <EOL> raise TypeError ( "<STR_LIT>" ) <EOL> for col in columns_list : <EOL> if not isinstance ( col , int ) : <EOL> raise TypeError ( "<STR_LIT>" ) <EOL> if col < <NUM_LIT:0> or col >= num_cols : <EOL> raise ValueError ( "<STR_LIT>" % ( col , num_cols ) ) <EOL> if not isinstance ( seed , int ) : <EOL> raise TypeError ( "<STR_LIT>" ) <EOL> if seed < <NUM_LIT:0> : <EOL> raise ValueError ( "<STR_LIT>" ) <EOL> if optional_settings : <EOL> if not isinstance ( optional_settings , list ) : <EOL> raise TypeError ( "<STR_LIT>" ) <EOL> for col_setting in optional_settings : <EOL> if isinstance ( col_setting , dict ) : <EOL> for key , value in six . iteritems ( col_setting ) : <EOL> if key not in supported_arguments : <EOL> raise KeyError ( "<STR_LIT>" % key ) <EOL> else : <EOL> optional_settings = [ None ] * len ( columns_list ) <EOL> random . seed ( seed ) <EOL> X = numpy . zeros ( ( num_rows , len ( columns_list ) ) ) <EOL> get_next_seed = lambda : random . randrange ( <NUM_LIT> ) <EOL> for c in range ( len ( columns_list ) ) : <EOL> col = columns_list [ c ] <EOL> for row in range ( num_rows ) : <EOL> X [ row , c ] = su . simple_predictive_sample ( M_c , X_L , X_D , [ ] , <EOL> [ ( row , col ) ] , get_next_seed , n = <NUM_LIT:1> ) [ <NUM_LIT:0> ] [ <NUM_LIT:0> ] <EOL> if isinstance ( optional_settings [ c ] , dict ) : <EOL> if has_key ( optional_settings [ c ] , '<STR_LIT>' ) : <EOL> proportion = optional_settings [ c ] [ '<STR_LIT>' ] <EOL> X = add_missing_data_to_column ( X , c , proportion ) <EOL> assert X . shape [ <NUM_LIT:0> ] == num_rows <EOL> assert X . shape [ <NUM_LIT:1> ] == len ( columns_list ) <EOL> return X </s>
