<s> """<STR_LIT>""" <EOL> from __future__ import division , print_function <EOL> import math <EOL> import numpy <EOL> import pandas <EOL> import pylab <EOL> from scipy . stats . stats import pearsonr <EOL> from sklearn . neighbors import NearestNeighbors <EOL> from . commonutils import map_on_cluster , check_sample_weight <EOL> __author__ = '<STR_LIT>' <EOL> __all__ = [ '<STR_LIT>' ] <EOL> def _count_probabilities ( primary_weights , secondary_weights , knn ) : <EOL> """<STR_LIT>""" <EOL> primary_probabilities = primary_weights / numpy . sum ( primary_weights ) <EOL> secondary_weights = numpy . array ( secondary_weights ) <EOL> knn_probabilities = numpy . take ( secondary_weights , knn ) <EOL> knn_probabilities /= knn_probabilities . sum ( axis = <NUM_LIT:1> , keepdims = True ) <EOL> secondary_probabilities = numpy . bincount ( knn . flatten ( ) , weights = knn_probabilities . flatten ( ) ) <EOL> return primary_probabilities * secondary_probabilities <EOL> def generate_toymc ( data , size , knn = <NUM_LIT:4> , power = <NUM_LIT> , <EOL> reweighting_iterations = <NUM_LIT:5> , sample_weight = None , random_state = numpy . random ) : <EOL> """<STR_LIT>""" <EOL> data = pandas . DataFrame ( data ) <EOL> input_length = len ( data ) <EOL> sample_weight = check_sample_weight ( data , sample_weight = sample_weight ) <EOL> sample_weight /= numpy . sum ( sample_weight ) <EOL> if input_length <= <NUM_LIT:2> : <EOL> return data , len ( data ) <EOL> assert knn > <NUM_LIT:0> , "<STR_LIT>" <EOL> neighbors_helper = NearestNeighbors ( n_neighbors = knn , algorithm = '<STR_LIT>' , ) <EOL> neighbors_helper . fit ( data ) <EOL> neighbours = neighbors_helper . kneighbors ( data , return_distance = False ) [ : , <NUM_LIT:1> : ] <EOL> secondary_weights = numpy . ones ( len ( neighbours ) , dtype = float ) <EOL> for _ in range ( reweighting_iterations ) : <EOL> probabilities = _count_probabilities ( sample_weight , secondary_weights , neighbours ) <EOL> secondary_weights *= ( ( sample_weight / probabilities ) ** <NUM_LIT:0.5> ) <EOL> k_1 = random_state . choice ( input_length , p = sample_weight , size = size ) <EOL> k_2 = numpy . zeros ( size , dtype = int ) <EOL> for i , first_neighbour in enumerate ( k_1 ) : <EOL> neighs = neighbours [ first_neighbour ] <EOL> neigh_weights = numpy . take ( secondary_weights , neighs ) <EOL> neigh_weights /= numpy . sum ( neigh_weights ) <EOL> k_2 [ i ] = random_state . choice ( neighs , p = neigh_weights ) <EOL> numpied_df = data . values <EOL> t_1 = <NUM_LIT> * random_state . random ( size ) ** power <EOL> t_2 = <NUM_LIT:1.> - t_1 <EOL> new_data = t_1 [ : , numpy . newaxis ] * numpied_df [ k_1 , : ] + t_2 [ : , numpy . newaxis ] * numpied_df [ k_2 , : ] <EOL> return pandas . DataFrame ( new_data , columns = data . columns ) , <NUM_LIT:0> <EOL> def prepare_toymc ( group , clustering_features , stayed_features , size_factor ) : <EOL> """<STR_LIT>""" <EOL> group_values , df = group <EOL> toymc_part , n_copied = generate_toymc ( df [ stayed_features ] , int ( len ( df ) * size_factor ) , knn = None ) <EOL> for i , col in enumerate ( clustering_features ) : <EOL> toymc_part [ col ] = group_values [ i ] <EOL> return toymc_part , n_copied <EOL> def generate_toymc_with_special_features ( data , size , knn = <NUM_LIT:4> , clustering_features = None , ipc_profile = None ) : <EOL> """<STR_LIT>""" <EOL> if clustering_features is None : <EOL> clustering_features = [ ] <EOL> stayed_features = [ col for col in data . columns if col not in clustering_features ] <EOL> size_factor = float ( size ) / len ( data ) <EOL> copied_groups = <NUM_LIT:0> <EOL> if len ( clustering_features ) == <NUM_LIT:0> : <EOL> result , copied = generate_toymc ( data , size = size , knn = knn ) <EOL> else : <EOL> grouped = data . groupby ( clustering_features ) <EOL> print ( "<STR_LIT>" ) <EOL> n_groups = len ( grouped ) <EOL> results = map_on_cluster ( ipc_profile , prepare_toymc , grouped , [ clustering_features ] * n_groups , <EOL> [ stayed_features ] * n_groups , [ size_factor ] * n_groups ) <EOL> toymc_parts , copied_list = zip ( * results ) <EOL> copied = numpy . sum ( copied_list ) <EOL> copied_groups = numpy . sum ( numpy . array ( copied_list ) != <NUM_LIT:0> ) <EOL> result = pandas . concat ( toymc_parts ) <EOL> if copied > <NUM_LIT:0> : <EOL> print ( "<STR_LIT>" % <EOL> ( copied , copied_groups , len ( result ) ) ) <EOL> return result <EOL> def compare_toymc ( data , clustering_features = None ) : <EOL> """<STR_LIT>""" <EOL> assert isinstance ( data , pandas . DataFrame ) , '<STR_LIT>' <EOL> from IPython . display import display_html <EOL> def print_title ( title ) : <EOL> display_html ( "<STR_LIT>" . format ( title ) , raw = True ) <EOL> toy_data = generate_toymc_with_special_features ( data , len ( data ) , clustering_features = clustering_features ) <EOL> numpy . set_printoptions ( precision = <NUM_LIT:4> , suppress = True ) <EOL> n_cols = <NUM_LIT:3> <EOL> n_rows = ( len ( data . columns ) + n_cols - <NUM_LIT:1> ) // n_cols <EOL> pylab . figure ( figsize = ( <NUM_LIT> , <NUM_LIT:5> * n_rows ) ) <EOL> for i , column in enumerate ( data . columns , <NUM_LIT:1> ) : <EOL> pylab . subplot ( n_rows , n_cols , i ) <EOL> pylab . title ( column ) <EOL> pylab . hist ( [ data [ column ] . values , toy_data [ column ] . values ] , histtype = '<STR_LIT>' , bins = <NUM_LIT:20> ) <EOL> pylab . show ( ) <EOL> print_title ( "<STR_LIT>" ) <EOL> mean_index = [ ] <EOL> mean_rows = [ ] <EOL> for column in data . columns : <EOL> mean_index . append ( column ) <EOL> mean1 = numpy . mean ( data [ column ] ) <EOL> mean2 = numpy . mean ( toy_data [ column ] ) <EOL> std1 = numpy . std ( data [ column ] ) <EOL> std2 = numpy . std ( toy_data [ column ] ) <EOL> mean_rows . append ( [ mean1 , mean2 , mean2 - mean1 , abs ( ( mean1 - mean2 ) * <NUM_LIT> / mean1 ) , <EOL> std1 , std2 , std2 - std1 , abs ( ( std2 - std1 ) * <NUM_LIT> / std1 ) ] ) <EOL> display_html ( pandas . DataFrame ( mean_rows , index = mean_index , <EOL> columns = [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] ) ) <EOL> print_title ( '<STR_LIT>' ) <EOL> cov_index = [ ] <EOL> cov_rows = [ ] <EOL> for i , first_column in enumerate ( data . columns , ) : <EOL> for second_column in data . columns [ i + <NUM_LIT:1> : ] : <EOL> cov_index . append ( ( first_column , second_column ) ) <EOL> data_cov = pearsonr ( data [ first_column ] , data [ second_column ] ) [ <NUM_LIT:0> ] <EOL> toy_cov = pearsonr ( toy_data [ first_column ] , toy_data [ second_column ] ) [ <NUM_LIT:0> ] <EOL> cov_rows . append ( [ data_cov , toy_cov , toy_cov - data_cov , abs ( ( data_cov - toy_cov ) * <NUM_LIT> / data_cov ) ] ) <EOL> display_html ( pandas . DataFrame ( cov_rows , index = cov_index , columns = [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] ) ) <EOL> for col , first_column in enumerate ( data . columns [ : <NUM_LIT:4> ] ) : <EOL> for second_column in data . columns [ col + <NUM_LIT:1> : <NUM_LIT:4> ] : <EOL> x_min = numpy . min ( data [ first_column ] ) <EOL> x_max = numpy . max ( data [ first_column ] ) <EOL> y_min = numpy . min ( data [ second_column ] ) <EOL> y_max = numpy . max ( data [ second_column ] ) <EOL> pylab . figure ( figsize = ( <NUM_LIT:12> , <NUM_LIT:5> ) ) <EOL> pylab . subplot ( <NUM_LIT> ) <EOL> pylab . plot ( data [ first_column ] , data [ second_column ] , '<STR_LIT:.>' , alpha = <NUM_LIT:0.1> ) <EOL> pylab . xlim ( x_min , x_max ) , pylab . ylim ( y_min , y_max ) <EOL> pylab . title ( "<STR_LIT>" ) <EOL> pylab . subplot ( <NUM_LIT> ) <EOL> pylab . plot ( toy_data [ first_column ] , toy_data [ second_column ] , '<STR_LIT:.>' , alpha = <NUM_LIT:0.1> ) <EOL> pylab . ylim ( y_min , y_max ) , pylab . xlim ( x_min , x_max ) <EOL> pylab . title ( "<STR_LIT>" ) <EOL> pylab . suptitle ( "<STR_LIT>" . format ( first_column , second_column ) ) <EOL> pylab . show ( ) </s>
