<s> import pandas <EOL> import util <EOL> import matplotlib . pyplot as plt <EOL> import scipy as sp <EOL> import scipy . stats <EOL> import numpy as np <EOL> import os <EOL> cur_dir = os . path . dirname ( os . path . abspath ( __file__ ) ) <EOL> def from_custom_file ( data_file , learn_options ) : <EOL> print "<STR_LIT>" % data_file <EOL> data = pandas . read_csv ( data_file ) <EOL> mandatory_columns = [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] <EOL> for col in mandatory_columns : <EOL> assert col in data . columns , "<STR_LIT>" % mandatory_columns <EOL> Xdf = pandas . DataFrame ( data ) <EOL> Xdf [ '<STR_LIT>' ] = Xdf [ '<STR_LIT>' ] <EOL> Xdf = Xdf . set_index ( [ '<STR_LIT>' , '<STR_LIT>' ] ) <EOL> Xdf [ '<STR_LIT>' ] = Xdf [ '<STR_LIT>' ] <EOL> Xdf . index . names = [ '<STR_LIT>' , '<STR_LIT>' ] <EOL> Xdf [ '<STR_LIT>' ] = [ '<STR_LIT>' % i for i in range ( Xdf . shape [ <NUM_LIT:0> ] ) ] <EOL> Xdf = Xdf . set_index ( '<STR_LIT>' , append = True ) <EOL> Y = None <EOL> gene_position = Xdf [ [ '<STR_LIT>' , '<STR_LIT>' ] ] <EOL> target_genes = np . unique ( Xdf . index . levels [ <NUM_LIT:1> ] ) <EOL> learn_options = set_V2_target_names ( learn_options ) <EOL> return Xdf , Y , gene_position , target_genes <EOL> def from_file ( data_file , learn_options , data_file2 = None , data_file3 = None ) : <EOL> if learn_options [ "<STR_LIT>" ] == <NUM_LIT:1> : <EOL> print "<STR_LIT>" % learn_options [ "<STR_LIT>" ] <EOL> assert not learn_options [ "<STR_LIT>" ] is not None , "<STR_LIT>" <EOL> annotations , gene_position , target_genes , Xdf , Y = read_V1_data ( data_file , learn_options ) <EOL> learn_options [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> learn_options [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> learn_options [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> elif learn_options [ "<STR_LIT>" ] == <NUM_LIT:2> : <EOL> Xdf , drugs_to_genes , target_genes , Y , gene_position = read_V2_data ( data_file , learn_options ) <EOL> xx = Xdf [ '<STR_LIT>' ] . values <EOL> yy = Y [ '<STR_LIT>' ] . values <EOL> rr , pp = sp . stats . pearsonr ( xx , yy ) <EOL> assert rr > <NUM_LIT:0> , "<STR_LIT>" <EOL> learn_options = set_V2_target_names ( learn_options ) <EOL> elif learn_options [ "<STR_LIT>" ] == <NUM_LIT:3> : <EOL> learn_options [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> learn_options [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> learn_options [ '<STR_LIT>' ] = None <EOL> Xdf , Y , gene_position , target_genes = mergeV1_V2 ( data_file , data_file2 , learn_options ) <EOL> elif learn_options [ "<STR_LIT>" ] == <NUM_LIT:4> : <EOL> learn_options [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> learn_options [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> learn_options [ '<STR_LIT>' ] = None <EOL> Xdf , Y , gene_position , target_genes = merge_all ( data_file , data_file2 , data_file3 , learn_options ) <EOL> elif learn_options [ '<STR_LIT>' ] == <NUM_LIT:5> : <EOL> learn_options [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> learn_options [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> learn_options [ '<STR_LIT>' ] = None <EOL> gene_position , target_genes , Xdf , Y = read_xu_et_al ( data_file3 ) <EOL> Xdf [ "<STR_LIT>" ] = Xdf [ "<STR_LIT>" ] . apply ( lambda x : x [ <NUM_LIT:0> : <NUM_LIT:30> ] ) <EOL> return Xdf , Y , gene_position , target_genes <EOL> def set_V2_target_names ( learn_options ) : <EOL> if '<STR_LIT>' not in learn_options . keys ( ) : <EOL> learn_options [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> if '<STR_LIT>' not in learn_options . keys ( ) : <EOL> learn_options [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> learn_options [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> return learn_options <EOL> def combine_organisms ( human_data , mouse_data ) : <EOL> cd13 = human_data . xs ( '<STR_LIT>' , level = '<STR_LIT>' , drop_level = False ) <EOL> X_CD13 , Y_CD13 = util . get_data ( cd13 , y_names = [ '<STR_LIT>' , '<STR_LIT>' ] ) <EOL> cd33 = human_data . xs ( '<STR_LIT>' , level = '<STR_LIT>' , drop_level = False ) <EOL> X_CD33 , Y_CD33 = util . get_data ( cd33 , y_names = [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] ) <EOL> cd15 = human_data . xs ( '<STR_LIT>' , level = '<STR_LIT>' , drop_level = False ) <EOL> X_CD15 , Y_CD15 = util . get_data ( cd15 , y_names = [ '<STR_LIT>' ] ) <EOL> mouse_X = pandas . DataFrame ( ) <EOL> mouse_Y = pandas . DataFrame ( ) <EOL> for k in mouse_data . index . levels [ <NUM_LIT:1> ] : <EOL> X , Y = util . get_data ( mouse_data . xs ( k , level = '<STR_LIT>' , drop_level = False ) , [ "<STR_LIT>" ] , target_gene = k , organism = '<STR_LIT>' ) <EOL> mouse_X = pandas . concat ( [ mouse_X , X ] , axis = <NUM_LIT:0> ) <EOL> mouse_Y = pandas . concat ( [ mouse_Y , Y ] , axis = <NUM_LIT:0> ) <EOL> X = pandas . concat ( [ X_CD13 , X_CD15 , X_CD33 , mouse_X ] , axis = <NUM_LIT:0> ) <EOL> Y = pandas . concat ( [ Y_CD13 , Y_CD15 , Y_CD33 , mouse_Y ] , axis = <NUM_LIT:0> ) <EOL> return X , Y <EOL> def read_V1_data ( data_file , learn_options , AML_file = cur_dir + "<STR_LIT>" ) : <EOL> if data_file is None : <EOL> data_file = cur_dir + "<STR_LIT>" <EOL> human_data = pandas . read_excel ( data_file , sheetname = <NUM_LIT:0> , index_col = [ <NUM_LIT:0> , <NUM_LIT:1> ] ) <EOL> mouse_data = pandas . read_excel ( data_file , sheetname = <NUM_LIT:1> , index_col = [ <NUM_LIT:0> , <NUM_LIT:1> ] ) <EOL> Xdf , Y = combine_organisms ( human_data , mouse_data ) <EOL> annotations = pandas . read_csv ( AML_file , delimiter = '<STR_LIT:\t>' , index_col = [ <NUM_LIT:0> , <NUM_LIT:4> ] ) <EOL> annotations . index . names = Xdf . index . names <EOL> gene_position = pandas . merge ( Xdf , annotations , how = "<STR_LIT>" , left_index = True , right_index = True ) <EOL> gene_position = util . impute_gene_position ( gene_position ) <EOL> gene_position = gene_position [ [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] ] <EOL> Y = Y . loc [ gene_position . index ] <EOL> Xdf = Xdf . loc [ gene_position . index ] <EOL> Y [ '<STR_LIT:test>' ] = <NUM_LIT:1> <EOL> target_genes = Y [ '<STR_LIT>' ] . unique ( ) <EOL> Y . index . names = [ '<STR_LIT>' , '<STR_LIT>' ] <EOL> assert Xdf . index . equals ( Y . index ) , "<STR_LIT>" <EOL> if learn_options is not None and learn_options [ "<STR_LIT>" ] : <EOL> print "<STR_LIT>" <EOL> print "<STR_LIT>" <EOL> print "<STR_LIT>" <EOL> Y [ '<STR_LIT>' ] = Y [ '<STR_LIT>' ] < <NUM_LIT> <EOL> print "<STR_LIT>" <EOL> import ipdb <EOL> ipdb . set_trace ( ) <EOL> return annotations , gene_position , target_genes , Xdf , Y <EOL> def rank_transform ( x ) : <EOL> return <NUM_LIT:1.0> - sp . stats . mstats . rankdata ( x ) / sp . stats . mstats . rankdata ( x ) . max ( ) <EOL> def read_xu_et_al ( data_file , learn_options = None , verbose = True , subsetting = '<STR_LIT>' ) : <EOL> if data_file is None : <EOL> data_file = '<STR_LIT>' <EOL> datasets = [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] <EOL> aggregated = None <EOL> for d in datasets : <EOL> data_efficient = pandas . read_excel ( data_file , sheetname = '<STR_LIT>' % d , skiprows = <NUM_LIT:2> ) <EOL> data_inefficient = pandas . read_excel ( data_file , sheetname = '<STR_LIT>' % d , skiprows = <NUM_LIT:2> ) <EOL> data_efficient [ '<STR_LIT>' ] = <NUM_LIT:1.> <EOL> data_inefficient [ '<STR_LIT>' ] = <NUM_LIT:0.> <EOL> exp_data = pandas . concat ( ( data_efficient , data_inefficient ) ) <EOL> exp_data [ '<STR_LIT>' ] = exp_data . groupby ( '<STR_LIT>' ) [ '<STR_LIT>' ] . transform ( rank_transform ) <EOL> exp_data [ '<STR_LIT>' ] = exp_data . groupby ( '<STR_LIT>' ) [ '<STR_LIT>' ] . transform ( rank_transform ) <EOL> if aggregated is None : <EOL> aggregated = exp_data <EOL> else : <EOL> aggregated = pandas . concat ( ( aggregated , exp_data ) ) <EOL> if subsetting == '<STR_LIT>' : <EOL> aggregated [ "<STR_LIT>" ] = aggregated [ "<STR_LIT>" ] . apply ( lambda x : x [ <NUM_LIT:6> : - <NUM_LIT:4> ] ) <EOL> else : <EOL> aggregated [ "<STR_LIT>" ] = aggregated [ "<STR_LIT>" ] . apply ( lambda x : x [ <NUM_LIT:10> : ] ) <EOL> aggregated [ "<STR_LIT>" ] = aggregated [ "<STR_LIT>" ] . apply ( lambda x : x . upper ( ) ) <EOL> aggregated . rename ( columns = { "<STR_LIT>" : '<STR_LIT>' , '<STR_LIT>' : '<STR_LIT>' , '<STR_LIT>' : '<STR_LIT>' } , inplace = True ) <EOL> aggregated [ '<STR_LIT>' ] . loc [ aggregated [ '<STR_LIT>' ] == '<STR_LIT:+>' ] = '<STR_LIT>' <EOL> aggregated [ '<STR_LIT>' ] . loc [ aggregated [ '<STR_LIT>' ] == '<STR_LIT:->' ] = '<STR_LIT>' <EOL> aggregated [ '<STR_LIT>' ] = aggregated [ [ '<STR_LIT>' , '<STR_LIT>' ] ] . mean ( axis = <NUM_LIT:1> ) <EOL> df = aggregated <EOL> df = df . rename ( columns = { '<STR_LIT>' : '<STR_LIT>' , '<STR_LIT>' : '<STR_LIT>' } ) <EOL> df [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> df [ '<STR_LIT:test>' ] = <NUM_LIT:1> <EOL> df = df . set_index ( [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] ) <EOL> df [ '<STR_LIT>' ] = df . index . get_level_values ( <NUM_LIT:0> ) <EOL> df [ '<STR_LIT>' ] = df . index . get_level_values ( <NUM_LIT:1> ) <EOL> df [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> df [ '<STR_LIT>' ] = df [ '<STR_LIT>' ] <EOL> df [ '<STR_LIT>' ] = df [ '<STR_LIT>' ] <EOL> df [ '<STR_LIT>' ] = df [ '<STR_LIT>' ] <EOL> df [ '<STR_LIT>' ] = <NUM_LIT:0> <EOL> df [ '<STR_LIT>' ] = <NUM_LIT:0> <EOL> target_genes = np . unique ( df [ '<STR_LIT>' ] . values ) <EOL> return df [ [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] ] , target_genes , df [ [ '<STR_LIT>' , '<STR_LIT>' ] ] , df [ [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT:test>' , '<STR_LIT>' ] ] <EOL> def read_V2_data ( data_file , learn_options = None , verbose = True ) : <EOL> if data_file is None : <EOL> data_file = cur_dir + "<STR_LIT>" <EOL> data = pandas . read_excel ( data_file , sheetname = "<STR_LIT>" , skiprows = range ( <NUM_LIT:0> , <NUM_LIT:6> + <NUM_LIT:1> ) , index_col = [ <NUM_LIT:0> , <NUM_LIT:4> ] ) <EOL> Xdf = pandas . DataFrame ( ) <EOL> known_pairs = { '<STR_LIT>' : [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] , <EOL> '<STR_LIT>' : [ '<STR_LIT>' ] , <EOL> '<STR_LIT>' : [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] } <EOL> drugs_to_genes = { '<STR_LIT>' : [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] , <EOL> '<STR_LIT>' : [ '<STR_LIT>' ] , <EOL> '<STR_LIT>' : [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] } <EOL> if learn_options is not None : <EOL> assert not ( learn_options [ '<STR_LIT>' ] and learn_options [ '<STR_LIT>' ] ) , "<STR_LIT>" <EOL> if learn_options [ '<STR_LIT>' ] : <EOL> drugs_to_genes [ '<STR_LIT>' ] . extend ( [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] ) <EOL> elif learn_options [ '<STR_LIT>' ] : <EOL> drugs_to_genes [ '<STR_LIT>' ] . extend ( [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] ) <EOL> drugs_to_genes [ '<STR_LIT>' ] . extend ( [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] ) <EOL> drugs_to_genes [ '<STR_LIT>' ] . extend ( [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] ) <EOL> count = <NUM_LIT:0> <EOL> for drug in drugs_to_genes . keys ( ) : <EOL> genes = drugs_to_genes [ drug ] <EOL> for g in genes : <EOL> Xtmp = data . copy ( ) . xs ( g , level = '<STR_LIT>' , drop_level = False ) <EOL> Xtmp [ '<STR_LIT>' ] = drug <EOL> Xtmp [ '<STR_LIT>' ] = Xtmp [ drug ] . copy ( ) <EOL> if g in known_pairs [ drug ] : <EOL> Xtmp [ '<STR_LIT:test>' ] = <NUM_LIT:1.> <EOL> else : <EOL> Xtmp [ '<STR_LIT:test>' ] = <NUM_LIT:0.> <EOL> count = count + Xtmp . shape [ <NUM_LIT:0> ] <EOL> Xdf = pandas . concat ( [ Xdf , Xtmp ] , axis = <NUM_LIT:0> ) <EOL> if verbose : <EOL> print "<STR_LIT>" % ( Xtmp . shape [ <NUM_LIT:0> ] , g , count ) <EOL> Xdf = Xdf . set_index ( '<STR_LIT>' , append = True ) <EOL> Y = pandas . DataFrame ( Xdf . pop ( "<STR_LIT>" ) ) <EOL> Y . columns . names = [ "<STR_LIT>" ] <EOL> test_gene = pandas . DataFrame ( Xdf . pop ( '<STR_LIT:test>' ) ) <EOL> target = pandas . DataFrame ( Xdf . index . get_level_values ( '<STR_LIT>' ) . values , index = Y . index , columns = [ "<STR_LIT>" ] ) <EOL> Y = pandas . concat ( ( Y , target , test_gene ) , axis = <NUM_LIT:1> ) <EOL> target_genes = Y [ '<STR_LIT>' ] . unique ( ) <EOL> gene_position = Xdf [ [ "<STR_LIT>" , "<STR_LIT>" ] ] . copy ( ) <EOL> y_rank = pandas . DataFrame ( ) <EOL> y_threshold = pandas . DataFrame ( ) <EOL> y_quant = pandas . DataFrame ( ) <EOL> for drug in drugs_to_genes . keys ( ) : <EOL> gene_list = drugs_to_genes [ drug ] <EOL> for gene in gene_list : <EOL> ytmp = pandas . DataFrame ( Y . xs ( ( gene , drug ) , level = [ "<STR_LIT>" , "<STR_LIT>" ] , drop_level = False ) [ '<STR_LIT>' ] ) <EOL> y_ranktmp , y_rank_raw , y_thresholdtmp , y_quanttmp = util . get_ranks ( ytmp , thresh = <NUM_LIT> , prefix = "<STR_LIT>" , flip = False ) <EOL> y_rank = pandas . concat ( ( y_rank , y_ranktmp ) , axis = <NUM_LIT:0> ) <EOL> y_threshold = pandas . concat ( ( y_threshold , y_thresholdtmp ) , axis = <NUM_LIT:0> ) <EOL> y_quant = pandas . concat ( ( y_quant , y_quanttmp ) , axis = <NUM_LIT:0> ) <EOL> yall = pandas . concat ( ( y_rank , y_threshold , y_quant ) , axis = <NUM_LIT:1> ) <EOL> Y = pandas . merge ( Y , yall , how = '<STR_LIT>' , left_index = True , right_index = True ) <EOL> y_rank = pandas . DataFrame ( ) <EOL> y_threshold = pandas . DataFrame ( ) <EOL> y_quant = pandas . DataFrame ( ) <EOL> for drug in drugs_to_genes . keys ( ) : <EOL> ytmp = pandas . DataFrame ( Y . xs ( drug , level = "<STR_LIT>" , drop_level = False ) [ '<STR_LIT>' ] ) <EOL> y_ranktmp , y_rank_raw , y_thresholdtmp , y_quanttmp = util . get_ranks ( ytmp , thresh = <NUM_LIT> , prefix = "<STR_LIT>" , flip = False ) <EOL> y_rank = pandas . concat ( ( y_rank , y_ranktmp ) , axis = <NUM_LIT:0> ) <EOL> y_threshold = pandas . concat ( ( y_threshold , y_thresholdtmp ) , axis = <NUM_LIT:0> ) <EOL> y_quant = pandas . concat ( ( y_quant , y_quanttmp ) , axis = <NUM_LIT:0> ) <EOL> yall = pandas . concat ( ( y_rank , y_threshold , y_quant ) , axis = <NUM_LIT:1> ) <EOL> Y = pandas . merge ( Y , yall , how = '<STR_LIT>' , left_index = True , right_index = True ) <EOL> PLOT = False <EOL> if PLOT : <EOL> labels = [ "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" ] <EOL> for label in labels : <EOL> plt . figure ( ) <EOL> plt . plot ( Xdf [ '<STR_LIT>' ] . values , Y [ label ] . values , '<STR_LIT:.>' ) <EOL> r , pearp = sp . stats . pearsonr ( Xdf [ '<STR_LIT>' ] . values . flatten ( ) , Y [ label ] . values . flatten ( ) ) <EOL> plt . title ( label + '<STR_LIT>' % ( r , pearp ) ) <EOL> plt . xlabel ( "<STR_LIT>" ) <EOL> plt . ylabel ( label ) <EOL> gene_position = util . impute_gene_position ( gene_position ) <EOL> if learn_options is not None and learn_options [ "<STR_LIT>" ] == "<STR_LIT>" : <EOL> print "<STR_LIT>" <EOL> data = pandas . read_excel ( data_file , sheetname = "<STR_LIT>" , skiprows = range ( <NUM_LIT:0> , <NUM_LIT:6> + <NUM_LIT:1> ) , index_col = [ <NUM_LIT:0> , <NUM_LIT:4> ] ) <EOL> data . index . names = [ "<STR_LIT>" , "<STR_LIT>" ] <EOL> experiments = { } <EOL> experiments [ '<STR_LIT>' ] = [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] <EOL> experiments [ '<STR_LIT>' ] = [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] <EOL> experiments [ '<STR_LIT>' ] = [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] <EOL> variance = None <EOL> for drug in drugs_to_genes . keys ( ) : <EOL> data_tmp = data . iloc [ data . index . get_level_values ( '<STR_LIT>' ) . isin ( drugs_to_genes [ drug ] ) ] [ experiments [ drug ] ] <EOL> data_tmp [ "<STR_LIT>" ] = drug <EOL> data_tmp = data_tmp . set_index ( '<STR_LIT>' , append = True ) <EOL> data_tmp [ "<STR_LIT>" ] = np . var ( data_tmp . values , axis = <NUM_LIT:1> ) <EOL> if variance is None : <EOL> variance = data_tmp [ "<STR_LIT>" ] . copy ( ) <EOL> else : <EOL> variance = pandas . concat ( ( variance , data_tmp [ "<STR_LIT>" ] ) , axis = <NUM_LIT:0> ) <EOL> orig_index = Y . index . copy ( ) <EOL> Y = pandas . merge ( Y , pandas . DataFrame ( variance ) , how = "<STR_LIT>" , left_index = True , right_index = True ) <EOL> Y = Y . ix [ orig_index ] <EOL> print "<STR_LIT>" <EOL> assert Xdf . index . equals ( Y . index ) , "<STR_LIT>" <EOL> return Xdf , drugs_to_genes , target_genes , Y , gene_position <EOL> def merge_all ( data_file = None , data_file2 = None , data_file3 = None , learn_options = None ) : <EOL> Xdf , Y , gene_position , target_genes = mergeV1_V2 ( data_file , data_file2 , learn_options ) <EOL> gene_position_xu , target_genes_xu , Xdf_xu , Y_xu = read_xu_et_al ( data_file3 , learn_options ) <EOL> Xdf = pandas . concat ( ( Xdf , Xdf_xu ) ) <EOL> Y = pandas . concat ( ( Y , Y_xu ) ) <EOL> gene_position = pandas . concat ( ( gene_position , gene_position_xu ) ) <EOL> target_genes = np . concatenate ( ( target_genes , target_genes_xu ) ) <EOL> return Xdf , Y , gene_position , target_genes <EOL> def mergeV1_V2 ( data_file , data_file2 , learn_options ) : <EOL> '''<STR_LIT>''' <EOL> assert not learn_options [ '<STR_LIT>' ] , "<STR_LIT>" <EOL> annotations , gene_position1 , target_genes1 , Xdf1 , Y1 = read_V1_data ( data_file , learn_options ) <EOL> Xdf2 , drugs_to_genes , target_genes2 , Y2 , gene_position2 = read_V2_data ( data_file2 ) <EOL> Y1 . rename ( columns = { '<STR_LIT>' : learn_options [ "<STR_LIT>" ] } , inplace = True ) <EOL> Y1 . rename ( columns = { '<STR_LIT>' : learn_options [ "<STR_LIT>" ] } , inplace = True ) <EOL> Y1 [ "<STR_LIT>" ] = [ "<STR_LIT>" for x in range ( Y1 . shape [ <NUM_LIT:0> ] ) ] <EOL> Y1 = Y1 . set_index ( '<STR_LIT>' , append = True ) <EOL> Y1 . index . names = [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] <EOL> Y_cols_to_keep = np . unique ( [ '<STR_LIT>' , '<STR_LIT:test>' , '<STR_LIT>' , '<STR_LIT>' ] ) <EOL> Y1 = Y1 [ Y_cols_to_keep ] <EOL> Y2 = Y2 [ Y_cols_to_keep ] <EOL> Xdf1 [ "<STR_LIT>" ] = [ "<STR_LIT>" for x in range ( Xdf1 . shape [ <NUM_LIT:0> ] ) ] <EOL> Xdf1 = Xdf1 . set_index ( '<STR_LIT>' , append = True ) <EOL> X_cols_to_keep = [ '<STR_LIT>' , '<STR_LIT>' ] <EOL> Xdf1 = Xdf1 [ X_cols_to_keep ] <EOL> Xdf2 = Xdf2 [ X_cols_to_keep ] <EOL> gene_position1 [ "<STR_LIT>" ] = [ "<STR_LIT>" for x in range ( gene_position1 . shape [ <NUM_LIT:0> ] ) ] <EOL> gene_position1 = gene_position1 . set_index ( '<STR_LIT>' , append = True ) <EOL> gene_position1 . index . names = [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] <EOL> cols_to_keep = [ u'<STR_LIT>' , u'<STR_LIT>' ] <EOL> gene_position1 = gene_position1 [ cols_to_keep ] <EOL> gene_position2 = gene_position2 [ cols_to_keep ] <EOL> Y = pandas . concat ( ( Y1 , Y2 ) , axis = <NUM_LIT:0> ) <EOL> Xdf = pandas . concat ( ( Xdf1 , Xdf2 ) , axis = <NUM_LIT:0> ) <EOL> gene_position = pandas . concat ( ( gene_position1 , gene_position2 ) ) <EOL> target_genes = np . concatenate ( ( target_genes1 , target_genes2 ) ) <EOL> save_to_file = False <EOL> if save_to_file : <EOL> Y . index . names = [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] <EOL> assert np . all ( Xdf . index . values == Y . index . values ) , "<STR_LIT>" <EOL> onedupind = np . where ( Y . index . duplicated ( ) ) [ <NUM_LIT:0> ] [ <NUM_LIT:0> ] <EOL> alldupind = np . where ( Y . index . get_level_values ( <NUM_LIT:0> ) . values == Y . index [ onedupind ] [ <NUM_LIT:0> ] ) [ <NUM_LIT:0> ] <EOL> assert len ( alldupind ) == <NUM_LIT:2> , "<STR_LIT>" <EOL> newindex = Y . index . tolist ( ) <EOL> newindex [ onedupind ] = ( newindex [ onedupind ] [ <NUM_LIT:0> ] , newindex [ onedupind ] [ <NUM_LIT:1> ] , "<STR_LIT>" ) <EOL> Y . index = pandas . MultiIndex . from_tuples ( newindex , names = Y . index . names ) <EOL> Xdf . index = pandas . MultiIndex . from_tuples ( newindex , names = Y . index . names ) <EOL> XandY = pandas . merge ( Xdf , Y , how = "<STR_LIT>" , left_index = True , right_index = True ) <EOL> gene_position_tmp = gene_position . copy ( ) <EOL> gene_position_tmp . index . names = [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] <EOL> gene_position_tmp . index = pandas . MultiIndex . from_tuples ( newindex , names = Y . index . names ) <EOL> XandY = pandas . merge ( XandY , gene_position_tmp , how = "<STR_LIT>" , left_index = True , right_index = True ) <EOL> XandY [ "<STR_LIT>" ] = XandY [ "<STR_LIT>" ] . apply ( lambda x : x [ <NUM_LIT:0> : <NUM_LIT:30> ] ) <EOL> XandY . to_csv ( r'<STR_LIT>' ) <EOL> return Xdf , Y , gene_position , target_genes <EOL> def get_V1_genes ( data_file = None ) : <EOL> annotations , gene_position , target_genes , Xdf , Y = read_V1_data ( data_file , learn_options = None ) <EOL> return target_genes <EOL> def get_V2_genes ( data_file = None ) : <EOL> Xdf , drugs_to_genes , target_genes , Y , gene_position = read_V2_data ( data_file , verbose = False ) <EOL> return target_genes <EOL> def get_V3_genes ( data_fileV1 = None , data_fileV2 = None ) : <EOL> target_genes = np . concatenate ( ( get_V1_genes ( data_fileV1 ) , get_V2_genes ( data_fileV2 ) ) ) <EOL> return target_genes <EOL> def get_xu_genes ( data_file = None ) : <EOL> return read_xu_et_al ( data_file ) [ <NUM_LIT:1> ] <EOL> def get_mouse_genes ( data_file = None ) : <EOL> annotations , gene_position , target_genes , Xdf , Y = read_V1_data ( data_file , learn_options = None ) <EOL> return Xdf [ Xdf [ '<STR_LIT>' ] == '<STR_LIT>' ] [ '<STR_LIT>' ] . unique ( ) <EOL> def get_human_genes ( data_file = None ) : <EOL> annotations , gene_position , target_genes , Xdf , Y = read_V1_data ( data_file , learn_options = None ) <EOL> mouse_genes = Xdf [ Xdf [ '<STR_LIT>' ] == '<STR_LIT>' ] [ '<STR_LIT>' ] . unique ( ) <EOL> all_genes = get_V3_genes ( None , None ) <EOL> return np . setdiff1d ( all_genes , mouse_genes ) </s>
