<s> import logging <EOL> import random <EOL> import numpy as np <EOL> import pandas as pd <EOL> from ramp . builders import build_target_safe <EOL> class BasicFolds ( object ) : <EOL> def __init__ ( self , num_folds , data , repeat = <NUM_LIT:1> , seed = None ) : <EOL> self . num_folds = num_folds <EOL> self . data = data <EOL> self . seed = seed <EOL> self . repeat = repeat <EOL> def __iter__ ( self ) : <EOL> n = len ( self . data ) <EOL> index = self . data . index <EOL> indices = range ( n ) <EOL> foldsize = n / self . num_folds <EOL> folds = [ ] <EOL> if self . seed is not None : <EOL> np . random . seed ( self . seed ) <EOL> for i in range ( self . repeat ) : <EOL> np . random . shuffle ( indices ) <EOL> for i in range ( self . num_folds ) : <EOL> test = index [ indices [ i * foldsize : ( i + <NUM_LIT:1> ) * foldsize ] ] <EOL> train = index - test <EOL> assert not ( train & test ) <EOL> fold = ( pd . Index ( train ) , pd . Index ( test ) ) <EOL> yield fold <EOL> class WatertightFolds ( BasicFolds ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , num_folds , data , leakage_column , ** kwargs ) : <EOL> super ( WatertightFolds , self ) . __init__ ( num_folds , data , ** kwargs ) <EOL> self . leakage_column = leakage_column <EOL> def __iter__ ( self ) : <EOL> n = len ( self . data ) <EOL> index = self . data . index <EOL> indices = range ( n ) <EOL> foldsize = n / self . num_folds <EOL> folds = [ ] <EOL> if self . seed is not None : <EOL> np . random . seed ( self . seed ) <EOL> for i in range ( self . repeat ) : <EOL> watertight_bins = self . data . groupby ( self . leakage_column ) [ self . data . columns [ <NUM_LIT:0> ] ] . count ( ) <EOL> watertight_bins = watertight_bins . reindex ( np . random . permutation ( watertight_bins . index ) ) <EOL> watertight_bins_cum = watertight_bins . cumsum ( ) <EOL> for i in range ( self . num_folds ) : <EOL> test_bins = watertight_bins_cum [ ( watertight_bins_cum > i * foldsize ) & <EOL> ( watertight_bins_cum <= ( i + <NUM_LIT:1> ) * foldsize ) ] <EOL> test = index [ self . data [ self . leakage_column ] . isin ( test_bins . index ) ] <EOL> train = index - test <EOL> assert not ( train & test ) <EOL> if np . abs ( len ( test ) - foldsize ) > <NUM_LIT> * foldsize : <EOL> logging . warn ( "<STR_LIT>" . format ( target = foldsize , actual = len ( test ) ) ) <EOL> fold = ( pd . Index ( train ) , pd . Index ( test ) ) <EOL> yield fold <EOL> class BinaryTargetFolds ( object ) : <EOL> def __init__ ( self , target , data , seed = None ) : <EOL> self . seed = seed <EOL> self . target = target <EOL> self . data = data <EOL> self . folds = None <EOL> self . y = None <EOL> def compute_folds ( self ) : <EOL> raise NotImplementedError <EOL> def build_target ( self ) : <EOL> y , ff = build_target_safe ( self . target , self . data ) <EOL> self . y = y <EOL> self . negatives = y [ ~ y . astype ( '<STR_LIT:bool>' ) ] . index <EOL> self . positives = y [ y . astype ( '<STR_LIT:bool>' ) ] . index <EOL> self . n_positives = len ( self . positives ) <EOL> self . n_negatives = len ( self . negatives ) <EOL> def randomize ( self ) : <EOL> if self . seed is not None : <EOL> np . random . seed ( self . seed ) <EOL> neg = pd . Index ( np . random . permutation ( self . negatives ) ) <EOL> pos = pd . Index ( np . random . permutation ( self . positives ) ) <EOL> return neg , pos <EOL> def __iter__ ( self ) : <EOL> if self . y is None : <EOL> self . build_target ( ) <EOL> if self . folds is None : <EOL> self . compute_folds ( ) <EOL> for fold in self . folds : <EOL> yield fold <EOL> class BalancedFolds ( BinaryTargetFolds ) : <EOL> def __init__ ( self , num_folds , target , data , seed = None ) : <EOL> self . num_folds = num_folds <EOL> super ( BalancedFolds , self ) . __init__ ( target , data , seed ) <EOL> def compute_folds ( self ) : <EOL> neg , pos = self . randomize ( ) <EOL> nn = len ( neg ) / self . num_folds <EOL> np = len ( pos ) / self . num_folds <EOL> folds = [ ] <EOL> for i in range ( self . num_folds ) : <EOL> s = i * nn <EOL> e = ( i + <NUM_LIT:1> ) * nn <EOL> train_neg , test_neg = neg [ : s ] + neg [ e : ] , neg [ s : e ] <EOL> s = i * np <EOL> e = ( i + <NUM_LIT:1> ) * np <EOL> train_pos , test_pos = pos [ : s ] + pos [ e : ] , pos [ s : e ] <EOL> fold = ( train_neg + train_pos , test_neg + test_pos ) <EOL> folds . append ( fold ) <EOL> self . folds = folds <EOL> class BootstrapFolds ( BalancedFolds ) : <EOL> def __init__ ( self , num_folds , target , data , <EOL> pos_train = None , pos_test = None , neg_train = None , neg_test = None , <EOL> train_pos_percent = None , test_pos_percent = None , train_percent = None , <EOL> seed = None ) : <EOL> super ( BootstrapFolds , self ) . __init__ ( num_folds , target , data , seed ) <EOL> if ( any ( [ pos_train , pos_test , neg_train , neg_test ] ) <EOL> and not all ( [ pos_train , pos_test , neg_train , neg_test ] ) ) : <EOL> raise ValueError ( "<STR_LIT>" ) <EOL> if ( any ( [ train_pos_percent , test_pos_percent , train_percent ] ) <EOL> and not all ( [ train_pos_percent , test_pos_percent , train_percent ] ) ) : <EOL> raise ValueError ( "<STR_LIT>" ) <EOL> if ( train_pos_percent is None ) == ( pos_test is None ) : <EOL> raise ValueError ( "<STR_LIT>" ) <EOL> self . pos_train = pos_train <EOL> self . neg_train = neg_train <EOL> self . pos_test = pos_test <EOL> self . neg_test = neg_test <EOL> self . train_pos_percent = train_pos_percent <EOL> self . test_pos_percent = test_pos_percent <EOL> self . train_percent = train_percent <EOL> self . using_percents = False <EOL> if self . test_pos_percent is not None : <EOL> self . using_percents = True <EOL> def from_sizes ( self ) : <EOL> train_neg = pd . Index ( np . random . choice ( self . negatives , self . neg_train , replace = True ) ) <EOL> test_neg = pd . Index ( np . random . choice ( self . negatives - train_neg , self . neg_test , replace = True ) ) <EOL> train_pos = pd . Index ( np . random . choice ( self . positives , self . pos_train , replace = True ) ) <EOL> test_pos = pd . Index ( np . random . choice ( self . positives - train_pos , self . pos_test , replace = True ) ) <EOL> return train_neg , test_neg , train_pos , test_pos <EOL> def from_percents ( self ) : <EOL> rp = self . train_percent * self . n_positives <EOL> tp = ( <NUM_LIT:1> - self . train_percent ) * self . n_positives <EOL> rn = rp / self . train_pos_percent - rp <EOL> tn = tp / self . test_pos_percent - tp <EOL> train_neg = pd . Index ( np . random . choice ( self . negatives , <EOL> int ( rn ) , <EOL> replace = True ) ) <EOL> test_neg = pd . Index ( np . random . choice ( self . negatives - train_neg , <EOL> int ( tn ) , <EOL> replace = True ) ) <EOL> train_pos = pd . Index ( np . random . choice ( self . positives , <EOL> int ( rp ) , <EOL> replace = True ) ) <EOL> test_pos = pd . Index ( np . random . choice ( self . positives - train_pos , <EOL> int ( tp ) , <EOL> replace = True ) ) <EOL> return train_neg , test_neg , train_pos , test_pos <EOL> def compute_folds ( self ) : <EOL> if self . seed is not None : <EOL> np . random . seed ( self . seed ) <EOL> folds = [ ] <EOL> for i in range ( self . num_folds ) : <EOL> if self . using_percents : <EOL> train_neg , test_neg , train_pos , test_pos = self . from_percents ( ) <EOL> else : <EOL> train_neg , test_neg , train_pos , test_pos = self . from_sizes ( ) <EOL> fold = ( train_neg . append ( train_pos ) , test_neg . append ( test_pos ) ) <EOL> folds . append ( fold ) <EOL> self . folds = folds <EOL> class BootstrapFoldsByPercentPositive ( BootstrapFolds ) : <EOL> def __init__ ( self , num_folds , target , data , <EOL> train_pos_percent , test_pos_percent , train_percent , <EOL> seed = None ) : <EOL> super ( BootstrapFoldsByPercentPositive , <EOL> self ) . __init__ ( num_folds , target , data , seed = seed , <EOL> train_pos_percent = train_pos_percent , <EOL> test_pos_percent = test_pos_percent , <EOL> train_percent = train_percent ) <EOL> class BootstrapFoldsBySize ( BootstrapFolds ) : <EOL> def __init__ ( self , num_folds , target , data , <EOL> pos_train = None , pos_test = None , neg_train = None , neg_test = None , <EOL> seed = None ) : <EOL> super ( BootstrapFoldsBySize , self ) . __init__ ( num_folds , target , data , seed = seed , <EOL> pos_train = pos_train , <EOL> pos_test = pos_test , <EOL> neg_train = neg_train , <EOL> neg_test = neg_test ) <EOL> make_default_folds = BasicFolds </s>
