<s> from collections import defaultdict <EOL> import logging <EOL> logger = logging . getLogger ( __name__ ) <EOL> LOG_STEP = <NUM_LIT> <EOL> class FeatureEvaluator ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , features ) : <EOL> self . features = features <EOL> def fit ( self , X , y = None ) : <EOL> self . alive_features = tuple ( self . features ) <EOL> return self <EOL> def fit_transform ( self , X , y = None ) : <EOL> self . fit ( X , y ) <EOL> return self . transform ( X ) <EOL> def transform ( self , X , y = None ) : <EOL> for d in X : <EOL> yield tuple ( ( f ( d ) for f in self . alive_features ) ) <EOL> class TolerantFeatureEvaluator ( object ) : <EOL> """<STR_LIT>""" <EOL> FEATURE_STRICT_UNTIL = <NUM_LIT:100> <EOL> FEATURE_MAX_ERRORS_ALLOWED = <NUM_LIT:5> <EOL> class NoFeaturesLeftError ( Exception ) : <EOL> pass <EOL> def __init__ ( self , features ) : <EOL> self . features = features <EOL> self . fitted = False <EOL> def fit ( self , X , y = None ) : <EOL> self . _fit_failure_stats = { <EOL> '<STR_LIT>' : [ ] , <EOL> '<STR_LIT>' : defaultdict ( list ) <EOL> } <EOL> self . alive_features = self . features [ : ] <EOL> dataset = X <EOL> last_sample_idx = - <NUM_LIT:1> <EOL> while dataset : <EOL> self . _samples_to_retry = [ ] <EOL> for i , d in enumerate ( dataset , last_sample_idx + <NUM_LIT:1> ) : <EOL> for feature in self . alive_features [ : ] : <EOL> try : <EOL> feature ( d ) <EOL> except Exception as e : <EOL> self . process_failure ( [ ] , e , feature , d , i ) <EOL> break <EOL> last_sample_idx = i <EOL> dataset = self . _samples_to_retry <EOL> self . alive_features = tuple ( self . alive_features ) <EOL> self . fitted = True <EOL> return self <EOL> def transform ( self , X , y = None ) : <EOL> for d in X : <EOL> yield tuple ( ( f ( d ) for f in self . alive_features ) ) <EOL> def fit_transform ( self , X , y = None ) : <EOL> self . _fit_failure_stats = { <EOL> '<STR_LIT>' : [ ] , <EOL> '<STR_LIT>' : defaultdict ( list ) <EOL> } <EOL> self . alive_features = self . features [ : ] <EOL> result = [ ] <EOL> dataset = X <EOL> last_sample_idx = - <NUM_LIT:1> <EOL> while dataset : <EOL> self . _samples_to_retry = [ ] <EOL> for i , d in enumerate ( dataset , last_sample_idx + <NUM_LIT:1> ) : <EOL> r = [ ] <EOL> for feature in self . alive_features [ : ] : <EOL> try : <EOL> r . append ( feature ( d ) ) <EOL> except Exception as e : <EOL> self . process_failure ( result , e , feature , d , i ) <EOL> break <EOL> else : <EOL> result . append ( r ) <EOL> last_sample_idx = i <EOL> dataset = self . _samples_to_retry <EOL> self . alive_features = tuple ( self . alive_features ) <EOL> self . fitted = True <EOL> return ( tuple ( r ) for r in result ) <EOL> def process_failure ( self , partial_eval , error , feature , dpoint , d_index ) : <EOL> logger . warning ( u'<STR_LIT>' % ( feature , <EOL> type ( error ) , error ) ) <EOL> self . _fit_failure_stats [ '<STR_LIT>' ] . append ( <EOL> dpoint . get ( '<STR_LIT>' , '<STR_LIT>' ) ) <EOL> feature_errors = self . _fit_failure_stats [ '<STR_LIT>' ] [ feature ] <EOL> feature_errors . append ( dpoint ) <EOL> if d_index < self . FEATURE_STRICT_UNTIL : <EOL> self . exclude_feature ( feature , partial_eval ) <EOL> elif len ( feature_errors ) > self . FEATURE_MAX_ERRORS_ALLOWED : <EOL> self . exclude_feature ( feature , partial_eval ) <EOL> def exclude_feature ( self , feature , partial_evaluation ) : <EOL> idx = self . alive_features . index ( feature ) <EOL> self . alive_features . remove ( feature ) <EOL> if not self . alive_features : <EOL> raise self . NoFeaturesLeftError ( ) <EOL> for evaluation in partial_evaluation : <EOL> evaluation . pop ( idx ) <EOL> discarded_samples = self . _fit_failure_stats [ '<STR_LIT>' ] [ feature ] <EOL> self . _samples_to_retry += discarded_samples </s>
