<s> from __future__ import division , print_function <EOL> from functools import partial <EOL> import os <EOL> import sys <EOL> import numpy as np <EOL> from sklearn . base import BaseEstimator , TransformerMixin <EOL> from . features import Features , DEFAULT_VARFRAC <EOL> from . utils import positive_int , portion , nonnegative_float , confirm_outfile <EOL> _do_nothing = lambda * a , ** k : None <EOL> DEFAULT_BLANK_HANDLER = '<STR_LIT>' <EOL> DEFAULT_BLANK_THRESH = <NUM_LIT:1000> <EOL> class FillBlanks ( BaseEstimator , TransformerMixin ) : <EOL> '''<STR_LIT>''' <EOL> def __init__ ( self , copy = True , blank_thresh = DEFAULT_BLANK_THRESH ) : <EOL> self . copy = copy <EOL> self . blank_thresh = blank_thresh <EOL> def fit ( self , X , y = None ) : <EOL> "<STR_LIT>" <EOL> return self <EOL> def transform ( self , X , y = None , copy = None ) : <EOL> copy = copy if copy is not None else self . copy <EOL> n_f , dim = X . shape <EOL> if copy : <EOL> X = X . copy ( ) <EOL> mag = <NUM_LIT:3> * ( <NUM_LIT:2> * self . blank_thresh / dim ) <EOL> blank_idx = np . sum ( X , axis = <NUM_LIT:1> ) < self . blank_thresh <EOL> X [ blank_idx , : ] = mag * np . random . rand ( blank_idx . sum ( ) , dim ) <EOL> return X <EOL> class ZeroBlanks ( BaseEstimator , TransformerMixin ) : <EOL> '''<STR_LIT>''' <EOL> def __init__ ( self , copy = True , blank_thresh = DEFAULT_BLANK_THRESH ) : <EOL> self . copy = copy <EOL> self . blank_thresh = blank_thresh <EOL> def fit ( self , X , y = None ) : <EOL> "<STR_LIT>" <EOL> return self <EOL> def transform ( self , X , y = None , copy = None ) : <EOL> copy = copy if copy is not None else self . copy <EOL> if copy : <EOL> X = X . copy ( ) <EOL> X [ np . sum ( X , axis = <NUM_LIT:1> ) < self . blank_thresh , : ] = <NUM_LIT:0> <EOL> return X <EOL> BLANK_HANDLERS = { <EOL> '<STR_LIT>' : FillBlanks , <EOL> '<STR_LIT>' : ZeroBlanks , <EOL> '<STR_LIT>' : None , <EOL> } <EOL> def handle_blanks ( features , blank_thresh = DEFAULT_BLANK_THRESH , <EOL> blank_handler = DEFAULT_BLANK_HANDLER , <EOL> inplace = False ) : <EOL> '''<STR_LIT>''' <EOL> if blank_handler not in BLANK_HANDLERS : <EOL> msg = "<STR_LIT>" . format ( <EOL> blank_handler , "<STR_LIT:U+002CU+0020>" . join ( map ( repr , BLANK_HANDLERS ) ) ) <EOL> raise ValueError ( msg ) <EOL> if blank_handler == '<STR_LIT>' : <EOL> feats = [ <EOL> f [ np . sum ( f , axis = <NUM_LIT:1> ) >= blank_thresh , : ] for f in features . features <EOL> ] <EOL> return features . _replace_bags ( feats , inplace = inplace ) <EOL> handler = BLANK_HANDLERS [ blank_handler ] ( blank_thresh = blank_thresh ) <EOL> r = features . _apply_transform ( handler , fit_first = True , inplace = inplace ) <EOL> if not inplace : <EOL> return r <EOL> def add_spatial_info ( features , add_x = True , add_y = True , inplace = False , <EOL> dtype = None ) : <EOL> '''<STR_LIT>''' <EOL> if not add_x and not add_y : <EOL> return None if inplace else features <EOL> indices = [ ] <EOL> if add_x : <EOL> indices . append ( <NUM_LIT:0> ) <EOL> if add_y : <EOL> indices . append ( <NUM_LIT:1> ) <EOL> if dtype is None : <EOL> dtype = features . dtype <EOL> if dtype . kind != '<STR_LIT:f>' : <EOL> dtype = np . float32 <EOL> spatial = np . asarray ( np . vstack ( features . frames ) [ : , indices ] , dtype = dtype ) <EOL> spatial /= spatial . max ( axis = <NUM_LIT:0> ) <EOL> new_feats = np . hstack ( ( features . _features , spatial ) ) <EOL> if inplace : <EOL> features . _features = new_feats <EOL> features . _refresh_features ( ) <EOL> else : <EOL> return Features ( <EOL> new_feats , n_pts = features . _n_pts , <EOL> categories = features . categories , names = features . names , <EOL> ** dict ( ( k , features . data [ k ] ) for k in features . _extra_names ) ) <EOL> def process_image_features ( features , verbose = False , inplace = False , <EOL> blank_thresh = DEFAULT_BLANK_THRESH , blank_handler = DEFAULT_BLANK_HANDLER , <EOL> do_pca = True , pca = None , <EOL> pca_k = None , pca_varfrac = DEFAULT_VARFRAC , pca_random = False , <EOL> pca_whiten = False , <EOL> add_x = True , add_y = True , <EOL> standardize_feats = True , scaler = None , <EOL> ret_pca = False , ret_scaler = False ) : <EOL> '''<STR_LIT>''' <EOL> pr = partial ( print , file = sys . stderr ) if verbose else _do_nothing <EOL> if blank_handler not in ( None , "<STR_LIT:none>" ) : <EOL> pr ( "<STR_LIT>" ) <EOL> ret = handle_blanks ( features , blank_thresh = blank_thresh , <EOL> blank_handler = blank_handler , inplace = inplace ) <EOL> if not inplace : <EOL> features = ret <EOL> if do_pca : <EOL> pr ( "<STR_LIT>" ) <EOL> old_dim = features . dim <EOL> ret = features . pca ( <EOL> pca = pca , ret_pca = True , k = pca_k , varfrac = pca_varfrac , <EOL> randomize = pca_random , whiten = pca_whiten , inplace = inplace ) <EOL> if inplace : <EOL> pca = ret <EOL> else : <EOL> features , pca = ret <EOL> new_dim = features . dim <EOL> pr ( "<STR_LIT>" . format ( old_dim , new_dim ) ) <EOL> if add_x or add_y : <EOL> pr ( "<STR_LIT>" ) <EOL> ret = add_spatial_info ( features , add_x , add_y , inplace = inplace ) <EOL> if not inplace : <EOL> features = ret <EOL> if standardize_feats : <EOL> pr ( "<STR_LIT>" ) <EOL> ret = features . standardize ( scaler = scaler , ret_scaler = True , <EOL> inplace = inplace ) <EOL> if inplace : <EOL> scaler = ret <EOL> else : <EOL> features , scaler = ret <EOL> if not ret_pca and not ret_scaler : <EOL> return features <EOL> ret = [ features ] <EOL> if ret_pca : <EOL> ret . append ( pca ) <EOL> if ret_scaler : <EOL> ret . append ( scaler ) <EOL> return ret <EOL> def parse_args ( args = None ) : <EOL> import argparse <EOL> class ActionNoYes ( argparse . Action ) : <EOL> def __init__ ( self , opt_name , off_name = None , dest = None , <EOL> default = True , required = False , help = None ) : <EOL> if off_name is None : <EOL> off_name = '<STR_LIT>' + opt_name <EOL> self . off_name = '<STR_LIT>' + off_name <EOL> if dest is None : <EOL> dest = opt_name . replace ( '<STR_LIT:->' , '<STR_LIT:_>' ) <EOL> super ( ActionNoYes , self ) . __init__ ( <EOL> [ '<STR_LIT>' + opt_name , '<STR_LIT>' + off_name ] , <EOL> dest , nargs = <NUM_LIT:0> , const = None , <EOL> default = default , required = required , help = help ) <EOL> def __call__ ( self , parser , namespace , values , option_string = None ) : <EOL> setattr ( namespace , self . dest , option_string != self . off_name ) <EOL> _def = "<STR_LIT>" <EOL> parser = argparse . ArgumentParser ( <EOL> description = "<STR_LIT>" ) <EOL> blanks = parser . add_argument_group ( "<STR_LIT>" ) <EOL> blanks . add_argument ( '<STR_LIT>' , dest = '<STR_LIT>' , <EOL> default = DEFAULT_BLANK_THRESH , type = nonnegative_float , <EOL> help = "<STR_LIT>" + _def ) <EOL> blanks . add_argument ( '<STR_LIT>' , <EOL> choices = list ( BLANK_HANDLERS ) + [ '<STR_LIT:none>' ] , default = DEFAULT_BLANK_HANDLER , <EOL> help = "<STR_LIT>" + _def ) <EOL> pca = parser . add_argument_group ( "<STR_LIT>" ) <EOL> pca . _add_action ( ActionNoYes ( '<STR_LIT>' , '<STR_LIT>' , default = True , <EOL> help = "<STR_LIT>" ) ) <EOL> dim = pca . add_mutually_exclusive_group ( ) <EOL> dim . add_argument ( '<STR_LIT>' , type = positive_int , default = None , <EOL> help = "<STR_LIT>" ) <EOL> dim . add_argument ( '<STR_LIT>' , default = DEFAULT_VARFRAC , <EOL> type = portion , metavar = '<STR_LIT>' , <EOL> help = "<STR_LIT>" + _def ) <EOL> pca . _add_action ( ActionNoYes ( '<STR_LIT>' , default = False , <EOL> help = "<STR_LIT>" ) ) <EOL> pca . _add_action ( ActionNoYes ( '<STR_LIT>' , default = False , <EOL> help = "<STR_LIT>" <EOL> "<STR_LIT>" ) ) <EOL> spa = parser . add_argument_group ( '<STR_LIT>' ) <EOL> spa . _add_action ( ActionNoYes ( '<STR_LIT>' , default = True , <EOL> help = "<STR_LIT>" ) ) <EOL> spa . _add_action ( ActionNoYes ( '<STR_LIT>' , default = True , <EOL> help = "<STR_LIT>" ) ) <EOL> std = parser . add_argument_group ( '<STR_LIT>' ) <EOL> std . _add_action ( ActionNoYes ( '<STR_LIT>' , default = True , <EOL> help = "<STR_LIT>" ) ) <EOL> parser . _add_action ( ActionNoYes ( '<STR_LIT>' , '<STR_LIT>' , default = True , <EOL> help = "<STR_LIT>" ) ) <EOL> parser . add_argument ( '<STR_LIT>' , <EOL> help = "<STR_LIT>" ) <EOL> parser . add_argument ( '<STR_LIT>' , help = "<STR_LIT>" ) <EOL> args = parser . parse_args ( args ) <EOL> load = args . load_file <EOL> save = args . save_file <EOL> del args . load_file , args . save_file <EOL> return args , load , save <EOL> def main ( ) : <EOL> args , load_file , save_file = parse_args ( ) <EOL> confirm_outfile ( save_file ) <EOL> pr = partial ( print , file = sys . stderr ) if args . verbose else _do_nothing <EOL> pr ( "<STR_LIT>" . format ( load_file ) ) <EOL> kwargs = { '<STR_LIT>' : True , '<STR_LIT>' : np . float32 } <EOL> if os . path . isdir ( load_file ) : <EOL> orig , attrs = Features . load_from_perbag ( load_file , ** kwargs ) <EOL> else : <EOL> orig , attrs = Features . load_from_hdf5 ( load_file , ** kwargs ) <EOL> new , pca , scaler = process_image_features ( <EOL> orig , ret_pca = True , ret_scaler = True , ** vars ( args ) ) <EOL> if pca is not None : <EOL> attrs [ '<STR_LIT>' ] = pca . mean_ <EOL> attrs [ '<STR_LIT>' ] = pca . components_ <EOL> if scaler is not None : <EOL> attrs [ '<STR_LIT>' ] = scaler . mean_ <EOL> attrs [ '<STR_LIT>' ] = scaler . std_ <EOL> pr ( "<STR_LIT>" . format ( save_file ) ) <EOL> new . save_as_hdf5 ( save_file , process_args = repr ( vars ( args ) ) , ** attrs ) <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> main ( ) </s>
