<s> import sys <EOL> import os <EOL> import logging <EOL> import json <EOL> from optparse import OptionParser <EOL> import progressbar <EOL> import numpy as np <EOL> from scipy . stats . stats import scoreatpercentile <EOL> from scipy . stats import norm as Gaussian <EOL> import cpa <EOL> import cpa . dbconnect <EOL> import cpa . util <EOL> logger = logging . getLogger ( __name__ ) <EOL> def np_load ( filename ) : <EOL> "<STR_LIT>" <EOL> with open ( filename , '<STR_LIT:rb>' ) as f : <EOL> x = np . load ( f ) <EOL> f . close ( ) <EOL> return x <EOL> def make_progress_bar ( text = None ) : <EOL> widgets = ( [ '<STR_LIT>' % text ] if text else [ ] ) + [ progressbar . Percentage ( ) , '<STR_LIT:U+0020>' , <EOL> progressbar . Bar ( ) , '<STR_LIT:U+0020>' , <EOL> progressbar . ETA ( ) ] <EOL> return progressbar . ProgressBar ( widgets = widgets ) <EOL> def _check_directory ( dir , resume ) : <EOL> if os . path . exists ( dir ) : <EOL> if not resume : <EOL> logger . error ( '<STR_LIT>' + dir ) <EOL> sys . exit ( <NUM_LIT:1> ) <EOL> else : <EOL> os . makedirs ( dir ) <EOL> import abc <EOL> class BaseNormalization ( object ) : <EOL> __metaclass__ = abc . ABCMeta <EOL> _cached_colmask = None <EOL> def __init__ ( self , cache , param_dir ) : <EOL> self . cache = cache <EOL> self . dir = os . path . join ( cache . cache_dir , param_dir ) <EOL> self . _colmask_filename = os . path . join ( self . dir , '<STR_LIT>' ) <EOL> def _params_filename ( self , plate ) : <EOL> return os . path . join ( self . dir , '<STR_LIT>' , <EOL> unicode ( plate ) + '<STR_LIT>' ) <EOL> @ property <EOL> def _colmask ( self ) : <EOL> if self . _cached_colmask is None : <EOL> self . _cached_colmask = np_load ( self . _colmask_filename ) <EOL> return self . _cached_colmask <EOL> @ abc . abstractmethod <EOL> def normalize ( self , plate , data ) : <EOL> pass <EOL> @ property <EOL> def colnames ( self ) : <EOL> """<STR_LIT>""" <EOL> return [ col <EOL> for col , keep in zip ( self . cache . colnames , self . _colmask ) <EOL> if keep ] <EOL> @ property <EOL> def colnames_excluded ( self ) : <EOL> """<STR_LIT>""" <EOL> return [ col <EOL> for col , keep in zip ( self . cache . colnames , self . _colmask ) <EOL> if not keep ] <EOL> def _create_cache ( self , predicate , resume = False ) : <EOL> self . _create_cache_params ( predicate , resume ) <EOL> self . _create_cache_colmask ( predicate ) <EOL> def _get_controls ( self , predicate ) : <EOL> """<STR_LIT>""" <EOL> plates_and_images = { } <EOL> for row in cpa . db . execute ( "<STR_LIT>" % <EOL> ( cpa . properties . plate_id , <EOL> '<STR_LIT:U+002CU+0020>' . join ( cpa . dbconnect . image_key_columns ( ) ) , <EOL> cpa . properties . image_table , predicate ) ) : <EOL> plate = row [ <NUM_LIT:0> ] <EOL> imKey = tuple ( row [ <NUM_LIT:1> : ] ) <EOL> plates_and_images . setdefault ( plate , [ ] ) . append ( imKey ) <EOL> return plates_and_images <EOL> def _create_cache_colmask ( self , predicate ) : <EOL> colmask = None <EOL> for plate , imKeys in self . _get_controls ( predicate ) . items ( ) : <EOL> params = np_load ( self . _params_filename ( plate ) ) <EOL> if len ( params ) == <NUM_LIT:0> : <EOL> continue <EOL> nonzero = self . _check_param_zero ( params ) <EOL> if colmask is None : <EOL> colmask = nonzero <EOL> else : <EOL> colmask &= nonzero <EOL> np . save ( self . _colmask_filename , colmask ) <EOL> @ abc . abstractmethod <EOL> def _compute_params ( self , features ) : <EOL> pass <EOL> @ abc . abstractproperty <EOL> def _null_param ( self ) : <EOL> """<STR_LIT>""" <EOL> pass <EOL> @ abc . abstractproperty <EOL> def _check_param_zero ( self , params ) : <EOL> """<STR_LIT>""" <EOL> pass <EOL> def _create_cache_params_1 ( self , plate , imKeys , filename ) : <EOL> features = self . cache . load ( imKeys ) [ <NUM_LIT:0> ] <EOL> if len ( features ) == <NUM_LIT:0> : <EOL> logger . warning ( '<STR_LIT>' % str ( plate ) ) <EOL> params = self . _null_param <EOL> else : <EOL> params = self . _compute_params ( features ) <EOL> np . save ( filename , params ) <EOL> def _create_cache_params ( self , predicate , resume = False ) : <EOL> controls = self . _get_controls ( predicate ) <EOL> for i , ( plate , imKeys ) in enumerate ( make_progress_bar ( '<STR_LIT>' ) ( controls . items ( ) ) ) : <EOL> filename = self . _params_filename ( plate ) <EOL> if i == <NUM_LIT:0> : <EOL> _check_directory ( os . path . dirname ( filename ) , resume ) <EOL> if resume and os . path . exists ( filename ) : <EOL> continue <EOL> self . _create_cache_params_1 ( plate , imKeys , filename ) <EOL> class DummyNormalization ( BaseNormalization ) : <EOL> def __init__ ( self , cache , param_dir = '<STR_LIT>' ) : <EOL> super ( DummyNormalization , self ) . __init__ ( cache , param_dir ) <EOL> def normalize ( self , plate , data ) : <EOL> return data <EOL> def _null_param ( self ) : <EOL> return np . zeros ( ( <NUM_LIT:0> , len ( self . cache . colnames ) ) ) <EOL> def _check_param_zero ( self , params ) : <EOL> return False <EOL> def _compute_params ( self , features ) : <EOL> return np . zeros ( ( <NUM_LIT:0> , len ( features ) ) ) <EOL> @ property <EOL> def colnames ( self ) : <EOL> return self . cache . colnames <EOL> class StdNormalization ( BaseNormalization ) : <EOL> def __init__ ( self , cache , param_dir = '<STR_LIT>' ) : <EOL> super ( StdNormalization , self ) . __init__ ( cache , param_dir ) <EOL> def normalize ( self , plate , data ) : <EOL> params = np_load ( self . _params_filename ( plate ) ) <EOL> assert data . shape [ <NUM_LIT:1> ] == params . shape [ <NUM_LIT:1> ] <EOL> data = data [ : , self . _colmask ] <EOL> params = params [ : , self . _colmask ] <EOL> shift = params [ <NUM_LIT:0> ] <EOL> scale = params [ <NUM_LIT:1> ] <EOL> assert np . all ( scale > <NUM_LIT:0> ) <EOL> return ( data - shift ) / scale <EOL> def _compute_params ( self , features ) : <EOL> m = features . shape [ <NUM_LIT:1> ] <EOL> params = np . ones ( ( <NUM_LIT:2> , m ) ) * np . nan <EOL> for j in xrange ( m ) : <EOL> params [ <NUM_LIT:0> , j ] = np . mean ( features [ : , j ] ) <EOL> params [ <NUM_LIT:1> , j ] = np . std ( features [ : , j ] ) <EOL> return params <EOL> def _null_param ( self ) : <EOL> return np . zeros ( ( <NUM_LIT:0> , len ( self . cache . colnames ) ) ) <EOL> def _check_param_zero ( self , params ) : <EOL> return params [ <NUM_LIT:1> ] != <NUM_LIT:0> <EOL> class RobustStdNormalization ( StdNormalization ) : <EOL> def __init__ ( self , cache , param_dir = '<STR_LIT>' ) : <EOL> super ( RobustStdNormalization , self ) . __init__ ( cache , param_dir ) <EOL> def _compute_params ( self , features ) : <EOL> m = features . shape [ <NUM_LIT:1> ] <EOL> params = np . ones ( ( <NUM_LIT:2> , m ) ) * np . nan <EOL> c = Gaussian . ppf ( <NUM_LIT:3> / <NUM_LIT> ) <EOL> for j in xrange ( m ) : <EOL> d = np . median ( features [ : , j ] ) <EOL> params [ <NUM_LIT:0> , j ] = d <EOL> params [ <NUM_LIT:1> , j ] = np . median ( np . fabs ( features [ : , j ] - d ) / c ) <EOL> return params <EOL> class RobustLinearNormalization ( BaseNormalization ) : <EOL> def __init__ ( self , cache , param_dir = '<STR_LIT>' , lower_q = <NUM_LIT:1> , upper_q = <NUM_LIT> ) : <EOL> super ( RobustLinearNormalization , self ) . __init__ ( cache , param_dir ) <EOL> self . lower_q = lower_q <EOL> self . upper_q = upper_q <EOL> def normalize ( self , plate , data ) : <EOL> percentiles = np_load ( self . _params_filename ( plate ) ) <EOL> assert data . shape [ <NUM_LIT:1> ] == percentiles . shape [ <NUM_LIT:1> ] <EOL> data = data [ : , self . _colmask ] <EOL> percentiles = percentiles [ : , self . _colmask ] <EOL> divisor = ( percentiles [ <NUM_LIT:1> ] - percentiles [ <NUM_LIT:0> ] ) <EOL> assert np . all ( divisor > <NUM_LIT:0> ) <EOL> return ( data - percentiles [ <NUM_LIT:0> ] ) / divisor <EOL> def _compute_params ( self , features ) : <EOL> m = features . shape [ <NUM_LIT:1> ] <EOL> percentiles = np . ones ( ( <NUM_LIT:2> , m ) ) * np . nan <EOL> for j in xrange ( m ) : <EOL> percentiles [ <NUM_LIT:0> , j ] = scoreatpercentile ( features [ : , j ] , self . lower_q ) <EOL> percentiles [ <NUM_LIT:1> , j ] = scoreatpercentile ( features [ : , j ] , self . upper_q ) <EOL> return percentiles <EOL> def _null_param ( self ) : <EOL> return np . zeros ( ( <NUM_LIT:0> , len ( self . cache . colnames ) ) ) <EOL> def _check_param_zero ( self , params ) : <EOL> return params [ <NUM_LIT:1> ] != params [ <NUM_LIT:0> ] <EOL> normalizations = dict ( ( c . __name__ , c ) <EOL> for c in [ RobustLinearNormalization , <EOL> RobustStdNormalization , <EOL> StdNormalization , <EOL> DummyNormalization ] ) <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> logging . basicConfig ( level = logging . INFO ) <EOL> parser = OptionParser ( "<STR_LIT>" ) <EOL> parser . add_option ( '<STR_LIT>' , '<STR_LIT>' , dest = '<STR_LIT>' , action = '<STR_LIT:store>' , default = '<STR_LIT>' , help = '<STR_LIT>' ) <EOL> options , args = parser . parse_args ( ) <EOL> if len ( args ) != <NUM_LIT:3> : <EOL> parser . error ( '<STR_LIT>' ) <EOL> properties_file , cache_dir , predicate = args <EOL> cpa . properties . LoadFile ( properties_file ) <EOL> from cpa . profiling . cache import Cache <EOL> cache = Cache ( cache_dir ) <EOL> normalizer = normalizations [ options . method ] ( cache ) <EOL> normalizer . _create_cache ( predicate ) </s>
