<s> """<STR_LIT>""" <EOL> import numpy as np <EOL> from . utils import floatX <EOL> from . random import get_rng <EOL> class Initializer ( object ) : <EOL> """<STR_LIT>""" <EOL> def __call__ ( self , shape ) : <EOL> """<STR_LIT>""" <EOL> return self . sample ( shape ) <EOL> def sample ( self , shape ) : <EOL> """<STR_LIT>""" <EOL> raise NotImplementedError ( ) <EOL> class Normal ( Initializer ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , std = <NUM_LIT> , mean = <NUM_LIT:0.0> ) : <EOL> self . std = std <EOL> self . mean = mean <EOL> def sample ( self , shape ) : <EOL> return floatX ( get_rng ( ) . normal ( self . mean , self . std , size = shape ) ) <EOL> class Uniform ( Initializer ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , range = <NUM_LIT> , std = None , mean = <NUM_LIT:0.0> ) : <EOL> if std is not None : <EOL> a = mean - np . sqrt ( <NUM_LIT:3> ) * std <EOL> b = mean + np . sqrt ( <NUM_LIT:3> ) * std <EOL> else : <EOL> try : <EOL> a , b = range <EOL> except TypeError : <EOL> a , b = - range , range <EOL> self . range = ( a , b ) <EOL> def sample ( self , shape ) : <EOL> return floatX ( get_rng ( ) . uniform ( <EOL> low = self . range [ <NUM_LIT:0> ] , high = self . range [ <NUM_LIT:1> ] , size = shape ) ) <EOL> class Glorot ( Initializer ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , initializer , gain = <NUM_LIT:1.0> , c01b = False ) : <EOL> if gain == '<STR_LIT:relu>' : <EOL> gain = np . sqrt ( <NUM_LIT:2> ) <EOL> self . initializer = initializer <EOL> self . gain = gain <EOL> self . c01b = c01b <EOL> def sample ( self , shape ) : <EOL> if self . c01b : <EOL> if len ( shape ) != <NUM_LIT:4> : <EOL> raise RuntimeError ( <EOL> "<STR_LIT>" ) <EOL> n1 , n2 = shape [ <NUM_LIT:0> ] , shape [ <NUM_LIT:3> ] <EOL> receptive_field_size = shape [ <NUM_LIT:1> ] * shape [ <NUM_LIT:2> ] <EOL> else : <EOL> if len ( shape ) < <NUM_LIT:2> : <EOL> raise RuntimeError ( <EOL> "<STR_LIT>" ) <EOL> n1 , n2 = shape [ : <NUM_LIT:2> ] <EOL> receptive_field_size = np . prod ( shape [ <NUM_LIT:2> : ] ) <EOL> std = self . gain * np . sqrt ( <NUM_LIT> / ( ( n1 + n2 ) * receptive_field_size ) ) <EOL> return self . initializer ( std = std ) . sample ( shape ) <EOL> class GlorotNormal ( Glorot ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , gain = <NUM_LIT:1.0> , c01b = False ) : <EOL> super ( GlorotNormal , self ) . __init__ ( Normal , gain , c01b ) <EOL> class GlorotUniform ( Glorot ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , gain = <NUM_LIT:1.0> , c01b = False ) : <EOL> super ( GlorotUniform , self ) . __init__ ( Uniform , gain , c01b ) <EOL> class He ( Initializer ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , initializer , gain = <NUM_LIT:1.0> , c01b = False ) : <EOL> if gain == '<STR_LIT:relu>' : <EOL> gain = np . sqrt ( <NUM_LIT:2> ) <EOL> self . initializer = initializer <EOL> self . gain = gain <EOL> self . c01b = c01b <EOL> def sample ( self , shape ) : <EOL> if self . c01b : <EOL> if len ( shape ) != <NUM_LIT:4> : <EOL> raise RuntimeError ( <EOL> "<STR_LIT>" ) <EOL> fan_in = np . prod ( shape [ : <NUM_LIT:3> ] ) <EOL> else : <EOL> if len ( shape ) == <NUM_LIT:2> : <EOL> fan_in = shape [ <NUM_LIT:0> ] <EOL> elif len ( shape ) > <NUM_LIT:2> : <EOL> fan_in = np . prod ( shape [ <NUM_LIT:1> : ] ) <EOL> else : <EOL> raise RuntimeError ( <EOL> "<STR_LIT>" ) <EOL> std = self . gain * np . sqrt ( <NUM_LIT:1.0> / fan_in ) <EOL> return self . initializer ( std = std ) . sample ( shape ) <EOL> class HeNormal ( He ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , gain = <NUM_LIT:1.0> , c01b = False ) : <EOL> super ( HeNormal , self ) . __init__ ( Normal , gain , c01b ) <EOL> class HeUniform ( He ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , gain = <NUM_LIT:1.0> , c01b = False ) : <EOL> super ( HeUniform , self ) . __init__ ( Uniform , gain , c01b ) <EOL> class Constant ( Initializer ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , val = <NUM_LIT:0.0> ) : <EOL> self . val = val <EOL> def sample ( self , shape ) : <EOL> return floatX ( np . ones ( shape ) * self . val ) <EOL> class Sparse ( Initializer ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , sparsity = <NUM_LIT:0.1> , std = <NUM_LIT> ) : <EOL> self . sparsity = sparsity <EOL> self . std = std <EOL> def sample ( self , shape ) : <EOL> if len ( shape ) != <NUM_LIT:2> : <EOL> raise RuntimeError ( <EOL> "<STR_LIT>" ) <EOL> w = floatX ( np . zeros ( shape ) ) <EOL> n_inputs , n_outputs = shape <EOL> size = int ( self . sparsity * n_inputs ) <EOL> for k in range ( n_outputs ) : <EOL> indices = np . arange ( n_inputs ) <EOL> get_rng ( ) . shuffle ( indices ) <EOL> indices = indices [ : size ] <EOL> values = floatX ( get_rng ( ) . normal ( <NUM_LIT:0.0> , self . std , size = size ) ) <EOL> w [ indices , k ] = values <EOL> return w <EOL> class Orthogonal ( Initializer ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , gain = <NUM_LIT:1.0> ) : <EOL> if gain == '<STR_LIT:relu>' : <EOL> gain = np . sqrt ( <NUM_LIT:2> ) <EOL> self . gain = gain <EOL> def sample ( self , shape ) : <EOL> if len ( shape ) < <NUM_LIT:2> : <EOL> raise RuntimeError ( "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> flat_shape = ( shape [ <NUM_LIT:0> ] , np . prod ( shape [ <NUM_LIT:1> : ] ) ) <EOL> a = get_rng ( ) . normal ( <NUM_LIT:0.0> , <NUM_LIT:1.0> , flat_shape ) <EOL> u , _ , v = np . linalg . svd ( a , full_matrices = False ) <EOL> q = u if u . shape == flat_shape else v <EOL> q = q . reshape ( shape ) <EOL> return floatX ( self . gain * q ) </s>
