<s> from . import NeuralLayer <EOL> from var import NeuralVariable <EOL> from deepy . utils import build_activation , FLOATX <EOL> import numpy as np <EOL> import theano <EOL> import theano . tensor as T <EOL> from collections import OrderedDict <EOL> OUTPUT_TYPES = [ "<STR_LIT>" , "<STR_LIT>" ] <EOL> INPUT_TYPES = [ "<STR_LIT>" , "<STR_LIT>" ] <EOL> class LSTM ( NeuralLayer ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , hidden_size , input_type = "<STR_LIT>" , output_type = "<STR_LIT>" , <EOL> inner_activation = "<STR_LIT>" , outer_activation = "<STR_LIT>" , <EOL> inner_init = None , outer_init = None , steps = None , <EOL> go_backwards = False , <EOL> persistent_state = False , batch_size = <NUM_LIT:0> , <EOL> reset_state_for_input = None , forget_bias = <NUM_LIT:1> , <EOL> mask = None , <EOL> second_input = None , second_input_size = None ) : <EOL> super ( LSTM , self ) . __init__ ( "<STR_LIT>" ) <EOL> self . _hidden_size = hidden_size <EOL> self . _input_type = input_type <EOL> self . _output_type = output_type <EOL> self . _inner_activation = inner_activation <EOL> self . _outer_activation = outer_activation <EOL> self . _inner_init = inner_init <EOL> self . _outer_init = outer_init <EOL> self . _steps = steps <EOL> self . persistent_state = persistent_state <EOL> self . reset_state_for_input = reset_state_for_input <EOL> self . batch_size = batch_size <EOL> self . go_backwards = go_backwards <EOL> mask = mask . tensor if type ( mask ) == NeuralVariable else mask <EOL> self . mask = mask . dimshuffle ( ( <NUM_LIT:1> , <NUM_LIT:0> ) ) if mask else None <EOL> self . _sequence_map = OrderedDict ( ) <EOL> if type ( second_input ) == NeuralVariable : <EOL> second_input_size = second_input . dim ( ) <EOL> second_input = second_input . tensor <EOL> self . second_input = second_input <EOL> self . second_input_size = second_input_size <EOL> self . forget_bias = forget_bias <EOL> if input_type not in INPUT_TYPES : <EOL> raise Exception ( "<STR_LIT>" % input_type ) <EOL> if output_type not in OUTPUT_TYPES : <EOL> raise Exception ( "<STR_LIT>" % output_type ) <EOL> if self . persistent_state and not self . batch_size : <EOL> raise Exception ( "<STR_LIT>" ) <EOL> if mask and input_type == "<STR_LIT>" : <EOL> raise Exception ( "<STR_LIT>" ) <EOL> def _auto_reset_memories ( self , x , h , m ) : <EOL> reset_matrix = T . neq ( x [ : , self . reset_state_for_input ] , <NUM_LIT:1> ) . dimshuffle ( <NUM_LIT:0> , '<STR_LIT:x>' ) <EOL> h = h * reset_matrix <EOL> m = m * reset_matrix <EOL> return h , m <EOL> def step ( self , * vars ) : <EOL> sequence_map = dict ( zip ( self . _sequence_map . keys ( ) , vars [ : len ( self . _sequence_map ) ] ) ) <EOL> h_tm1 , c_tm1 = vars [ - <NUM_LIT:2> : ] <EOL> if self . reset_state_for_input != None : <EOL> h_tm1 , c_tm1 = self . _auto_reset_memories ( sequence_map [ "<STR_LIT:x>" ] , h_tm1 , c_tm1 ) <EOL> if self . _input_type == "<STR_LIT>" : <EOL> xi_t , xf_t , xo_t , xc_t = map ( sequence_map . get , [ "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" ] ) <EOL> else : <EOL> xi_t , xf_t , xo_t , xc_t = <NUM_LIT:0> , <NUM_LIT:0> , <NUM_LIT:0> , <NUM_LIT:0> <EOL> if "<STR_LIT>" in sequence_map : <EOL> xi2 , xf2 , xo2 , xc2 = map ( sequence_map . get , [ "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" ] ) <EOL> xi_t += xi2 <EOL> xf_t += xf2 <EOL> xo_t += xo2 <EOL> xc_t += xc2 <EOL> i_t = self . _inner_act ( xi_t + T . dot ( h_tm1 , self . U_i ) + self . b_i ) <EOL> f_t = self . _inner_act ( xf_t + T . dot ( h_tm1 , self . U_f ) + self . b_f ) <EOL> c_t = f_t * c_tm1 + i_t * self . _outer_act ( xc_t + T . dot ( h_tm1 , self . U_c ) + self . b_c ) <EOL> o_t = self . _inner_act ( xo_t + T . dot ( h_tm1 , self . U_o ) + self . b_o ) <EOL> h_t = o_t * self . _outer_act ( c_t ) <EOL> if "<STR_LIT>" in sequence_map : <EOL> mask = sequence_map [ "<STR_LIT>" ] . dimshuffle ( <NUM_LIT:0> , '<STR_LIT:x>' ) <EOL> h_t = h_t * mask + h_tm1 * ( <NUM_LIT:1> - mask ) <EOL> c_t = c_t * mask + c_tm1 * ( <NUM_LIT:1> - mask ) <EOL> return h_t , c_t <EOL> def produce_input_sequences ( self , x , mask = None , second_input = None ) : <EOL> self . _sequence_map . clear ( ) <EOL> if self . _input_type == "<STR_LIT>" : <EOL> xi = T . dot ( x , self . W_i ) <EOL> xf = T . dot ( x , self . W_f ) <EOL> xc = T . dot ( x , self . W_c ) <EOL> xo = T . dot ( x , self . W_o ) <EOL> self . _sequence_map . update ( [ ( "<STR_LIT>" , xi ) , ( "<STR_LIT>" , xf ) , ( "<STR_LIT>" , xc ) , ( "<STR_LIT>" , xo ) ] ) <EOL> if self . reset_state_for_input != None : <EOL> self . _sequence_map [ "<STR_LIT:x>" ] = x <EOL> if mask : <EOL> self . _sequence_map [ "<STR_LIT>" ] = mask <EOL> elif self . mask : <EOL> self . _sequence_map [ "<STR_LIT>" ] = self . mask <EOL> if self . second_input and not second_input : <EOL> second_input = self . second_input <EOL> if second_input : <EOL> xi2 = T . dot ( second_input , self . W_i2 ) <EOL> xf2 = T . dot ( second_input , self . W_f2 ) <EOL> xc2 = T . dot ( second_input , self . W_c2 ) <EOL> xo2 = T . dot ( second_input , self . W_o2 ) <EOL> self . _sequence_map . update ( [ ( "<STR_LIT>" , xi2 ) , ( "<STR_LIT>" , xf2 ) , ( "<STR_LIT>" , xc2 ) , ( "<STR_LIT>" , xo2 ) ] ) <EOL> return self . _sequence_map . values ( ) <EOL> def produce_initial_states ( self , x ) : <EOL> if self . persistent_state : <EOL> return self . state_h , self . state_m <EOL> else : <EOL> h0 = T . alloc ( np . cast [ FLOATX ] ( <NUM_LIT:0.> ) , x . shape [ <NUM_LIT:0> ] , self . _hidden_size ) <EOL> m0 = h0 <EOL> return h0 , m0 <EOL> def compute_tensor ( self , x ) : <EOL> h0 , m0 = self . produce_initial_states ( x ) <EOL> if self . _input_type == "<STR_LIT>" : <EOL> x = x . dimshuffle ( ( <NUM_LIT:1> , <NUM_LIT:0> , <NUM_LIT:2> ) ) <EOL> sequences = self . produce_input_sequences ( x ) <EOL> else : <EOL> h0 = x <EOL> sequences = self . produce_input_sequences ( None ) <EOL> [ hiddens , memories ] , _ = theano . scan ( <EOL> self . step , <EOL> sequences = sequences , <EOL> outputs_info = [ h0 , m0 ] , <EOL> n_steps = self . _steps , <EOL> go_backwards = self . go_backwards <EOL> ) <EOL> if self . persistent_state : <EOL> self . register_updates ( ( self . state_h , hiddens [ - <NUM_LIT:1> ] ) ) <EOL> self . register_updates ( ( self . state_m , memories [ - <NUM_LIT:1> ] ) ) <EOL> if self . _output_type == "<STR_LIT>" : <EOL> return hiddens [ - <NUM_LIT:1> ] <EOL> elif self . _output_type == "<STR_LIT>" : <EOL> return hiddens . dimshuffle ( ( <NUM_LIT:1> , <NUM_LIT:0> , <NUM_LIT:2> ) ) <EOL> def prepare ( self ) : <EOL> self . _setup_params ( ) <EOL> self . _setup_functions ( ) <EOL> def _setup_functions ( self ) : <EOL> self . _inner_act = build_activation ( self . _inner_activation ) <EOL> self . _outer_act = build_activation ( self . _outer_activation ) <EOL> def _setup_params ( self ) : <EOL> self . output_dim = self . _hidden_size <EOL> self . W_i = self . create_weight ( self . input_dim , self . _hidden_size , "<STR_LIT>" , initializer = self . _outer_init ) <EOL> self . U_i = self . create_weight ( self . _hidden_size , self . _hidden_size , "<STR_LIT>" , initializer = self . _inner_init ) <EOL> self . b_i = self . create_bias ( self . _hidden_size , "<STR_LIT:i>" ) <EOL> self . W_f = self . create_weight ( self . input_dim , self . _hidden_size , "<STR_LIT>" , initializer = self . _outer_init ) <EOL> self . U_f = self . create_weight ( self . _hidden_size , self . _hidden_size , "<STR_LIT>" , initializer = self . _inner_init ) <EOL> self . b_f = self . create_bias ( self . _hidden_size , "<STR_LIT:f>" ) <EOL> if self . forget_bias > <NUM_LIT:0> : <EOL> self . b_f . set_value ( np . ones ( ( self . _hidden_size , ) , dtype = FLOATX ) ) <EOL> self . W_c = self . create_weight ( self . input_dim , self . _hidden_size , "<STR_LIT>" , initializer = self . _outer_init ) <EOL> self . U_c = self . create_weight ( self . _hidden_size , self . _hidden_size , "<STR_LIT>" , initializer = self . _inner_init ) <EOL> self . b_c = self . create_bias ( self . _hidden_size , "<STR_LIT:c>" ) <EOL> self . W_o = self . create_weight ( self . input_dim , self . _hidden_size , "<STR_LIT>" , initializer = self . _outer_init ) <EOL> self . U_o = self . create_weight ( self . _hidden_size , self . _hidden_size , "<STR_LIT>" , initializer = self . _inner_init ) <EOL> self . b_o = self . create_bias ( self . _hidden_size , suffix = "<STR_LIT:o>" ) <EOL> if self . _input_type == "<STR_LIT>" : <EOL> self . register_parameters ( self . W_i , self . U_i , self . b_i , <EOL> self . W_c , self . U_c , self . b_c , <EOL> self . W_f , self . U_f , self . b_f , <EOL> self . W_o , self . U_o , self . b_o ) <EOL> else : <EOL> self . register_parameters ( self . U_i , self . b_i , <EOL> self . U_c , self . b_c , <EOL> self . U_f , self . b_f , <EOL> self . U_o , self . b_o ) <EOL> if self . second_input_size : <EOL> self . W_i2 = self . create_weight ( self . second_input_size , self . _hidden_size , "<STR_LIT>" , initializer = self . _outer_init ) <EOL> self . W_f2 = self . create_weight ( self . second_input_size , self . _hidden_size , "<STR_LIT>" , initializer = self . _outer_init ) <EOL> self . W_c2 = self . create_weight ( self . second_input_size , self . _hidden_size , "<STR_LIT>" , initializer = self . _outer_init ) <EOL> self . W_o2 = self . create_weight ( self . second_input_size , self . _hidden_size , "<STR_LIT>" , initializer = self . _outer_init ) <EOL> self . register_parameters ( self . W_i2 , self . W_f2 , self . W_c2 , self . W_o2 ) <EOL> if self . persistent_state : <EOL> self . state_h = self . create_matrix ( self . batch_size , self . _hidden_size , "<STR_LIT>" ) <EOL> self . state_m = self . create_matrix ( self . batch_size , self . _hidden_size , "<STR_LIT>" ) <EOL> self . register_free_parameters ( self . state_h , self . state_m ) <EOL> else : <EOL> self . state_h = None <EOL> self . state_m = None </s>
