<s> from __future__ import print_function <EOL> import six . moves . cPickle as pickle <EOL> from collections import OrderedDict <EOL> import copy <EOL> import gzip <EOL> import os <EOL> import urllib <EOL> import random <EOL> import stat <EOL> import subprocess <EOL> import sys <EOL> import timeit <EOL> import numpy <EOL> import theano <EOL> from theano import tensor as T <EOL> import sys <EOL> sys . setrecursionlimit ( <NUM_LIT> ) <EOL> PREFIX = os . getenv ( <EOL> '<STR_LIT>' , <EOL> os . path . join ( os . path . split ( os . path . abspath ( os . path . dirname ( __file__ ) ) ) [ <NUM_LIT:0> ] , <EOL> '<STR_LIT:data>' ) ) <EOL> def shuffle ( lol , seed ) : <EOL> '''<STR_LIT>''' <EOL> for l in lol : <EOL> random . seed ( seed ) <EOL> random . shuffle ( l ) <EOL> def contextwin ( l , win ) : <EOL> '''<STR_LIT>''' <EOL> assert ( win % <NUM_LIT:2> ) == <NUM_LIT:1> <EOL> assert win >= <NUM_LIT:1> <EOL> l = list ( l ) <EOL> lpadded = win // <NUM_LIT:2> * [ - <NUM_LIT:1> ] + l + win // <NUM_LIT:2> * [ - <NUM_LIT:1> ] <EOL> out = [ lpadded [ i : ( i + win ) ] for i in range ( len ( l ) ) ] <EOL> assert len ( out ) == len ( l ) <EOL> return out <EOL> def atisfold ( fold ) : <EOL> assert fold in range ( <NUM_LIT:5> ) <EOL> filename = os . path . join ( PREFIX , '<STR_LIT>' + str ( fold ) + '<STR_LIT>' ) <EOL> f = gzip . open ( filename , '<STR_LIT:rb>' ) <EOL> try : <EOL> train_set , valid_set , test_set , dicts = pickle . load ( f , encoding = '<STR_LIT>' ) <EOL> except : <EOL> train_set , valid_set , test_set , dicts = pickle . load ( f ) <EOL> return train_set , valid_set , test_set , dicts <EOL> def conlleval ( p , g , w , filename , script_path ) : <EOL> '''<STR_LIT>''' <EOL> out = '<STR_LIT>' <EOL> for sl , sp , sw in zip ( g , p , w ) : <EOL> out += '<STR_LIT>' <EOL> for wl , wp , w in zip ( sl , sp , sw ) : <EOL> out += w + '<STR_LIT:U+0020>' + wl + '<STR_LIT:U+0020>' + wp + '<STR_LIT:\n>' <EOL> out += '<STR_LIT>' <EOL> f = open ( filename , '<STR_LIT:w>' ) <EOL> f . writelines ( out ) <EOL> f . close ( ) <EOL> return get_perf ( filename , script_path ) <EOL> def download ( origin , destination ) : <EOL> '''<STR_LIT>''' <EOL> print ( '<STR_LIT>' % origin ) <EOL> urllib . urlretrieve ( origin , destination ) <EOL> def get_perf ( filename , folder ) : <EOL> '''<STR_LIT>''' <EOL> _conlleval = os . path . join ( folder , '<STR_LIT>' ) <EOL> if not os . path . isfile ( _conlleval ) : <EOL> url = '<STR_LIT>' <EOL> download ( url , _conlleval ) <EOL> os . chmod ( _conlleval , stat . S_IRWXU ) <EOL> proc = subprocess . Popen ( [ "<STR_LIT>" , <EOL> _conlleval ] , <EOL> stdin = subprocess . PIPE , <EOL> stdout = subprocess . PIPE ) <EOL> stdout , _ = proc . communicate ( '<STR_LIT>' . join ( open ( filename ) . readlines ( ) ) . encode ( '<STR_LIT:utf-8>' ) ) <EOL> stdout = stdout . decode ( '<STR_LIT:utf-8>' ) <EOL> out = None <EOL> for line in stdout . split ( '<STR_LIT:\n>' ) : <EOL> if '<STR_LIT>' in line : <EOL> out = line . split ( ) <EOL> break <EOL> if out is None : <EOL> print ( stdout . split ( '<STR_LIT:\n>' ) ) <EOL> precision = float ( out [ <NUM_LIT:6> ] [ : - <NUM_LIT:2> ] ) <EOL> recall = float ( out [ <NUM_LIT:8> ] [ : - <NUM_LIT:2> ] ) <EOL> f1score = float ( out [ <NUM_LIT:10> ] ) <EOL> return { '<STR_LIT:p>' : precision , '<STR_LIT:r>' : recall , '<STR_LIT>' : f1score } <EOL> class RNNSLU ( object ) : <EOL> '''<STR_LIT>''' <EOL> def __init__ ( self , nh , nc , ne , de , cs ) : <EOL> '''<STR_LIT>''' <EOL> self . emb = theano . shared ( name = '<STR_LIT>' , <EOL> value = <NUM_LIT> * numpy . random . uniform ( - <NUM_LIT:1.0> , <NUM_LIT:1.0> , <EOL> ( ne + <NUM_LIT:1> , de ) ) <EOL> . astype ( theano . config . floatX ) ) <EOL> self . wx = theano . shared ( name = '<STR_LIT>' , <EOL> value = <NUM_LIT> * numpy . random . uniform ( - <NUM_LIT:1.0> , <NUM_LIT:1.0> , <EOL> ( de * cs , nh ) ) <EOL> . astype ( theano . config . floatX ) ) <EOL> self . wh = theano . shared ( name = '<STR_LIT>' , <EOL> value = <NUM_LIT> * numpy . random . uniform ( - <NUM_LIT:1.0> , <NUM_LIT:1.0> , <EOL> ( nh , nh ) ) <EOL> . astype ( theano . config . floatX ) ) <EOL> self . w = theano . shared ( name = '<STR_LIT:w>' , <EOL> value = <NUM_LIT> * numpy . random . uniform ( - <NUM_LIT:1.0> , <NUM_LIT:1.0> , <EOL> ( nh , nc ) ) <EOL> . astype ( theano . config . floatX ) ) <EOL> self . bh = theano . shared ( name = '<STR_LIT>' , <EOL> value = numpy . zeros ( nh , <EOL> dtype = theano . config . floatX ) ) <EOL> self . b = theano . shared ( name = '<STR_LIT:b>' , <EOL> value = numpy . zeros ( nc , <EOL> dtype = theano . config . floatX ) ) <EOL> self . h0 = theano . shared ( name = '<STR_LIT>' , <EOL> value = numpy . zeros ( nh , <EOL> dtype = theano . config . floatX ) ) <EOL> self . params = [ self . emb , self . wx , self . wh , self . w , <EOL> self . bh , self . b , self . h0 ] <EOL> idxs = T . imatrix ( ) <EOL> x = self . emb [ idxs ] . reshape ( ( idxs . shape [ <NUM_LIT:0> ] , de * cs ) ) <EOL> y_sentence = T . ivector ( '<STR_LIT>' ) <EOL> def recurrence ( x_t , h_tm1 ) : <EOL> h_t = T . nnet . sigmoid ( T . dot ( x_t , self . wx ) <EOL> + T . dot ( h_tm1 , self . wh ) + self . bh ) <EOL> s_t = T . nnet . softmax ( T . dot ( h_t , self . w ) + self . b ) <EOL> return [ h_t , s_t ] <EOL> [ h , s ] , _ = theano . scan ( fn = recurrence , <EOL> sequences = x , <EOL> outputs_info = [ self . h0 , None ] , <EOL> n_steps = x . shape [ <NUM_LIT:0> ] ) <EOL> p_y_given_x_sentence = s [ : , <NUM_LIT:0> , : ] <EOL> y_pred = T . argmax ( p_y_given_x_sentence , axis = <NUM_LIT:1> ) <EOL> lr = T . scalar ( '<STR_LIT>' ) <EOL> sentence_nll = - T . mean ( T . log ( p_y_given_x_sentence ) <EOL> [ T . arange ( x . shape [ <NUM_LIT:0> ] ) , y_sentence ] ) <EOL> sentence_gradients = T . grad ( sentence_nll , self . params ) <EOL> sentence_updates = OrderedDict ( ( p , p - lr * g ) <EOL> for p , g in <EOL> zip ( self . params , sentence_gradients ) ) <EOL> self . classify = theano . function ( inputs = [ idxs ] , outputs = y_pred ) <EOL> self . sentence_train = theano . function ( inputs = [ idxs , y_sentence , lr ] , <EOL> outputs = sentence_nll , <EOL> updates = sentence_updates ) <EOL> self . normalize = theano . function ( inputs = [ ] , <EOL> updates = { self . emb : <EOL> self . emb / <EOL> T . sqrt ( ( self . emb ** <NUM_LIT:2> ) <EOL> . sum ( axis = <NUM_LIT:1> ) ) <EOL> . dimshuffle ( <NUM_LIT:0> , '<STR_LIT:x>' ) } ) <EOL> def train ( self , x , y , window_size , learning_rate ) : <EOL> cwords = contextwin ( x , window_size ) <EOL> words = list ( map ( lambda x : numpy . asarray ( x ) . astype ( '<STR_LIT>' ) , cwords ) ) <EOL> labels = y <EOL> self . sentence_train ( words , labels , learning_rate ) <EOL> self . normalize ( ) <EOL> def save ( self , folder ) : <EOL> for param in self . params : <EOL> numpy . save ( os . path . join ( folder , <EOL> param . name + '<STR_LIT>' ) , param . get_value ( ) ) <EOL> def load ( self , folder ) : <EOL> for param in self . params : <EOL> param . set_value ( numpy . load ( os . path . join ( folder , <EOL> param . name + '<STR_LIT>' ) ) ) <EOL> def main ( param = None ) : <EOL> if not param : <EOL> param = { <EOL> '<STR_LIT>' : <NUM_LIT:3> , <EOL> '<STR_LIT:data>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : <NUM_LIT> , <EOL> '<STR_LIT>' : <NUM_LIT:1> , <EOL> '<STR_LIT>' : True , <EOL> '<STR_LIT>' : <NUM_LIT:7> , <EOL> '<STR_LIT>' : <NUM_LIT:200> , <EOL> '<STR_LIT>' : <NUM_LIT> , <EOL> '<STR_LIT>' : <NUM_LIT:50> , <EOL> '<STR_LIT>' : <NUM_LIT> , <EOL> '<STR_LIT>' : False } <EOL> print ( param ) <EOL> folder_name = os . path . basename ( __file__ ) . split ( '<STR_LIT:.>' ) [ <NUM_LIT:0> ] <EOL> folder = os . path . join ( os . path . dirname ( __file__ ) , folder_name ) <EOL> if not os . path . exists ( folder ) : <EOL> os . mkdir ( folder ) <EOL> train_set , valid_set , test_set , dic = atisfold ( param [ '<STR_LIT>' ] ) <EOL> idx2label = dict ( ( k , v ) for v , k in dic [ '<STR_LIT>' ] . items ( ) ) <EOL> idx2word = dict ( ( k , v ) for v , k in dic [ '<STR_LIT>' ] . items ( ) ) <EOL> train_lex , train_ne , train_y = train_set <EOL> valid_lex , valid_ne , valid_y = valid_set <EOL> test_lex , test_ne , test_y = test_set <EOL> vocsize = len ( dic [ '<STR_LIT>' ] ) <EOL> nclasses = len ( dic [ '<STR_LIT>' ] ) <EOL> nsentences = len ( train_lex ) <EOL> groundtruth_valid = [ map ( lambda x : idx2label [ x ] , y ) for y in valid_y ] <EOL> words_valid = [ map ( lambda x : idx2word [ x ] , w ) for w in valid_lex ] <EOL> groundtruth_test = [ map ( lambda x : idx2label [ x ] , y ) for y in test_y ] <EOL> words_test = [ map ( lambda x : idx2word [ x ] , w ) for w in test_lex ] <EOL> numpy . random . seed ( param [ '<STR_LIT>' ] ) <EOL> random . seed ( param [ '<STR_LIT>' ] ) <EOL> rnn = RNNSLU ( nh = param [ '<STR_LIT>' ] , <EOL> nc = nclasses , <EOL> ne = vocsize , <EOL> de = param [ '<STR_LIT>' ] , <EOL> cs = param [ '<STR_LIT>' ] ) <EOL> best_f1 = - numpy . inf <EOL> param [ '<STR_LIT>' ] = param [ '<STR_LIT>' ] <EOL> for e in range ( param [ '<STR_LIT>' ] ) : <EOL> shuffle ( [ train_lex , train_ne , train_y ] , param [ '<STR_LIT>' ] ) <EOL> param [ '<STR_LIT>' ] = e <EOL> tic = timeit . default_timer ( ) <EOL> for i , ( x , y ) in enumerate ( zip ( train_lex , train_y ) ) : <EOL> rnn . train ( x , y , param [ '<STR_LIT>' ] , param [ '<STR_LIT>' ] ) <EOL> print ( '<STR_LIT>' % ( <EOL> e , ( i + <NUM_LIT:1> ) * <NUM_LIT> / nsentences ) , end = '<STR_LIT:U+0020>' ) <EOL> print ( '<STR_LIT>' % ( timeit . default_timer ( ) - tic ) , end = '<STR_LIT>' ) <EOL> sys . stdout . flush ( ) <EOL> predictions_test = [ map ( lambda x : idx2label [ x ] , <EOL> rnn . classify ( numpy . asarray ( <EOL> contextwin ( x , param [ '<STR_LIT>' ] ) ) . astype ( '<STR_LIT>' ) ) ) <EOL> for x in test_lex ] <EOL> predictions_valid = [ map ( lambda x : idx2label [ x ] , <EOL> rnn . classify ( numpy . asarray ( <EOL> contextwin ( x , param [ '<STR_LIT>' ] ) ) . astype ( '<STR_LIT>' ) ) ) <EOL> for x in valid_lex ] <EOL> res_test = conlleval ( predictions_test , <EOL> groundtruth_test , <EOL> words_test , <EOL> folder + '<STR_LIT>' , <EOL> folder ) <EOL> res_valid = conlleval ( predictions_valid , <EOL> groundtruth_valid , <EOL> words_valid , <EOL> folder + '<STR_LIT>' , <EOL> folder ) <EOL> if res_valid [ '<STR_LIT>' ] > best_f1 : <EOL> if param [ '<STR_LIT>' ] : <EOL> rnn . save ( folder ) <EOL> best_rnn = copy . deepcopy ( rnn ) <EOL> best_f1 = res_valid [ '<STR_LIT>' ] <EOL> if param [ '<STR_LIT>' ] : <EOL> print ( '<STR_LIT>' , e , <EOL> '<STR_LIT>' , res_valid [ '<STR_LIT>' ] , <EOL> '<STR_LIT>' , res_test [ '<STR_LIT>' ] ) <EOL> param [ '<STR_LIT>' ] , param [ '<STR_LIT>' ] = res_valid [ '<STR_LIT>' ] , res_test [ '<STR_LIT>' ] <EOL> param [ '<STR_LIT>' ] , param [ '<STR_LIT>' ] = res_valid [ '<STR_LIT:p>' ] , res_test [ '<STR_LIT:p>' ] <EOL> param [ '<STR_LIT>' ] , param [ '<STR_LIT>' ] = res_valid [ '<STR_LIT:r>' ] , res_test [ '<STR_LIT:r>' ] <EOL> param [ '<STR_LIT>' ] = e <EOL> subprocess . call ( [ '<STR_LIT>' , folder + '<STR_LIT>' , <EOL> folder + '<STR_LIT>' ] ) <EOL> subprocess . call ( [ '<STR_LIT>' , folder + '<STR_LIT>' , <EOL> folder + '<STR_LIT>' ] ) <EOL> else : <EOL> if param [ '<STR_LIT>' ] : <EOL> print ( '<STR_LIT>' ) <EOL> if param [ '<STR_LIT>' ] and abs ( param [ '<STR_LIT>' ] - param [ '<STR_LIT>' ] ) >= <NUM_LIT:10> : <EOL> param [ '<STR_LIT>' ] *= <NUM_LIT:0.5> <EOL> rnn = best_rnn <EOL> if param [ '<STR_LIT>' ] < <NUM_LIT> : <EOL> break <EOL> print ( '<STR_LIT>' , param [ '<STR_LIT>' ] , <EOL> '<STR_LIT>' , param [ '<STR_LIT>' ] , <EOL> '<STR_LIT>' , param [ '<STR_LIT>' ] , <EOL> '<STR_LIT>' , folder ) <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> main ( ) </s>
