<s> import numpy <EOL> import copy <EOL> class HomogeneousData ( ) : <EOL> def __init__ ( self , data , batch_size = <NUM_LIT> , maxlen = None ) : <EOL> self . batch_size = <NUM_LIT> <EOL> self . data = data <EOL> self . batch_size = batch_size <EOL> self . maxlen = maxlen <EOL> self . prepare ( ) <EOL> self . reset ( ) <EOL> def prepare ( self ) : <EOL> self . caps = self . data [ <NUM_LIT:0> ] <EOL> self . feats = self . data [ <NUM_LIT:1> ] <EOL> self . feats2 = self . data [ <NUM_LIT:2> ] <EOL> self . lengths = [ len ( cc . split ( ) ) for cc in self . caps ] <EOL> self . len_unique = numpy . unique ( self . lengths ) <EOL> if self . maxlen : <EOL> self . len_unique = [ ll for ll in self . len_unique if ll <= self . maxlen ] <EOL> self . len_indices = dict ( ) <EOL> self . len_counts = dict ( ) <EOL> for ll in self . len_unique : <EOL> self . len_indices [ ll ] = numpy . where ( self . lengths == ll ) [ <NUM_LIT:0> ] <EOL> self . len_counts [ ll ] = len ( self . len_indices [ ll ] ) <EOL> self . len_curr_counts = copy . copy ( self . len_counts ) <EOL> def reset ( self ) : <EOL> self . len_curr_counts = copy . copy ( self . len_counts ) <EOL> self . len_unique = numpy . random . permutation ( self . len_unique ) <EOL> self . len_indices_pos = dict ( ) <EOL> for ll in self . len_unique : <EOL> self . len_indices_pos [ ll ] = <NUM_LIT:0> <EOL> self . len_indices [ ll ] = numpy . random . permutation ( self . len_indices [ ll ] ) <EOL> self . len_idx = - <NUM_LIT:1> <EOL> def next ( self ) : <EOL> count = <NUM_LIT:0> <EOL> while True : <EOL> self . len_idx = numpy . mod ( self . len_idx + <NUM_LIT:1> , len ( self . len_unique ) ) <EOL> if self . len_curr_counts [ self . len_unique [ self . len_idx ] ] > <NUM_LIT:0> : <EOL> break <EOL> count += <NUM_LIT:1> <EOL> if count >= len ( self . len_unique ) : <EOL> break <EOL> if count >= len ( self . len_unique ) : <EOL> self . reset ( ) <EOL> raise StopIteration ( ) <EOL> curr_batch_size = numpy . minimum ( self . batch_size , self . len_curr_counts [ self . len_unique [ self . len_idx ] ] ) <EOL> curr_pos = self . len_indices_pos [ self . len_unique [ self . len_idx ] ] <EOL> curr_indices = self . len_indices [ self . len_unique [ self . len_idx ] ] [ curr_pos : curr_pos + curr_batch_size ] <EOL> self . len_indices_pos [ self . len_unique [ self . len_idx ] ] += curr_batch_size <EOL> self . len_curr_counts [ self . len_unique [ self . len_idx ] ] -= curr_batch_size <EOL> caps = [ self . caps [ ii ] for ii in curr_indices ] <EOL> feats = [ self . feats [ ii ] for ii in curr_indices ] <EOL> feats2 = [ self . feats2 [ ii ] for ii in curr_indices ] <EOL> return caps , feats , feats2 <EOL> def __iter__ ( self ) : <EOL> return self <EOL> def prepare_data ( seqs_x , seqs_y , seqs_z , worddict , maxlen = None , n_words = <NUM_LIT> ) : <EOL> """<STR_LIT>""" <EOL> seqsX = [ ] <EOL> seqsY = [ ] <EOL> seqsZ = [ ] <EOL> for cc in seqs_x : <EOL> seqsX . append ( [ worddict [ w ] if worddict [ w ] < n_words else <NUM_LIT:1> for w in cc . split ( ) ] ) <EOL> for cc in seqs_y : <EOL> seqsY . append ( [ worddict [ w ] if worddict [ w ] < n_words else <NUM_LIT:1> for w in cc . split ( ) ] ) <EOL> for cc in seqs_z : <EOL> seqsZ . append ( [ worddict [ w ] if worddict [ w ] < n_words else <NUM_LIT:1> for w in cc . split ( ) ] ) <EOL> seqs_x = seqsX <EOL> seqs_y = seqsY <EOL> seqs_z = seqsZ <EOL> lengths_x = [ len ( s ) for s in seqs_x ] <EOL> lengths_y = [ len ( s ) for s in seqs_y ] <EOL> lengths_z = [ len ( s ) for s in seqs_z ] <EOL> if maxlen != None : <EOL> new_seqs_x = [ ] <EOL> new_seqs_y = [ ] <EOL> new_seqs_z = [ ] <EOL> new_lengths_x = [ ] <EOL> new_lengths_y = [ ] <EOL> new_lengths_z = [ ] <EOL> for l_x , s_x , l_y , s_y , l_z , s_z in zip ( lengths_x , seqs_x , lengths_y , seqs_y , lengths_z , seqs_z ) : <EOL> if l_x < maxlen and l_y < maxlen and l_z < maxlen : <EOL> new_seqs_x . append ( s_x ) <EOL> new_lengths_x . append ( l_x ) <EOL> new_seqs_y . append ( s_y ) <EOL> new_lengths_y . append ( l_y ) <EOL> new_seqs_z . append ( s_z ) <EOL> new_lengths_z . append ( l_z ) <EOL> lengths_x = new_lengths_x <EOL> seqs_x = new_seqs_x <EOL> lengths_y = new_lengths_y <EOL> seqs_y = new_seqs_y <EOL> lengths_z = new_lengths_z <EOL> seqs_z = new_seqs_z <EOL> if len ( lengths_x ) < <NUM_LIT:1> or len ( lengths_y ) < <NUM_LIT:1> or len ( lengths_z ) < <NUM_LIT:1> : <EOL> return None , None , None , None , None , None <EOL> n_samples = len ( seqs_x ) <EOL> maxlen_x = numpy . max ( lengths_x ) + <NUM_LIT:1> <EOL> maxlen_y = numpy . max ( lengths_y ) + <NUM_LIT:1> <EOL> maxlen_z = numpy . max ( lengths_z ) + <NUM_LIT:1> <EOL> x = numpy . zeros ( ( maxlen_x , n_samples ) ) . astype ( '<STR_LIT>' ) <EOL> y = numpy . zeros ( ( maxlen_y , n_samples ) ) . astype ( '<STR_LIT>' ) <EOL> z = numpy . zeros ( ( maxlen_z , n_samples ) ) . astype ( '<STR_LIT>' ) <EOL> x_mask = numpy . zeros ( ( maxlen_x , n_samples ) ) . astype ( '<STR_LIT>' ) <EOL> y_mask = numpy . zeros ( ( maxlen_y , n_samples ) ) . astype ( '<STR_LIT>' ) <EOL> z_mask = numpy . zeros ( ( maxlen_z , n_samples ) ) . astype ( '<STR_LIT>' ) <EOL> for idx , [ s_x , s_y , s_z ] in enumerate ( zip ( seqs_x , seqs_y , seqs_z ) ) : <EOL> x [ : lengths_x [ idx ] , idx ] = s_x <EOL> x_mask [ : lengths_x [ idx ] + <NUM_LIT:1> , idx ] = <NUM_LIT:1.> <EOL> y [ : lengths_y [ idx ] , idx ] = s_y <EOL> y_mask [ : lengths_y [ idx ] + <NUM_LIT:1> , idx ] = <NUM_LIT:1.> <EOL> z [ : lengths_z [ idx ] , idx ] = s_z <EOL> z_mask [ : lengths_z [ idx ] + <NUM_LIT:1> , idx ] = <NUM_LIT:1.> <EOL> return x , x_mask , y , y_mask , z , z_mask <EOL> def grouper ( text ) : <EOL> """<STR_LIT>""" <EOL> source = text [ <NUM_LIT:1> : ] [ : - <NUM_LIT:1> ] <EOL> forward = text [ <NUM_LIT:2> : ] <EOL> backward = text [ : - <NUM_LIT:2> ] <EOL> X = ( source , forward , backward ) <EOL> return X </s>
