<s> """<STR_LIT>""" <EOL> __docformat__ = '<STR_LIT>' <EOL> __authors__ = ( "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> __contact__ = "<STR_LIT>" <EOL> import numpy <EOL> import os , gc <EOL> import tables <EOL> import copy <EOL> import logging <EOL> import threading <EOL> import Queue <EOL> import collections <EOL> from tm_dataset import PytablesBitextFetcher , PytablesBitextIterator <EOL> def prepare_data ( seqs_x , seqs_y , maxlen = None , n_words_src = <NUM_LIT> , n_words = <NUM_LIT> ) : <EOL> lengths_x = [ len ( s ) for s in seqs_x ] <EOL> lengths_y = [ len ( s ) for s in seqs_y ] <EOL> if maxlen != None : <EOL> new_seqs_x = [ ] <EOL> new_seqs_y = [ ] <EOL> new_lengths_x = [ ] <EOL> new_lengths_y = [ ] <EOL> for l_x , s_x , l_y , s_y in zip ( lengths_x , seqs_x , lengths_y , seqs_y ) : <EOL> if l_x < maxlen and l_y < maxlen : <EOL> new_seqs_x . append ( s_x ) <EOL> new_lengths_x . append ( l_x ) <EOL> new_seqs_y . append ( s_y ) <EOL> new_lengths_y . append ( l_y ) <EOL> lengths_x = new_lengths_x <EOL> seqs_x = new_seqs_x <EOL> lengths_y = new_lengths_y <EOL> seqs_y = new_seqs_y <EOL> if len ( lengths_x ) < <NUM_LIT:1> or len ( lengths_y ) < <NUM_LIT:1> : <EOL> return None , None , None , None <EOL> n_samples = len ( seqs_x ) <EOL> maxlen_x = numpy . max ( lengths_x ) + <NUM_LIT:1> <EOL> maxlen_y = numpy . max ( lengths_y ) + <NUM_LIT:1> <EOL> x = numpy . zeros ( ( maxlen_x , n_samples ) ) . astype ( '<STR_LIT>' ) <EOL> y = numpy . zeros ( ( maxlen_y , n_samples ) ) . astype ( '<STR_LIT>' ) <EOL> x_mask = numpy . zeros ( ( maxlen_x , n_samples ) ) . astype ( '<STR_LIT>' ) <EOL> y_mask = numpy . zeros ( ( maxlen_y , n_samples ) ) . astype ( '<STR_LIT>' ) <EOL> for idx , [ s_x , s_y ] in enumerate ( zip ( seqs_x , seqs_y ) ) : <EOL> s_x [ numpy . where ( s_x >= n_words_src - <NUM_LIT:1> ) ] = <NUM_LIT:1> <EOL> s_y [ numpy . where ( s_y >= n_words - <NUM_LIT:1> ) ] = <NUM_LIT:1> <EOL> x [ : lengths_x [ idx ] , idx ] = s_x <EOL> x_mask [ : lengths_x [ idx ] + <NUM_LIT:1> , idx ] = <NUM_LIT:1.> <EOL> y [ : lengths_y [ idx ] , idx ] = s_y <EOL> y_mask [ : lengths_y [ idx ] + <NUM_LIT:1> , idx ] = <NUM_LIT:1.> <EOL> return x , x_mask , y , y_mask <EOL> def load_data ( batch_size = <NUM_LIT> ) : <EOL> '''<STR_LIT>''' <EOL> path_src = '<STR_LIT>' <EOL> path_trg = '<STR_LIT>' <EOL> print '<STR_LIT>' <EOL> train = PytablesBitextIterator ( batch_size , path_trg , path_src , use_infinite_loop = False ) <EOL> valid = None <EOL> test = None <EOL> return train , valid , test </s>
