def read_data(source_path, target_path, buckets, EOS_ID, max_size=None):
 data_set = [[] for _ in buckets]
 with tf.gfile.GFile(source_path, mode='r') as source_file:
  with tf.gfile.GFile(target_path, mode='r') as target_file:
   (source, target) = (source_file.readline(), target_file.readline())
   counter = 0
   while (source and target and ((not max_size) or (counter < max_size))):
    counter += 1
    if ((counter % 100000) == 0):
     print(('  reading data line %d' % counter))
     sys.stdout.flush()
    source_ids = [int(x) for x in source.split()]
    target_ids = [int(x) for x in target.split()]
    target_ids.append(EOS_ID)
    for (bucket_id, (source_size, target_size)) in enumerate(buckets):
     if ((len(source_ids) < source_size) and (len(target_ids) < target_size)):
      data_set[bucket_id].append([source_ids, target_ids])
      break
    (source, target) = (source_file.readline(), target_file.readline())
 return data_set

def read_data ( source_path , target_path , buckets , EOS_ID , max_size = None ) : data_set = [ [ ] for _ in buckets ] with tf . gfile . GFile ( source_path , mode = 'r' ) as source_file : with tf . gfile . GFile ( target_path , mode = 'r' ) as target_file : ( source , target ) = ( source_file . readline ( ) , target_file . readline ( ) ) counter = 0 while ( source and target and ( ( not max_size ) or ( counter < max_size ) ) ) : counter += 1 if ( ( counter % 100000 ) == 0 ) : print ( ( '  reading data line %d' % counter ) ) sys . stdout . flush ( ) source_ids = [ int ( x ) for x in source . split ( ) ] target_ids = [ int ( x ) for x in target . split ( ) ] target_ids . append ( EOS_ID ) for ( bucket_id , ( source_size , target_size ) ) in enumerate ( buckets ) : if ( ( len ( source_ids ) < source_size ) and ( len ( target_ids ) < target_size ) ) : data_set [ bucket_id ] . append ( [ source_ids , target_ids ] ) break ( source , target ) = ( source_file . readline ( ) , target_file . readline ( ) ) return data_set
read a given number of 32-bits unsigned integers from the given file with the given endianness .