<gh_stars>1-10
import numpy as np
import tensorflow as tf
import helpers

tf.reset_default_graph() 
sess = tf.InteractiveSession() 

#Vocabulary size.
PAD = 0
EOS = 1

vocab_size = 10
input_embedding_size = 20 #character length

encoder_hidden_units = 20 #num neurons
decoder_hidden_units = encoder_hidden_units * 2
#in original paper, they used same number of neurons for both encoder and decoder, but we use twice
#as many so decoded output is different, the target value is the original input in this example

#input placeholders
encoder_inputs = tf.placeholder(shape=(None, None), dtype=tf.int32, name='encoder_inputs')
encoder_inputs_length = tf.placeholder(shape=(None,), dtype=tf.int32, name='encoder_inputs_length')
decoder_targets = tf.placeholder(shape=(None, None), dtype=tf.int32, name='decoder_targets')

#randomly initialized embedding matrix that can fit input sequence
#used to convert sequences to vectors (embeddings) for both encoder and decoder of the right size
#reshaping is a thing, in TF you gotta make sure you tensors are the right shape (num dimensions)
embeddings = tf.Variable(tf.random_uniform([vocab_size, input_embedding_size], -1.0, 1.0), dtype=tf.float32)
encoder_inputs_embedded = tf.nn.embedding_lookup(embeddings, encoder_inputs)

#####API LOCATION ERROR HERE
from tensorflow.python.ops.rnn_cell import LSTMCell, LSTMStateTuple
#from tensorflow.contrib.rnn.python.ops.core_rnn_cell_impl import LSTMCell, LSTMStateTuple

encoder_cell_fw = LSTMCell(encoder_hidden_units)
encoder_cell_bw = LSTMCell(encoder_hidden_units)

((encoder_fw_outputs,encoder_bw_outputs),(encoder_fw_final_state,encoder_bw_final_state)) = (
    tf.nn.bidirectional_dynamic_rnn(cell_fw=encoder_cell_fw,
                                    cell_bw=encoder_cell_bw,
                                    inputs=encoder_inputs_embedded,
                                    sequence_length=encoder_inputs_length,
                                    dtype=tf.float32, time_major=True)
    )

#Concatenates tensors along one dimension.
##############print("? = ",encoder_bw_outputs)
encoder_outputs = tf.concat((encoder_fw_outputs, encoder_bw_outputs), 2)

#letters h and c are commonly used to denote "output value" and "cell state". 
#http://colah.github.io/posts/2015-08-Understanding-LSTMs/ 
#Those tensors represent combined internal state of the cell, and should be passed together. 

encoder_final_state_c = tf.concat(
    (encoder_fw_final_state.c, encoder_bw_final_state.c), 1)

encoder_final_state_h = tf.concat(
    (encoder_fw_final_state.h, encoder_bw_final_state.h), 1)

#TF Tuple used by LSTM Cells for state_size, zero_state, and output state.
encoder_final_state = LSTMStateTuple(
    c=encoder_final_state_c,
    h=encoder_final_state_h
)

decoder_cell = LSTMCell(decoder_hidden_units)
encoder_max_time, batch_size = tf.unstack(tf.shape(encoder_inputs))
decoder_lengths = encoder_inputs_length + 3
# +2 additional steps, +1 leading <EOS> token for decoder inputs

#manually specifying since we are going to implement attention details for the decoder in a sec
#weights
W = tf.Variable(tf.random_uniform([decoder_hidden_units, vocab_size], -1, 1), dtype=tf.float32)
#bias
b = tf.Variable(tf.zeros([vocab_size]), dtype=tf.float32)

#create padded inputs for the decoder from the word embeddings

#were telling the program to tests a condition, and trigger an error if the condition is false.
assert EOS == 1 and PAD == 0

eos_time_slice = tf.ones([batch_size], dtype=tf.int32, name='EOS')
pad_time_slice = tf.zeros([batch_size], dtype=tf.int32, name='PAD')

#retrieves rows of the params tensor. The behavior is similar to using indexing with arrays in numpy
eos_step_embedded = tf.nn.embedding_lookup(embeddings, eos_time_slice)
pad_step_embedded = tf.nn.embedding_lookup(embeddings, pad_time_slice)

#manually specifying loop function through time - to get initial cell state and input to RNN
#normally we'd just use dynamic_rnn, but lets get detailed here with raw_rnn

#we define and return these values, no operations occur here
def loop_fn_initial():
    initial_elements_finished = (0 >= decoder_lengths)  # all False at the initial step
    #end of sentence
    initial_input = eos_step_embedded
    #last time steps cell state
    initial_cell_state = encoder_final_state
    #none
    initial_cell_output = None
    #none
    initial_loop_state = None  # we don't need to pass any additional information
    return (initial_elements_finished,
            initial_input,
            initial_cell_state,
            initial_cell_output,
            initial_loop_state)

#attention mechanism --choose which previously generated token to pass as input in the next timestep
def loop_fn_transition(time, previous_output, previous_state, previous_loop_state):
    
    def get_next_input():
        #dot product between previous ouput and weights, then + biases
        output_logits = tf.add(tf.matmul(previous_output, W), b)
        #Logits simply means that the function operates on the unscaled output of 
        #earlier layers and that the relative scale to understand the units is linear. 
        #It means, in particular, the sum of the inputs may not equal 1, that the values are not probabilities 
        #(you might have an input of 5).
        #prediction value at current time step
        
        #Returns the index with the largest value across axes of a tensor.
        #This is Attention!!!!
        prediction = tf.argmax(output_logits, axis=1)
        #embed prediction for the next input
        next_input = tf.nn.embedding_lookup(embeddings, prediction)
        return next_input
    elements_finished = (time >= decoder_lengths) # this operation produces boolean tensor of [batch_size]
                                                  # defining if corresponding sequence has ended

    
    
    #Computes the "logical and" of elements across dimensions of a tensor.
    finished = tf.reduce_all(elements_finished) # -> boolean scalar
    #Return either fn1() or fn2() based on the boolean predicate pred.
    input = tf.cond(finished, lambda: pad_step_embedded, get_next_input)
    
    #set previous to current
    state = previous_state
    output = previous_output
    loop_state = None

    return (elements_finished, 
            input,
            state,
            output,
            loop_state)

def loop_fn(time, previous_output, previous_state, previous_loop_state):
    if previous_state is None:    # time == 0
        assert previous_output is None and previous_state is None
        return loop_fn_initial()
    else:
        return loop_fn_transition(time, previous_output, previous_state, previous_loop_state)

#Creates an RNN specified by RNNCell cell and loop function loop_fn.
#This function is a more primitive version of dynamic_rnn that provides more direct access to the 
#inputs each iteration. It also provides more control over when to start and finish reading the sequence, 
#and what to emit for the output.
#ta = tensor array
decoder_outputs_ta, decoder_final_state, _ = tf.nn.raw_rnn(decoder_cell, loop_fn)
decoder_outputs = decoder_outputs_ta.stack()

decoder_outputs

#to convert output to human readable prediction
#we will reshape output tensor

#Unpacks the given dimension of a rank-R tensor into rank-(R-1) tensors.
#reduces dimensionality
decoder_max_steps, decoder_batch_size, decoder_dim = tf.unstack(tf.shape(decoder_outputs))
#flettened output tensor
decoder_outputs_flat = tf.reshape(decoder_outputs, (-1, decoder_dim))
#pass flattened tensor through decoder
decoder_logits_flat = tf.add(tf.matmul(decoder_outputs_flat, W), b)
#prediction vals
decoder_logits = tf.reshape(decoder_logits_flat, (decoder_max_steps, decoder_batch_size, vocab_size))

#final prediction
decoder_prediction = tf.argmax(decoder_logits, 2)

#cross entropy loss
#one hot encode the target values so we don't rank just differentiate
stepwise_cross_entropy = tf.nn.softmax_cross_entropy_with_logits(
    labels=tf.one_hot(decoder_targets, depth=vocab_size, dtype=tf.float32),
    logits=decoder_logits,
)

#loss function
loss = tf.reduce_mean(stepwise_cross_entropy)
#train it 
train_op = tf.train.AdamOptimizer().minimize(loss)

sess.run(tf.global_variables_initializer())

batch_size = 2

# batches = helpers.random_sequences(length_from=3, length_to=8, vocab_lower=2, vocab_upper=10, batch_size=batch_size)
train_features_batches = helpers.loadDataFile("../MNIST_data/train_features.txt")
train_labels_batches = helpers.loadDataFile("../MNIST_data/train_labels.txt")


print("train_features_batches:", train_features_batches)

print('head of the train_features_batches:')
for seq in next(train_features_batches.__iter__())[:10]:
    print(seq)

print("train_labels_batches:", train_features_batches)

print('head of the train_labels_batches:')
for seq in next(train_labels_batches.__iter__())[:10]:
    print(seq)

#############

def next_feed(train_features_batches, train_labels_batches):
    train_features_batche = train_features_batches
    train_labels_batche = train_labels_batches
    encoder_inputs_, encoder_input_lengths_ = helpers.batch(train_features_batche)
    decoder_targets_, _ = helpers.batch(
        [(sequence) + [EOS] + [PAD] * 2 for sequence in train_labels_batche]
    )
    return {
        encoder_inputs: encoder_inputs_,
        encoder_inputs_length: encoder_input_lengths_,
        decoder_targets: decoder_targets_,
    }

loss_track = []

max_batches = 30
batches_in_epoch = 10

import time
try:
    start = time.time()
    config = tf.ConfigProto()
    config.gpu_options.allocator_type = 'BFC'
    config.gpu_options.per_process_gpu_memory_fraction = 0.90
    linesofar = 0
    for batch in range(max_batches):
        train_features_batches = helpers.next_batch_k("../MNIST_data/features.txt", batch_size, linesofar)
        train_labels_batches = helpers.next_batch_k("../MNIST_data/labels.txt", batch_size, linesofar)
        fd = next_feed(train_features_batches, train_labels_batches)
        _, l = sess.run([train_op, loss], fd)
        loss_track.append(l)
        linesofar += batch_size

        if batch == 0 or batch % batches_in_epoch == 0:
            print('batch {}'.format(batch))
            if(batch != 0):
                print("Time used: ", time.time() - start)
            print('  minibatch loss: {}'.format(sess.run(loss, fd)))
            predict_ = sess.run(decoder_prediction, fd)
            for i, (inp, pred) in enumerate(zip(fd[encoder_inputs].T, predict_.T)):
                print('  sample {}:'.format(i + 1))
                print('    input     > {}'.format(inp))
                print('    predicted > {}'.format(pred))
                if i >= 2:
                    break
            print()
            start = time.time()
    print("Optimization Finished!")

    # start prediction
    start = time.time()
    test_features_batches = helpers.loadDataFile("../MNIST_data/test_features.txt")
    test_labels_batches = helpers.loadDataFile("../MNIST_data/test_labels.txt")
    test_fd = next_feed(test_features_batches, test_labels_batches)
    print("Time used: ", time.time() - start)
    print('  tests batch loss: {}'.format(sess.run(loss, test_fd)))
    test_predict_ = sess.run(decoder_prediction, test_fd)

    for i, (inp, pred) in enumerate(zip(test_fd[encoder_inputs].T, test_predict_.T)):
        print('  sample {}:'.format(i + 1))
        print('    input     > {}'.format(inp))
        print('    predicted > {}'.format(pred))
        if i >= 2:
            break
    print()
    print("Prediction Finished!")
            
except KeyboardInterrupt:
    print('training interrupted')

import numpy as np
import tensorflow as tf
import helpers

tf.reset_default_graph() 
sess = tf.InteractiveSession() 

#Vocabulary size.
PAD = 0
EOS = 1

vocab_size = 10
input_embedding_size = 20 #character length

encoder_hidden_units = 20 #num neurons
decoder_hidden_units = encoder_hidden_units * 2
#in original paper, they used same number of neurons for both encoder and decoder, but we use twice
#as many so decoded output is different, the target value is the original input in this example

#input placeholders
encoder_inputs = tf.placeholder(shape=(None, None), dtype=tf.int32, name='encoder_inputs')
encoder_inputs_length = tf.placeholder(shape=(None,), dtype=tf.int32, name='encoder_inputs_length')
decoder_targets = tf.placeholder(shape=(None, None), dtype=tf.int32, name='decoder_targets')

#randomly initialized embedding matrix that can fit input sequence
#used to convert sequences to vectors (embeddings) for both encoder and decoder of the right size
#reshaping is a thing, in TF you gotta make sure you tensors are the right shape (num dimensions)
embeddings = tf.Variable(tf.random_uniform([vocab_size, input_embedding_size], -1.0, 1.0), dtype=tf.float32)
encoder_inputs_embedded = tf.nn.embedding_lookup(embeddings, encoder_inputs)

#####API LOCATION ERROR HERE
from tensorflow.python.ops.rnn_cell import LSTMCell, LSTMStateTuple
#from tensorflow.contrib.rnn.python.ops.core_rnn_cell_impl import LSTMCell, LSTMStateTuple

encoder_cell_fw = LSTMCell(encoder_hidden_units)
encoder_cell_bw = LSTMCell(encoder_hidden_units)

((encoder_fw_outputs,encoder_bw_outputs),(encoder_fw_final_state,encoder_bw_final_state)) = (
    tf.nn.bidirectional_dynamic_rnn(cell_fw=encoder_cell_fw,
                                    cell_bw=encoder_cell_bw,
                                    inputs=encoder_inputs_embedded,
                                    sequence_length=encoder_inputs_length,
                                    dtype=tf.float32, time_major=True)
    )

#Concatenates tensors along one dimension.
##############print("? = ",encoder_bw_outputs)
encoder_outputs = tf.concat((encoder_fw_outputs, encoder_bw_outputs), 2)

#letters h and c are commonly used to denote "output value" and "cell state". 
#http://colah.github.io/posts/2015-08-Understanding-LSTMs/ 
#Those tensors represent combined internal state of the cell, and should be passed together. 

encoder_final_state_c = tf.concat(
    (encoder_fw_final_state.c, encoder_bw_final_state.c), 1)

encoder_final_state_h = tf.concat(
    (encoder_fw_final_state.h, encoder_bw_final_state.h), 1)

#TF Tuple used by LSTM Cells for state_size, zero_state, and output state.
encoder_final_state = LSTMStateTuple(
    c=encoder_final_state_c,
    h=encoder_final_state_h
)

decoder_cell = LSTMCell(decoder_hidden_units)
encoder_max_time, batch_size = tf.unstack(tf.shape(encoder_inputs))
decoder_lengths = encoder_inputs_length + 3
# +2 additional steps, +1 leading <EOS> token for decoder inputs

#manually specifying since we are going to implement attention details for the decoder in a sec
#weights
W = tf.Variable(tf.random_uniform([decoder_hidden_units, vocab_size], -1, 1), dtype=tf.float32)
#bias
b = tf.Variable(tf.zeros([vocab_size]), dtype=tf.float32)

#create padded inputs for the decoder from the word embeddings

#were telling the program to tests a condition, and trigger an error if the condition is false.
assert EOS == 1 and PAD == 0

eos_time_slice = tf.ones([batch_size], dtype=tf.int32, name='EOS')
pad_time_slice = tf.zeros([batch_size], dtype=tf.int32, name='PAD')

#retrieves rows of the params tensor. The behavior is similar to using indexing with arrays in numpy
eos_step_embedded = tf.nn.embedding_lookup(embeddings, eos_time_slice)
pad_step_embedded = tf.nn.embedding_lookup(embeddings, pad_time_slice)

#manually specifying loop function through time - to get initial cell state and input to RNN
#normally we'd just use dynamic_rnn, but lets get detailed here with raw_rnn

#we define and return these values, no operations occur here
def loop_fn_initial():
    initial_elements_finished = (0 >= decoder_lengths)  # all False at the initial step
    #end of sentence
    initial_input = eos_step_embedded
    #last time steps cell state
    initial_cell_state = encoder_final_state
    #none
    initial_cell_output = None
    #none
    initial_loop_state = None  # we don't need to pass any additional information
    return (initial_elements_finished,
            initial_input,
            initial_cell_state,
            initial_cell_output,
            initial_loop_state)

#attention mechanism --choose which previously generated token to pass as input in the next timestep
def loop_fn_transition(time, previous_output, previous_state, previous_loop_state):
    
    def get_next_input():
        #dot product between previous ouput and weights, then + biases
        output_logits = tf.add(tf.matmul(previous_output, W), b)
        #Logits simply means that the function operates on the unscaled output of 
        #earlier layers and that the relative scale to understand the units is linear. 
        #It means, in particular, the sum of the inputs may not equal 1, that the values are not probabilities 
        #(you might have an input of 5).
        #prediction value at current time step
        
        #Returns the index with the largest value across axes of a tensor.
        #This is Attention!!!!
        prediction = tf.argmax(output_logits, axis=1)
        #embed prediction for the next input
        next_input = tf.nn.embedding_lookup(embeddings, prediction)
        return next_input
    elements_finished = (time >= decoder_lengths) # this operation produces boolean tensor of [batch_size]
                                                  # defining if corresponding sequence has ended

    
    
    #Computes the "logical and" of elements across dimensions of a tensor.
    finished = tf.reduce_all(elements_finished) # -> boolean scalar
    #Return either fn1() or fn2() based on the boolean predicate pred.
    input = tf.cond(finished, lambda: pad_step_embedded, get_next_input)
    
    #set previous to current
    state = previous_state
    output = previous_output
    loop_state = None

    return (elements_finished, 
            input,
            state,
            output,
            loop_state)

def loop_fn(time, previous_output, previous_state, previous_loop_state):
    if previous_state is None:    # time == 0
        assert previous_output is None and previous_state is None
        return loop_fn_initial()
    else:
        return loop_fn_transition(time, previous_output, previous_state, previous_loop_state)

#Creates an RNN specified by RNNCell cell and loop function loop_fn.
#This function is a more primitive version of dynamic_rnn that provides more direct access to the 
#inputs each iteration. It also provides more control over when to start and finish reading the sequence, 
#and what to emit for the output.
#ta = tensor array
decoder_outputs_ta, decoder_final_state, _ = tf.nn.raw_rnn(decoder_cell, loop_fn)
decoder_outputs = decoder_outputs_ta.stack()

decoder_outputs

#to convert output to human readable prediction
#we will reshape output tensor

#Unpacks the given dimension of a rank-R tensor into rank-(R-1) tensors.
#reduces dimensionality
decoder_max_steps, decoder_batch_size, decoder_dim = tf.unstack(tf.shape(decoder_outputs))
#flettened output tensor
decoder_outputs_flat = tf.reshape(decoder_outputs, (-1, decoder_dim))
#pass flattened tensor through decoder
decoder_logits_flat = tf.add(tf.matmul(decoder_outputs_flat, W), b)
#prediction vals
decoder_logits = tf.reshape(decoder_logits_flat, (decoder_max_steps, decoder_batch_size, vocab_size))

#final prediction
decoder_prediction = tf.argmax(decoder_logits, 2)

#cross entropy loss
#one hot encode the target values so we don't rank just differentiate
stepwise_cross_entropy = tf.nn.softmax_cross_entropy_with_logits(
    labels=tf.one_hot(decoder_targets, depth=vocab_size, dtype=tf.float32),
    logits=decoder_logits,
)

#loss function
loss = tf.reduce_mean(stepwise_cross_entropy)
#train it 
train_op = tf.train.AdamOptimizer().minimize(loss)

sess.run(tf.global_variables_initializer())

batch_size = 2

# batches = helpers.random_sequences(length_from=3, length_to=8, vocab_lower=2, vocab_upper=10, batch_size=batch_size)
train_features_batches = helpers.loadDataFile("../MNIST_data/train_features.txt")
train_labels_batches = helpers.loadDataFile("../MNIST_data/train_labels.txt")


print("train_features_batches:", train_features_batches)

print('head of the train_features_batches:')
for seq in next(train_features_batches.__iter__())[:10]:
    print(seq)

print("train_labels_batches:", train_features_batches)

print('head of the train_labels_batches:')
for seq in next(train_labels_batches.__iter__())[:10]:
    print(seq)

#############

def next_feed(train_features_batches, train_labels_batches):
    train_features_batche = train_features_batches
    train_labels_batche = train_labels_batches
    encoder_inputs_, encoder_input_lengths_ = helpers.batch(train_features_batche)
    decoder_targets_, _ = helpers.batch(
        [(sequence) + [EOS] + [PAD] * 2 for sequence in train_labels_batche]
    )
    return {
        encoder_inputs: encoder_inputs_,
        encoder_inputs_length: encoder_input_lengths_,
        decoder_targets: decoder_targets_,
    }

loss_track = []

max_batches = 30
batches_in_epoch = 10

import time
try:
    start = time.time()
    config = tf.ConfigProto()
    config.gpu_options.allocator_type = 'BFC'
    config.gpu_options.per_process_gpu_memory_fraction = 0.90
    linesofar = 0
    for batch in range(max_batches):
        train_features_batches = helpers.next_batch_k("../MNIST_data/features.txt", batch_size, linesofar)
        train_labels_batches = helpers.next_batch_k("../MNIST_data/labels.txt", batch_size, linesofar)
        fd = next_feed(train_features_batches, train_labels_batches)
        _, l = sess.run([train_op, loss], fd)
        loss_track.append(l)
        linesofar += batch_size

        if batch == 0 or batch % batches_in_epoch == 0:
            print('batch {}'.format(batch))
            if(batch != 0):
                print("Time used: ", time.time() - start)
            print('  minibatch loss: {}'.format(sess.run(loss, fd)))
            predict_ = sess.run(decoder_prediction, fd)
            for i, (inp, pred) in enumerate(zip(fd[encoder_inputs].T, predict_.T)):
                print('  sample {}:'.format(i + 1))
                print('    input     > {}'.format(inp))
                print('    predicted > {}'.format(pred))
                if i >= 2:
                    break
            print()
            start = time.time()
    print("Optimization Finished!")

    # start prediction
    start = time.time()
    test_features_batches = helpers.loadDataFile("../MNIST_data/test_features.txt")
    test_labels_batches = helpers.loadDataFile("../MNIST_data/test_labels.txt")
    test_fd = next_feed(test_features_batches, test_labels_batches)
    print("Time used: ", time.time() - start)
    print('  tests batch loss: {}'.format(sess.run(loss, test_fd)))
    test_predict_ = sess.run(decoder_prediction, test_fd)

    for i, (inp, pred) in enumerate(zip(test_fd[encoder_inputs].T, test_predict_.T)):
        print('  sample {}:'.format(i + 1))
        print('    input     > {}'.format(inp))
        print('    predicted > {}'.format(pred))
        if i >= 2:
            break
    print()
    print("Prediction Finished!")
            
except KeyboardInterrupt:
    print('training interrupted')
#Vocabulary size. #character length #num neurons #in original paper, they used same number of neurons for both encoder and decoder, but we use twice #as many so decoded output is different, the target value is the original input in this example #input placeholders #randomly initialized embedding matrix that can fit input sequence #used to convert sequences to vectors (embeddings) for both encoder and decoder of the right size #reshaping is a thing, in TF you gotta make sure you tensors are the right shape (num dimensions) #####API LOCATION ERROR HERE #from tensorflow.contrib.rnn.python.ops.core_rnn_cell_impl import LSTMCell, LSTMStateTuple #Concatenates tensors along one dimension. ##############print("? = ",encoder_bw_outputs) #letters h and c are commonly used to denote "output value" and "cell state". #http://colah.github.io/posts/2015-08-Understanding-LSTMs/ #Those tensors represent combined internal state of the cell, and should be passed together. #TF Tuple used by LSTM Cells for state_size, zero_state, and output state. # +2 additional steps, +1 leading <EOS> token for decoder inputs #manually specifying since we are going to implement attention details for the decoder in a sec #weights #bias #create padded inputs for the decoder from the word embeddings #were telling the program to tests a condition, and trigger an error if the condition is false. #retrieves rows of the params tensor. The behavior is similar to using indexing with arrays in numpy #manually specifying loop function through time - to get initial cell state and input to RNN #normally we'd just use dynamic_rnn, but lets get detailed here with raw_rnn #we define and return these values, no operations occur here # all False at the initial step #end of sentence #last time steps cell state #none #none # we don't need to pass any additional information #attention mechanism --choose which previously generated token to pass as input in the next timestep #dot product between previous ouput and weights, then + biases #Logits simply means that the function operates on the unscaled output of #earlier layers and that the relative scale to understand the units is linear. #It means, in particular, the sum of the inputs may not equal 1, that the values are not probabilities #(you might have an input of 5). #prediction value at current time step #Returns the index with the largest value across axes of a tensor. #This is Attention!!!! #embed prediction for the next input # this operation produces boolean tensor of [batch_size] # defining if corresponding sequence has ended #Computes the "logical and" of elements across dimensions of a tensor. # -> boolean scalar #Return either fn1() or fn2() based on the boolean predicate pred. #set previous to current # time == 0 #Creates an RNN specified by RNNCell cell and loop function loop_fn. #This function is a more primitive version of dynamic_rnn that provides more direct access to the #inputs each iteration. It also provides more control over when to start and finish reading the sequence, #and what to emit for the output. #ta = tensor array #to convert output to human readable prediction #we will reshape output tensor #Unpacks the given dimension of a rank-R tensor into rank-(R-1) tensors. #reduces dimensionality #flettened output tensor #pass flattened tensor through decoder #prediction vals #final prediction #cross entropy loss #one hot encode the target values so we don't rank just differentiate #loss function #train it # batches = helpers.random_sequences(length_from=3, length_to=8, vocab_lower=2, vocab_upper=10, batch_size=batch_size) ############# # start prediction