title: multiple runs of inference model using saved checkpoint produces stochastic errors - Tensorflow
body: <p>I am running Tensorflow 0.12.1 on a GPU. I have a trained Deep CNN model whose weights I've saved using a  checkpoint file. During inference, I reload the saved checkpoint using <code>restorer.restore(sess, tf.train.latest_checkpoint(FLAGS.train_dir))</code>.  The code seems to run without issues, but everytime I re-run the script, I'm getting screwed up outputs. AFAIK, I do not shuffle my test set inputs. The inputs are being loaded and fed to the network properly. It is just the output of different runs of the CNN on the same test set using the same order is producing very different outputs.  I'm perplexed! Also, how do I execute a graph loaded with saved checkpoint without running an <code>init_op</code> during inference? It seems my code requires all global and local variables to be initialized before execution. (I initialize first, and then only restore the checkpoint!).Here's a snippet of my code:</p>
  
  <pre><code>import tensorflow as tf
  import numpy as np
  import os
  import os.path
  from datetime import datetime
  import time
  import random
  import json
  
  from tensorflow.python.framework import ops
  from tensorflow.python.framework import dtypes
  
  
  from modelFCNN3 import model 
  
  def read_input(inp_queue,height=224,width=224,channels=3, mask=False):
    value = tf.read_file(inp_queue)
    image = tf.image.decode_png(value)
    image =  tf.image.resize_images(image, [height, width],method=2)
    image = tf.cast(image, tf.uint8)
    image.set_shape([height,width,channels])
    image = tf.reshape(image,[height,width,channels])
    if mask:
        image = tf.to_float(tf.greater_equal(image,128))
        image = tf.cast(image,tf.float32)
    else:
        image = tf.image.per_image_standardization(image)
        image = tf.cast(image,tf.float32)
    return image
  
  
  
  
  if __name__ == '__main__':
  
      tf.reset_default_graph()
  
      with open('X_test.json', 'r') as infile:
          X_test = json.load(infile)
  
      with open('y_test.json', 'r') as infile:
          y_test = json.load(infile)
  
      imagelist = ops.convert_to_tensor(X_test, dtype=dtypes.string)
      labellist = ops.convert_to_tensor(y_test, dtype=dtypes.string)
  
      input_queue = tf.train.slice_input_producer([imagelist, labellist],
                                              num_epochs=1,
                                              shuffle=False)
  
      image = read_input(input_queue[0],height=224,width=224,channels=3, mask=False)
  
      label = read_input(input_queue[1],height=224,width=224,channels=1, mask=True)
  
      images_batch, labels_batch = tf.train.batch([image, label], batch_size=FLAGS.batch_size,
          enqueue_many=False,shapes=None, allow_smaller_final_batch=True)
  
      global_step = tf.Variable(0, trainable=False)
      images = tf.placeholder_with_default(images_batch, shape=[None, 224,224,3])
      labels = tf.placeholder_with_default(labels_batch, shape=[None, 224,224,1])
  
      restorer = tf.train.Saver()
  
      logits = model(images).logits
      labels = tf.cast(labels,tf.int32)
      labels.set_shape([FLAGS.batch_size,224,224,1])
  
      valid_prediction = tf.argmax(tf.nn.softmax(logits), dimension=3)
      valid_prediction.set_shape([FLAGS.batch_size,224,224])
  
      meanIOU,update_op_mIOU= tf.contrib.metrics.streaming_mean_iou(tf.cast(valid_prediction,tf.int32), tf.squeeze(labels),FLAGS.num_classes)
  
      init = tf.global_variables_initializer()
      init_locals = tf.local_variables_initializer()
  
  
      with tf.Session() as sess:
  
          sess.run([init, init_locals])
  
          restorer.restore(sess, tf.train.latest_checkpoint(FLAGS.train_dir))
          print("Model restored.")
  
  
          coord = tf.train.Coordinator()
          threads = tf.train.start_queue_runners(coord=coord,sess=sess)
          summary_writer = tf.summary.FileWriter(FLAGS.train_dir, sess.graph)
  
          try:
              step = 0
              avg = []
              while not coord.should_stop():
                  myimg, predimg, mylbl= sess.run([images,valid_prediction,labels])
                  mIOU,_ = sess.run([meanIOU,update_op_mIOU])
                  avg.append(mIOU)
  
              step += 1
  
          except tf.errors.OutOfRangeError:
              print('Done training -- epoch limit reached')
  
          finally:
  
              coord.request_stop()
              coord.join(threads)
              sess.close()
  </code></pre>