title: Tensorflow starts fast and slows down during training
body: <p>Im an electrical engineering student and im trying to model an industrial plant based on the power in a resistor inside a boiller, the temperature of the water in the boiller and the water flow passing through the boiller using python 3.5 and tensorflow.
  The matter is that im a beginner at python and tensorflow and i wrote this code that works, but the trainning starts fast and rapidly start to slow down, and by the middle of the trainning its starts to taking ages between steps.
  I just need some help on the optimization, and of course, any tips are welcome! 
  Thank you very much!
  Here is the code:</p>
  
  <pre><code>import numpy as np
  import tensorflow as tf
  input_vec_size = 3
  step_size = 0.05
  batch_size = 3
  test_size = 16
  train_end = 1905
  eval_end = 290
  predict_end = 1396
  n_cores = 4
  def read_my_file_format(filename_queue):
      line_reader = tf.TextLineReader(skip_header_lines=1)
      _, csv_row = line_reader.read(filename_queue)
      record_defaults = [[0.0], [0.0], [0.0], [0.0]]
      time, power_in, temperature, flow = \
          tf.decode_csv(csv_row, record_defaults=record_defaults)
      features = tf.pack([
          power_in,
          temperature
          ])
      return features, flow
  def input_pipeline(directory, batch_size, n_cores, buffer_size, num_epochs=None):
      filename_queue = tf.train.string_input_producer(
        tf.train.match_filenames_once(directory),
        shuffle=True)
      features, flow = read_my_file_format(filename_queue)
      x, y = tf.train.batch(
           [features, flow], batch_size=batch_size, allow_smaller_final_batch=True)
  def init_weights(shape):
      return tf.Variable(tf.random_normal(shape, stddev=0.001))
  def init_bias(shape): #inicializa bias
      initial = tf.constant(0.001, shape=shape) #variancia 0.1
      return tf.Variable(initial)
  def model(X, w_h, w_h2, w_o, B, B2, B3, p_keep_input, p_keep_hidden): 
      X = tf.nn.dropout(X, p_keep_input)
      h = tf.nn.relu(tf.matmul(X, w_h)+B)
      h = tf.nn.dropout(h, p_keep_hidden)
      h2 = tf.nn.relu(tf.matmul(h, w_h2)+B2)
      h2 = tf.nn.dropout(h2, p_keep_hidden)
      return tf.matmul(h2, w_o)+B3
  
  X = tf.placeholder("float", [None, input_vec_size])
  Y = tf.placeholder("float", [None, 1])
  p_keep_hidden = tf.placeholder("float")
  p_keep_input = tf.placeholder("float")
  w_h = init_weights([input_vec_size, fclayer_size])
  w_h2= init_weights([fclayer_size, fclayer_size])
  w_o= init_weights([fclayer_size, 1])
  B = init_bias([fclayer_size])
  B2 = init_bias([fclayer_size])
  B3 = init_bias([1])
  py_x = model(X, w_h, w_h2, w_o, B, B2, B3, p_keep_input, p_keep_hidden)
  predict_op = py_x[0]
  cost = tf.reduce_mean(tf.square(predict_op - Y))
  train_op = tf.train.MomentumOptimizer(step_size, 0.5).minimize(cost)
  saver = tf.train.Saver()
  directory = "./train/*.csv"
  x, y = input_pipeline(directory, batch_size, n_cores, buffer_size, num_epochs=None)
  directory_eval = "./eval/*.csv"
  xe, ye = input_pipeline(directory_eval, test_size, n_cores, buffer_size, num_epochs=None)
  directory_predict = "./predict/*.csv"
  xp, yp = input_pipeline(directory_predict, test_size, n_cores, buffer_size, num_epochs=None)
  
  with tf.Session() as sess:
      tf.initialize_all_variables().run()
      coord = tf.train.Coordinator()
      threads = tf.train.start_queue_runners(coord=coord)
      print("==================================TREINAMENTO=================================")
      for iteraction in range(int(train_end/batch_size)):
          trX, trY = sess.run([x,y])
          for i in range(0, batch_size):
              features, features_past, features_past2 = sess.run(tf.unpack(trX[i])), sess.run(tf.unpack(trX[i-1])), sess.run(tf.unpack(trX[i-2]))
          power_in_i = features[0] - 4
          temperature_i = features[1]
          temperature_i1 = features_past[1]
          temperature_i2 = features_past2[1]
          trX_now = tf.pack([power_in_i, (temperature_i-temperature_i1), (temperature_i-temperature_i2)])
          trX_now = sess.run(trX_now)
          X_Batch, Y_Batch = trX_now.reshape([-1, input_vec_size]), trY[i].reshape([-1, 1])
          sess.run(train_op, feed_dict={X: X_Batch,
                                        Y: Y_Batch, p_keep_input: 0.95, p_keep_hidden: 0.7})
          if(i%batch_size == 0):
              predict_train = sess.run(tf.reshape(predict_op, [-1, 1]), feed_dict={X: X_Batch, p_keep_input: 1.0, p_keep_hidden: 1.0})
              train_cost = sess.run(cost, feed_dict={py_x: predict_train, Y: Y_Batch})
              print("Train Batch:", iteraction,"Sample:", batch_size*iteraction, "X:", X_Batch, "Y:", Y_Batch, "y_:",
                  predict_train, "Cost:", train_cost)
      saver.save(sess, "./model.ckpt")
      print('Variaveis salvas com sucesso')
      coord.request_stop()
      coord.join(threads)
      sess.close()
      print('=============================Fim do Treinamento=============================')
  with tf.Session() as sess:
      tf.initialize_all_variables().run()
      coord = tf.train.Coordinator()
      threads = tf.train.start_queue_runners(coord=coord)
  
      print("==============================VALIDAÇAO==================================")
      saver.restore(sess, "./model.ckpt")
      print("Model restored.")
      for iteraction in range(int(eval_end/test_size)):
          teX, teY = sess.run([xe, ye])
          for i in range(0, test_size):
              features, features_past, features_past2 = sess.run(tf.unpack(teX[i])), sess.run(tf.unpack(teX[i - 1])), sess.run(tf.unpack(teX[i-2]))
          power_in_i = features[0] - 4
          temperature_i = features[1]
          temperature_i1 = features_past[1]
          teX_now = tf.pack([power_in_i, (temperature_i - temperature_i1), (temperature_i-temperature_i2)])
          teX_now = sess.run(teX_now)
          X_Batch, Y_Batch = teX_now.reshape([-1, input_vec_size]), teY[i].reshape([-1, 1])
          predict_eval = sess.run(tf.reshape(predict_op, [-1, 1]), feed_dict={X: X_Batch, p_keep_input: 1.0, p_keep_hidden: 1.0})
          eval_cost = sess.run(cost, feed_dict={py_x: predict_eval, Y: Y_Batch})
          print("Eval Batch:", iteraction,"Sample:", batch_size*iteraction, "X:", X.eval(feed_dict={X: X_Batch}), "Y:", Y_Batch, "y_:",
               predict_eval, "Cost:", eval_cost)
      coord.request_stop()
      coord.join(threads)
      sess.close()
  print('=============================FIM DA VALIDAÇAO=============================')
  
  with tf.Session() as sess:
      tf.initialize_all_variables().run()
      coord = tf.train.Coordinator()
      threads = tf.train.start_queue_runners(coord=coord)
  
      print("==============================PREDIÇÃO==================================")
      saver.restore(sess, "./model.ckpt")
      print("Model restored.")
      predict_batch_mean = 0
      predict_mean = 0
      for iteraction in range(int(predict_end / test_size)):
          tpX, tpY = sess.run([xp, yp])
          for i in range(0, test_size):
              features, features_past, features_past2 = sess.run(tf.unpack(tpX[i])), sess.run(tf.unpack(tpX[i - 1])), sess.run(tf.unpack(tpX[i-2]))
              power_in_i = features[0]- 4
              temperature_i = features[1]
              temperature_i1 = features_past[1]
              tpX_now = tf.pack([power_in_i, (temperature_i - temperature_i1), (temperature_i-temperature_i2)])
              tpX_now = sess.run(tpX_now)
              X_Batch, Y_Batch = tpX_now.reshape([-1, input_vec_size]), tpY[i].reshape([-1, 1])
              prediction = sess.run(tf.reshape(predict_op, [-1, 1]), feed_dict={X: X_Batch, p_keep_input: 1.0, p_keep_hidden: 1.0})
              print("Predict Batch:", iteraction,"Sample:", batch_size*iteraction, "X:", X.eval(feed_dict={X: X_Batch}), "y_:",
                  prediction)
              predict_batch_mean = (predict_batch_mean + prediction)/i
          predict_mean = (predict_mean + predict_batch_mean)/iteraction
          print("Predicted Flow:", predict_mean)
      coord.request_stop()
      coord.join(threads)
      sess.close()
  </code></pre>