title: Online PPO: TensorFlow Session returns NaN
body: <p>I'm trying to train an online Proximal Policy Optimization model using tensorflow but after a while, the tensorflow session starts returning NaNs. This causes my agent to <code>step</code> using those nans and eventually the whole thing becomes a mess.</p>
  
  <p><strong>Short snippet from console:</strong></p>
  
  <pre class="lang-sh prettyprint-override"><code>Action Taken   [2.        1.3305835 0.9937418]
  Observation    [  0.69689728  -0.46114012 -11.39961704  -0.05004346  -0.05004346
     0.74720544   3.49857114   3.05071477  -1.10276782  -9.71530186]
  Reward Gained  -0.023699851569145534
  
  Action Taken   [2.         0.62562937 1.0081608 ]
  Observation    [ 0.71591491 -0.47488649 11.84026042 -0.05004346 -0.05004346  0.75886336
    3.49857114  3.07180685 -1.12458586 -9.84382414]
  Reward Gained  -0.015462812448075767
  
  Action Taken   [nan nan nan]
  Observation    [        nan         nan         nan -0.05004346 -0.05004346         nan
           nan         nan         nan         nan]
  Reward Gained  nan
  
  Action Taken   [nan nan nan]
  Observation    [        nan         nan         nan -0.05004346 -0.05004346         nan
           nan         nan         nan         nan]
  Reward Gained  nan
  </code></pre>
  
  <p><strong>My code [UPDATED]:</strong></p>
  
  <pre class="lang-py prettyprint-override"><code>import gym
  import numpy as np
  import tensorflow as tf
  import rocket_lander_gym
  
  EP_LEN = 200
  GAMMA = 0.9
  SL_LR = 1e-4
  CR_LR = 1e-4
  BATCH = 5
  ACTOR_UPDATE_STEPS = 20
  CRITIC_UPDATE_STEPS = 20
  STATE_DIM, ACT_DIM = 10, 3
  
  METHOD = [
      dict(name='kl_penalty', kl_target=0.01, lam=0.5),   
      dict(name='clip', epsilon=0.2),
  ][1]
  
  PRINT_DEBUG_MSG = True
  
  class PPO:
      def __init__(self):
          self.tfsess = tf.Session()
          self.tf_state = tf.placeholder(tf.float32, [None, STATE_DIM], 'state')
  
          # Critic (value network)
          with tf.variable_scope('critic'):
              # Layers
              l1 = tf.layers.dense(self.tf_state, 100, tf.nn.relu)
              # Value
              self.value = tf.layers.dense(l1, 1)
              # Discounted reward: reward in the furture
              self.tf_dreward = tf.placeholder(tf.float32, [None, 1], 'discounted_reward')
              # Advantage: determine quality of action
              self.advantage = self.tf_dreward - self.value
              # Loss function: minimize the advantage over time
              # The loss function is a mean squared error
              self.loss = tf.reduce_mean(tf.square(self.advantage))
              # Gradient descent using Adam optimizer
              self.train_opt = tf.train.AdamOptimizer(CR_LR)
              gradients, variables = zip(*self.train_opt.compute_gradients(self.loss))
              gradients, _ = tf.clip_by_global_norm(gradients, 1.0)
              self.train_opt = self.train_opt.apply_gradients(zip(gradients, variables))
  
          # Actor (policy network)
          pi, pi_params = self.tinynn('pi', trainable=True)
          old_pi, old_pi_params = self.tinynn('old_pi', trainable=False)
  
          # Sample actions from both the old and the new policy networks
          with tf.variable_scope('sample_action'):
              # Choose an action from the distribution learnt
              self.sample_operation = tf.squeeze(pi.sample(1), axis=0)
          with tf.variable_scope('update_old_pi'):
              # Choose an action from the distribution learnt
              self.update_old_pi_operation = [old_pi.assign(p) for p, old_pi in zip(pi_params, old_pi_params)]
  
          # Placeholder for the action and the advantage
          self.tf_action = tf.placeholder(tf.float32, [None, ACT_DIM], 'action')
          self.tf_advantage = tf.placeholder(tf.float32, [None, 1], 'advantage')
  
          # Compute loss function
          with tf.variable_scope('loss'):
              with tf.variable_scope('surrogate'):
                  ratio = pi.prob(self.tf_advantage) / old_pi.prob(self.tf_advantage)
                  surrogate = ratio * self.tf_advantage
  
              # KL penalty
              if METHOD['name'] == 'kl_penalty':
                  # Lambda
                  self.tf_lambda = tf.placeholder(tf.float32, None, 'lambda')
                  # Compute KL divergence between old and new policy
                  kl = tf.contrib.distributions.kl_divergence(old_pi, pi)
                  # Get mean
                  self.kl_mean = tf.reduce_mean(kl)
                  # Compute loss using surrogate
                  self.aloss = -(tf.reduce_mean(surrogate - self.tf_lambda * kl))
              else:
                  self.aloss = -tf.reduce_mean(tf.minimum(surrogate, tf.clip_by_value(ratio, 1.-METHOD['epsilon'],  1.+METHOD['epsilon']) * self.tf_advantage))
  
          # Minimize the loss using gradient descent
          with tf.variable_scope('atrain'):
              self.atrain_operation = tf.train.AdamOptimizer(SL_LR)
              gradients, variables = zip(*self.atrain_operation.compute_gradients(self.aloss))
              gradients, _ = tf.clip_by_global_norm(gradients, 1.0)
              self.atrain_operation = self.atrain_operation.apply_gradients(zip(gradients, variables))
  
          # Write to disk
          tf.summary.FileWriter("log/", self.tfsess.graph)
  
          # Run the session
          self.tfsess.run(tf.global_variables_initializer())
  
  
      def update(self, state, action, reward):
          self.tfsess.run(self.update_old_pi_operation)
          advantage = self.tfsess.run(self.advantage, {self.tf_state: state, self.tf_dreward: reward})
  
          # Update actor (policy)
          if METHOD['name'] == 'kl_penalty':
              for _ in range(ACTOR_UPDATE_STEPS):
                  _, kl = self.tfsess.run([self.atrain_operation, self.kl_mean], {self.tf_state: state, self.tf_action: action, tf_advantage: advantage, self.tf_lambda: METHOD['lam']})
                  if kl &gt; 4*METHOD['kl_target']:
                      break
              if kl &lt; METHOD['kl_target'] / 1.5:
                  # Adaptive lambda
                  METHOD['lam'] /= 2
              elif kl &gt; METHOD['kl_target'] * 1.5:
                  METHOD['lam'] *= 2
              # Lambda might explode, we need to clip it
              METHOD['lam'] = np.clip(METHOD['lam'], 1e-4, 10)
          else:
              [self.tfsess.run(self.atrain_operation, {self.tf_state: state, self.tf_action: action, self.tf_advantage: advantage}) for _ in range(ACTOR_UPDATE_STEPS)]
  
          # Update critic (value)
          [self.tfsess.run(self.train_opt, {self.tf_state: state, self.tf_dreward: reward}) for _ in range(CRITIC_UPDATE_STEPS)]
  
  
      def tinynn(self, name, trainable):
          with tf.variable_scope(name):
              l1 = tf.layers.dense(self.tf_state, 100, tf.nn.relu, trainable=trainable)
              mu = 2 * tf.layers.dense(l1, ACT_DIM, tf.nn.tanh, trainable=trainable)
              sigma = tf.layers.dense(l1, ACT_DIM, tf.nn.softplus, trainable=trainable)
              norm_dist = tf.distributions.Normal(loc=mu, scale=sigma)
          params = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope=name)
          return norm_dist, params
  
  
      def choose_action(self, state):
          state = state[np.newaxis, :]
          action = self.tfsess.run(self.sample_operation, {self.tf_state: state})[0]
          return np.clip(action, -1, 1)
  
  
      def get_value(self, state):
          if state.ndim &lt; 2: state = state[np.newaxis, :]
          return self.tfsess.run(self.value, {self.tf_state: state})[0, 0]
  
  
      def train(self, env, ppo, epochs, render=True):
          # Rewards
          all_ep_r = []
          # Training loop
          for ep in range(epochs):
              # Initial state
              s = env.reset()
              # States, actions and rewards
              buffer_s, buffer_a, buffer_r = [], [], []
              # Initial reward
              ep_r = 0
              # For a single episode
              for t in range(EP_LEN):
                  if render:
                      # Render the environment
                      env.render()
                  # Choose best action
                  a = ppo.choose_action(s)
                  # State,reward,done,info
                  s_, r, done, _ = env.step(a)
                  if PRINT_DEBUG_MSG:
                      print("Action Taken  ",a)
                      print("Observation   ",s_)
                      print("Reward Gained ",r, end='\n\n')
                  # Add to buffers
                  buffer_s.append(s)
                  buffer_a.append(a)
                  buffer_r.append((r+8)/8)    # normalize reward, find to be useful
                  s = s_
                  # Total reward
                  ep_r += r
  
                  # Update PPO
                  if (t+1) % BATCH == 0 or t == EP_LEN - 1:
                      # Get value
                      v_s_ = ppo.get_value(s_)
                      # Discounted reward
                      discounted_r = []
                      # Update rewards
                      for r in buffer_r[::-1]:
                          v_s_ = r + GAMMA * v_s_
                          discounted_r.append(v_s_)
                      discounted_r.reverse()
                      # Buffer states actions rewards
                      bs, ba, br = np.vstack(buffer_s), np.vstack(buffer_a), np.array(discounted_r)[:, np.newaxis]
                      buffer_s, buffer_a, buffer_r = [], [], []
                      ppo.update(bs, ba, br)
  
                  # Check if done
                  if done:
                      print("Simulation done.")
                      break
              # Append episode rewards
              if ep == 0: all_ep_r.append(ep_r)
              else: all_ep_r.append(all_ep_r[-1]*0.9 + ep_r*0.1)
              # Close the environment
              env.close()
          # Return all episode rewards
          return all_ep_r
  
  
  if __name__ == '__main__':
      ppo = PPO()
      env = gym.make('RocketLander-v0')
      reward = ppo.train(env, ppo, 100)
      print(reward)
  </code></pre>
  
  <p><strong>What I have tried:</strong></p>
  
  <ol>
  <li>I have tried lowering the learning rate of both my actor and critic    networks but the nans are still there.</li>
  <li>Reduced the <code>BATCH</code> number so that the PPO gets updated faster.</li>
  </ol>
  
  <p>I have been stuck with this problem for hours now, I can't find any solutions online. I'm also a newbie, if there are any dumb mistakes, please pardon me.</p>
  
  <p><strong>UPDATE: TRACEBACK</strong></p>
  
  <pre class="lang-sh prettyprint-override"><code>Traceback (most recent call last):
    File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1334, in _do_call
      return fn(*args)
    File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1319, in _run_fn
      options, feed_dict, fetch_list, target_list, run_metadata)
    File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1407, in _call_tf_sessionrun
      run_metadata)
  tensorflow.python.framework.errors_impl.InvalidArgumentError: Found Inf or NaN global norm. : Tensor had NaN values
       [[{{node atrain/VerifyFinite/CheckNumerics}} = CheckNumerics[T=DT_FLOAT, message="Found Inf or NaN global norm.", _device="/job:localhost/replica:0/task:0/device:CPU:0"](atrain/global_norm/global_norm)]]
  
  During handling of the above exception, another exception occurred:
  
  Traceback (most recent call last):
    File "main.py", line 209, in &lt;module&gt;
      reward = ppo.train(env, ppo, 100)
    File "main.py", line 191, in train
      ppo.update(bs, ba, br)
    File "main.py", line 118, in update
      [self.tfsess.run(self.atrain_operation, {self.tf_state: state, self.tf_action: action, self.tf_advantage: advantage}) for _ in range(ACTOR_UPDATE_STEPS)]
    File "main.py", line 118, in &lt;listcomp&gt;
      [self.tfsess.run(self.atrain_operation, {self.tf_state: state, self.tf_action: action, self.tf_advantage: advantage}) for _ in range(ACTOR_UPDATE_STEPS)]
    File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 929, in run
      run_metadata_ptr)
    File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1152, in _run
      feed_dict_tensor, options, run_metadata)
    File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1328, in _do_run
      run_metadata)
    File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1348, in _do_call
      raise type(e)(node_def, op, message)
  tensorflow.python.framework.errors_impl.InvalidArgumentError: Found Inf or NaN global norm. : Tensor had NaN values
       [[node atrain/VerifyFinite/CheckNumerics (defined at main.py:90)  = CheckNumerics[T=DT_FLOAT, message="Found Inf or NaN global norm.", _device="/job:localhost/replica:0/task:0/device:CPU:0"](atrain/global_norm/global_norm)]]
  
  Caused by op 'atrain/VerifyFinite/CheckNumerics', defined at:
    File "main.py", line 207, in &lt;module&gt;
      ppo = PPO()
    File "main.py", line 90, in __init__
      gradients, _ = tf.clip_by_global_norm(gradients, 1.0)
    File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/tensorflow/python/ops/clip_ops.py", line 265, in clip_by_global_norm
      "Found Inf or NaN global norm.")
    File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/tensorflow/python/ops/numerics.py", line 47, in verify_tensor_all_finite
      verify_input = array_ops.check_numerics(t, message=msg)
    File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/tensorflow/python/ops/gen_array_ops.py", line 817, in check_numerics
      "CheckNumerics", tensor=tensor, message=message, name=name)
    File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py", line 787, in _apply_op_helper
      op_def=op_def)
    File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/tensorflow/python/util/deprecation.py", line 488, in new_func
      return func(*args, **kwargs)
    File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 3274, in create_op
      op_def=op_def)
    File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 1770, in __init__
      self._traceback = tf_stack.extract_stack()
  
  InvalidArgumentError (see above for traceback): Found Inf or NaN global norm. : Tensor had NaN values
       [[node atrain/VerifyFinite/CheckNumerics (defined at main.py:90)  = CheckNumerics[T=DT_FLOAT, message="Found Inf or NaN global norm.", _device="/job:localhost/replica:0/task:0/device:CPU:0"](atrain/global_norm/global_norm)]]
  </code></pre>