def __init__(self, batch_env, step, is_training, should_log, config):
    """Create an instance of the PPO algorithm.

    Args:
      batch_env: In-graph batch environment.
      step: Integer tensor holding the current training step.
      is_training: Boolean tensor for whether the algorithm should train.
      should_log: Boolean tensor for whether summaries should be returned.
      config: Object containing the agent configuration as attributes.
    """
    self._batch_env = batch_env
    self._step = step
    self._is_training = is_training
    self._should_log = should_log
    self._config = config
    self._observ_filter = parts.StreamingNormalize(
        self._batch_env.observ[0], center=True, scale=True, clip=5,
        name='normalize_observ')
    self._reward_filter = parts.StreamingNormalize(
        self._batch_env.reward[0], center=False, scale=True, clip=10,
        name='normalize_reward')
    self._use_gpu = self._config.use_gpu and utility.available_gpus()
    policy_params, state = self._initialize_policy()
    self._initialize_memory(policy_params)
    # Initialize the optimizer and penalty.
    with tf.device('/gpu:0' if self._use_gpu else '/cpu:0'):
      self._optimizer = self._config.optimizer(self._config.learning_rate)
    self._penalty = tf.Variable(
        self._config.kl_init_penalty, False, dtype=tf.float32)
    # If the policy is stateful, allocate space to store its state.
    with tf.variable_scope('ppo_temporary'):
      with tf.device('/gpu:0'):
        if state is None:
          self._last_state = None
        else:
          var_like = lambda x: tf.Variable(lambda: tf.zeros_like(x), False)
          self._last_state = tools.nested.map(var_like, state)
    # Remember the action and policy parameters to write into the memory.
    with tf.variable_scope('ppo_temporary'):
      self._last_action = tf.Variable(
          tf.zeros_like(self._batch_env.action), False, name='last_action')
      self._last_policy = tools.nested.map(
          lambda x: tf.Variable(tf.zeros_like(x[:, 0], False)), policy_params)
def __init__ ( self , batch_env , step , is_training , should_log , config ) : self . _batch_env = batch_env self . _step = step self . _is_training = is_training self . _should_log = should_log self . _config = config self . _observ_filter = parts . StreamingNormalize ( self . _batch_env . observ [ 0 ] , center = True , scale = True , clip = 5 , name = 'normalize_observ' ) self . _reward_filter = parts . StreamingNormalize ( self . _batch_env . reward [ 0 ] , center = False , scale = True , clip = 10 , name = 'normalize_reward' ) self . _use_gpu = self . _config . use_gpu and utility . available_gpus ( ) policy_params , state = self . _initialize_policy ( ) self . _initialize_memory ( policy_params ) with tf . device ( '/gpu:0' if self . _use_gpu else '/cpu:0' ) : self . _optimizer = self . _config . optimizer ( self . _config . learning_rate ) self . _penalty = tf . Variable ( self . _config . kl_init_penalty , False , dtype = tf . float32 ) with tf . variable_scope ( 'ppo_temporary' ) : with tf . device ( '/gpu:0' ) : if state is None : self . _last_state = None else : var_like = lambda x : tf . Variable ( lambda : tf . zeros_like ( x ) , False ) self . _last_state = tools . nested . map ( var_like , state ) with tf . variable_scope ( 'ppo_temporary' ) : self . _last_action = tf . Variable ( tf . zeros_like ( self . _batch_env . action ) , False , name = 'last_action' ) self . _last_policy = tools . nested . map ( lambda x : tf . Variable ( tf . zeros_like ( x [ : , 0 ] , False ) ) , policy_params )
Create an instance of the PPO algorithm.

    Args:
      batch_env: In-graph batch environment.
      step: Integer tensor holding the current training step.
      is_training: Boolean tensor for whether the algorithm should train.
      should_log: Boolean tensor for whether summaries should be returned.
      config: Object containing the agent configuration as attributes.