<filename>baselines/atari_ddpg/training.py
#import os
import time
#from collections import deque
#import pickle

from baselines.atari_ddpg.ddpg import DDPG
#from baselines.atari_ddpg.util import mpi_mean, mpi_std, mpi_sum
import baselines.common.tf_util as U
#import numpy
from baselines import logger
import numpy as np
#import tensorflow as tf
#from mpi4py import MPI
MAX_TRAINING_STEPS = 20*60*60/3
TOTAL_FRAMES = 50000000
from baselines.atari_ddpg.logger import Logger as hist_logging
import random
def train(env, env_id, nb_epochs, nb_epoch_cycles, render_eval, reward_scale, render, param_noise, actor, critic,
    normalize_returns, normalize_observations, critic_l2_reg, entropy_weight, actor_lr, critic_lr, action_noise,
    popart, gamma, layer_norm, batch_norm, clip_norm, nb_train_steps, nb_rollout_steps, nb_eval_steps, batch_size, memory,
    tau=0.01, eval_env=None, param_noise_adaption_interval=50):
    #rank = MPI.COMM_WORLD.Get_rank()

    #assert (np.abs(env.action_space.low) == env.action_space.high).all()  # we assume symmetric actions.
    #max_action = env.action_space.high
    max_action = 0
    logger.info('scaling actions by {} before executing in env'.format(max_action))
    agent = DDPG(actor, critic, memory, env.observation_space.shape, env.action_space.n,
        gamma=gamma, tau=tau, normalize_returns=normalize_returns, normalize_observations=normalize_observations,
        batch_size=batch_size, action_noise=action_noise, param_noise=param_noise, critic_l2_reg=critic_l2_reg,
        actor_lr=actor_lr, critic_lr=critic_lr, enable_popart=popart, clip_norm=clip_norm, entropy_weight = entropy_weight,
        reward_scale=reward_scale)
    logger.info('Using agent with the following configuration:')
    logger.info(str(agent.__dict__.items()))
    hist_logger = hist_logging('./logs')
    ln = 0
    bn = 0
    if layer_norm == True:
        ln = 1
    if batch_norm == True:
        bn = 1

    name_ = env_id + '_actor_learning_rate_' + np.str(actor_lr) + '_critic_lr_' + np.str(critic_lr) + '_entropy_weight_' + np.str(entropy_weight) + '_layer_norm_' + np.str(ln)  + '_batch_norm_' + np.str(bn) +  '_clip_norm_' + np.str(clip_norm) + '_'+ str(random.randint(1000, 1200))

    '''
    step = 0
    episode = 0

    eval_episode_rewards_history = deque(maxlen=100)
    episode_rewards_history = deque(maxlen=100)
    '''
    with U.single_threaded_session() as sess:
        # Prepare everything.
        agent.initialize(sess)
        sess.graph.finalize()

        agent.reset()
        obs = env.reset()
        '''
        if eval_env is not None:
            eval_obs = eval_env.reset()
        done = False
        episode_reward = 0.
        episode_step = 0
        episodes = 0
        t = 0

        epoch = 0
        start_time = time.time()

        epoch_episode_rewards = []
        epoch_episode_steps = []
        #epoch_episode_eval_rewards = []
        #epoch_episode_eval_steps = []
        #epoch_start_time = time.time()
        epoch_actions = []
        epoch_qs = []
        epoch_episodes = 0
        '''
        num_frames = 0
        num_games = 0
        current_game_frames = 0
        reward_per_episode = 0
        last_time = time.time()
        last_frame_count = 0.0

        while num_frames <= TOTAL_FRAMES+1:
            num_frames += 1
            current_game_frames += 1
            action, q = agent.pi(obs, apply_noise=True, compute_Q=True)
            a  = np.random.choice(np.arange(action.shape[1]), p=action.ravel())
            new_obs, r, done, info = env.step(a)
            #r = max(-1,min(1, r)) #reward clippint -1 to 1

            i2 = time.time()
            agent.store_transition(obs, action, r, new_obs, done)
            reward_per_episode += r
            if memory.count() >  10000:#REPLAY_START_SIZE
                i1 = time.time()
                cl, al, entropy_cost, actor_grads, critic_grads, actor_norm, critic_norm = agent.train()
                i2 = time.time()

                if num_frames %100 ==0:
                    print("Time for update:", i2-i1)
                    print("Time for update:", i2 -i1, "critic cost:", cl, "actor_cost:", al, "entropy:",  entropy_cost, "actor norm:", actor_norm, "critic norm:",critic_norm, "action:", action)
            obs = new_obs

            if done or current_game_frames > MAX_TRAINING_STEPS:
                obs = env.reset()
                num_games += 1
                print("episode:", num_games, "Total reward:", reward_per_episode, "Number of steps:", current_game_frames)
                #hist_logger.log_histogram(name_, reward_per_episode, num_games)
                hist_logger.log_scalar(name_, reward_per_episode, num_frames)
                #hist_logger.log_scalar(name_ + 'ACTOR_NORM', actor_norm, num_frames)
                current_game_frames = 0
                reward_per_episode = 0
                #if memory.count() > 10000:
                #    for num_up  in range(25):
                #        t1 = time.time()
                #        cl, al, entropy_cost, actor_grads = agent.train(train_only_actor=True)
                #        t2= time.time()
                #        print("Time for only actor update:", t2-t1, "critic_cost:", cl, "actor_cost:", al, "entropy:",  entropy_cost, "actor norm:", numpy.linalg.norm(actor_grads))

            if num_frames % 10000 == 0:
                new_time = time.time()
                diff = new_time - last_time
                last_time = new_time
                elapsed_frames = num_frames - last_frame_count
                last_frame_count = num_frames
                print("frames: ",num_frames,"    games: ",num_games,"    speed: ",(elapsed_frames/diff)," frames/second")

        '''
        for epoch in range(nb_epochs):
            for cycle in range(nb_epoch_cycles):
                # Perform rollouts.
                for t_rollout in range(nb_rollout_steps):
                    # Predict next action.
                    action, q = agent.pi(obs, apply_noise=True, compute_Q=True)

                    a  = np.random.choice(np.arange(action.shape[1]), p=action.ravel())
                    # Execute next action.
                    if rank == 0 and render:
                        env.render()
                    new_obs, r, done, info = env.step(a)
                    #if self.replay_buffer.count() >  REPLAY_START_SIZE:
                    t += 1
                    if rank == 0 and render:
                        env.render()
                    episode_reward += r
                    episode_step += 1

                    # Book-keeping.
                    epoch_actions.append(action)
                    epoch_qs.append(q)
                    agent.store_transition(obs, action, r, new_obs, done)
                    if memory.count() >  10000:#REPLAY_START_SIZE:
                        cl, al, entropy_cost = agent.train()
                    obs = new_obs

                    if done:
                        print(action)
                        # Episode done.
                        epoch_episode_rewards.append(episode_reward)
                        episode_rewards_history.append(episode_reward)
                        epoch_episode_steps.append(episode_step)
                        episode_reward = 0.
                        episode_step = 0
                        epoch_episodes += 1
                        episodes += 1

                        agent.reset()
                        obs = env.reset()
                # Train.
                epoch_actor_losses = []
                epoch_actor_entropy_losses = []
                epoch_critic_losses = []
                epoch_adaptive_distances = []
                for t_train in range(nb_train_steps):
                    # Adapt param noise, if necessary.


                    #if memory.nb_entries >= batch_size and t % param_noise_adaption_interval == 0:
                    #    distance = agent.adapt_param_noise()
                    #    epoch_adaptive_distances.append(distance)

                    cl, al, entropy_cost = agent.train()
                    epoch_critic_losses.append(cl)
                    epoch_actor_losses.append(al)
                    epoch_actor_entropy_losses.append(entropy_cost)
                    agent.update_target_net()

                # Evaluate.
                eval_episode_rewards = []
                eval_qs = []
                if eval_env is not None:
                    eval_episode_reward = 0.
                    for t_rollout in range(nb_eval_steps):
                        eval_action, eval_q = agent.pi(eval_obs, apply_noise=False, compute_Q=True)
                        #eval_obs, eval_r, eval_done, eval_info = eval_env.step(max_action * eval_action)
                        eval_obs, eval_r, eval_done, eval_info = eval_env.step(eval_action)
                        if render_eval:
                            eval_env.render()
                        eval_episode_reward += eval_r

                        eval_qs.append(eval_q)
                        if eval_done:
                            eval_obs = eval_env.reset()
                            eval_episode_rewards.append(eval_episode_reward)
                            eval_episode_rewards_history.append(eval_episode_reward)
                            eval_episode_reward = 0.

            # Log stats.
            #epoch_train_duration = time.time() - epoch_start_time
            duration = time.time() - start_time
            stats = agent.get_stats()
            combined_stats = {}
            for key in sorted(stats.keys()):
                combined_stats[key] = mpi_mean(stats[key])

            # Rollout statistics.
            combined_stats['rollout/return'] = mpi_mean(epoch_episode_rewards)
            combined_stats['rollout/return_history'] = mpi_mean(np.mean(episode_rewards_history))
            combined_stats['rollout/episode_steps'] = mpi_mean(epoch_episode_steps)
            combined_stats['rollout/episodes'] = mpi_sum(epoch_episodes)
            combined_stats['rollout/actions_mean'] = mpi_mean(epoch_actions)
            combined_stats['rollout/actions_std'] = mpi_std(epoch_actions)
            combined_stats['rollout/Q_mean'] = mpi_mean(epoch_qs)

            # Train statistics.
            combined_stats['train/loss_actor'] = mpi_mean(epoch_actor_losses)
            combined_stats['train/loss_actor_entropy'] = mpi_mean(epoch_actor_entropy_losses)
            combined_stats['train/loss_critic'] = mpi_mean(epoch_critic_losses)
            combined_stats['train/param_noise_distance'] = mpi_mean(epoch_adaptive_distances)

            # Evaluation statistics.
            if eval_env is not None:
                combined_stats['eval/return'] = mpi_mean(eval_episode_rewards)
                combined_stats['eval/return_history'] = mpi_mean(np.mean(eval_episode_rewards_history))
                combined_stats['eval/Q'] = mpi_mean(eval_qs)
                combined_stats['eval/episodes'] = mpi_mean(len(eval_episode_rewards))

            # Total statistics.
            combined_stats['total/duration'] = mpi_mean(duration)
            combined_stats['total/steps_per_second'] = mpi_mean(float(t) / float(duration))
            combined_stats['total/episodes'] = mpi_mean(episodes)
            combined_stats['total/epochs'] = epoch + 1
            combined_stats['total/steps'] = t

            for key in sorted(combined_stats.keys()):
                logger.record_tabular(key, combined_stats[key])
            logger.dump_tabular()
            logger.info('')
            logdir = logger.get_dir()
            if rank == 0 and logdir:
                if hasattr(env, 'get_state'):
                    with open(os.path.join(logdir, 'env_state.pkl'), 'wb') as f:
                        pickle.dump(env.get_state(), f)
                if eval_env and hasattr(eval_env, 'get_state'):
                    with open(os.path.join(logdir, 'eval_env_state.pkl'), 'wb') as f:
                        pickle.dump(eval_env.get_state(), f)
            '''

<filename>baselines/atari_ddpg/training.py
#import os
import time
#from collections import deque
#import pickle

from baselines.atari_ddpg.ddpg import DDPG
#from baselines.atari_ddpg.util import mpi_mean, mpi_std, mpi_sum
import baselines.common.tf_util as U
#import numpy
from baselines import logger
import numpy as np
#import tensorflow as tf
#from mpi4py import MPI
MAX_TRAINING_STEPS = 20*60*60/3
TOTAL_FRAMES = 50000000
from baselines.atari_ddpg.logger import Logger as hist_logging
import random
def train(env, env_id, nb_epochs, nb_epoch_cycles, render_eval, reward_scale, render, param_noise, actor, critic,
    normalize_returns, normalize_observations, critic_l2_reg, entropy_weight, actor_lr, critic_lr, action_noise,
    popart, gamma, layer_norm, batch_norm, clip_norm, nb_train_steps, nb_rollout_steps, nb_eval_steps, batch_size, memory,
    tau=0.01, eval_env=None, param_noise_adaption_interval=50):
    #rank = MPI.COMM_WORLD.Get_rank()

    #assert (np.abs(env.action_space.low) == env.action_space.high).all()  # we assume symmetric actions.
    #max_action = env.action_space.high
    max_action = 0
    logger.info('scaling actions by {} before executing in env'.format(max_action))
    agent = DDPG(actor, critic, memory, env.observation_space.shape, env.action_space.n,
        gamma=gamma, tau=tau, normalize_returns=normalize_returns, normalize_observations=normalize_observations,
        batch_size=batch_size, action_noise=action_noise, param_noise=param_noise, critic_l2_reg=critic_l2_reg,
        actor_lr=actor_lr, critic_lr=critic_lr, enable_popart=popart, clip_norm=clip_norm, entropy_weight = entropy_weight,
        reward_scale=reward_scale)
    logger.info('Using agent with the following configuration:')
    logger.info(str(agent.__dict__.items()))
    hist_logger = hist_logging('./logs')
    ln = 0
    bn = 0
    if layer_norm == True:
        ln = 1
    if batch_norm == True:
        bn = 1

    name_ = env_id + '_actor_learning_rate_' + np.str(actor_lr) + '_critic_lr_' + np.str(critic_lr) + '_entropy_weight_' + np.str(entropy_weight) + '_layer_norm_' + np.str(ln)  + '_batch_norm_' + np.str(bn) +  '_clip_norm_' + np.str(clip_norm) + '_'+ str(random.randint(1000, 1200))

    '''
    step = 0
    episode = 0

    eval_episode_rewards_history = deque(maxlen=100)
    episode_rewards_history = deque(maxlen=100)
    '''
    with U.single_threaded_session() as sess:
        # Prepare everything.
        agent.initialize(sess)
        sess.graph.finalize()

        agent.reset()
        obs = env.reset()
        '''
        if eval_env is not None:
            eval_obs = eval_env.reset()
        done = False
        episode_reward = 0.
        episode_step = 0
        episodes = 0
        t = 0

        epoch = 0
        start_time = time.time()

        epoch_episode_rewards = []
        epoch_episode_steps = []
        #epoch_episode_eval_rewards = []
        #epoch_episode_eval_steps = []
        #epoch_start_time = time.time()
        epoch_actions = []
        epoch_qs = []
        epoch_episodes = 0
        '''
        num_frames = 0
        num_games = 0
        current_game_frames = 0
        reward_per_episode = 0
        last_time = time.time()
        last_frame_count = 0.0

        while num_frames <= TOTAL_FRAMES+1:
            num_frames += 1
            current_game_frames += 1
            action, q = agent.pi(obs, apply_noise=True, compute_Q=True)
            a  = np.random.choice(np.arange(action.shape[1]), p=action.ravel())
            new_obs, r, done, info = env.step(a)
            #r = max(-1,min(1, r)) #reward clippint -1 to 1

            i2 = time.time()
            agent.store_transition(obs, action, r, new_obs, done)
            reward_per_episode += r
            if memory.count() >  10000:#REPLAY_START_SIZE
                i1 = time.time()
                cl, al, entropy_cost, actor_grads, critic_grads, actor_norm, critic_norm = agent.train()
                i2 = time.time()

                if num_frames %100 ==0:
                    print("Time for update:", i2-i1)
                    print("Time for update:", i2 -i1, "critic cost:", cl, "actor_cost:", al, "entropy:",  entropy_cost, "actor norm:", actor_norm, "critic norm:",critic_norm, "action:", action)
            obs = new_obs

            if done or current_game_frames > MAX_TRAINING_STEPS:
                obs = env.reset()
                num_games += 1
                print("episode:", num_games, "Total reward:", reward_per_episode, "Number of steps:", current_game_frames)
                #hist_logger.log_histogram(name_, reward_per_episode, num_games)
                hist_logger.log_scalar(name_, reward_per_episode, num_frames)
                #hist_logger.log_scalar(name_ + 'ACTOR_NORM', actor_norm, num_frames)
                current_game_frames = 0
                reward_per_episode = 0
                #if memory.count() > 10000:
                #    for num_up  in range(25):
                #        t1 = time.time()
                #        cl, al, entropy_cost, actor_grads = agent.train(train_only_actor=True)
                #        t2= time.time()
                #        print("Time for only actor update:", t2-t1, "critic_cost:", cl, "actor_cost:", al, "entropy:",  entropy_cost, "actor norm:", numpy.linalg.norm(actor_grads))

            if num_frames % 10000 == 0:
                new_time = time.time()
                diff = new_time - last_time
                last_time = new_time
                elapsed_frames = num_frames - last_frame_count
                last_frame_count = num_frames
                print("frames: ",num_frames,"    games: ",num_games,"    speed: ",(elapsed_frames/diff)," frames/second")

        '''
        for epoch in range(nb_epochs):
            for cycle in range(nb_epoch_cycles):
                # Perform rollouts.
                for t_rollout in range(nb_rollout_steps):
                    # Predict next action.
                    action, q = agent.pi(obs, apply_noise=True, compute_Q=True)

                    a  = np.random.choice(np.arange(action.shape[1]), p=action.ravel())
                    # Execute next action.
                    if rank == 0 and render:
                        env.render()
                    new_obs, r, done, info = env.step(a)
                    #if self.replay_buffer.count() >  REPLAY_START_SIZE:
                    t += 1
                    if rank == 0 and render:
                        env.render()
                    episode_reward += r
                    episode_step += 1

                    # Book-keeping.
                    epoch_actions.append(action)
                    epoch_qs.append(q)
                    agent.store_transition(obs, action, r, new_obs, done)
                    if memory.count() >  10000:#REPLAY_START_SIZE:
                        cl, al, entropy_cost = agent.train()
                    obs = new_obs

                    if done:
                        print(action)
                        # Episode done.
                        epoch_episode_rewards.append(episode_reward)
                        episode_rewards_history.append(episode_reward)
                        epoch_episode_steps.append(episode_step)
                        episode_reward = 0.
                        episode_step = 0
                        epoch_episodes += 1
                        episodes += 1

                        agent.reset()
                        obs = env.reset()
                # Train.
                epoch_actor_losses = []
                epoch_actor_entropy_losses = []
                epoch_critic_losses = []
                epoch_adaptive_distances = []
                for t_train in range(nb_train_steps):
                    # Adapt param noise, if necessary.


                    #if memory.nb_entries >= batch_size and t % param_noise_adaption_interval == 0:
                    #    distance = agent.adapt_param_noise()
                    #    epoch_adaptive_distances.append(distance)

                    cl, al, entropy_cost = agent.train()
                    epoch_critic_losses.append(cl)
                    epoch_actor_losses.append(al)
                    epoch_actor_entropy_losses.append(entropy_cost)
                    agent.update_target_net()

                # Evaluate.
                eval_episode_rewards = []
                eval_qs = []
                if eval_env is not None:
                    eval_episode_reward = 0.
                    for t_rollout in range(nb_eval_steps):
                        eval_action, eval_q = agent.pi(eval_obs, apply_noise=False, compute_Q=True)
                        #eval_obs, eval_r, eval_done, eval_info = eval_env.step(max_action * eval_action)
                        eval_obs, eval_r, eval_done, eval_info = eval_env.step(eval_action)
                        if render_eval:
                            eval_env.render()
                        eval_episode_reward += eval_r

                        eval_qs.append(eval_q)
                        if eval_done:
                            eval_obs = eval_env.reset()
                            eval_episode_rewards.append(eval_episode_reward)
                            eval_episode_rewards_history.append(eval_episode_reward)
                            eval_episode_reward = 0.

            # Log stats.
            #epoch_train_duration = time.time() - epoch_start_time
            duration = time.time() - start_time
            stats = agent.get_stats()
            combined_stats = {}
            for key in sorted(stats.keys()):
                combined_stats[key] = mpi_mean(stats[key])

            # Rollout statistics.
            combined_stats['rollout/return'] = mpi_mean(epoch_episode_rewards)
            combined_stats['rollout/return_history'] = mpi_mean(np.mean(episode_rewards_history))
            combined_stats['rollout/episode_steps'] = mpi_mean(epoch_episode_steps)
            combined_stats['rollout/episodes'] = mpi_sum(epoch_episodes)
            combined_stats['rollout/actions_mean'] = mpi_mean(epoch_actions)
            combined_stats['rollout/actions_std'] = mpi_std(epoch_actions)
            combined_stats['rollout/Q_mean'] = mpi_mean(epoch_qs)

            # Train statistics.
            combined_stats['train/loss_actor'] = mpi_mean(epoch_actor_losses)
            combined_stats['train/loss_actor_entropy'] = mpi_mean(epoch_actor_entropy_losses)
            combined_stats['train/loss_critic'] = mpi_mean(epoch_critic_losses)
            combined_stats['train/param_noise_distance'] = mpi_mean(epoch_adaptive_distances)

            # Evaluation statistics.
            if eval_env is not None:
                combined_stats['eval/return'] = mpi_mean(eval_episode_rewards)
                combined_stats['eval/return_history'] = mpi_mean(np.mean(eval_episode_rewards_history))
                combined_stats['eval/Q'] = mpi_mean(eval_qs)
                combined_stats['eval/episodes'] = mpi_mean(len(eval_episode_rewards))

            # Total statistics.
            combined_stats['total/duration'] = mpi_mean(duration)
            combined_stats['total/steps_per_second'] = mpi_mean(float(t) / float(duration))
            combined_stats['total/episodes'] = mpi_mean(episodes)
            combined_stats['total/epochs'] = epoch + 1
            combined_stats['total/steps'] = t

            for key in sorted(combined_stats.keys()):
                logger.record_tabular(key, combined_stats[key])
            logger.dump_tabular()
            logger.info('')
            logdir = logger.get_dir()
            if rank == 0 and logdir:
                if hasattr(env, 'get_state'):
                    with open(os.path.join(logdir, 'env_state.pkl'), 'wb') as f:
                        pickle.dump(env.get_state(), f)
                if eval_env and hasattr(eval_env, 'get_state'):
                    with open(os.path.join(logdir, 'eval_env_state.pkl'), 'wb') as f:
                        pickle.dump(eval_env.get_state(), f)
            '''

#import os #from collections import deque #import pickle #from baselines.atari_ddpg.util import mpi_mean, mpi_std, mpi_sum #import numpy #import tensorflow as tf #from mpi4py import MPI #rank = MPI.COMM_WORLD.Get_rank() #assert (np.abs(env.action_space.low) == env.action_space.high).all()  # we assume symmetric actions. #max_action = env.action_space.high step = 0     episode = 0      eval_episode_rewards_history = deque(maxlen=100)     episode_rewards_history = deque(maxlen=100) # Prepare everything. if eval_env is not None:             eval_obs = eval_env.reset()         done = False         episode_reward = 0.         episode_step = 0         episodes = 0         t = 0          epoch = 0         start_time = time.time()          epoch_episode_rewards = []         epoch_episode_steps = []         #epoch_episode_eval_rewards = []         #epoch_episode_eval_steps = []         #epoch_start_time = time.time()         epoch_actions = []         epoch_qs = []         epoch_episodes = 0 #r = max(-1,min(1, r)) #reward clippint -1 to 1 #REPLAY_START_SIZE #hist_logger.log_histogram(name_, reward_per_episode, num_games) #hist_logger.log_scalar(name_ + 'ACTOR_NORM', actor_norm, num_frames) #if memory.count() > 10000: #    for num_up  in range(25): #        t1 = time.time() #        cl, al, entropy_cost, actor_grads = agent.train(train_only_actor=True) #        t2= time.time() #        print("Time for only actor update:", t2-t1, "critic_cost:", cl, "actor_cost:", al, "entropy:",  entropy_cost, "actor norm:", numpy.linalg.norm(actor_grads)) for epoch in range(nb_epochs):             for cycle in range(nb_epoch_cycles):                 # Perform rollouts.                 for t_rollout in range(nb_rollout_steps):                     # Predict next action.                     action, q = agent.pi(obs, apply_noise=True, compute_Q=True)                      a  = np.random.choice(np.arange(action.shape[1]), p=action.ravel())                     # Execute next action.                     if rank == 0 and render:                         env.render()                     new_obs, r, done, info = env.step(a)                     #if self.replay_buffer.count() >  REPLAY_START_SIZE:                     t += 1                     if rank == 0 and render:                         env.render()                     episode_reward += r                     episode_step += 1                      # Book-keeping.                     epoch_actions.append(action)                     epoch_qs.append(q)                     agent.store_transition(obs, action, r, new_obs, done)                     if memory.count() >  10000:#REPLAY_START_SIZE:                         cl, al, entropy_cost = agent.train()                     obs = new_obs                      if done:                         print(action)                         # Episode done.                         epoch_episode_rewards.append(episode_reward)                         episode_rewards_history.append(episode_reward)                         epoch_episode_steps.append(episode_step)                         episode_reward = 0.                         episode_step = 0                         epoch_episodes += 1                         episodes += 1                          agent.reset()                         obs = env.reset()                 # Train.                 epoch_actor_losses = []                 epoch_actor_entropy_losses = []                 epoch_critic_losses = []                 epoch_adaptive_distances = []                 for t_train in range(nb_train_steps):                     # Adapt param noise, if necessary.                       #if memory.nb_entries >= batch_size and t % param_noise_adaption_interval == 0:                     #    distance = agent.adapt_param_noise()                     #    epoch_adaptive_distances.append(distance)                      cl, al, entropy_cost = agent.train()                     epoch_critic_losses.append(cl)                     epoch_actor_losses.append(al)                     epoch_actor_entropy_losses.append(entropy_cost)                     agent.update_target_net()                  # Evaluate.                 eval_episode_rewards = []                 eval_qs = []                 if eval_env is not None:                     eval_episode_reward = 0.                     for t_rollout in range(nb_eval_steps):                         eval_action, eval_q = agent.pi(eval_obs, apply_noise=False, compute_Q=True)                         #eval_obs, eval_r, eval_done, eval_info = eval_env.step(max_action * eval_action)                         eval_obs, eval_r, eval_done, eval_info = eval_env.step(eval_action)                         if render_eval:                             eval_env.render()                         eval_episode_reward += eval_r                          eval_qs.append(eval_q)                         if eval_done:                             eval_obs = eval_env.reset()                             eval_episode_rewards.append(eval_episode_reward)                             eval_episode_rewards_history.append(eval_episode_reward)                             eval_episode_reward = 0.              # Log stats.             #epoch_train_duration = time.time() - epoch_start_time             duration = time.time() - start_time             stats = agent.get_stats()             combined_stats = {}             for key in sorted(stats.keys()):                 combined_stats[key] = mpi_mean(stats[key])              # Rollout statistics.             combined_stats['rollout/return'] = mpi_mean(epoch_episode_rewards)             combined_stats['rollout/return_history'] = mpi_mean(np.mean(episode_rewards_history))             combined_stats['rollout/episode_steps'] = mpi_mean(epoch_episode_steps)             combined_stats['rollout/episodes'] = mpi_sum(epoch_episodes)             combined_stats['rollout/actions_mean'] = mpi_mean(epoch_actions)             combined_stats['rollout/actions_std'] = mpi_std(epoch_actions)             combined_stats['rollout/Q_mean'] = mpi_mean(epoch_qs)              # Train statistics.             combined_stats['train/loss_actor'] = mpi_mean(epoch_actor_losses)             combined_stats['train/loss_actor_entropy'] = mpi_mean(epoch_actor_entropy_losses)             combined_stats['train/loss_critic'] = mpi_mean(epoch_critic_losses)             combined_stats['train/param_noise_distance'] = mpi_mean(epoch_adaptive_distances)              # Evaluation statistics.             if eval_env is not None:                 combined_stats['eval/return'] = mpi_mean(eval_episode_rewards)                 combined_stats['eval/return_history'] = mpi_mean(np.mean(eval_episode_rewards_history))                 combined_stats['eval/Q'] = mpi_mean(eval_qs)                 combined_stats['eval/episodes'] = mpi_mean(len(eval_episode_rewards))              # Total statistics.             combined_stats['total/duration'] = mpi_mean(duration)             combined_stats['total/steps_per_second'] = mpi_mean(float(t) / float(duration))             combined_stats['total/episodes'] = mpi_mean(episodes)             combined_stats['total/epochs'] = epoch + 1             combined_stats['total/steps'] = t              for key in sorted(combined_stats.keys()):                 logger.record_tabular(key, combined_stats[key])             logger.dump_tabular()             logger.info('')             logdir = logger.get_dir()             if rank == 0 and logdir:                 if hasattr(env, 'get_state'):                     with open(os.path.join(logdir, 'env_state.pkl'), 'wb') as f:                         pickle.dump(env.get_state(), f)                 if eval_env and hasattr(eval_env, 'get_state'):                     with open(os.path.join(logdir, 'eval_env_state.pkl'), 'wb') as f:                         pickle.dump(eval_env.get_state(), f)