import gym
import random
import torch
import numpy as np
from collections import deque
import matplotlib.pyplot as plt
#%matplotlib inline
from unityagents import UnityEnvironment
import numpy as np
import argparse
import sys


env = UnityEnvironment(file_name="Banana_Windows_x86_64/Banana.exe")

# get the default brain
brain_name = env.brain_names[0]
brain = env.brains[brain_name]

# reset the environment
env_info = env.reset(train_mode=True)[brain_name]

# number of agents in the environment
print('Number of agents:', len(env_info.agents))

# number of actions
action_size = brain.vector_action_space_size
print('Number of actions:', action_size)

# examine the state space 
state = env_info.vector_observations[0]
print('States look like:', state)
state_size = len(state)
print('States have length:', state_size)

#from dqn_agent import Agent
def dqn(n_episodes=2000, max_t=5000, eps_start=1.0, eps_end=0.01, eps_decay=0.995):
    """Deep Q-Learning.
    
    Params
    ======
        n_episodes (int): maximum number of training episodes
        max_t (int): maximum number of timesteps per episode
        eps_start (float): starting value of epsilon, for epsilon-greedy action selection
        eps_end (float): minimum value of epsilon
        eps_decay (float): multiplicative factor (per episode) for decreasing epsilon
    """
    scores = []                        # list containing scores from each episode
    
    scores_window = deque(maxlen=100)  # last 100 scores
    eps = eps_start                    # initialize epsilon
    for i_episode in range(1, n_episodes+1):
        env_info = env.reset(train_mode=True)[brain_name]
        state = env_info.vector_observations[0]
        score = 0
        #for t in range(max_t):
        while True:
            action = agent.act(state, eps)
            action = action.astype(int)
            env_info = env.step(action)[brain_name]
            next_state = env_info.vector_observations[0]   
            # get the next state
            reward = env_info.rewards[0]   # get the reward
            done = env_info.local_done[0]
            #next_state, reward, done, _ = env.step(action)
            agent.step(state, action, reward, next_state, done)
            state = next_state
            score += reward
            if done:
                break 
        scores_window.append(score)       # save most recent score
        scores.append(score)              # save most recent score
        eps = max(eps_end, eps_decay*eps) # decrease epsilon
        print('\rEpisode {}\tAverage Score: {:.2f}'.format(i_episode, np.mean(scores_window)), end="")
        if i_episode % 100 == 0:
            print('\rEpisode {}\tAverage Score: {:.2f}'.format(i_episode, np.mean(scores_window)))
            #print(len(agent.memory.memory), max(agent.memory.memory)[0], agent.memory.memory[0][0])
        # if np.mean(scores_window)>=200.0:
        #     print('\nEnvironment solved in {:d} episodes!\tAverage Score: {:.2f}'.format(i_episode-100, np.mean(scores_window)))
        #     torch.save(agent.qnetwork_local.state_dict(), 'checkpoint.pth')
        #     break
    return scores

def main():
    #try:
    scores = dqn()
    # except:
    #     print('****'*8)
    #     scores = dqn()

    # plot the scores
    fig = plt.figure()
    ax = fig.add_subplot(111)
    plt.plot(np.arange(len(scores)), scores)
    plt.ylabel('Score')
    plt.xlabel('Episode #')
    plt.show()
    torch.save(agent.qnetwork_local.state_dict(), 'checkpoint.pth')

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("network", help="you can choose to use DDQN by input DDQN or use DuelingDQN by input Duel here")
    parser.add_argument("priority", help="you can either train mode with priority exp replay or without it")
    args = parser.parse_args()
    if args.network not in ('DDQN', 'Duel'):
        print('No such network, please make sure you choose either DDQN or Duel as an input for the network argument')
        sys.exit()
    if args.priority == 'priority':
        from dqn_agent_test import Agent
    else:
        from dqn_agent import Agent
    agent = Agent(state_size=37, action_size=4, seed=0, mode = args.network)
    main()
import gym
import random
import torch
import numpy as np
from collections import deque
import matplotlib.pyplot as plt
#%matplotlib inline
from unityagents import UnityEnvironment
import numpy as np
import argparse
import sys


env = UnityEnvironment(file_name="Banana_Windows_x86_64/Banana.exe")

# get the default brain
brain_name = env.brain_names[0]
brain = env.brains[brain_name]

# reset the environment
env_info = env.reset(train_mode=True)[brain_name]

# number of agents in the environment
print('Number of agents:', len(env_info.agents))

# number of actions
action_size = brain.vector_action_space_size
print('Number of actions:', action_size)

# examine the state space 
state = env_info.vector_observations[0]
print('States look like:', state)
state_size = len(state)
print('States have length:', state_size)

#from dqn_agent import Agent
def dqn(n_episodes=2000, max_t=5000, eps_start=1.0, eps_end=0.01, eps_decay=0.995):
    """Deep Q-Learning.
    
    Params
    ======
        n_episodes (int): maximum number of training episodes
        max_t (int): maximum number of timesteps per episode
        eps_start (float): starting value of epsilon, for epsilon-greedy action selection
        eps_end (float): minimum value of epsilon
        eps_decay (float): multiplicative factor (per episode) for decreasing epsilon
    """
    scores = []                        # list containing scores from each episode
    
    scores_window = deque(maxlen=100)  # last 100 scores
    eps = eps_start                    # initialize epsilon
    for i_episode in range(1, n_episodes+1):
        env_info = env.reset(train_mode=True)[brain_name]
        state = env_info.vector_observations[0]
        score = 0
        #for t in range(max_t):
        while True:
            action = agent.act(state, eps)
            action = action.astype(int)
            env_info = env.step(action)[brain_name]
            next_state = env_info.vector_observations[0]   
            # get the next state
            reward = env_info.rewards[0]   # get the reward
            done = env_info.local_done[0]
            #next_state, reward, done, _ = env.step(action)
            agent.step(state, action, reward, next_state, done)
            state = next_state
            score += reward
            if done:
                break 
        scores_window.append(score)       # save most recent score
        scores.append(score)              # save most recent score
        eps = max(eps_end, eps_decay*eps) # decrease epsilon
        print('\rEpisode {}\tAverage Score: {:.2f}'.format(i_episode, np.mean(scores_window)), end="")
        if i_episode % 100 == 0:
            print('\rEpisode {}\tAverage Score: {:.2f}'.format(i_episode, np.mean(scores_window)))
            #print(len(agent.memory.memory), max(agent.memory.memory)[0], agent.memory.memory[0][0])
        # if np.mean(scores_window)>=200.0:
        #     print('\nEnvironment solved in {:d} episodes!\tAverage Score: {:.2f}'.format(i_episode-100, np.mean(scores_window)))
        #     torch.save(agent.qnetwork_local.state_dict(), 'checkpoint.pth')
        #     break
    return scores

def main():
    #try:
    scores = dqn()
    # except:
    #     print('****'*8)
    #     scores = dqn()

    # plot the scores
    fig = plt.figure()
    ax = fig.add_subplot(111)
    plt.plot(np.arange(len(scores)), scores)
    plt.ylabel('Score')
    plt.xlabel('Episode #')
    plt.show()
    torch.save(agent.qnetwork_local.state_dict(), 'checkpoint.pth')

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("network", help="you can choose to use DDQN by input DDQN or use DuelingDQN by input Duel here")
    parser.add_argument("priority", help="you can either train mode with priority exp replay or without it")
    args = parser.parse_args()
    if args.network not in ('DDQN', 'Duel'):
        print('No such network, please make sure you choose either DDQN or Duel as an input for the network argument')
        sys.exit()
    if args.priority == 'priority':
        from dqn_agent_test import Agent
    else:
        from dqn_agent import Agent
    agent = Agent(state_size=37, action_size=4, seed=0, mode = args.network)
    main()
#%matplotlib inline # get the default brain # reset the environment # number of agents in the environment # number of actions # examine the state space #from dqn_agent import Agent Deep Q-Learning.          Params     ======         n_episodes (int): maximum number of training episodes         max_t (int): maximum number of timesteps per episode         eps_start (float): starting value of epsilon, for epsilon-greedy action selection         eps_end (float): minimum value of epsilon         eps_decay (float): multiplicative factor (per episode) for decreasing epsilon # list containing scores from each episode # last 100 scores # initialize epsilon #for t in range(max_t): # get the next state # get the reward #next_state, reward, done, _ = env.step(action) # save most recent score # save most recent score # decrease epsilon #print(len(agent.memory.memory), max(agent.memory.memory)[0], agent.memory.memory[0][0]) # if np.mean(scores_window)>=200.0: #     print('\nEnvironment solved in {:d} episodes!\tAverage Score: {:.2f}'.format(i_episode-100, np.mean(scores_window))) #     torch.save(agent.qnetwork_local.state_dict(), 'checkpoint.pth') #     break #try: # except: #     print('****'*8) #     scores = dqn() # plot the scores #')