# -*- coding: utf-8 -*-
from __future__ import absolute_import
from __future__ import division

import numpy as np

from keras.utils.generic_utils import Progbar
from six.moves import xrange

class Agent(object):
    """Base Agent class

    Parameters
    ----------
    model : :obj:`Model`
        A learning model. Ex: neural network or table
    memory : :obj:`Memory`
    """
    def __init__(self, model, memory):
        self.model = model
        self.memory = memory


class DiscreteAgent(Agent):
    """Single Discrete action Agent

    Parameters
    ----------
    model : :obj:`Model`
        A learning model. Ex: neural network o} table
    memory : :obj:`Memory`
        Model's memory for storing experiences for replay and such.
    epsilon : callable
        A rule to define if model explore or exploit
        TODO: generalize this to a class that controls if it should explore and
        define custom explorations rules

    """
    def __init__(self, model, memory, epsilon=None):
        super(DiscreteAgent, self).__init__(model, memory)
        if epsilon is None:
            epsilon = lambda *args: .1
        self.epsilon = epsilon

    def compile(self, *args, **kwargs):
        self.model.compile(*args, **kwargs)
        if 'experience' in kwargs:
            experience = kwargs['experience']
        else:
            experience = None
        self.memory.reset(experience)
#    def compile(self, optimizer="sgd", loss="mse", policy_rule="max",
#                experience=None):
#        self.model.compile(optimizer, loss, policy_rule)
#        self.memory.reset(experience)

    def values(self, observation, train=False):
        return self.model.values(observation, train)

    def max_values(self, observation, train=False):
        return self.model.max_values(observation, train)

    def policy(self, observation, train=False):
        if train and np.random.rand() <= self.epsilon():
            return [np.random.randint(0, self.num_actions)]
        else:
            return self.model.policy(observation, train)

    def update(self, batch_size=1, exp_batch_size=0, gamma=0.9, callback=None):
        inputs, targets, actions = self.get_batch(
            self.model, batch_size=batch_size, exp_batch_size=exp_batch_size,
            gamma=gamma, callback=callback)
        loss = self.model.update(inputs, targets, actions)
        return loss

    @property
    def num_actions(self):
        return self.model.num_actions

    @property
    def input_shape(self):
        return self.model.input_shape

    def reset(self):
        self.memory.reset()

    def remember(self, prev_state, action, reward, next_state, game_over):
        self.memory.remember(prev_state, action, reward,
                             next_state, game_over)

    def get_batch(self, model, batch_size=1, exp_batch_size=0,
                  gamma=0.9, callback=None):
        return self.memory.get_batch(model, batch_size, exp_batch_size,
                                     gamma, callback)

    def learn(self, env, epoch=1, batch_size=1, exp_batch_size=0,
              gamma=0.9, reset_memory=False, verbose=1, callbacks=None):
        """Train Agent to play Enviroment env

        Parameters
        ----------
        env : :obj:`Enviroment`
            The enviroment the agent learn to play
        epoch : int
            number of complete episodes to play
        batch_size : int
            number of experiences to replay per step
        exp_batch_size : int
            number of experiences to replay from the consolidated
            :attr:`ExperienceReplayexperience.experience`.
        gamma : float
            discount factor
        reset_memory : bool
            if we should restart :attr:`ExperienceReplay.memory` before
            starting the game.
        verbose : int
            controls how much should we print
        callbacks : list of callables
            TODO: Add callback support

        """
        print("Learning started!")
        print("[Environment]: {}".format(env.description))
        print("[Model]: {}".format(self.model.description))
        print("[Memory]: {}".format(self.memory.description))
        if reset_memory:
            self.reset()
        progbar = Progbar(epoch)

        for e in xrange(epoch):
            # reset environment on each epoch
            env.reset()
            game_over = False
            loss = 0
            rewards = 0

            # get initial observation, start game
            obs_t = env.observe()

            # Run an episonde
            while not game_over:
                obs_tm1 = obs_t
                action = self.policy(obs_tm1, train=True)

                # apply action, get rewards and new state
                obs_t, reward, game_over = env.update(action)
                rewards += reward

                # store experience
                self.remember(obs_tm1, action, reward, obs_t, game_over)

                # adapt model
                loss += self.update(batch_size=batch_size,
                                    exp_batch_size=exp_batch_size,
                                    gamma=gamma)
            if verbose == 1:
                progbar.add(1, values=[("loss", loss), ("rewards", rewards)])

    def play(self, env, epoch=1, batch_size=1, visualize=None, verbose=1):
        print("Free play started!")
        frames = np.zeros((0, ) + env.observe_image().shape[1:])
        frames = frames.transpose(0, 2, 3, 1)
        progbar = Progbar(epoch)

        for e in xrange(epoch):
            # reset environment on each epoch
            env.reset()
            game_over = False
            loss = 0
            rewards = 0

            # get initial observation, start game
            obs_t = env.observe()
            while not game_over:
                obs_tm1 = obs_t

                # get next action
                action = self.policy(obs_tm1, train=False)

                # apply action, get rewareds and new state
                obs_t, reward, game_over = env.update(action)
                rewards += reward

                frame_t = env.observe_image().transpose(0, 2, 3, 1)
                frames = np.concatenate([frames, frame_t], axis=0)

            if verbose == 1:
                progbar.add(1, values=[("loss", loss), ("rewards", rewards)])


        if visualize:
            from agnez.video import make_gif
            print("Making gif!")
            frames = np.repeat(frames, 3, axis=-1)
            make_gif(frames[:visualize['n_frames']],
                     filepath=visualize['filepath'], gray=visualize['gray'], interpolation='none')
            print("See your gif at {}".format(visualize['filepath']))

# -*- coding: utf-8 -*-
from __future__ import absolute_import
from __future__ import division

import numpy as np

from keras.utils.generic_utils import Progbar
from six.moves import xrange

class Agent(object):
    """Base Agent class

    Parameters
    ----------
    model : :obj:`Model`
        A learning model. Ex: neural network or table
    memory : :obj:`Memory`
    """
    def __init__(self, model, memory):
        self.model = model
        self.memory = memory


class DiscreteAgent(Agent):
    """Single Discrete action Agent

    Parameters
    ----------
    model : :obj:`Model`
        A learning model. Ex: neural network o} table
    memory : :obj:`Memory`
        Model's memory for storing experiences for replay and such.
    epsilon : callable
        A rule to define if model explore or exploit
        TODO: generalize this to a class that controls if it should explore and
        define custom explorations rules

    """
    def __init__(self, model, memory, epsilon=None):
        super(DiscreteAgent, self).__init__(model, memory)
        if epsilon is None:
            epsilon = lambda *args: .1
        self.epsilon = epsilon

    def compile(self, *args, **kwargs):
        self.model.compile(*args, **kwargs)
        if 'experience' in kwargs:
            experience = kwargs['experience']
        else:
            experience = None
        self.memory.reset(experience)
#    def compile(self, optimizer="sgd", loss="mse", policy_rule="max",
#                experience=None):
#        self.model.compile(optimizer, loss, policy_rule)
#        self.memory.reset(experience)

    def values(self, observation, train=False):
        return self.model.values(observation, train)

    def max_values(self, observation, train=False):
        return self.model.max_values(observation, train)

    def policy(self, observation, train=False):
        if train and np.random.rand() <= self.epsilon():
            return [np.random.randint(0, self.num_actions)]
        else:
            return self.model.policy(observation, train)

    def update(self, batch_size=1, exp_batch_size=0, gamma=0.9, callback=None):
        inputs, targets, actions = self.get_batch(
            self.model, batch_size=batch_size, exp_batch_size=exp_batch_size,
            gamma=gamma, callback=callback)
        loss = self.model.update(inputs, targets, actions)
        return loss

    @property
    def num_actions(self):
        return self.model.num_actions

    @property
    def input_shape(self):
        return self.model.input_shape

    def reset(self):
        self.memory.reset()

    def remember(self, prev_state, action, reward, next_state, game_over):
        self.memory.remember(prev_state, action, reward,
                             next_state, game_over)

    def get_batch(self, model, batch_size=1, exp_batch_size=0,
                  gamma=0.9, callback=None):
        return self.memory.get_batch(model, batch_size, exp_batch_size,
                                     gamma, callback)

    def learn(self, env, epoch=1, batch_size=1, exp_batch_size=0,
              gamma=0.9, reset_memory=False, verbose=1, callbacks=None):
        """Train Agent to play Enviroment env

        Parameters
        ----------
        env : :obj:`Enviroment`
            The enviroment the agent learn to play
        epoch : int
            number of complete episodes to play
        batch_size : int
            number of experiences to replay per step
        exp_batch_size : int
            number of experiences to replay from the consolidated
            :attr:`ExperienceReplayexperience.experience`.
        gamma : float
            discount factor
        reset_memory : bool
            if we should restart :attr:`ExperienceReplay.memory` before
            starting the game.
        verbose : int
            controls how much should we print
        callbacks : list of callables
            TODO: Add callback support

        """
        print("Learning started!")
        print("[Environment]: {}".format(env.description))
        print("[Model]: {}".format(self.model.description))
        print("[Memory]: {}".format(self.memory.description))
        if reset_memory:
            self.reset()
        progbar = Progbar(epoch)

        for e in xrange(epoch):
            # reset environment on each epoch
            env.reset()
            game_over = False
            loss = 0
            rewards = 0

            # get initial observation, start game
            obs_t = env.observe()

            # Run an episonde
            while not game_over:
                obs_tm1 = obs_t
                action = self.policy(obs_tm1, train=True)

                # apply action, get rewards and new state
                obs_t, reward, game_over = env.update(action)
                rewards += reward

                # store experience
                self.remember(obs_tm1, action, reward, obs_t, game_over)

                # adapt model
                loss += self.update(batch_size=batch_size,
                                    exp_batch_size=exp_batch_size,
                                    gamma=gamma)
            if verbose == 1:
                progbar.add(1, values=[("loss", loss), ("rewards", rewards)])

    def play(self, env, epoch=1, batch_size=1, visualize=None, verbose=1):
        print("Free play started!")
        frames = np.zeros((0, ) + env.observe_image().shape[1:])
        frames = frames.transpose(0, 2, 3, 1)
        progbar = Progbar(epoch)

        for e in xrange(epoch):
            # reset environment on each epoch
            env.reset()
            game_over = False
            loss = 0
            rewards = 0

            # get initial observation, start game
            obs_t = env.observe()
            while not game_over:
                obs_tm1 = obs_t

                # get next action
                action = self.policy(obs_tm1, train=False)

                # apply action, get rewareds and new state
                obs_t, reward, game_over = env.update(action)
                rewards += reward

                frame_t = env.observe_image().transpose(0, 2, 3, 1)
                frames = np.concatenate([frames, frame_t], axis=0)

            if verbose == 1:
                progbar.add(1, values=[("loss", loss), ("rewards", rewards)])


        if visualize:
            from agnez.video import make_gif
            print("Making gif!")
            frames = np.repeat(frames, 3, axis=-1)
            make_gif(frames[:visualize['n_frames']],
                     filepath=visualize['filepath'], gray=visualize['gray'], interpolation='none')
            print("See your gif at {}".format(visualize['filepath']))

# -*- coding: utf-8 -*- Base Agent class      Parameters     ----------     model : :obj:`Model`         A learning model. Ex: neural network or table     memory : :obj:`Memory` Single Discrete action Agent      Parameters     ----------     model : :obj:`Model`         A learning model. Ex: neural network o} table     memory : :obj:`Memory`         Model's memory for storing experiences for replay and such.     epsilon : callable         A rule to define if model explore or exploit         TODO: generalize this to a class that controls if it should explore and         define custom explorations rules #    def compile(self, optimizer="sgd", loss="mse", policy_rule="max", #                experience=None): #        self.model.compile(optimizer, loss, policy_rule) #        self.memory.reset(experience) Train Agent to play Enviroment env          Parameters         ----------         env : :obj:`Enviroment`             The enviroment the agent learn to play         epoch : int             number of complete episodes to play         batch_size : int             number of experiences to replay per step         exp_batch_size : int             number of experiences to replay from the consolidated             :attr:`ExperienceReplayexperience.experience`.         gamma : float             discount factor         reset_memory : bool             if we should restart :attr:`ExperienceReplay.memory` before             starting the game.         verbose : int             controls how much should we print         callbacks : list of callables             TODO: Add callback support # reset environment on each epoch # get initial observation, start game # Run an episonde # apply action, get rewards and new state # store experience # adapt model # reset environment on each epoch # get initial observation, start game # get next action # apply action, get rewareds and new state