import tensorflow as tf
import numpy as np
import os
from TOOLS.Logger import Logger
from ENVS.AbstractEnv import AbsEnv
from .model import AbsEvaluateModel, AbsControlModel


class DataBuffer(object):
    def __init__(self, obs_dim: int, act_dim: int, buffer_size: int, gamma: float):
        self.obs_t_buffer = np.zeros((buffer_size, obs_dim), dtype=np.float32)
        self.act_t_buffer = np.zeros((buffer_size, act_dim), dtype=np.float32)
        self.rew_t_buffer = np.zeros((buffer_size, ), dtype=np.float32)
        self.obs_n_buffer = np.zeros((buffer_size, obs_dim), dtype=np.float32)
        self.don_n_buffer = np.zeros((buffer_size, ), dtype=np.float32)
        self.rep_t_buffer = np.zeros((buffer_size, ), dtype=np.float32)

        self.gamma = gamma

        self.current_index = 0
        self.path_start_index = 0
        self.buffer_size = buffer_size

    def store(self, obs_t, act_t, rew_t, obs_n, don_n):
        cur_index = self.current_index
        self.obs_t_buffer[cur_index] = obs_t
        self.act_t_buffer[cur_index] = act_t
        self.rew_t_buffer[cur_index] = rew_t
        self.obs_n_buffer[cur_index] = obs_n
        self.don_n_buffer[cur_index] = don_n

        self.current_index += 1

    def finish_path(self, last_value):
        stage_index = slice(self.path_start_index, self.current_index)
        stage_rew = self.rew_t_buffer[stage_index]
        discount_rep = []
        cur_discount_rep = last_value[0]
        for rew_t in reversed(stage_rew):
            cur_discount_rep = self.gamma * cur_discount_rep + rew_t
            discount_rep.append(cur_discount_rep)
        self.rep_t_buffer[stage_index] = np.array(discount_rep[::-1])

        self.path_start_index = self.current_index

    def get_data(self):
        assert self.current_index == self.buffer_size
        self.current_index, self.path_start_index = 0, 0
        return self.obs_t_buffer, self.act_t_buffer, self.rep_t_buffer, self.don_n_buffer


class PPOTDLamda(object):
    def __init__(self, env: AbsEnv, policy_model: AbsControlModel, evaluate_model: AbsEvaluateModel,
                 model_save_dir: str, exp_name: str, gamma: float, logger: Logger, clip_ratio: float,
                 policy_lr: float, evaluate_lr: float, is_ou_noise: bool, act_lim: np.ndarray):
        self.env = env
        self.policy_model = policy_model
        self.evaluate_model = evaluate_model
        self.target_dir_path = os.path.join(model_save_dir, exp_name)
        if not os.path.exists(self.target_dir_path):
            os.mkdir(self.target_dir_path)
        self.target_file_path = os.path.join(self.target_dir_path, 'PPOTDLamda_Model.ckpt')
        self.gamma = gamma
        self.logger = logger
        self.clip_ratio = clip_ratio
        self.policy_lr = policy_lr
        self.evaluate_lr = evaluate_lr
        self.is_ou_noise = is_ou_noise
        self.act_lim = act_lim

        self.obs_t_ph, self.act_ph, self.adv_ph, self.partial_real_rep_ph, self.val, self.act_t, \
            self.update_pre_policy_params, self.adv_loss, self.eva_loss, self.train_adv_op, \
            self.train_eva_op = self.build_model()

    def build_model(self):
        # +++++++++++++++ 占位符 ++++++++++++++++++
        obs_t_ph = tf.placeholder(dtype=tf.float32, shape=(None, self.env.obs_dim), name='Obs_PH')
        act_ph = tf.placeholder(dtype=tf.float32, shape=(None, self.env.act_dim), name='Act_PH')
        adv_ph = tf.placeholder(dtype=tf.float32, shape=(None, 1), name='Adv_PH')
        partial_real_rep_ph = tf.placeholder(dtype=tf.float32, shape=(None, 1), name='Partial_Real_Repay_PH')
        # +++++++++++++++ 占位符 ++++++++++++++++++
        # +++++++++++++++ 模  型 ++++++++++++++++++
        val = self.evaluate_model.build_model(obs_t_ph)
        with tf.variable_scope('MainPolicyModel'):
            main_pi = self.policy_model.build_model(obs_t_ph, trainable=True)
        with tf.variable_scope('PrePolicyModel'):
            pre_pi = self.policy_model.build_model(obs_t_ph, trainable=False)
        main_policy_params = [i for i in tf.global_variables() if 'MainPolicyModel' in i.name]
        pre_policy_params = [i for i in tf.global_variables() if 'PrePolicyModel' in i.name]
        update_pre_policy_params = [tf.assign(pre_param, main_param)
                                    for main_param, pre_param in zip(main_policy_params, pre_policy_params)]
        act_t = tf.clip_by_value(tf.squeeze(main_pi.sample(1), axis=0), -self.act_lim, self.act_lim)
        pro_ph_act = main_pi.prob(act_ph)
        pre_pro_ph_act = tf.stop_gradient(pre_pi.prob(act_ph))
        # +++++++++++++++ 模  型 ++++++++++++++++++
        # +++++++++++++++ 算  法 ++++++++++++++++++
        ratio = pro_ph_act / pre_pro_ph_act
        adv_adj = ratio * adv_ph
        adv_loss = -tf.reduce_mean(tf.minimum(adv_adj, tf.clip_by_value(ratio,
                                                                        1-self.clip_ratio, 1+self.clip_ratio) * adv_ph))
        train_adv_op = tf.train.AdamOptimizer(learning_rate=self.policy_lr).minimize(adv_loss,
                                                                                     var_list=main_policy_params)
        eva_loss = tf.reduce_mean(tf.square(partial_real_rep_ph - val))
        train_eva_op = tf.train.AdamOptimizer(learning_rate=self.evaluate_lr).minimize(eva_loss)
        # +++++++++++++++ 算  法 ++++++++++++++++++
        return obs_t_ph, act_ph, adv_ph, partial_real_rep_ph, val, act_t, update_pre_policy_params, adv_loss, eva_loss,\
            train_adv_op, train_eva_op

    def train(self, learning_epoch, max_iter_per_epoch, mini_batch, retrain_label, update_num, save_freq):
        sess = tf.Session()
        model_saver = tf.train.Saver()
        if retrain_label:
            model_saver.restore(sess, tf.train.latest_checkpoint(self.target_dir_path))
        else:
            sess.run(tf.global_variables_initializer())
            sess.run(self.update_pre_policy_params)
        data_buffer = DataBuffer(self.env.obs_dim, self.env.act_dim, max_iter_per_epoch, self.gamma)
        for epoch in range(learning_epoch):
            obs_t = self.env.reset()
            ep_repay = 0
            for epoch_control_index in range(max_iter_per_epoch):
                act_t = sess.run(self.act_t, feed_dict={self.obs_t_ph: obs_t.reshape(1, -1)})[0]
                rew_t, obs_n, don_n, _ = self.env.step(act_t)
                rew_t = (rew_t + 8) / 8
                ep_repay += rew_t
                data_buffer.store(obs_t, act_t, rew_t, obs_n, don_n)
                obs_t = obs_n

                if (don_n or epoch_control_index % mini_batch == 0 or epoch_control_index == max_iter_per_epoch-1)\
                        and epoch_control_index > 0:
                    if don_n:
                        last_value = 0
                    else:
                        last_value = sess.run(self.val, feed_dict={self.obs_t_ph: obs_n.reshape(1, -1)})[0]
                    data_buffer.finish_path(last_value)

                    if don_n:
                        self.logger.to_log('游戏正常结束，总得分为：%.2f' % ep_repay)
                    elif epoch_control_index == max_iter_per_epoch - 1:
                        self.logger.to_log('游戏达到最大控制次数，总得分为：%.2f' % ep_repay)

                    if don_n:
                        obs_t = self.env.reset()
                        ep_repay = 0
            obs_t_buffer, act_t_buffer, rep_t_buffer, don_n_buffer = data_buffer.get_data()
            rep_t_buffer = np.reshape(rep_t_buffer, (len(rep_t_buffer), 1))
            don_n_buffer = np.reshape(don_n_buffer, (len(don_n_buffer), 1))
            val_t_buffer = sess.run(self.val, feed_dict={self.obs_t_ph: obs_t_buffer})
            adv_t_buffer = rep_t_buffer - val_t_buffer
            '''
            mean = np.mean(adv_t_buffer)
            std = np.std(adv_t_buffer)
            adv_t_buffer = (adv_t_buffer-mean) / std
            '''
            adv_loss_list = []
            eva_loss_list = []
            for _ in range(update_num):
                _, adv_loss = sess.run([self.train_adv_op, self.adv_loss],
                                       feed_dict={self.obs_t_ph: obs_t_buffer, self.act_ph: act_t_buffer,
                                                  self.adv_ph: adv_t_buffer})
                _, eva_loss = sess.run([self.train_eva_op, self.eva_loss],
                                       feed_dict={self.obs_t_ph: obs_t_buffer,
                                                  self.partial_real_rep_ph: rep_t_buffer})
                adv_loss_list.append(adv_loss)
                eva_loss_list.append(eva_loss)
            self.logger.to_log('[%d]广义优势损失为：%.4f    估值损失为：%.4f' % (epoch, np.mean(adv_loss_list),
                                                                                np.mean(eva_loss_list)))
            sess.run(self.update_pre_policy_params)
            if epoch > 1 and epoch % save_freq == 0:
                model_saver.save(sess, self.target_file_path, global_step=epoch)
        self.logger.to_log('学习过程结束，存储最终的模型参数')
        model_saver.save(sess, self.target_file_path, global_step=learning_epoch)



import tensorflow as tf
import numpy as np
import os
from TOOLS.Logger import Logger
from ENVS.AbstractEnv import AbsEnv
from .model import AbsEvaluateModel, AbsControlModel


class DataBuffer(object):
    def __init__(self, obs_dim: int, act_dim: int, buffer_size: int, gamma: float):
        self.obs_t_buffer = np.zeros((buffer_size, obs_dim), dtype=np.float32)
        self.act_t_buffer = np.zeros((buffer_size, act_dim), dtype=np.float32)
        self.rew_t_buffer = np.zeros((buffer_size, ), dtype=np.float32)
        self.obs_n_buffer = np.zeros((buffer_size, obs_dim), dtype=np.float32)
        self.don_n_buffer = np.zeros((buffer_size, ), dtype=np.float32)
        self.rep_t_buffer = np.zeros((buffer_size, ), dtype=np.float32)

        self.gamma = gamma

        self.current_index = 0
        self.path_start_index = 0
        self.buffer_size = buffer_size

    def store(self, obs_t, act_t, rew_t, obs_n, don_n):
        cur_index = self.current_index
        self.obs_t_buffer[cur_index] = obs_t
        self.act_t_buffer[cur_index] = act_t
        self.rew_t_buffer[cur_index] = rew_t
        self.obs_n_buffer[cur_index] = obs_n
        self.don_n_buffer[cur_index] = don_n

        self.current_index += 1

    def finish_path(self, last_value):
        stage_index = slice(self.path_start_index, self.current_index)
        stage_rew = self.rew_t_buffer[stage_index]
        discount_rep = []
        cur_discount_rep = last_value[0]
        for rew_t in reversed(stage_rew):
            cur_discount_rep = self.gamma * cur_discount_rep + rew_t
            discount_rep.append(cur_discount_rep)
        self.rep_t_buffer[stage_index] = np.array(discount_rep[::-1])

        self.path_start_index = self.current_index

    def get_data(self):
        assert self.current_index == self.buffer_size
        self.current_index, self.path_start_index = 0, 0
        return self.obs_t_buffer, self.act_t_buffer, self.rep_t_buffer, self.don_n_buffer


class PPOTDLamda(object):
    def __init__(self, env: AbsEnv, policy_model: AbsControlModel, evaluate_model: AbsEvaluateModel,
                 model_save_dir: str, exp_name: str, gamma: float, logger: Logger, clip_ratio: float,
                 policy_lr: float, evaluate_lr: float, is_ou_noise: bool, act_lim: np.ndarray):
        self.env = env
        self.policy_model = policy_model
        self.evaluate_model = evaluate_model
        self.target_dir_path = os.path.join(model_save_dir, exp_name)
        if not os.path.exists(self.target_dir_path):
            os.mkdir(self.target_dir_path)
        self.target_file_path = os.path.join(self.target_dir_path, 'PPOTDLamda_Model.ckpt')
        self.gamma = gamma
        self.logger = logger
        self.clip_ratio = clip_ratio
        self.policy_lr = policy_lr
        self.evaluate_lr = evaluate_lr
        self.is_ou_noise = is_ou_noise
        self.act_lim = act_lim

        self.obs_t_ph, self.act_ph, self.adv_ph, self.partial_real_rep_ph, self.val, self.act_t, \
            self.update_pre_policy_params, self.adv_loss, self.eva_loss, self.train_adv_op, \
            self.train_eva_op = self.build_model()

    def build_model(self):
        # +++++++++++++++ 占位符 ++++++++++++++++++
        obs_t_ph = tf.placeholder(dtype=tf.float32, shape=(None, self.env.obs_dim), name='Obs_PH')
        act_ph = tf.placeholder(dtype=tf.float32, shape=(None, self.env.act_dim), name='Act_PH')
        adv_ph = tf.placeholder(dtype=tf.float32, shape=(None, 1), name='Adv_PH')
        partial_real_rep_ph = tf.placeholder(dtype=tf.float32, shape=(None, 1), name='Partial_Real_Repay_PH')
        # +++++++++++++++ 占位符 ++++++++++++++++++
        # +++++++++++++++ 模  型 ++++++++++++++++++
        val = self.evaluate_model.build_model(obs_t_ph)
        with tf.variable_scope('MainPolicyModel'):
            main_pi = self.policy_model.build_model(obs_t_ph, trainable=True)
        with tf.variable_scope('PrePolicyModel'):
            pre_pi = self.policy_model.build_model(obs_t_ph, trainable=False)
        main_policy_params = [i for i in tf.global_variables() if 'MainPolicyModel' in i.name]
        pre_policy_params = [i for i in tf.global_variables() if 'PrePolicyModel' in i.name]
        update_pre_policy_params = [tf.assign(pre_param, main_param)
                                    for main_param, pre_param in zip(main_policy_params, pre_policy_params)]
        act_t = tf.clip_by_value(tf.squeeze(main_pi.sample(1), axis=0), -self.act_lim, self.act_lim)
        pro_ph_act = main_pi.prob(act_ph)
        pre_pro_ph_act = tf.stop_gradient(pre_pi.prob(act_ph))
        # +++++++++++++++ 模  型 ++++++++++++++++++
        # +++++++++++++++ 算  法 ++++++++++++++++++
        ratio = pro_ph_act / pre_pro_ph_act
        adv_adj = ratio * adv_ph
        adv_loss = -tf.reduce_mean(tf.minimum(adv_adj, tf.clip_by_value(ratio,
                                                                        1-self.clip_ratio, 1+self.clip_ratio) * adv_ph))
        train_adv_op = tf.train.AdamOptimizer(learning_rate=self.policy_lr).minimize(adv_loss,
                                                                                     var_list=main_policy_params)
        eva_loss = tf.reduce_mean(tf.square(partial_real_rep_ph - val))
        train_eva_op = tf.train.AdamOptimizer(learning_rate=self.evaluate_lr).minimize(eva_loss)
        # +++++++++++++++ 算  法 ++++++++++++++++++
        return obs_t_ph, act_ph, adv_ph, partial_real_rep_ph, val, act_t, update_pre_policy_params, adv_loss, eva_loss,\
            train_adv_op, train_eva_op

    def train(self, learning_epoch, max_iter_per_epoch, mini_batch, retrain_label, update_num, save_freq):
        sess = tf.Session()
        model_saver = tf.train.Saver()
        if retrain_label:
            model_saver.restore(sess, tf.train.latest_checkpoint(self.target_dir_path))
        else:
            sess.run(tf.global_variables_initializer())
            sess.run(self.update_pre_policy_params)
        data_buffer = DataBuffer(self.env.obs_dim, self.env.act_dim, max_iter_per_epoch, self.gamma)
        for epoch in range(learning_epoch):
            obs_t = self.env.reset()
            ep_repay = 0
            for epoch_control_index in range(max_iter_per_epoch):
                act_t = sess.run(self.act_t, feed_dict={self.obs_t_ph: obs_t.reshape(1, -1)})[0]
                rew_t, obs_n, don_n, _ = self.env.step(act_t)
                rew_t = (rew_t + 8) / 8
                ep_repay += rew_t
                data_buffer.store(obs_t, act_t, rew_t, obs_n, don_n)
                obs_t = obs_n

                if (don_n or epoch_control_index % mini_batch == 0 or epoch_control_index == max_iter_per_epoch-1)\
                        and epoch_control_index > 0:
                    if don_n:
                        last_value = 0
                    else:
                        last_value = sess.run(self.val, feed_dict={self.obs_t_ph: obs_n.reshape(1, -1)})[0]
                    data_buffer.finish_path(last_value)

                    if don_n:
                        self.logger.to_log('游戏正常结束，总得分为：%.2f' % ep_repay)
                    elif epoch_control_index == max_iter_per_epoch - 1:
                        self.logger.to_log('游戏达到最大控制次数，总得分为：%.2f' % ep_repay)

                    if don_n:
                        obs_t = self.env.reset()
                        ep_repay = 0
            obs_t_buffer, act_t_buffer, rep_t_buffer, don_n_buffer = data_buffer.get_data()
            rep_t_buffer = np.reshape(rep_t_buffer, (len(rep_t_buffer), 1))
            don_n_buffer = np.reshape(don_n_buffer, (len(don_n_buffer), 1))
            val_t_buffer = sess.run(self.val, feed_dict={self.obs_t_ph: obs_t_buffer})
            adv_t_buffer = rep_t_buffer - val_t_buffer
            '''
            mean = np.mean(adv_t_buffer)
            std = np.std(adv_t_buffer)
            adv_t_buffer = (adv_t_buffer-mean) / std
            '''
            adv_loss_list = []
            eva_loss_list = []
            for _ in range(update_num):
                _, adv_loss = sess.run([self.train_adv_op, self.adv_loss],
                                       feed_dict={self.obs_t_ph: obs_t_buffer, self.act_ph: act_t_buffer,
                                                  self.adv_ph: adv_t_buffer})
                _, eva_loss = sess.run([self.train_eva_op, self.eva_loss],
                                       feed_dict={self.obs_t_ph: obs_t_buffer,
                                                  self.partial_real_rep_ph: rep_t_buffer})
                adv_loss_list.append(adv_loss)
                eva_loss_list.append(eva_loss)
            self.logger.to_log('[%d]广义优势损失为：%.4f    估值损失为：%.4f' % (epoch, np.mean(adv_loss_list),
                                                                                np.mean(eva_loss_list)))
            sess.run(self.update_pre_policy_params)
            if epoch > 1 and epoch % save_freq == 0:
                model_saver.save(sess, self.target_file_path, global_step=epoch)
        self.logger.to_log('学习过程结束，存储最终的模型参数')
        model_saver.save(sess, self.target_file_path, global_step=learning_epoch)



# +++++++++++++++ 占位符 ++++++++++++++++++ # +++++++++++++++ 占位符 ++++++++++++++++++ # +++++++++++++++ 模  型 ++++++++++++++++++ # +++++++++++++++ 模  型 ++++++++++++++++++ # +++++++++++++++ 算  法 ++++++++++++++++++ # +++++++++++++++ 算  法 ++++++++++++++++++ mean = np.mean(adv_t_buffer)
             std = np.std(adv_t_buffer)
             adv_t_buffer = (adv_t_buffer-mean) / std