<reponame>o-evgeny/FinRL
# common library
import pandas as pd
import numpy as np
import time
import gym
import torch as th
#import tensorflow


# RL models from stable-baselines
# from stable_baselines import SAC
# from stable_baselines import TD3

from stable_baselines3.ppo import MlpPolicy
from stable_baselines3.common.vec_env import DummyVecEnv

from stable_baselines3 import DDPG
from stable_baselines3.common.noise import (
    NormalActionNoise,
    OrnsteinUhlenbeckActionNoise,
)

from finrl.config import config
from finrl.preprocessing.data import data_split
#from finrl.env.env_stocktrading import StockTradingEnv
#from finrl.env.env_stocktrading import StockTradingEnvShort as StockTradingEnv
#from finrl.env.env_stocktrading import StockTradingEnvShortPNLStop as StockTradingEnv
#from finrl.env.env_discr import StockTradingEnvShortPNL as StockTradingEnv
#from finrl.env.env_cont import StockTradingEnvShortPNL as StockTradingEnv
from finrl.env.env_cont2 import StockTradingEnvShortPNL as StockTradingEnv


from stable_baselines3 import A2C
from stable_baselines3 import PPO
from stable_baselines3 import TD3
from stable_baselines3 import SAC
from stable_baselines3.td3.policies import MlpPolicy as TD3Pol
from stable_baselines3.sac.policies import MlpPolicy as SACPol
from stable_baselines3.ppo.policies import MlpPolicy as PPOPol
from stable_baselines3.a2c.policies import MlpPolicy as A2CPol
from stable_baselines3.common.noise import (
    NormalActionNoise,
    OrnsteinUhlenbeckActionNoise,
)

from stable_baselines3.common.torch_layers import MlpExtractor


MODELS = {"a2c": A2C, "ddpg": DDPG, "td3": TD3, "sac": SAC, "ppo": PPO}

MODEL_KWARGS = {x: config.__dict__[f"{x.upper()}_PARAMS"] for x in MODELS.keys()}

NOISE = {
    "normal": NormalActionNoise,
    "ornstein_uhlenbeck": OrnsteinUhlenbeckActionNoise,
}


class DRLAgent:
    """Provides implementations for DRL algorithms

    Attributes
    ----------
        env: gym environment class
            user-defined class

    Methods
    -------
        train_PPO()
            the implementation for PPO algorithm
        train_A2C()
            the implementation for A2C algorithm
        train_DDPG()
            the implementation for DDPG algorithm
        train_TD3()
            the implementation for TD3 algorithm
        train_SAC()
            the implementation for SAC algorithm
        DRL_prediction()
            make a prediction in a test dataset and get results
    """

    @staticmethod
    def DRL_prediction(model, environment):
        test_env, test_obs = environment.get_sb_env()
        """make a prediction"""
        account_memory = []
        actions_memory = []
        test_env.reset()
        for i in range(len(environment.df.index.unique())):
            action, _states = model.predict(test_obs)
            #account_memory = test_env.env_method(method_name="save_asset_memory")
            #actions_memory = test_env.env_method(method_name="save_action_memory")
            test_obs, rewards, dones, info = test_env.step(action)
            if i == (len(environment.df.index.unique()) - 2):
              account_memory = test_env.env_method(method_name="save_asset_memory")
              actions_memory = test_env.env_method(method_name="save_action_memory")
            if dones[0]:
                print("hit end!")
                break
        return account_memory[0], actions_memory[0]

    def __init__(self, env):
        self.env = env

    def get_model(
        self,
        model_name,
        policy="MlpPolicy",
        policy_kwargs=None,
        model_kwargs=None,
        verbose=1,
    ):
        if model_name not in MODELS:
            raise NotImplementedError("NotImplementedError")

        if model_kwargs is None:
            model_kwargs = MODEL_KWARGS[model_name]

        if "action_noise" in model_kwargs:
            n_actions = self.env.action_space.shape[-1]
            model_kwargs["action_noise"] = NOISE[model_kwargs["action_noise"]](
                mean=np.zeros(n_actions), sigma=0.15 * np.ones(n_actions)
            )
        print(model_kwargs)
        model = MODELS[model_name](
            policy=policy,
            env=self.env,
            tensorboard_log=f"{config.TENSORBOARD_LOG_DIR}/{model_name}",
            verbose=verbose,
            policy_kwargs=policy_kwargs,
            **model_kwargs,
        )
        return model

    def train_model(self, model, tb_log_name, total_timesteps=5000):
        model = model.learn(total_timesteps=total_timesteps, tb_log_name=tb_log_name)
        return model


class DRLEnsembleAgent:
    @staticmethod
    def get_model(model_name,
                    env,
                    policy="MlpPolicy",
                    policy_kwargs=dict(activation_fn=th.nn.SELU,net_arch=[256,dict(pi=[256,256], vf=[256,256])]),#features_extractor_class=CustomCNN,features_extractor_kwargs=dict(features_dim=128))#,net_arch=dict(pi=[32,32], qf=[128,128])),
                    model_kwargs=None,
                    sigma_noise=0.3,
                    verbose=1):

        if model_name not in MODELS:
            raise NotImplementedError("NotImplementedError")

        if model_kwargs is None:
            temp_model_kwargs = MODEL_KWARGS[model_name]
        else:
            temp_model_kwargs = model_kwargs.copy()

        if "action_noise" in temp_model_kwargs:
            n_actions = env.action_space.shape[-1]
            temp_model_kwargs["action_noise"] = NOISE[temp_model_kwargs["action_noise"]](
                mean=np.zeros(n_actions), sigma=sigma_noise * np.ones(n_actions)
            )
        print(temp_model_kwargs)
        model = MODELS[model_name](
            policy=policy,
            env=env,
            tensorboard_log=f"{config.TENSORBOARD_LOG_DIR}/{model_name}",
            verbose=verbose,
            policy_kwargs=policy_kwargs,
            **temp_model_kwargs,
        )
        return model

    @staticmethod
    def train_model(model, model_name, tb_log_name, iter_num, total_timesteps=5000):
        model = model.learn(total_timesteps=total_timesteps, tb_log_name=tb_log_name)
        #self.model=model
        #model.save(f"{config.TRAINED_MODEL_DIR}/{model_name.upper()}_{total_timesteps//1000}k_{iter_num}")
        #model.save('PPO_010421')
        return model

    @staticmethod
    def get_validation_sharpe(iteration,model_name,best_selection_method='reward'):
        if best_selection_method=='sharpe':
        ###Calculate SharСВУ№pe ratio based on validation results###
            df_total_value = pd.read_csv('results/account_value_validation_{}_{}.csv'.format(model_name,iteration))
            if df_total_value['daily_return'].std()!=0:
                sharpe = (252 ** 0.5) * df_total_value['daily_return'].mean() / \
                        df_total_value['daily_return'].std()
                return sharpe
            else:
                return 0
        elif best_selection_method=='reward':
            df_total_value = pd.read_csv('results/account_value_validation_{}_{}.csv'.format(model_name,iteration))
            return df_total_value['account_value'].values[-1] - df_total_value['account_value'].values[0]

    def __init__(self,df,
                train_period,val_test_period,
                rebalance_window, validation_window,
                stock_dim,
                hmax,                
                initial_amount,
                buy_cost_pct,
                sell_cost_pct,
                reward_scaling,
                state_space,
                action_space,
                tech_indicator_list,
                print_verbosity,
                sigma_noise=0.1,
                prediction=False,
                 
                
                
                ):

        self.df=df
        self.train_period = train_period
        self.val_test_period = val_test_period

        self.unique_trade_date = df[(df.date > val_test_period[0])&(df.date <= val_test_period[1])].date.unique()
        self.rebalance_window = rebalance_window
        self.validation_window = validation_window

        self.stock_dim = stock_dim
        self.hmax = hmax
        self.initial_amount = initial_amount
        self.buy_cost_pct = buy_cost_pct
        self.sell_cost_pct = sell_cost_pct
        self.reward_scaling = reward_scaling
        self.state_space = state_space
        self.action_space = action_space
        self.tech_indicator_list = tech_indicator_list
        self.print_verbosity = print_verbosity
        self.sigma_noise=sigma_noise
        self.model=0
        self.prediction=prediction


    def DRL_validation(self, model, test_data, test_env, test_obs):
        ###validation process###
        for i in range(len(test_data.index.unique())):
            action, _states = model.predict(test_obs)
            test_obs, rewards, dones, info = test_env.step(action)

    def DRL_prediction(self,model,name,last_state,iter_num,turbulence_threshold,initial):
        ### make a prediction based on trained model###

        ## trading env
        trade_data = data_split(self.df, start=self.unique_trade_date[iter_num - self.rebalance_window], end=self.unique_trade_date[iter_num])
        trade_env = DummyVecEnv([lambda: StockTradingEnv(trade_data,
                                                        self.stock_dim,
                                                        self.hmax,
                                                        self.initial_amount,
                                                        self.buy_cost_pct,
                                                        self.sell_cost_pct,
                                                        self.reward_scaling,
                                                        self.state_space,
                                                        self.action_space,
                                                        self.tech_indicator_list,
                                                        turbulence_threshold=turbulence_threshold,
                                                        initial=initial,
                                                        previous_state=last_state,
                                                        model_name=name,
                                                        mode='trade',
                                                        iteration=iter_num,
                                                        print_verbosity=self.print_verbosity)])

        #pol=model.policy
        #f=open('pol.txt','w')
        #f.write(str(model.policy.state_dict()))
        #f.close()
               
        #policy.save('PPOPol')
        #model.save('test')
        #model=PPO.load('test')
        #self.model=model
        model=self.model
        trade_obs = trade_env.reset()
        #print('MINUS ACTION!!!!!!!!!!!!!!!!!!!')
        for i in range(len(trade_data.index.unique())):
            f=open('test.txt','a')
            
            action, _states = model.predict(trade_obs)
            #action=-action
            
            f.write('\ndata:'+str(i)+'\nobs:'+str(trade_obs)+'\nact:'+str(action)+'\n===')
            f.close()
            
            trade_obs, rewards, dones, info = trade_env.step(action)

            if i == (len(trade_data.index.unique()) - 2):
                # print(env_test.render())
                last_state = trade_env.render()
                f=open('test.txt','a')
                f.write('\nlastObs:'+str(last_state))
                f.close()

        df_last_state = pd.DataFrame({'last_state': last_state})
        df_last_state.to_csv('results/last_state_{}_{}.csv'.format(name, i), index=False)
        return last_state

    def run_ensemble_strategy(self,timesteps_dict,best_select_method='sharpe',A2C_model_kwargs=None,PPO_model_kwargs=None,DDPG_model_kwargs=None,TD3_model_kwargs=None,SAC_model_kwargs=None,turb=None):
        """Ensemble Strategy that combines PPO, A2C and DDPG"""
        print("============Start Ensemble Strategy============")
        # for ensemble model, it's necessary to feed the last state
        # of the previous model to the current model as the initial state
        last_state_ensemble = []
        self.best_select_method=best_select_method
        ppo_sharpe_list = []
        ddpg_sharpe_list = []
        a2c_sharpe_list = []
        td3_sharpe_list=[]
        sac_sharpe_list=[]

        model_use = []
        validation_start_date_list = []
        validation_end_date_list = []
        iteration_list = []
        if turb!=None:
            insample_turbulence = self.df[(self.df.date<self.train_period[1]) & (self.df.date>=self.train_period[0])]
            insample_turbulence_threshold = np.quantile(insample_turbulence.turbulence.values, turb)
        else:
            insample_turbulence_threshold=100
            

        start = time.time()
        for i in range(self.rebalance_window + self.validation_window, len(self.unique_trade_date), self.rebalance_window):
            validation_start_date = self.unique_trade_date[i - self.rebalance_window - self.validation_window]
            validation_end_date = self.unique_trade_date[i - self.rebalance_window]

            validation_start_date_list.append(validation_start_date)
            validation_end_date_list.append(validation_end_date)
            iteration_list.append(i)

            print("============================================")
            ## initial state is empty
            if i - self.rebalance_window - self.validation_window == 0:
                # inital state
                initial = True
            else:
                # previous state
                initial = False

            # Tuning trubulence index based on historical data
            # Turbulence lookback window is one quarter (63 days)
            end_date_index = self.df.index[self.df["date"] == self.unique_trade_date[i - self.rebalance_window - self.validation_window]].to_list()[-1]
            start_date_index = end_date_index - self.rebalance_window + 1
            if turb!=None:

                historical_turbulence = self.df.iloc[start_date_index:(end_date_index + 1), :]

                historical_turbulence = historical_turbulence.drop_duplicates(subset=['date'])

                historical_turbulence_mean = np.mean(historical_turbulence.turbulence.values)

                #print(historical_turbulence_mean)

                if historical_turbulence_mean > insample_turbulence_threshold:
                    # if the mean of the historical data is greater than the 90% quantile of insample turbulence data
                    # then we assume that the current market is volatile,
                    # therefore we set the 90% quantile of insample turbulence data as the turbulence threshold
                    # meaning the current turbulence can't exceed the 90% quantile of insample turbulence data
                    turbulence_threshold = insample_turbulence_threshold
                else:
                    # if the mean of the historical data is less than the 90% quantile of insample turbulence data
                    # then we tune up the turbulence_threshold, meaning we lower the risk
                    turbulence_threshold = np.quantile(insample_turbulence.turbulence.values, 1)

                turbulence_threshold = np.quantile(insample_turbulence.turbulence.values, 0.99) 
                print("turbulence_threshold: ", turbulence_threshold)
            else:
                turbulence_threshold=100

            ############## Environment Setup starts ##############
            ## training env
            train = data_split(self.df, start=self.train_period[0], end=self.unique_trade_date[i - self.rebalance_window - self.validation_window])
            print(self.train_period[0],self.unique_trade_date[i - self.rebalance_window - self.validation_window])
            self.train_env = DummyVecEnv([lambda: StockTradingEnv(train,
                                                                self.stock_dim,
                                                                self.hmax,
                                                                self.initial_amount,
                                                                self.buy_cost_pct,
                                                                self.sell_cost_pct,
                                                                self.reward_scaling,
                                                                self.state_space,
                                                                self.action_space,
                                                                self.tech_indicator_list,
                                                                print_verbosity=self.print_verbosity)])

            validation = data_split(self.df, start=self.unique_trade_date[i - self.rebalance_window - self.validation_window],
                                    end=self.unique_trade_date[i - self.rebalance_window])
            ############## Environment Setup ends ##############

            ############## Training and Validation starts ##############
            print("======Model training from: ", self.train_period[0], "to ",
                  self.unique_trade_date[i - self.rebalance_window - self.validation_window])
            # print("training: ",len(data_split(df, start=20090000, end=test.datadate.unique()[i-rebalance_window]) ))
            # print("==============Model Training===========")
            if A2C_model_kwargs==None:
                model_a2c=-100
                sharpe_a2c=-1e18
            else:
                print("======A2C Training========")

            
                model_a2c = self.get_model("a2c",self.train_env,policy="MlpPolicy",model_kwargs=A2C_model_kwargs)

                model_a2c = self.train_model(model_a2c, "a2c", tb_log_name="a2c_{}".format(i), iter_num = i, total_timesteps=timesteps_dict['a2c']) #100_000

                print("======A2C Validation from: ", validation_start_date, "to ",validation_end_date)
                val_env_a2c = DummyVecEnv([lambda: StockTradingEnv(validation,
                                                                    self.stock_dim,
                                                                    self.hmax,
                                                                    self.initial_amount,
                                                                    self.buy_cost_pct,
                                                                    self.sell_cost_pct,
                                                                    self.reward_scaling,
                                                                    self.state_space,
                                                                    self.action_space,
                                                                    self.tech_indicator_list,
                                                                    turbulence_threshold=turbulence_threshold,
                                                                    iteration=i,
                                                                    model_name='A2C',
                                                                    mode='validation',
                                                                    print_verbosity=self.print_verbosity)])
                val_obs_a2c = val_env_a2c.reset()
                self.DRL_validation(model=model_a2c,test_data=validation,test_env=val_env_a2c,test_obs=val_obs_a2c)
                sharpe_a2c = self.get_validation_sharpe(i,model_name="A2C")
                print("A2C Sharpe Ratio: ", sharpe_a2c)
            if PPO_model_kwargs==None:
                model_ppo=-100
                sharpe_ppo=-1e18
            else:
                
                print("======PPO Training========")
                model_ppo = self.get_model("ppo",self.train_env,policy="MlpPolicy",model_kwargs=PPO_model_kwargs)
                model_ppo = self.train_model(model_ppo, "ppo", tb_log_name="ppo_{}".format(i), iter_num = i, total_timesteps=timesteps_dict['ppo']) #100_000
                print("======PPO Validation from: ", validation_start_date, "to ",validation_end_date)
                val_env_ppo = DummyVecEnv([lambda: StockTradingEnv(validation,
                                                                    self.stock_dim,
                                                                    self.hmax,
                                                                    self.initial_amount,
                                                                    self.buy_cost_pct,
                                                                    self.sell_cost_pct,
                                                                    self.reward_scaling,
                                                                    self.state_space,
                                                                    self.action_space,
                                                                    self.tech_indicator_list,
                                                                    turbulence_threshold=turbulence_threshold,
                                                                    iteration=i,
                                                                    model_name='PPO',
                                                                    mode='validation',
                                                                    print_verbosity=self.print_verbosity)])
                val_obs_ppo = val_env_ppo.reset()
                self.DRL_validation(model=model_ppo,test_data=validation,test_env=val_env_ppo,test_obs=val_obs_ppo)
                sharpe_ppo = self.get_validation_sharpe(i,model_name="PPO")
                print("PPO Sharpe Ratio: ", sharpe_ppo)
            if DDPG_model_kwargs==None:
                model_ddpg=-100
                sharpe_ddpg=-1e18
            else:
                print("======DDPG Training========")
                model_ddpg = self.get_model("ddpg",self.train_env,policy="MlpPolicy",model_kwargs=DDPG_model_kwargs)
                model_ddpg = self.train_model(model_ddpg, "ddpg", tb_log_name="ddpg_{}".format(i), iter_num = i, total_timesteps=timesteps_dict['ddpg'])  #50_000
                print("======DDPG Validation from: ", validation_start_date, "to ",validation_end_date)
                val_env_ddpg = DummyVecEnv([lambda: StockTradingEnv(validation,
                                                                    self.stock_dim,
                                                                    self.hmax,
                                                                    self.initial_amount,
                                                                    self.buy_cost_pct,
                                                                    self.sell_cost_pct,
                                                                    self.reward_scaling,
                                                                    self.state_space,
                                                                    self.action_space,
                                                                    self.tech_indicator_list,
                                                                    turbulence_threshold=turbulence_threshold,
                                                                    iteration=i,
                                                                    model_name='DDPG',
                                                                    mode='validation',
                                                                    print_verbosity=self.print_verbosity)])
                val_obs_ddpg = val_env_ddpg.reset()
                self.DRL_validation(model=model_ddpg,test_data=validation,test_env=val_env_ddpg,test_obs=val_obs_ddpg)
                sharpe_ddpg = self.get_validation_sharpe(i,model_name="DDPG")
                print("DDPG Sharpe Ratio: ",sharpe_ddpg)
            
            if TD3_model_kwargs==None:
                model_td3=-100
                sharpe_td3=-1e18
            else:
                
                print("======TD3 Training========")
                model_td3 = self.get_model("td3",self.train_env,policy="MlpPolicy",model_kwargs=TD3_model_kwargs)
                model_td3 = self.train_model(model_td3, "td3", tb_log_name="td3_{}".format(i), iter_num = i, total_timesteps=timesteps_dict['td3'])  
                print("======TD3 Validation from: ", validation_start_date, "to ",validation_end_date)
                val_env_td3 = DummyVecEnv([lambda: StockTradingEnv(validation,
                                                                    self.stock_dim,
                                                                    self.hmax,
                                                                    self.initial_amount,
                                                                    self.buy_cost_pct,
                                                                    self.sell_cost_pct,
                                                                    self.reward_scaling,
                                                                    self.state_space,
                                                                    self.action_space,
                                                                    self.tech_indicator_list,
                                                                    turbulence_threshold=turbulence_threshold,
                                                                    iteration=i,
                                                                    model_name='TD3',
                                                                    mode='validation',
                                                                    print_verbosity=self.print_verbosity)])
                val_obs_td3 = val_env_td3.reset()
                self.DRL_validation(model=model_td3,test_data=validation,test_env=val_env_td3,test_obs=val_obs_td3)
                sharpe_td3 = self.get_validation_sharpe(i,model_name="TD3")
                print("TD3 Sharpe Ratio: ",sharpe_td3)
            
            if SAC_model_kwargs==None:
                model_sac=-100
                sharpe_sac=-1e18
            else:
                
                print("======SAC Training========")
                model_sac = self.get_model("sac",self.train_env,policy="MlpPolicy",model_kwargs=SAC_model_kwargs)
                model_sac = self.train_model(model_sac, "sac", tb_log_name="sac_{}".format(i), iter_num = i, total_timesteps=timesteps_dict['sac'])  
                print("======SAC Validation from: ", validation_start_date, "to ",validation_end_date)
                val_env_sac = DummyVecEnv([lambda: StockTradingEnv(validation,
                                                                    self.stock_dim,
                                                                    self.hmax,
                                                                    self.initial_amount,
                                                                    self.buy_cost_pct,
                                                                    self.sell_cost_pct,
                                                                    self.reward_scaling,
                                                                    self.state_space,
                                                                    self.action_space,
                                                                    self.tech_indicator_list,
                                                                    turbulence_threshold=turbulence_threshold,
                                                                    iteration=i,
                                                                    model_name='SAC',
                                                                    mode='validation',
                                                                    print_verbosity=self.print_verbosity)])
                val_obs_sac = val_env_sac.reset()
                self.DRL_validation(model=model_sac,test_data=validation,test_env=val_env_sac,test_obs=val_obs_sac)
                sharpe_sac = self.get_validation_sharpe(i,model_name="SAC")
                print("SAC Sharpe Ratio: ",sharpe_sac)
            
            
            ppo_sharpe_list.append(sharpe_ppo)
            a2c_sharpe_list.append(sharpe_a2c)
            ddpg_sharpe_list.append(sharpe_ddpg)
            td3_sharpe_list.append(sharpe_td3)
            sac_sharpe_list.append(sharpe_sac)
#sharpe_ddpg=-100;sharpe_sac=-100; sharpe_td3=-100; #model_ddpg=-100;model_sac=-100;model_td3=-100;
#
            print("======Best Model Retraining from: ", self.train_period[0], "to ",
                  self.unique_trade_date[i - self.rebalance_window])
            # Environment setup for model retraining up to first trade date
            #train_full = data_split(self.df, start=self.train_period[0], end=self.unique_trade_date[i - self.rebalance_window])
            #self.train_full_env = DummyVecEnv([lambda: StockTradingEnv(train_full,
            #                                                    self.stock_dim,
            #                                                    self.hmax,
            #                                                    self.initial_amount,
            #                                                    self.buy_cost_pct,
            #                                                    self.sell_cost_pct,
            #                                                    self.reward_scaling,
            #                                                    self.state_space,
            #                                                    self.action_space,
            #                                                    self.tech_indicator_list,
            #                                                    print_verbosity=self.print_verbosity)])
            # Model Selection based on sharpe ratio
            
            sharpes={'a2c':sharpe_a2c,'ppo':sharpe_ppo,'ddpg':sharpe_ddpg,'td3':sharpe_td3, 'sac':sharpe_sac}
            sharpes_models={'a2c':model_a2c,'ppo':model_ppo,'ddpg':model_ddpg,'td3':model_td3, 'sac':model_sac}
            best_sharpe=max(sharpes,key=sharpes.get)
            model_use.append(best_sharpe)
            model_ensemble=sharpes_models[best_sharpe]
            
            
            #if (sharpe_ppo >= sharpe_a2c) & (sharpe_ppo >= sharpe_ddpg):
            #    model_use.append('PPO')
            #    model_ensemble = model_ppo

                #model_ensemble = self.get_model("ppo",self.train_full_env,policy="MlpPolicy",model_kwargs=PPO_model_kwargs)
                #model_ensemble = self.train_model(model_ensemble, "ensemble", tb_log_name="ensemble_{}".format(i), iter_num = i, total_timesteps=timesteps_dict['ppo']) #100_000
           # elif (sharpe_a2c > sharpe_ppo) & (sharpe_a2c > sharpe_ddpg):
           #     model_use.append('A2C')
           #     model_ensemble = model_a2c

                #model_ensemble = self.get_model("a2c",self.train_full_env,policy="MlpPolicy",model_kwargs=A2C_model_kwargs)
                #model_ensemble = self.train_model(model_ensemble, "ensemble", tb_log_name="ensemble_{}".format(i), iter_num = i, total_timesteps=timesteps_dict['a2c']) #100_000
            #else:
            #    model_use.append('DDPG')
            #    model_ensemble = model_ddpg

                #model_ensemble = self.get_model("ddpg",self.train_full_env,policy="MlpPolicy",model_kwargs=DDPG_model_kwargs)
                #model_ensemble = self.train_model(model_ensemble, "ensemble", tb_log_name="ensemble_{}".format(i), iter_num = i, total_timesteps=timesteps_dict['ddpg']) #50_000

            ############## Training and Validation ends ##############

            ############## Trading starts ##############
            print("======Trading from: ", self.unique_trade_date[i - self.rebalance_window], "to ", self.unique_trade_date[i])
            #print("Used Model: ", model_ensemble)
            #import copy
            self.model=model_ensemble
            if self.prediction:
                last_state_ensemble = self.DRL_prediction(model=model_ensemble, name="ensemble",
                                                         last_state=last_state_ensemble, iter_num=i,
                                                         turbulence_threshold = turbulence_threshold,
                                                         initial=initial)
            
            ############## Trading ends ##############

        end = time.time()
        print("Ensemble Strategy took: ", (end - start) / 60, " minutes")

        df_summary = pd.DataFrame([iteration_list,validation_start_date_list,validation_end_date_list,model_use,a2c_sharpe_list,ppo_sharpe_list,ddpg_sharpe_list,td3_sharpe_list,sac_sharpe_list]).T
        df_summary.columns = ['Iter','Val Start','Val End','Model Used','A2C Sharpe','PPO Sharpe','DDPG Sharpe','TD3 Sharpe','SAC Sharpe']

        return df_summary
    
    
    def predict(self,obs):
        return self.model.predict(obs)
    def getModel(self):
        return self.model




# common library
import pandas as pd
import numpy as np
import time
import gym
import torch as th
#import tensorflow


# RL models from stable-baselines
# from stable_baselines import SAC
# from stable_baselines import TD3

from stable_baselines3.ppo import MlpPolicy
from stable_baselines3.common.vec_env import DummyVecEnv

from stable_baselines3 import DDPG
from stable_baselines3.common.noise import (
    NormalActionNoise,
    OrnsteinUhlenbeckActionNoise,
)

from finrl.config import config
from finrl.preprocessing.data import data_split
#from finrl.env.env_stocktrading import StockTradingEnv
#from finrl.env.env_stocktrading import StockTradingEnvShort as StockTradingEnv
#from finrl.env.env_stocktrading import StockTradingEnvShortPNLStop as StockTradingEnv
#from finrl.env.env_discr import StockTradingEnvShortPNL as StockTradingEnv
#from finrl.env.env_cont import StockTradingEnvShortPNL as StockTradingEnv
from finrl.env.env_cont2 import StockTradingEnvShortPNL as StockTradingEnv


from stable_baselines3 import A2C
from stable_baselines3 import PPO
from stable_baselines3 import TD3
from stable_baselines3 import SAC
from stable_baselines3.td3.policies import MlpPolicy as TD3Pol
from stable_baselines3.sac.policies import MlpPolicy as SACPol
from stable_baselines3.ppo.policies import MlpPolicy as PPOPol
from stable_baselines3.a2c.policies import MlpPolicy as A2CPol
from stable_baselines3.common.noise import (
    NormalActionNoise,
    OrnsteinUhlenbeckActionNoise,
)

from stable_baselines3.common.torch_layers import MlpExtractor


MODELS = {"a2c": A2C, "ddpg": DDPG, "td3": TD3, "sac": SAC, "ppo": PPO}

MODEL_KWARGS = {x: config.__dict__[f"{x.upper()}_PARAMS"] for x in MODELS.keys()}

NOISE = {
    "normal": NormalActionNoise,
    "ornstein_uhlenbeck": OrnsteinUhlenbeckActionNoise,
}


class DRLAgent:
    """Provides implementations for DRL algorithms

    Attributes
    ----------
        env: gym environment class
            user-defined class

    Methods
    -------
        train_PPO()
            the implementation for PPO algorithm
        train_A2C()
            the implementation for A2C algorithm
        train_DDPG()
            the implementation for DDPG algorithm
        train_TD3()
            the implementation for TD3 algorithm
        train_SAC()
            the implementation for SAC algorithm
        DRL_prediction()
            make a prediction in a test dataset and get results
    """

    @staticmethod
    def DRL_prediction(model, environment):
        test_env, test_obs = environment.get_sb_env()
        """make a prediction"""
        account_memory = []
        actions_memory = []
        test_env.reset()
        for i in range(len(environment.df.index.unique())):
            action, _states = model.predict(test_obs)
            #account_memory = test_env.env_method(method_name="save_asset_memory")
            #actions_memory = test_env.env_method(method_name="save_action_memory")
            test_obs, rewards, dones, info = test_env.step(action)
            if i == (len(environment.df.index.unique()) - 2):
              account_memory = test_env.env_method(method_name="save_asset_memory")
              actions_memory = test_env.env_method(method_name="save_action_memory")
            if dones[0]:
                print("hit end!")
                break
        return account_memory[0], actions_memory[0]

    def __init__(self, env):
        self.env = env

    def get_model(
        self,
        model_name,
        policy="MlpPolicy",
        policy_kwargs=None,
        model_kwargs=None,
        verbose=1,
    ):
        if model_name not in MODELS:
            raise NotImplementedError("NotImplementedError")

        if model_kwargs is None:
            model_kwargs = MODEL_KWARGS[model_name]

        if "action_noise" in model_kwargs:
            n_actions = self.env.action_space.shape[-1]
            model_kwargs["action_noise"] = NOISE[model_kwargs["action_noise"]](
                mean=np.zeros(n_actions), sigma=0.15 * np.ones(n_actions)
            )
        print(model_kwargs)
        model = MODELS[model_name](
            policy=policy,
            env=self.env,
            tensorboard_log=f"{config.TENSORBOARD_LOG_DIR}/{model_name}",
            verbose=verbose,
            policy_kwargs=policy_kwargs,
            **model_kwargs,
        )
        return model

    def train_model(self, model, tb_log_name, total_timesteps=5000):
        model = model.learn(total_timesteps=total_timesteps, tb_log_name=tb_log_name)
        return model


class DRLEnsembleAgent:
    @staticmethod
    def get_model(model_name,
                    env,
                    policy="MlpPolicy",
                    policy_kwargs=dict(activation_fn=th.nn.SELU,net_arch=[256,dict(pi=[256,256], vf=[256,256])]),#features_extractor_class=CustomCNN,features_extractor_kwargs=dict(features_dim=128))#,net_arch=dict(pi=[32,32], qf=[128,128])),
                    model_kwargs=None,
                    sigma_noise=0.3,
                    verbose=1):

        if model_name not in MODELS:
            raise NotImplementedError("NotImplementedError")

        if model_kwargs is None:
            temp_model_kwargs = MODEL_KWARGS[model_name]
        else:
            temp_model_kwargs = model_kwargs.copy()

        if "action_noise" in temp_model_kwargs:
            n_actions = env.action_space.shape[-1]
            temp_model_kwargs["action_noise"] = NOISE[temp_model_kwargs["action_noise"]](
                mean=np.zeros(n_actions), sigma=sigma_noise * np.ones(n_actions)
            )
        print(temp_model_kwargs)
        model = MODELS[model_name](
            policy=policy,
            env=env,
            tensorboard_log=f"{config.TENSORBOARD_LOG_DIR}/{model_name}",
            verbose=verbose,
            policy_kwargs=policy_kwargs,
            **temp_model_kwargs,
        )
        return model

    @staticmethod
    def train_model(model, model_name, tb_log_name, iter_num, total_timesteps=5000):
        model = model.learn(total_timesteps=total_timesteps, tb_log_name=tb_log_name)
        #self.model=model
        #model.save(f"{config.TRAINED_MODEL_DIR}/{model_name.upper()}_{total_timesteps//1000}k_{iter_num}")
        #model.save('PPO_010421')
        return model

    @staticmethod
    def get_validation_sharpe(iteration,model_name,best_selection_method='reward'):
        if best_selection_method=='sharpe':
        ###Calculate SharСВУ№pe ratio based on validation results###
            df_total_value = pd.read_csv('results/account_value_validation_{}_{}.csv'.format(model_name,iteration))
            if df_total_value['daily_return'].std()!=0:
                sharpe = (252 ** 0.5) * df_total_value['daily_return'].mean() / \
                        df_total_value['daily_return'].std()
                return sharpe
            else:
                return 0
        elif best_selection_method=='reward':
            df_total_value = pd.read_csv('results/account_value_validation_{}_{}.csv'.format(model_name,iteration))
            return df_total_value['account_value'].values[-1] - df_total_value['account_value'].values[0]

    def __init__(self,df,
                train_period,val_test_period,
                rebalance_window, validation_window,
                stock_dim,
                hmax,                
                initial_amount,
                buy_cost_pct,
                sell_cost_pct,
                reward_scaling,
                state_space,
                action_space,
                tech_indicator_list,
                print_verbosity,
                sigma_noise=0.1,
                prediction=False,
                 
                
                
                ):

        self.df=df
        self.train_period = train_period
        self.val_test_period = val_test_period

        self.unique_trade_date = df[(df.date > val_test_period[0])&(df.date <= val_test_period[1])].date.unique()
        self.rebalance_window = rebalance_window
        self.validation_window = validation_window

        self.stock_dim = stock_dim
        self.hmax = hmax
        self.initial_amount = initial_amount
        self.buy_cost_pct = buy_cost_pct
        self.sell_cost_pct = sell_cost_pct
        self.reward_scaling = reward_scaling
        self.state_space = state_space
        self.action_space = action_space
        self.tech_indicator_list = tech_indicator_list
        self.print_verbosity = print_verbosity
        self.sigma_noise=sigma_noise
        self.model=0
        self.prediction=prediction


    def DRL_validation(self, model, test_data, test_env, test_obs):
        ###validation process###
        for i in range(len(test_data.index.unique())):
            action, _states = model.predict(test_obs)
            test_obs, rewards, dones, info = test_env.step(action)

    def DRL_prediction(self,model,name,last_state,iter_num,turbulence_threshold,initial):
        ### make a prediction based on trained model###

        ## trading env
        trade_data = data_split(self.df, start=self.unique_trade_date[iter_num - self.rebalance_window], end=self.unique_trade_date[iter_num])
        trade_env = DummyVecEnv([lambda: StockTradingEnv(trade_data,
                                                        self.stock_dim,
                                                        self.hmax,
                                                        self.initial_amount,
                                                        self.buy_cost_pct,
                                                        self.sell_cost_pct,
                                                        self.reward_scaling,
                                                        self.state_space,
                                                        self.action_space,
                                                        self.tech_indicator_list,
                                                        turbulence_threshold=turbulence_threshold,
                                                        initial=initial,
                                                        previous_state=last_state,
                                                        model_name=name,
                                                        mode='trade',
                                                        iteration=iter_num,
                                                        print_verbosity=self.print_verbosity)])

        #pol=model.policy
        #f=open('pol.txt','w')
        #f.write(str(model.policy.state_dict()))
        #f.close()
               
        #policy.save('PPOPol')
        #model.save('test')
        #model=PPO.load('test')
        #self.model=model
        model=self.model
        trade_obs = trade_env.reset()
        #print('MINUS ACTION!!!!!!!!!!!!!!!!!!!')
        for i in range(len(trade_data.index.unique())):
            f=open('test.txt','a')
            
            action, _states = model.predict(trade_obs)
            #action=-action
            
            f.write('\ndata:'+str(i)+'\nobs:'+str(trade_obs)+'\nact:'+str(action)+'\n===')
            f.close()
            
            trade_obs, rewards, dones, info = trade_env.step(action)

            if i == (len(trade_data.index.unique()) - 2):
                # print(env_test.render())
                last_state = trade_env.render()
                f=open('test.txt','a')
                f.write('\nlastObs:'+str(last_state))
                f.close()

        df_last_state = pd.DataFrame({'last_state': last_state})
        df_last_state.to_csv('results/last_state_{}_{}.csv'.format(name, i), index=False)
        return last_state

    def run_ensemble_strategy(self,timesteps_dict,best_select_method='sharpe',A2C_model_kwargs=None,PPO_model_kwargs=None,DDPG_model_kwargs=None,TD3_model_kwargs=None,SAC_model_kwargs=None,turb=None):
        """Ensemble Strategy that combines PPO, A2C and DDPG"""
        print("============Start Ensemble Strategy============")
        # for ensemble model, it's necessary to feed the last state
        # of the previous model to the current model as the initial state
        last_state_ensemble = []
        self.best_select_method=best_select_method
        ppo_sharpe_list = []
        ddpg_sharpe_list = []
        a2c_sharpe_list = []
        td3_sharpe_list=[]
        sac_sharpe_list=[]

        model_use = []
        validation_start_date_list = []
        validation_end_date_list = []
        iteration_list = []
        if turb!=None:
            insample_turbulence = self.df[(self.df.date<self.train_period[1]) & (self.df.date>=self.train_period[0])]
            insample_turbulence_threshold = np.quantile(insample_turbulence.turbulence.values, turb)
        else:
            insample_turbulence_threshold=100
            

        start = time.time()
        for i in range(self.rebalance_window + self.validation_window, len(self.unique_trade_date), self.rebalance_window):
            validation_start_date = self.unique_trade_date[i - self.rebalance_window - self.validation_window]
            validation_end_date = self.unique_trade_date[i - self.rebalance_window]

            validation_start_date_list.append(validation_start_date)
            validation_end_date_list.append(validation_end_date)
            iteration_list.append(i)

            print("============================================")
            ## initial state is empty
            if i - self.rebalance_window - self.validation_window == 0:
                # inital state
                initial = True
            else:
                # previous state
                initial = False

            # Tuning trubulence index based on historical data
            # Turbulence lookback window is one quarter (63 days)
            end_date_index = self.df.index[self.df["date"] == self.unique_trade_date[i - self.rebalance_window - self.validation_window]].to_list()[-1]
            start_date_index = end_date_index - self.rebalance_window + 1
            if turb!=None:

                historical_turbulence = self.df.iloc[start_date_index:(end_date_index + 1), :]

                historical_turbulence = historical_turbulence.drop_duplicates(subset=['date'])

                historical_turbulence_mean = np.mean(historical_turbulence.turbulence.values)

                #print(historical_turbulence_mean)

                if historical_turbulence_mean > insample_turbulence_threshold:
                    # if the mean of the historical data is greater than the 90% quantile of insample turbulence data
                    # then we assume that the current market is volatile,
                    # therefore we set the 90% quantile of insample turbulence data as the turbulence threshold
                    # meaning the current turbulence can't exceed the 90% quantile of insample turbulence data
                    turbulence_threshold = insample_turbulence_threshold
                else:
                    # if the mean of the historical data is less than the 90% quantile of insample turbulence data
                    # then we tune up the turbulence_threshold, meaning we lower the risk
                    turbulence_threshold = np.quantile(insample_turbulence.turbulence.values, 1)

                turbulence_threshold = np.quantile(insample_turbulence.turbulence.values, 0.99) 
                print("turbulence_threshold: ", turbulence_threshold)
            else:
                turbulence_threshold=100

            ############## Environment Setup starts ##############
            ## training env
            train = data_split(self.df, start=self.train_period[0], end=self.unique_trade_date[i - self.rebalance_window - self.validation_window])
            print(self.train_period[0],self.unique_trade_date[i - self.rebalance_window - self.validation_window])
            self.train_env = DummyVecEnv([lambda: StockTradingEnv(train,
                                                                self.stock_dim,
                                                                self.hmax,
                                                                self.initial_amount,
                                                                self.buy_cost_pct,
                                                                self.sell_cost_pct,
                                                                self.reward_scaling,
                                                                self.state_space,
                                                                self.action_space,
                                                                self.tech_indicator_list,
                                                                print_verbosity=self.print_verbosity)])

            validation = data_split(self.df, start=self.unique_trade_date[i - self.rebalance_window - self.validation_window],
                                    end=self.unique_trade_date[i - self.rebalance_window])
            ############## Environment Setup ends ##############

            ############## Training and Validation starts ##############
            print("======Model training from: ", self.train_period[0], "to ",
                  self.unique_trade_date[i - self.rebalance_window - self.validation_window])
            # print("training: ",len(data_split(df, start=20090000, end=test.datadate.unique()[i-rebalance_window]) ))
            # print("==============Model Training===========")
            if A2C_model_kwargs==None:
                model_a2c=-100
                sharpe_a2c=-1e18
            else:
                print("======A2C Training========")

            
                model_a2c = self.get_model("a2c",self.train_env,policy="MlpPolicy",model_kwargs=A2C_model_kwargs)

                model_a2c = self.train_model(model_a2c, "a2c", tb_log_name="a2c_{}".format(i), iter_num = i, total_timesteps=timesteps_dict['a2c']) #100_000

                print("======A2C Validation from: ", validation_start_date, "to ",validation_end_date)
                val_env_a2c = DummyVecEnv([lambda: StockTradingEnv(validation,
                                                                    self.stock_dim,
                                                                    self.hmax,
                                                                    self.initial_amount,
                                                                    self.buy_cost_pct,
                                                                    self.sell_cost_pct,
                                                                    self.reward_scaling,
                                                                    self.state_space,
                                                                    self.action_space,
                                                                    self.tech_indicator_list,
                                                                    turbulence_threshold=turbulence_threshold,
                                                                    iteration=i,
                                                                    model_name='A2C',
                                                                    mode='validation',
                                                                    print_verbosity=self.print_verbosity)])
                val_obs_a2c = val_env_a2c.reset()
                self.DRL_validation(model=model_a2c,test_data=validation,test_env=val_env_a2c,test_obs=val_obs_a2c)
                sharpe_a2c = self.get_validation_sharpe(i,model_name="A2C")
                print("A2C Sharpe Ratio: ", sharpe_a2c)
            if PPO_model_kwargs==None:
                model_ppo=-100
                sharpe_ppo=-1e18
            else:
                
                print("======PPO Training========")
                model_ppo = self.get_model("ppo",self.train_env,policy="MlpPolicy",model_kwargs=PPO_model_kwargs)
                model_ppo = self.train_model(model_ppo, "ppo", tb_log_name="ppo_{}".format(i), iter_num = i, total_timesteps=timesteps_dict['ppo']) #100_000
                print("======PPO Validation from: ", validation_start_date, "to ",validation_end_date)
                val_env_ppo = DummyVecEnv([lambda: StockTradingEnv(validation,
                                                                    self.stock_dim,
                                                                    self.hmax,
                                                                    self.initial_amount,
                                                                    self.buy_cost_pct,
                                                                    self.sell_cost_pct,
                                                                    self.reward_scaling,
                                                                    self.state_space,
                                                                    self.action_space,
                                                                    self.tech_indicator_list,
                                                                    turbulence_threshold=turbulence_threshold,
                                                                    iteration=i,
                                                                    model_name='PPO',
                                                                    mode='validation',
                                                                    print_verbosity=self.print_verbosity)])
                val_obs_ppo = val_env_ppo.reset()
                self.DRL_validation(model=model_ppo,test_data=validation,test_env=val_env_ppo,test_obs=val_obs_ppo)
                sharpe_ppo = self.get_validation_sharpe(i,model_name="PPO")
                print("PPO Sharpe Ratio: ", sharpe_ppo)
            if DDPG_model_kwargs==None:
                model_ddpg=-100
                sharpe_ddpg=-1e18
            else:
                print("======DDPG Training========")
                model_ddpg = self.get_model("ddpg",self.train_env,policy="MlpPolicy",model_kwargs=DDPG_model_kwargs)
                model_ddpg = self.train_model(model_ddpg, "ddpg", tb_log_name="ddpg_{}".format(i), iter_num = i, total_timesteps=timesteps_dict['ddpg'])  #50_000
                print("======DDPG Validation from: ", validation_start_date, "to ",validation_end_date)
                val_env_ddpg = DummyVecEnv([lambda: StockTradingEnv(validation,
                                                                    self.stock_dim,
                                                                    self.hmax,
                                                                    self.initial_amount,
                                                                    self.buy_cost_pct,
                                                                    self.sell_cost_pct,
                                                                    self.reward_scaling,
                                                                    self.state_space,
                                                                    self.action_space,
                                                                    self.tech_indicator_list,
                                                                    turbulence_threshold=turbulence_threshold,
                                                                    iteration=i,
                                                                    model_name='DDPG',
                                                                    mode='validation',
                                                                    print_verbosity=self.print_verbosity)])
                val_obs_ddpg = val_env_ddpg.reset()
                self.DRL_validation(model=model_ddpg,test_data=validation,test_env=val_env_ddpg,test_obs=val_obs_ddpg)
                sharpe_ddpg = self.get_validation_sharpe(i,model_name="DDPG")
                print("DDPG Sharpe Ratio: ",sharpe_ddpg)
            
            if TD3_model_kwargs==None:
                model_td3=-100
                sharpe_td3=-1e18
            else:
                
                print("======TD3 Training========")
                model_td3 = self.get_model("td3",self.train_env,policy="MlpPolicy",model_kwargs=TD3_model_kwargs)
                model_td3 = self.train_model(model_td3, "td3", tb_log_name="td3_{}".format(i), iter_num = i, total_timesteps=timesteps_dict['td3'])  
                print("======TD3 Validation from: ", validation_start_date, "to ",validation_end_date)
                val_env_td3 = DummyVecEnv([lambda: StockTradingEnv(validation,
                                                                    self.stock_dim,
                                                                    self.hmax,
                                                                    self.initial_amount,
                                                                    self.buy_cost_pct,
                                                                    self.sell_cost_pct,
                                                                    self.reward_scaling,
                                                                    self.state_space,
                                                                    self.action_space,
                                                                    self.tech_indicator_list,
                                                                    turbulence_threshold=turbulence_threshold,
                                                                    iteration=i,
                                                                    model_name='TD3',
                                                                    mode='validation',
                                                                    print_verbosity=self.print_verbosity)])
                val_obs_td3 = val_env_td3.reset()
                self.DRL_validation(model=model_td3,test_data=validation,test_env=val_env_td3,test_obs=val_obs_td3)
                sharpe_td3 = self.get_validation_sharpe(i,model_name="TD3")
                print("TD3 Sharpe Ratio: ",sharpe_td3)
            
            if SAC_model_kwargs==None:
                model_sac=-100
                sharpe_sac=-1e18
            else:
                
                print("======SAC Training========")
                model_sac = self.get_model("sac",self.train_env,policy="MlpPolicy",model_kwargs=SAC_model_kwargs)
                model_sac = self.train_model(model_sac, "sac", tb_log_name="sac_{}".format(i), iter_num = i, total_timesteps=timesteps_dict['sac'])  
                print("======SAC Validation from: ", validation_start_date, "to ",validation_end_date)
                val_env_sac = DummyVecEnv([lambda: StockTradingEnv(validation,
                                                                    self.stock_dim,
                                                                    self.hmax,
                                                                    self.initial_amount,
                                                                    self.buy_cost_pct,
                                                                    self.sell_cost_pct,
                                                                    self.reward_scaling,
                                                                    self.state_space,
                                                                    self.action_space,
                                                                    self.tech_indicator_list,
                                                                    turbulence_threshold=turbulence_threshold,
                                                                    iteration=i,
                                                                    model_name='SAC',
                                                                    mode='validation',
                                                                    print_verbosity=self.print_verbosity)])
                val_obs_sac = val_env_sac.reset()
                self.DRL_validation(model=model_sac,test_data=validation,test_env=val_env_sac,test_obs=val_obs_sac)
                sharpe_sac = self.get_validation_sharpe(i,model_name="SAC")
                print("SAC Sharpe Ratio: ",sharpe_sac)
            
            
            ppo_sharpe_list.append(sharpe_ppo)
            a2c_sharpe_list.append(sharpe_a2c)
            ddpg_sharpe_list.append(sharpe_ddpg)
            td3_sharpe_list.append(sharpe_td3)
            sac_sharpe_list.append(sharpe_sac)
#sharpe_ddpg=-100;sharpe_sac=-100; sharpe_td3=-100; #model_ddpg=-100;model_sac=-100;model_td3=-100;
#
            print("======Best Model Retraining from: ", self.train_period[0], "to ",
                  self.unique_trade_date[i - self.rebalance_window])
            # Environment setup for model retraining up to first trade date
            #train_full = data_split(self.df, start=self.train_period[0], end=self.unique_trade_date[i - self.rebalance_window])
            #self.train_full_env = DummyVecEnv([lambda: StockTradingEnv(train_full,
            #                                                    self.stock_dim,
            #                                                    self.hmax,
            #                                                    self.initial_amount,
            #                                                    self.buy_cost_pct,
            #                                                    self.sell_cost_pct,
            #                                                    self.reward_scaling,
            #                                                    self.state_space,
            #                                                    self.action_space,
            #                                                    self.tech_indicator_list,
            #                                                    print_verbosity=self.print_verbosity)])
            # Model Selection based on sharpe ratio
            
            sharpes={'a2c':sharpe_a2c,'ppo':sharpe_ppo,'ddpg':sharpe_ddpg,'td3':sharpe_td3, 'sac':sharpe_sac}
            sharpes_models={'a2c':model_a2c,'ppo':model_ppo,'ddpg':model_ddpg,'td3':model_td3, 'sac':model_sac}
            best_sharpe=max(sharpes,key=sharpes.get)
            model_use.append(best_sharpe)
            model_ensemble=sharpes_models[best_sharpe]
            
            
            #if (sharpe_ppo >= sharpe_a2c) & (sharpe_ppo >= sharpe_ddpg):
            #    model_use.append('PPO')
            #    model_ensemble = model_ppo

                #model_ensemble = self.get_model("ppo",self.train_full_env,policy="MlpPolicy",model_kwargs=PPO_model_kwargs)
                #model_ensemble = self.train_model(model_ensemble, "ensemble", tb_log_name="ensemble_{}".format(i), iter_num = i, total_timesteps=timesteps_dict['ppo']) #100_000
           # elif (sharpe_a2c > sharpe_ppo) & (sharpe_a2c > sharpe_ddpg):
           #     model_use.append('A2C')
           #     model_ensemble = model_a2c

                #model_ensemble = self.get_model("a2c",self.train_full_env,policy="MlpPolicy",model_kwargs=A2C_model_kwargs)
                #model_ensemble = self.train_model(model_ensemble, "ensemble", tb_log_name="ensemble_{}".format(i), iter_num = i, total_timesteps=timesteps_dict['a2c']) #100_000
            #else:
            #    model_use.append('DDPG')
            #    model_ensemble = model_ddpg

                #model_ensemble = self.get_model("ddpg",self.train_full_env,policy="MlpPolicy",model_kwargs=DDPG_model_kwargs)
                #model_ensemble = self.train_model(model_ensemble, "ensemble", tb_log_name="ensemble_{}".format(i), iter_num = i, total_timesteps=timesteps_dict['ddpg']) #50_000

            ############## Training and Validation ends ##############

            ############## Trading starts ##############
            print("======Trading from: ", self.unique_trade_date[i - self.rebalance_window], "to ", self.unique_trade_date[i])
            #print("Used Model: ", model_ensemble)
            #import copy
            self.model=model_ensemble
            if self.prediction:
                last_state_ensemble = self.DRL_prediction(model=model_ensemble, name="ensemble",
                                                         last_state=last_state_ensemble, iter_num=i,
                                                         turbulence_threshold = turbulence_threshold,
                                                         initial=initial)
            
            ############## Trading ends ##############

        end = time.time()
        print("Ensemble Strategy took: ", (end - start) / 60, " minutes")

        df_summary = pd.DataFrame([iteration_list,validation_start_date_list,validation_end_date_list,model_use,a2c_sharpe_list,ppo_sharpe_list,ddpg_sharpe_list,td3_sharpe_list,sac_sharpe_list]).T
        df_summary.columns = ['Iter','Val Start','Val End','Model Used','A2C Sharpe','PPO Sharpe','DDPG Sharpe','TD3 Sharpe','SAC Sharpe']

        return df_summary
    
    
    def predict(self,obs):
        return self.model.predict(obs)
    def getModel(self):
        return self.model
# common library #import tensorflow # RL models from stable-baselines # from stable_baselines import SAC # from stable_baselines import TD3 #from finrl.env.env_stocktrading import StockTradingEnv #from finrl.env.env_stocktrading import StockTradingEnvShort as StockTradingEnv #from finrl.env.env_stocktrading import StockTradingEnvShortPNLStop as StockTradingEnv #from finrl.env.env_discr import StockTradingEnvShortPNL as StockTradingEnv #from finrl.env.env_cont import StockTradingEnvShortPNL as StockTradingEnv Provides implementations for DRL algorithms      Attributes     ----------         env: gym environment class             user-defined class      Methods     -------         train_PPO()             the implementation for PPO algorithm         train_A2C()             the implementation for A2C algorithm         train_DDPG()             the implementation for DDPG algorithm         train_TD3()             the implementation for TD3 algorithm         train_SAC()             the implementation for SAC algorithm         DRL_prediction()             make a prediction in a test dataset and get results make a prediction #account_memory = test_env.env_method(method_name="save_asset_memory") #actions_memory = test_env.env_method(method_name="save_action_memory") #features_extractor_class=CustomCNN,features_extractor_kwargs=dict(features_dim=128))#,net_arch=dict(pi=[32,32], qf=[128,128])), #self.model=model #model.save(f"{config.TRAINED_MODEL_DIR}/{model_name.upper()}_{total_timesteps//1000}k_{iter_num}") #model.save('PPO_010421') ###Calculate SharСВУ№pe ratio based on validation results### ###validation process### ### make a prediction based on trained model### ## trading env #pol=model.policy #f=open('pol.txt','w') #f.write(str(model.policy.state_dict())) #f.close() #policy.save('PPOPol') #model.save('test') #model=PPO.load('test') #self.model=model #print('MINUS ACTION!!!!!!!!!!!!!!!!!!!') #action=-action # print(env_test.render()) Ensemble Strategy that combines PPO, A2C and DDPG # for ensemble model, it's necessary to feed the last state # of the previous model to the current model as the initial state ## initial state is empty # inital state # previous state # Tuning trubulence index based on historical data # Turbulence lookback window is one quarter (63 days) #print(historical_turbulence_mean) # if the mean of the historical data is greater than the 90% quantile of insample turbulence data # then we assume that the current market is volatile, # therefore we set the 90% quantile of insample turbulence data as the turbulence threshold # meaning the current turbulence can't exceed the 90% quantile of insample turbulence data # if the mean of the historical data is less than the 90% quantile of insample turbulence data # then we tune up the turbulence_threshold, meaning we lower the risk ############## Environment Setup starts ############## ## training env ############## Environment Setup ends ############## ############## Training and Validation starts ############## # print("training: ",len(data_split(df, start=20090000, end=test.datadate.unique()[i-rebalance_window]) )) # print("==============Model Training===========") #100_000 #100_000 #50_000 #sharpe_ddpg=-100;sharpe_sac=-100; sharpe_td3=-100; #model_ddpg=-100;model_sac=-100;model_td3=-100; # # Environment setup for model retraining up to first trade date #train_full = data_split(self.df, start=self.train_period[0], end=self.unique_trade_date[i - self.rebalance_window]) #self.train_full_env = DummyVecEnv([lambda: StockTradingEnv(train_full, #                                                    self.stock_dim, #                                                    self.hmax, #                                                    self.initial_amount, #                                                    self.buy_cost_pct, #                                                    self.sell_cost_pct, #                                                    self.reward_scaling, #                                                    self.state_space, #                                                    self.action_space, #                                                    self.tech_indicator_list, #                                                    print_verbosity=self.print_verbosity)]) # Model Selection based on sharpe ratio #if (sharpe_ppo >= sharpe_a2c) & (sharpe_ppo >= sharpe_ddpg): #    model_use.append('PPO') #    model_ensemble = model_ppo #model_ensemble = self.get_model("ppo",self.train_full_env,policy="MlpPolicy",model_kwargs=PPO_model_kwargs) #model_ensemble = self.train_model(model_ensemble, "ensemble", tb_log_name="ensemble_{}".format(i), iter_num = i, total_timesteps=timesteps_dict['ppo']) #100_000 # elif (sharpe_a2c > sharpe_ppo) & (sharpe_a2c > sharpe_ddpg): #     model_use.append('A2C') #     model_ensemble = model_a2c #model_ensemble = self.get_model("a2c",self.train_full_env,policy="MlpPolicy",model_kwargs=A2C_model_kwargs) #model_ensemble = self.train_model(model_ensemble, "ensemble", tb_log_name="ensemble_{}".format(i), iter_num = i, total_timesteps=timesteps_dict['a2c']) #100_000 #else: #    model_use.append('DDPG') #    model_ensemble = model_ddpg #model_ensemble = self.get_model("ddpg",self.train_full_env,policy="MlpPolicy",model_kwargs=DDPG_model_kwargs) #model_ensemble = self.train_model(model_ensemble, "ensemble", tb_log_name="ensemble_{}".format(i), iter_num = i, total_timesteps=timesteps_dict['ddpg']) #50_000 ############## Training and Validation ends ############## ############## Trading starts ############## #print("Used Model: ", model_ensemble) #import copy ############## Trading ends ##############