<reponame>o-evgeny/FinRL
import numpy as np
import pandas as pd
from gym.utils import seeding
import gym
from gym import spaces
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import pickle
from stable_baselines3.common.vec_env import DummyVecEnv
#from stable_baselines3.common import logger
class StockTradingEnvShortPNL (gym.Env):
    """A stock trading environment for OpenAI gym"""
    metadata = {'render.modes': ['human']}

    def __init__(self, 
                df, 
                stock_dim,
                hmax,                
                initial_amount,
                buy_cost_pct,
                sell_cost_pct,
                reward_scaling,
                state_space,
                action_space,
                tech_indicator_list,
                turbulence_threshold=None,
                make_plots = False, 
                print_verbosity = 10,
                day = 0, 
                initial=True,
                previous_state=[],
                model_name = '',
                mode='',
                iteration=''):
        print("HELLLLOv5!!!")
        self.day = day
        self.df = df
        self.stock_dim = stock_dim
        self.hmax = hmax
        self.initial_amount = initial_amount
        self.buy_cost_pct = buy_cost_pct
        self.sell_cost_pct = sell_cost_pct
        self.reward_scaling = reward_scaling
        self.state_space = state_space
        self.action_space = action_space
        self.tech_indicator_list = tech_indicator_list
        #self.action_space = spaces.Box(low = -1, high = 1,shape = (1,),dtype=np.float32) 
        self.action_space = spaces.Discrete(9) 
        #self.action_space = spaces.MultiDiscrete([2,2,21,51])
        self.observation_space = spaces.Box(low=-np.inf, high=np.inf, shape = (self.state_space,))
        self.data = self.df.loc[self.day,:]
        self.terminal = False
        self.make_plots = make_plots
        self.print_verbosity = print_verbosity
        self.turbulence_threshold = None
        self.initial = initial
        self.previous_state = previous_state
        self.model_name=model_name
        self.mode=mode 
        self.iteration=iteration
        self.openedPos=0
        self.lastPos=0
        self.lastPrice=0
        self.longPosQty=0
        self.longPosBP=0
        self.shortPosQty=0
        self.shortPosBP=0
        self.shortPoses=[]
        # initalize state
        self.precision=10
        self.stoploss=0
        self.takeprofit=0
        self.state = self._initiate_state()
        
        # initialize reward
        self.reward = 0
        self.turbulence = 0
        self.cost = 0
        self.trades = 0
        self.episode = 0
        
        self.musor=[]
        
        # memorize all the total balance change
        self.asset_memory = [self.initial_amount]
        self.states0_memory= [self.state[0]]
        self.states1_memory= [self.state[1]]
        self.states3_memory= [self.state[3]]
        self.states4_memory= [self.state[4]]
        self.states5_memory= [self.state[5]]
        self.states6_memory= [self.state[6]]
        self.states7_memory= [self.state[7]]
        self.states8_memory= [self.state[8]]
        self.states9_memory= [self.state[9]]
        self.states10_memory= [self.state[10]]
        self.states11_memory= [self.state[11]]
        self.states12_memory= [self.state[12]]
        self.states13_memory= [self.state[13]]
        self.states14_memory= [self.state[14]]
        self.states15_memory= [self.state[15]]
        self.states16_memory=[0]
        self.states16a_memory= [self.state[16]]
        self.rewards_memory = []
        self.actions_memory=[]
        self.date_memory=[self._get_date()]
        self.action=0
        #self.reset()
        self._seed()
        

    def PNL(self,stopPrice=None):
        if stopPrice==None:
            if self.openedPos==1:

                return (self.state[1]-self.state[4])*self.state[3]
            elif self.openedPos==-1:
                return (self.state[6]-self.state[1])*self.state[5]
            else:
                #print('0 pos')
                return 0
        else:
            if self.openedPos==1:

                return (stopPrice-self.state[4])*self.state[3]
            elif self.openedPos==-1:
                return (self.state[6]-stopPrice)*self.state[5]
            else:
                #print('0 pos')
                return 0
            
    def _sell_stock(self, action):
        index=0
       # action=self.avShortQty()*abs(action)+self.avLongQty(
        if True:
                short_num_shares = 0
                if self.state[index+1]>0: 
                    # Sell only if the price is > 0 (no missing data in this particular date)
                    # perform sell action based on the sign of the action
                    if self.openedPos==1:
                        #if self.state[index+self.stock_dim+1] > 0:
                            # Sell only if current asset is > 0
                            action=(self.avShortQty()+self.avLongQty())*abs(action)
                            if abs(action)<self.state[3]:
                                #total_contract_value=self.state[3]*self.state[4] - abs(action)*self.state[1]
                                #avg_entry_price=total_contract_value/(self.state[3] - abs(action))
                                pnl_before=self.PNL()
                                self.state[0]+=self.state[4]*abs(action)
                                self.state[3] -= abs(action)
                                #self.state[4] = avg_entry_price
                                pnl_after=self.PNL()
                                self.state[0] += pnl_before - pnl_after
                                
                                self.trades+=1
                                
                            elif abs(action)==self.state[3]:
                                pnl=self.PNL()
                                self.state[0]+=self.state[4]*self.state[3]
                                self.state[3]=0
                                self.state[4]=0
                                self.state[0]+=pnl
                                
                                self.openedPos=0
                                
                                self.trades+=1
                            
                            elif abs(action)>self.state[3]:
                                
                                pnl=self.PNL()
                                add_qty=abs(action)-self.state[3]
                                self.state[0]+=self.state[3]*self.state[4]
                                self.state[3]=0
                                self.state[4]=0
                                self.state[0]+=pnl
                                self.openedPos=0
                                
                                available_qty = max(0,self.state[0]//self.state[1])
                                if available_qty>0:
                                    short_qty = min(add_qty,available_qty)
                                    self.state[5]=short_qty
                                    self.state[6]=self.state[1]
                                    self.state[0]-= short_qty * self.state[1]

                                    self.openedPos=-1

                                self.trades+=1
                               

                    elif self.openedPos==0:
                        action=self.avShortQty()*abs(action)
                        available_qty = max(0,self.state[0] // self.state[1])
                        short_qty = min(available_qty, abs(action))
                        self.state[0] -= short_qty * self.state[1]
                        self.state[5]=short_qty
                        self.state[6]=self.state[1]
                        
                        self.openedPos=-1
                        
                        self.trades+=1
                        
                    elif self.openedPos==-1:
                        action=self.avShortQty()*abs(action)
                        pnl=self.PNL()
                        #self.state[0]+=pnl 
                        available_qty=max(0,(self.state[0]+pnl) // self.state[1])
                        add_qty=min(available_qty,abs(action))
                        if add_qty>0:
                            self.state[0] -= self.state[1] * add_qty ####
                            total_contract_value=self.state[5]*self.state[6] + abs(add_qty) * self.state[1]
                            avg_entry_price=total_contract_value/(self.state[5] + abs(add_qty))
                            self.state[5]+=add_qty
                            self.state[6]=avg_entry_price

                            self.trades+=1
                                                                  
        else:
            short_num_shares = 0
            print("sell action:",action)
        #print('states:',self.state[3],self.state[4],self.state[5],self.state[6])    
        return 0
    
                                                              
         

    
    def _buy_stock(self, action):
        index=0
        #action=self.avLongQty()*abs(action)
        if True:                                                          
            
                if self.state[index+1]>0: 
                    # Sell only if the price is > 0 (no missing data in this particular date)
                    # perform sell action based on the sign of the action
                    if self.openedPos==1:
                        action=self.avLongQty()*abs(action)
                        #if self.state[index+self.stock_dim+1] > 0:
                            # Sell only if current asset is > 0
                        pnl=self.PNL()
                        #self.state[0]+=pnl
                        available_qty=max(0,(self.state[0]+pnl) // self.state[1])
                        add_qty=min(available_qty,abs(action))
                        if add_qty>0:
                            self.state[0] -= self.state[1]*abs(add_qty) ######

                            total_contract_value=self.state[3]*self.state[4] + abs(add_qty) * self.state[1]
                            avg_entry_price=total_contract_value/(self.state[3] + abs(add_qty))
                            self.state[3]+=add_qty
                            self.state[4]=avg_entry_price
                            self.trades+=1
                                

                    elif self.openedPos==0:
                        action=self.avLongQty()*abs(action)
                        available_qty = max(0,self.state[0] // self.state[1])
                        long_qty = min(available_qty, abs(action))
                        self.state[0] -= long_qty * self.state[1]
                        self.state[3]=long_qty
                        self.state[4]=self.state[1]
                        
                        self.openedPos=1
                        
                        self.trades+=1
                        
                    elif self.openedPos==-1:
                        action=(self.avShortQty()+self.avLongQty())*abs(action)
                        if abs(action)<self.state[5]:
                            #total_contract_value=self.state[5]*self.state[6] - abs(action)*self.state[1]
                            #avg_entry_price=total_contract_value/(self.state[5] - abs(action))
                            pnl_before=self.PNL()
                            self.state[0]+=self.state[6]*abs(action)
                            #self.state[0]-=self.state[1]*abs(action) #######
                            self.state[5] -= abs(action)
                            #self.state[6] = avg_entry_price
                            pnl_after=self.PNL()
                            self.state[0] += pnl_before - pnl_after
                            self.trades+=1

                        elif abs(action)==self.state[5]:
                            pnl=self.PNL()
                            self.state[0]+=self.state[6]*abs(action)
                            self.state[5]=0
                            self.state[6]=0
                            self.state[0]+=pnl

                            self.openedPos=0

                            self.trades+=1

                        elif abs(action)>self.state[5]:
                            pnl=self.PNL()
                            add_qty=abs(action)-self.state[5]
                            self.state[0]+=self.state[5]*self.state[6]
                            self.state[5]=0
                            self.state[6]=0
                            self.state[0]+=pnl
                            self.openedPos=0

                            available_qty=max(0,self.state[0]//self.state[1])
                            if available_qty>0:
                                
                                long_qty=min(add_qty,available_qty)
                                self.state[3]=long_qty
                                self.state[4]=self.state[1]
                                self.state[0]-= long_qty * self.state[1]

                                self.openedPos=1

                            self.trades+=1
        
        else:
            short_num_shares = 0
            print("buy action:",action)
        #print('states:',self.state[3],self.state[4],self.state[5],self.state[6])
        return 0
    def avLongQty(self):
        pnl=self.PNL()
        short_value=self.state[5]*self.state[1]
        avQty=round((self.state[0]+pnl + short_value)/self.state[1],self.precision)
        return max(0,avQty)
    
    def avShortQty(self):
        pnl=self.PNL()
        long_value=self.state[3]*self.state[1]
        avQty=round((self.state[0]+pnl + long_value)/self.state[1],self.precision)
        return max(0,avQty)
    
    def shortPos(self,direct,pct_qty):
        
        if self.openedPos==0:
            if direct==0:
                return 0
            elif direct==1:
                shortQty=round(self.avShortQty()*pct_qty,self.precision)
                self.state[5]+=shortQty
                self.state[6]=self.state[1]
                self.state[0]-=shortQty*self.state[1]
                self.openedPos=-1
                self.trades+=1
                
        elif self.openedPos==1:
            avQty=self.avShortQty()
            if avQty>0:
                pnl=self.PNL()
                self.state[0]+=self.state[3]*self.state[4] + pnl
                self.state[3]=0
                self.state[4]=0
                
                self.state[5]=round(avQty * pct_qty,self.precision)
                self.state[6]=self.state[1]
                self.state[0]-=self.state[5]*self.state[6]
                self.openedPos=-1
                self.trades+=1
                
               
                    
        elif self.openedPos==-1:
            if direct==0:
                reduceShortQty=round(self.state[5]*pct_qty,self.precision)
                pnl_before=self.PNL()
                self.state[5]-=reduceShortQty
                self.state[0]+=self.state[6]*reduceShortQty
                pnl_after=self.PNL()
                self.state[0]+=pnl_before-pnl_after
                
                if self.state[5]==0:
                    self.state[6]=0
                    self.openedPos=0
                
                self.trades+=1
                
            elif direct==1:
                avQty=self.avShortQty()
                if avQty>0 and pct_qty>0:
                    shortQty=round(avQty * pct_qty,self.precision)
                    self.state[0] -= self.state[1]*shortQty
                    
                    total_contract_value=self.state[5]*self.state[6] + shortQty * self.state[1]
                    avg_entry_price=round(total_contract_value/(self.state[5] + shortQty),self.precision)
                    self.state[5]+=shortQty
                    self.state[6]=avg_entry_price
                    self.trades+=1
                    
    
    def longPos(self,direct,pct_qty):
        if self.openedPos==0:
            if direct==0:
                return 0
            elif direct==1:
                longQty=round(self.avLongQty()*pct_qty,self.precision)
                self.state[3]+=longQty
                self.state[4]=self.state[1]
                self.state[0]-=longQty*self.state[1]
                self.openedPos=1
                self.trades+=1
        elif self.openedPos==1:
            if direct==0:
                reduceLongQty=round(self.state[3]*pct_qty,self.precision)
                pnl_before=self.PNL()
                self.state[3]-=reduceLongQty
                self.state[0]+=self.state[4]*reduceLongQty
                pnl_after=self.PNL()
                self.state[0]+=pnl_before-pnl_after
                
                if self.state[3]==0:
                    self.state[4]=0
                    self.openedPos=0
                
                self.trades+=1
                
            elif direct==1:
                avQty=self.avLongQty()
                if avQty>0 and pct_qty>0:
                    longQty=round(avQty * pct_qty,self.precision)
                    self.state[0] -= self.state[1]*longQty
                    
                    total_contract_value=self.state[3]*self.state[4] + longQty * self.state[1]
                    avg_entry_price=round(total_contract_value/(self.state[3] + longQty),self.precision)
                    self.state[3]+=longQty
                    self.state[4]=avg_entry_price
                    self.trades+=1
                    
        elif self.openedPos==-1:
            avQty=self.avLongQty()
            if avQty>0:
                pnl=self.PNL()
                self.state[0]+=self.state[5]*self.state[6] + pnl
                self.state[5]=0
                self.state[6]=0
                
                self.state[3]=round(avQty * pct_qty,self.precision)
                self.state[4]=self.state[1]
                self.state[0]-=self.state[3]*self.state[4]
                self.openedPos=1
                self.trades+=1
            
    def closePos(self):
        #pnl=self.PNL()
        self.state[0]=self.get_end_asset()
        self.state[3]=0; self.state[4]=0
        self.state[5]=0; self.state[6]=0
        self.openedPos=0
        
    def long(self,action):
      qty=action*self.state[0]/self.state[1]
      self.state[0]-=qty*self.state[1]
      self.state[3]=qty
      self.state[4]=self.state[1]
      self.openedPos=1
      self.trades+=1
      
      
    def short(self,action):
      qty=action*self.state[0]/self.state[1]
      self.state[0]-=qty*self.state[1]
      self.state[5]=qty
      self.state[6]=self.state[1]
      self.openedPos=-1
      self.trades+=1
                                                          
                    
            
                                                              
    def get_end_asset(self):
        end_total_asset= self.state[0]+ \
                self.state[3]*self.state[4] + (self.state[1] - self.state[4])*self.state[3] +\
                self.state[6]*self.state[5] + (self.state[6]-self.state[1])*self.state[5]
        #print(end_total_asset, self.state[0],self.longPosBP,self.longPosQty,self.shortPosBP,self.shortPosQty)
        return end_total_asset
    def _make_plot(self):
        plt.plot(self.asset_memory,'r')
        plt.savefig('results/account_value_trade_{}.png'.format(self.episode))
        plt.close()

    def step(self, actions):
        self.terminal = self.day >= len(self.df.index.unique())-1
        if self.terminal:
            # print(f"Episode: {self.episode}")
            if self.make_plots:
                self._make_plot()            
            end_total_asset = self.get_end_asset()
            # sum(np.array(self.state[1:(self.stock_dim+1)])*np.array(self.state[(self.stock_dim+1):(self.stock_dim*2+1)]))
            #print(np.array(self.state[1:(self.stock_dim+1)]),np.array(self.state[(self.stock_dim+1):(self.stock_dim*2+1)]),np.array(self.state[1:(self.stock_dim+1)])*np.array(self.state[(self.stock_dim+1):(self.stock_dim*2+1)]))
            df_total_value = pd.DataFrame(self.asset_memory)
            tot_reward =  self.get_end_asset() - self.initial_amount
            #print("tot reward:", tot_reward,"params: state[0], self.state[3:6]", self.state[0],self.state[3],self.state[4],self.state[5],self.state[6])
            df_total_value.columns = ['account_value']
            df_total_value['date'] = self.date_memory
            df_total_value['daily_return']=df_total_value['account_value'].pct_change(1)
            #self.actions_memory.append('nan')
            #df_total_value['actions']=self.actions_memory
            df_total_value['state[0]']=self.states0_memory
            df_total_value['state[1]']=self.states1_memory
            df_total_value['state[3]']=self.states3_memory
            df_total_value['state[4]']=self.states4_memory
            df_total_value['state[5]']=self.states5_memory
            df_total_value['act']=self.states16_memory
            df_total_value['state[6]']=self.states6_memory
            df_total_value['begin_asset']=end_total_asset
            
            df_total_value['avlong']=self.states7_memory
            df_total_value['avshort']=self.states8_memory
            df_total_value['avsum']=self.states16a_memory
            
            df_total_value['gea']=self.states9_memory
            df_total_value['opPos']=self.states10_memory
            df_total_value['pct']=self.states11_memory
            df_total_value['open']=self.states12_memory
            df_total_value['low']=self.states13_memory
            df_total_value['high']=self.states14_memory
            df_total_value['stop']=self.states15_memory

            #df_total_value['actions']
            if df_total_value['daily_return'].std() !=0:
                sharpe = (252**0.5)*df_total_value['daily_return'].mean()/ \
                      df_total_value['daily_return'].std()
            df_rewards = pd.DataFrame(self.rewards_memory)
            df_rewards.columns = ['account_rewards']
            df_rewards['date'] = self.date_memory[:-1]
            if self.episode % self.print_verbosity == 0:
                print(f"day: {self.day}, episode: {self.episode}")
                print(f"begin_total_asset: {self.asset_memory[0]:0.10f}")
                print(f"end_total_asset: {end_total_asset:0.10f}")
                print(f"total_reward: {tot_reward:0.10f}")
                print(f"total_cost: {self.cost:0.10f}")
                print(f"total_trades: {self.trades}")
                if df_total_value['daily_return'].std() != 0:
                    print(f"Sharpe: {sharpe:0.3f}")
                print("=================================")

            if (self.model_name!='') and (self.mode!=''):
                df_actions = self.save_action_memory()
                df_actions.to_csv('results/actions_{}_{}_{}.csv'.format(self.mode,self.model_name, self.iteration))
                df_total_value.to_csv('results/account_value_{}_{}_{}.csv'.format(self.mode,self.model_name, self.iteration),index=False)
                df_rewards.to_csv('results/account_rewards_{}_{}_{}.csv'.format(self.mode,self.model_name, self.iteration),index=False)
                plt.plot(self.asset_memory,'r')
                plt.savefig('results/account_value_{}_{}_{}.png'.format(self.mode,self.model_name, self.iteration),index=False)
                plt.close()
                

            # Add outputs to logger interface
            #logger.record("environment/portfolio_value", end_total_asset)
            #logger.record("environment/total_reward", tot_reward)
            #logger.record("environment/total_reward_pct", (tot_reward / (end_total_asset - tot_reward)) * 100)
            #logger.record("environment/total_cost", self.cost)
            #logger.record("environment/total_trades", self.trades)
            #if self.reward<0:
            #    self.reward*=2
            return self.state, self.reward, self.terminal, {}
            #print(4,self.state)
        else:
            #print("start step:",actions,self.openedPos,round(self.state[0],2),round(self.state[1],2),round(self.state[3],2),round(self.state[4],2),round(self.state[5],2),round(self.state[6],2),round(self.state[7],2),round(self.state[8],2),round(self.state[9],2))
            #actions
            #actions = actions * self.hmax #actions initially is scaled between 0 to 1
            #actions=actions*10*abs(self.get_end_asset()//self.state[1],()
            t=actions
            begin_total_asset=self.get_end_asset()
            #if actions[0]==0:
            #    self.closePos()
            #stop=actions[1]
            #tp=actions[1]
            #actions=actions[0]
            #self.musor.append(actions)
            #if len(self.musor)%25==0:
            #    print("act::::",self.musor)
            #    self.musor=[]
                
            act=actions#[0]#-actions[1]
            if act>0 and act <5:
                act=1/act
                self.long(act)

            elif act>4:
                act=act-4
                act=1/act
                self.short(act)
              #self.short(abs(act))
            elif act==0:
                pass
                  
            #self.stoploss=0.1*(1+stop)/2#0.2 * actions[1]/50
            #self.stoploss=0.07*(1+tp)/2
            self.actions_memory.append(actions)
            #self.action=argsort_actions
            self.asset_memory.append(begin_total_asset)
            #self.states0_memory.append(self.state[0])
            self.states1_memory.append(self.state[1])
            prev_state1=self.state[1] #необходим что ниже рассчитать стоп по текущей цене(до обновления)
            #print("sss:",self.state[0])
            self.states3_memory.append(self.state[3])
            self.states4_memory.append(self.state[4])
            self.states5_memory.append(self.state[5])
            self.states6_memory.append(self.state[6])
            #print('states:',self.state[3],self.state[4],self.state[5],self.state[6])
            self.states7_memory.append(self.state[7])
            self.states8_memory.append(self.state[8])
            self.states9_memory.append(self.state[9])
            self.states10_memory.append(self.state[10])
            self.states11_memory.append(self.state[11])
            self.states12_memory.append(self.state[12])
            self.states13_memory.append(self.state[13])
            self.states14_memory.append(self.state[14])
            self.states16_memory.append(actions)
            self.states16a_memory.append(self.state[16])
            
            self.day += 1
            self.data = self.df.loc[self.day,:]    
            if self.turbulence_threshold is not None:     
                #self.turbulence = self.data['turbulence'].values[0]
                self.turbulence = self.data['turbulence']
            self.state =  self._update_state()
            
            
            self.state[13]=self.data.low
            self.state[14]=self.data.high
            #if self.openedPos==1:
            #    sl=prev_state1*(1-self.stoploss)
               
            #elif self.openedPos==-1:
            #    sl=prev_state1*(1+self.stoploss)
            #else:
            #    sl=-1
            self.states15_memory.append(self.stoploss)
            
            self.closePos()     
            end_total_asset = self.get_end_asset()
            self.states0_memory.append(self.state[0])

            
            self.date_memory.append(self._get_date())
            self.reward = end_total_asset - begin_total_asset
            #print('end tot ass:',end_total_asset,"step reward:", self.reward, "params: state[0], self.state[3:6]", self.state[0],self.state[3],self.state[4],self.state[5],self.state[6])
            #print("step reward:", self.reward)
            self.rewards_memory.append(self.reward)
            self.reward = self.reward/begin_total_asset
           # print("end step:",actions,self.openedPos,round(self.state[0],2),round(self.state[1],2),round(self.state[3],2),round(self.state[4],2),round(self.state[5],2),round(self.state[6],2),round(self.state[7],2),round(self.state[8],2),round(self.state[9],2))
            #if self.reward<0:
            #    self.reward*=1.3
            #print(end_total_asset,self.state)
        return self.state, self.reward*self.reward_scaling, self.terminal, {}

    def reset(self):  
        #initiate state
        self.state = self._initiate_state()
        if self.state[3]>0:
            self.openedPos=1
        elif self.state[5]>0:
            self.openedPos=-1
        if self.initial:
            self.asset_memory = [self.initial_amount]
            self.states0_memory= [self.state[0]]
            self.states1_memory= [self.state[1]]
            self.states3_memory= [self.state[3]]
            self.states4_memory= [self.state[4]]
            self.states5_memory= [self.state[5]]
            self.states6_memory= [self.state[6]]
            #print('states2:',self.state[3],self.state[4],self.state[5],self.state[6])
            self.states7_memory= [self.state[7]]
            self.states8_memory= [self.state[8]]
            self.states9_memory= [self.state[9]]
            self.states10_memory= [self.state[10]]
            self.states11_memory= [self.state[11]]
            self.states12_memory= [self.state[12]]
            self.states13_memory= [self.state[13]]
            self.states14_memory= [self.state[14]]
            self.states15_memory= [self.state[15]]
            self.states16_memory=[0]
            self.states16a_memory= [self.state[16]]
        else:
            #raise NotImplementedError
            previous_total_asset = self.previous_state[0]+ \
                self.previous_state[4]*self.previous_state[3] + (self.previous_state[1] - self.previous_state[4])*self.previous_state[3] +\
                self.previous_state[6]*self.previous_state[5] + (self.previous_state[6]-self.previous_state[1])*self.previous_state[5]
            previous_total_asset2 = self.state[0]+ \
                self.state[4]*self.state[3] + (self.state[1] - self.state[4])*self.state[3] +\
                self.state[6]*self.state[5] + (self.state[6]-self.state[1])*self.state[5]
           
            #self.previous_state[0]+ \
            #sum(np.array(self.state[1:(self.stock_dim+1)])*np.array(self.previous_state[(self.stock_dim+1):(self.stock_dim*2+1)]))
            self.asset_memory = [previous_total_asset]
            #print('pta:',previous_total_asset)
            #print('current:', self.state[0],self.state[1],self.state[2],self.state[3],self.state[4],self.state[5],self.state[6])
            #print('prev:', self.previous_state[0],self.previous_state[1],self.previous_state[2],self.previous_state[3],self.previous_state[4],self.previous_state[5],self.previous_state[6])
            #print('base:',self.previous_state[0]+ self.state[1:(self.stock_dim+1)]+np.array(self.previous_state[(self.stock_dim+1):(self.stock_dim*2+1)]),self.previous_state[0], self.state[1:(self.stock_dim+1)],np.array(self.previous_state[(self.stock_dim+1):(self.stock_dim*2+1)]))
        self.day = 0
        self.data = self.df.loc[self.day,:]
        self.turbulence = 0
        self.cost = 0
        self.trades = 0
        self.terminal = False 
        # self.iteration=self.iteration
        self.rewards_memory = []
        self.actions_memory=[]
        self.date_memory=[self._get_date()]
        #self.openedPos=0
        self.episode+=1
        #print(6,self.state)
        return self.state
    
    def render(self, mode='human',close=False):
        return self.state

    def _initiate_state(self):
        if self.initial:
            # For Initial State
            if len(self.df.tic.unique())>1:
                # for multiple stock
                state = [self.initial_amount] + \
                         self.data.close.values.tolist() + \
                         [0]*self.stock_dim  + \
                         sum([self.data[tech].values.tolist() for tech in self.tech_indicator_list ], [])
            else:
                # for single stock
                
                state = [self.initial_amount] + \
                        [self.data.close] + \
                        [0]*self.stock_dim  + \
                        [0]+\
                        [0]+\
                        [0]+\
                        [0]+\
                        [0] + [0]+ [0*self.initial_amount] + [0*self.openedPos] +[self.data.pct_ch] +\
                        [self.data.open]+[self.data.low]+[self.data.high]+\
                        [0]+\
                        [0*round(2 * self.initial_amount/self.data.close,self.precision)]+\
                        sum([[self.data[tech]] for tech in self.tech_indicator_list ], [])
                #print(0,'len:',len(state),state)
        else:
            #Using Previous State
            if len(self.df.tic.unique())>1:
                # for multiple stock
                state = [self.previous_state[0]] + \
                         self.data.close.values.tolist() + \
                         self.previous_state[(self.stock_dim+1):(self.stock_dim*2+1)]  + \
                         sum([self.data[tech].values.tolist() for tech in self.tech_indicator_list ], [])
            else:
                # for single stock
                state = [self.previous_state[0]] + \
                        [self.data.close] + \
                        self.previous_state[(self.stock_dim+1):(self.stock_dim*2+1)]  + \
                        [self.previous_state[3]] + [self.previous_state[4]] + [self.previous_state[5]] + [self.previous_state[6]] + \
                        [self.previous_state[7]]+[self.previous_state[8]] + [self.previous_state[9]] +[self.previous_state[10]]+[self.previous_state[11]] +\
                        [self.previous_state[12]]+[self.previous_state[13]]+[self.previous_state[14]]+\
                        [self.previous_state[15]]+\
                        [self.previous_state[16]]+\
                        sum([[self.data[tech]] for tech in self.tech_indicator_list ], [])
                #print(1,state)
        #print("initial:",self.initial)
        return state

    def _update_state(self):
        if len(self.df.tic.unique())>1:
            # for multiple stock
            state =  [self.state[0]] + \
                      self.data.close.values.tolist() + \
                      list(self.state[(self.stock_dim+1):(self.stock_dim*2+1)]) + \
                      sum([self.data[tech].values.tolist() for tech in self.tech_indicator_list ], [])

        else:
            
            self.state[13]=self.data.low
            self.state[14]=self.data.high
            #self.state[1]=self.data.close
            if False:#self.openedPos==-1:
                self.stoploss=self.state[1]*(1-self.stoploss)
                if self.state[13]<=self.stoploss and self.stoploss<=self.state[1]:
                    #self.states15_memory.append([self.stoploss,self.state[13],self.state[1],self.openedPos])
                    pnl=self.PNL(stopPrice=self.stoploss)
                    self.state[0]+=self.state[5]*self.state[6]+pnl#+self.PNL(stopPrice=self.stoploss)
                    self.state[5]=0
                    self.state[6]=0
                    self.openedPos=0
                    #self.stoploss=0
                    #print('stop from 1:', self.state[13],self.stoploss)
                
                    
            elif False:#self.openedPos==1:
                self.stoploss=self.state[1]*(1+self.stoploss)
                if self.state[14]>=self.stoploss and self.stoploss>=self.state[1]:
                    #self.states15_memory.append([self.stoploss,self.state[14],self.state[1],self.openedPos])
                    pnl=self.PNL(stopPrice=self.stoploss)
                    self.state[0]+=self.state[3]*self.state[4] + pnl#+self.PNL(stopPrice=self.stoploss)
                    self.state[3]=0
                    self.state[4]=0
                    self.openedPos=0
                    #self.stoploss=0

                    
                    #print('stop from -1:', self.state[14],self.stoploss)
            self.state[1]=self.data.close
            
            state =  [self.state[0]] + \
                     [self.data.close] + \
                     list(self.state[(self.stock_dim+1):(self.stock_dim*2+1)]) + \
                     [self.state[3]] + [self.state[4]] + [self.state[5]] + [self.state[6]] + [0*self.avLongQty()/(self.avLongQty()+self.avShortQty()+1e-10)] + [0*self.avShortQty()/(self.avLongQty()+self.avShortQty() +1e-10)]+ [0*self.get_end_asset()] + [0*self.openedPos] +[self.data.pct_ch] +\
                     [self.data.open]+[self.data.low]+[self.data.high]+\
                     [self.stoploss]+\
                     [0*(self.avLongQty()+self.avShortQty())]+\
                     sum([[self.data[tech]] for tech in self.tech_indicator_list ], [])
            #print(5,self.openedPos,self.action,state)
                          
        return state

    def _get_date(self):
        if len(self.df.tic.unique())>1:
            date = self.data.date.unique()[0]
        else:
            date = self.data.date
        return date

    def save_asset_memory(self):
        date_list = self.date_memory
        asset_list = self.asset_memory
        #print(len(date_list))
        #print(len(asset_list))
        df_account_value = pd.DataFrame({'date':date_list,'account_value':asset_list})
        return df_account_value

    def save_action_memory(self):
        if len(self.df.tic.unique())>1:
            # date and close price length must match actions length
            date_list = self.date_memory[:-1]
            df_date = pd.DataFrame(date_list)
            df_date.columns = ['date']
            
            action_list = self.actions_memory
            df_actions = pd.DataFrame(action_list)
            df_actions.columns = self.data.tic.values
            df_actions.index = df_date.date
            #df_actions = pd.DataFrame({'date':date_list,'actions':action_list})
        else:
            date_list = self.date_memory[:-1]
            action_list = self.actions_memory
            df_actions = pd.DataFrame({'date':date_list,'actions':action_list})
        return df_actions

    def _seed(self, seed=None):
        self.np_random, seed = seeding.np_random(seed)
        return [seed]


    def get_sb_env(self):
        e = DummyVecEnv([lambda: self])
        obs = e.reset()
        return e, obs


class StockTradingEnvShortPNLStop (gym.Env):
    """A stock trading environment for OpenAI gym"""
    metadata = {'render.modes': ['human']}

    def __init__(self, 
                df, 
                stock_dim,
                hmax,                
                initial_amount,
                buy_cost_pct,
                sell_cost_pct,
                reward_scaling,
                state_space,
                action_space,
                tech_indicator_list,
                turbulence_threshold=None,
                make_plots = False, 
                print_verbosity = 10,
                day = 0, 
                initial=True,
                previous_state=[],
                model_name = '',
                mode='',
                iteration=''):
        print("HELLLLOv5!!!")
        self.day = day
        self.df = df
        self.stock_dim = stock_dim
        self.hmax = hmax
        self.initial_amount = initial_amount
        self.buy_cost_pct = buy_cost_pct
        self.sell_cost_pct = sell_cost_pct
        self.reward_scaling = reward_scaling
        self.state_space = state_space
        self.action_space = action_space
        self.tech_indicator_list = tech_indicator_list
        self.action_space = spaces.Box(low = -1, high = 1,shape = (2,),dtype=np.float32) 
        #self.action_space = spaces.MultiDiscrete([2,2,21,51])
        self.observation_space = spaces.Box(low=-np.inf, high=np.inf, shape = (self.state_space,))
        self.data = self.df.loc[self.day,:]
        self.terminal = False
        self.make_plots = make_plots
        self.print_verbosity = print_verbosity
        self.turbulence_threshold = None
        self.initial = initial
        self.previous_state = previous_state
        self.model_name=model_name
        self.mode=mode 
        self.iteration=iteration
        self.openedPos=0
        self.lastPos=0
        self.lastPrice=0
        self.longPosQty=0
        self.longPosBP=0
        self.shortPosQty=0
        self.shortPosBP=0
        self.shortPoses=[]
        # initalize state
        self.precision=10
        self.stoploss=0
        self.takeprofit=0
        self.state = self._initiate_state()
        
        # initialize reward
        self.reward = 0
        self.turbulence = 0
        self.cost = 0
        self.trades = 0
        self.episode = 0
        
        self.musor=[]
        
        # memorize all the total balance change
        self.asset_memory = [self.initial_amount]
        self.states0_memory= [self.state[0]]
        self.states1_memory= [self.state[1]]
        self.states3_memory= [self.state[3]]
        self.states4_memory= [self.state[4]]
        self.states5_memory= [self.state[5]]
        self.states6_memory= [self.state[6]]
        self.states7_memory= [self.state[7]]
        self.states8_memory= [self.state[8]]
        self.states9_memory= [self.state[9]]
        self.states10_memory= [self.state[10]]
        self.states11_memory= [self.state[11]]
        self.states12_memory= [self.state[12]]
        self.states13_memory= [self.state[13]]
        self.states14_memory= [self.state[14]]
        self.states15_memory= [self.state[15]]
        self.states16_memory=[0]
        self.states16a_memory= [self.state[16]]
        self.rewards_memory = []
        self.actions_memory=[]
        self.date_memory=[self._get_date()]
        self.action=0
        #self.reset()
        self._seed()
        

    def PNL(self,stopPrice=None):
        if stopPrice==None:
            if self.openedPos==1:

                return (self.state[1]-self.state[4])*self.state[3]
            elif self.openedPos==-1:
                return (self.state[6]-self.state[1])*self.state[5]
            else:
                #print('0 pos')
                return 0
        else:
            if self.openedPos==1:

                return (stopPrice-self.state[4])*self.state[3]
            elif self.openedPos==-1:
                return (self.state[6]-stopPrice)*self.state[5]
            else:
                #print('0 pos')
                return 0
            
    def _sell_stock(self, action):
        index=0
       # action=self.avShortQty()*abs(action)+self.avLongQty(
        if True:
                short_num_shares = 0
                if self.state[index+1]>0: 
                    # Sell only if the price is > 0 (no missing data in this particular date)
                    # perform sell action based on the sign of the action
                    if self.openedPos==1:
                        #if self.state[index+self.stock_dim+1] > 0:
                            # Sell only if current asset is > 0
                            action=(self.avShortQty()+self.avLongQty())*abs(action)
                            if abs(action)<self.state[3]:
                                #total_contract_value=self.state[3]*self.state[4] - abs(action)*self.state[1]
                                #avg_entry_price=total_contract_value/(self.state[3] - abs(action))
                                pnl_before=self.PNL()
                                self.state[0]+=self.state[4]*abs(action)
                                self.state[3] -= abs(action)
                                #self.state[4] = avg_entry_price
                                pnl_after=self.PNL()
                                self.state[0] += pnl_before - pnl_after
                                
                                self.trades+=1
                                
                            elif abs(action)==self.state[3]:
                                pnl=self.PNL()
                                self.state[0]+=self.state[4]*self.state[3]
                                self.state[3]=0
                                self.state[4]=0
                                self.state[0]+=pnl
                                
                                self.openedPos=0
                                
                                self.trades+=1
                            
                            elif abs(action)>self.state[3]:
                                
                                pnl=self.PNL()
                                add_qty=abs(action)-self.state[3]
                                self.state[0]+=self.state[3]*self.state[4]
                                self.state[3]=0
                                self.state[4]=0
                                self.state[0]+=pnl
                                self.openedPos=0
                                
                                available_qty = max(0,self.state[0]//self.state[1])
                                if available_qty>0:
                                    short_qty = min(add_qty,available_qty)
                                    self.state[5]=short_qty
                                    self.state[6]=self.state[1]
                                    self.state[0]-= short_qty * self.state[1]

                                    self.openedPos=-1

                                self.trades+=1
                               

                    elif self.openedPos==0:
                        action=self.avShortQty()*abs(action)
                        available_qty = max(0,self.state[0] // self.state[1])
                        short_qty = min(available_qty, abs(action))
                        self.state[0] -= short_qty * self.state[1]
                        self.state[5]=short_qty
                        self.state[6]=self.state[1]
                        
                        self.openedPos=-1
                        
                        self.trades+=1
                        
                    elif self.openedPos==-1:
                        action=self.avShortQty()*abs(action)
                        pnl=self.PNL()
                        #self.state[0]+=pnl 
                        available_qty=max(0,(self.state[0]+pnl) // self.state[1])
                        add_qty=min(available_qty,abs(action))
                        if add_qty>0:
                            self.state[0] -= self.state[1] * add_qty ####
                            total_contract_value=self.state[5]*self.state[6] + abs(add_qty) * self.state[1]
                            avg_entry_price=total_contract_value/(self.state[5] + abs(add_qty))
                            self.state[5]+=add_qty
                            self.state[6]=avg_entry_price

                            self.trades+=1
                                                                  
        else:
            short_num_shares = 0
            print("sell action:",action)
        #print('states:',self.state[3],self.state[4],self.state[5],self.state[6])    
        return 0
    
                                                              
         

    
    def _buy_stock(self, action):
        index=0
        #action=self.avLongQty()*abs(action)
        if True:                                                          
            
                if self.state[index+1]>0: 
                    # Sell only if the price is > 0 (no missing data in this particular date)
                    # perform sell action based on the sign of the action
                    if self.openedPos==1:
                        action=self.avLongQty()*abs(action)
                        #if self.state[index+self.stock_dim+1] > 0:
                            # Sell only if current asset is > 0
                        pnl=self.PNL()
                        #self.state[0]+=pnl
                        available_qty=max(0,(self.state[0]+pnl) // self.state[1])
                        add_qty=min(available_qty,abs(action))
                        if add_qty>0:
                            self.state[0] -= self.state[1]*abs(add_qty) ######

                            total_contract_value=self.state[3]*self.state[4] + abs(add_qty) * self.state[1]
                            avg_entry_price=total_contract_value/(self.state[3] + abs(add_qty))
                            self.state[3]+=add_qty
                            self.state[4]=avg_entry_price
                            self.trades+=1
                                

                    elif self.openedPos==0:
                        action=self.avLongQty()*abs(action)
                        available_qty = max(0,self.state[0] // self.state[1])
                        long_qty = min(available_qty, abs(action))
                        self.state[0] -= long_qty * self.state[1]
                        self.state[3]=long_qty
                        self.state[4]=self.state[1]
                        
                        self.openedPos=1
                        
                        self.trades+=1
                        
                    elif self.openedPos==-1:
                        action=(self.avShortQty()+self.avLongQty())*abs(action)
                        if abs(action)<self.state[5]:
                            #total_contract_value=self.state[5]*self.state[6] - abs(action)*self.state[1]
                            #avg_entry_price=total_contract_value/(self.state[5] - abs(action))
                            pnl_before=self.PNL()
                            self.state[0]+=self.state[6]*abs(action)
                            #self.state[0]-=self.state[1]*abs(action) #######
                            self.state[5] -= abs(action)
                            #self.state[6] = avg_entry_price
                            pnl_after=self.PNL()
                            self.state[0] += pnl_before - pnl_after
                            self.trades+=1

                        elif abs(action)==self.state[5]:
                            pnl=self.PNL()
                            self.state[0]+=self.state[6]*abs(action)
                            self.state[5]=0
                            self.state[6]=0
                            self.state[0]+=pnl

                            self.openedPos=0

                            self.trades+=1

                        elif abs(action)>self.state[5]:
                            pnl=self.PNL()
                            add_qty=abs(action)-self.state[5]
                            self.state[0]+=self.state[5]*self.state[6]
                            self.state[5]=0
                            self.state[6]=0
                            self.state[0]+=pnl
                            self.openedPos=0

                            available_qty=max(0,self.state[0]//self.state[1])
                            if available_qty>0:
                                
                                long_qty=min(add_qty,available_qty)
                                self.state[3]=long_qty
                                self.state[4]=self.state[1]
                                self.state[0]-= long_qty * self.state[1]

                                self.openedPos=1

                            self.trades+=1
        
        else:
            short_num_shares = 0
            print("buy action:",action)
        #print('states:',self.state[3],self.state[4],self.state[5],self.state[6])
        return 0
    def avLongQty(self):
        pnl=self.PNL()
        short_value=self.state[5]*self.state[1]
        avQty=round((self.state[0]+pnl + short_value)/self.state[1],self.precision)
        return max(0,avQty)
    
    def avShortQty(self):
        pnl=self.PNL()
        long_value=self.state[3]*self.state[1]
        avQty=round((self.state[0]+pnl + long_value)/self.state[1],self.precision)
        return max(0,avQty)
    
    def shortPos(self,direct,pct_qty):
        
        if self.openedPos==0:
            if direct==0:
                return 0
            elif direct==1:
                shortQty=round(self.avShortQty()*pct_qty,self.precision)
                self.state[5]+=shortQty
                self.state[6]=self.state[1]
                self.state[0]-=shortQty*self.state[1]
                self.openedPos=-1
                self.trades+=1
                
        elif self.openedPos==1:
            avQty=self.avShortQty()
            if avQty>0:
                pnl=self.PNL()
                self.state[0]+=self.state[3]*self.state[4] + pnl
                self.state[3]=0
                self.state[4]=0
                
                self.state[5]=round(avQty * pct_qty,self.precision)
                self.state[6]=self.state[1]
                self.state[0]-=self.state[5]*self.state[6]
                self.openedPos=-1
                self.trades+=1
                
               
                    
        elif self.openedPos==-1:
            if direct==0:
                reduceShortQty=round(self.state[5]*pct_qty,self.precision)
                pnl_before=self.PNL()
                self.state[5]-=reduceShortQty
                self.state[0]+=self.state[6]*reduceShortQty
                pnl_after=self.PNL()
                self.state[0]+=pnl_before-pnl_after
                
                if self.state[5]==0:
                    self.state[6]=0
                    self.openedPos=0
                
                self.trades+=1
                
            elif direct==1:
                avQty=self.avShortQty()
                if avQty>0 and pct_qty>0:
                    shortQty=round(avQty * pct_qty,self.precision)
                    self.state[0] -= self.state[1]*shortQty
                    
                    total_contract_value=self.state[5]*self.state[6] + shortQty * self.state[1]
                    avg_entry_price=round(total_contract_value/(self.state[5] + shortQty),self.precision)
                    self.state[5]+=shortQty
                    self.state[6]=avg_entry_price
                    self.trades+=1
                    
    
    def longPos(self,direct,pct_qty):
        if self.openedPos==0:
            if direct==0:
                return 0
            elif direct==1:
                longQty=round(self.avLongQty()*pct_qty,self.precision)
                self.state[3]+=longQty
                self.state[4]=self.state[1]
                self.state[0]-=longQty*self.state[1]
                self.openedPos=1
                self.trades+=1
        elif self.openedPos==1:
            if direct==0:
                reduceLongQty=round(self.state[3]*pct_qty,self.precision)
                pnl_before=self.PNL()
                self.state[3]-=reduceLongQty
                self.state[0]+=self.state[4]*reduceLongQty
                pnl_after=self.PNL()
                self.state[0]+=pnl_before-pnl_after
                
                if self.state[3]==0:
                    self.state[4]=0
                    self.openedPos=0
                
                self.trades+=1
                
            elif direct==1:
                avQty=self.avLongQty()
                if avQty>0 and pct_qty>0:
                    longQty=round(avQty * pct_qty,self.precision)
                    self.state[0] -= self.state[1]*longQty
                    
                    total_contract_value=self.state[3]*self.state[4] + longQty * self.state[1]
                    avg_entry_price=round(total_contract_value/(self.state[3] + longQty),self.precision)
                    self.state[3]+=longQty
                    self.state[4]=avg_entry_price
                    self.trades+=1
                    
        elif self.openedPos==-1:
            avQty=self.avLongQty()
            if avQty>0:
                pnl=self.PNL()
                self.state[0]+=self.state[5]*self.state[6] + pnl
                self.state[5]=0
                self.state[6]=0
                
                self.state[3]=round(avQty * pct_qty,self.precision)
                self.state[4]=self.state[1]
                self.state[0]-=self.state[3]*self.state[4]
                self.openedPos=1
                self.trades+=1
            
    def closePos(self):
        #pnl=self.PNL()
        self.state[0]=self.get_end_asset()
        self.state[3]=0; self.state[4]=0
        self.state[5]=0; self.state[6]=0
        self.openedPos=0
        
    def long(self,action):
      qty=action*self.state[0]/self.state[1]
      self.state[0]-=qty*self.state[1]
      self.state[3]=qty
      self.state[4]=self.state[1]
      self.openedPos=1
      self.trades+=1
      
      
    def short(self,action):
      qty=action*self.state[0]/self.state[1]
      self.state[0]-=qty*self.state[1]
      self.state[5]=qty
      self.state[6]=self.state[1]
      self.openedPos=-1
      self.trades+=1
                                                          
                    
            
                                                              
    def get_end_asset(self):
        end_total_asset= self.state[0]+ \
                self.state[3]*self.state[4] + (self.state[1] - self.state[4])*self.state[3] +\
                self.state[6]*self.state[5] + (self.state[6]-self.state[1])*self.state[5]
        #print(end_total_asset, self.state[0],self.longPosBP,self.longPosQty,self.shortPosBP,self.shortPosQty)
        return end_total_asset
    def _make_plot(self):
        plt.plot(self.asset_memory,'r')
        plt.savefig('results/account_value_trade_{}.png'.format(self.episode))
        plt.close()

    def step(self, actions):
        self.terminal = self.day >= len(self.df.index.unique())-1
        if self.terminal:
            # print(f"Episode: {self.episode}")
            if self.make_plots:
                self._make_plot()            
            end_total_asset = self.get_end_asset()
            # sum(np.array(self.state[1:(self.stock_dim+1)])*np.array(self.state[(self.stock_dim+1):(self.stock_dim*2+1)]))
            #print(np.array(self.state[1:(self.stock_dim+1)]),np.array(self.state[(self.stock_dim+1):(self.stock_dim*2+1)]),np.array(self.state[1:(self.stock_dim+1)])*np.array(self.state[(self.stock_dim+1):(self.stock_dim*2+1)]))
            df_total_value = pd.DataFrame(self.asset_memory)
            tot_reward =  self.get_end_asset() - self.initial_amount
            #print("tot reward:", tot_reward,"params: state[0], self.state[3:6]", self.state[0],self.state[3],self.state[4],self.state[5],self.state[6])
            df_total_value.columns = ['account_value']
            df_total_value['date'] = self.date_memory
            df_total_value['daily_return']=df_total_value['account_value'].pct_change(1)
            #self.actions_memory.append('nan')
            #df_total_value['actions']=self.actions_memory
            df_total_value['state[0]']=self.states0_memory
            df_total_value['state[1]']=self.states1_memory
            df_total_value['state[3]']=self.states3_memory
            df_total_value['state[4]']=self.states4_memory
            df_total_value['state[5]']=self.states5_memory
            df_total_value['act']=self.states16_memory
            df_total_value['state[6]']=self.states6_memory
            df_total_value['begin_asset']=end_total_asset
            
            df_total_value['avlong']=self.states7_memory
            df_total_value['avshort']=self.states8_memory
            df_total_value['avsum']=self.states16a_memory
            
            df_total_value['gea']=self.states9_memory
            df_total_value['opPos']=self.states10_memory
            df_total_value['pct']=self.states11_memory
            df_total_value['open']=self.states12_memory
            df_total_value['low']=self.states13_memory
            df_total_value['high']=self.states14_memory
            df_total_value['stop']=self.states15_memory

            #df_total_value['actions']
            if df_total_value['daily_return'].std() !=0:
                sharpe = (252**0.5)*df_total_value['daily_return'].mean()/ \
                      df_total_value['daily_return'].std()
            df_rewards = pd.DataFrame(self.rewards_memory)
            df_rewards.columns = ['account_rewards']
            df_rewards['date'] = self.date_memory[:-1]
            if self.episode % self.print_verbosity == 0:
                print(f"day: {self.day}, episode: {self.episode}")
                print(f"begin_total_asset: {self.asset_memory[0]:0.10f}")
                print(f"end_total_asset: {end_total_asset:0.10f}")
                print(f"total_reward: {tot_reward:0.10f}")
                print(f"total_cost: {self.cost:0.10f}")
                print(f"total_trades: {self.trades}")
                if df_total_value['daily_return'].std() != 0:
                    print(f"Sharpe: {sharpe:0.3f}")
                print("=================================")

            if (self.model_name!='') and (self.mode!=''):
                df_actions = self.save_action_memory()
                df_actions.to_csv('results/actions_{}_{}_{}.csv'.format(self.mode,self.model_name, self.iteration))
                df_total_value.to_csv('results/account_value_{}_{}_{}.csv'.format(self.mode,self.model_name, self.iteration),index=False)
                df_rewards.to_csv('results/account_rewards_{}_{}_{}.csv'.format(self.mode,self.model_name, self.iteration),index=False)
                plt.plot(self.asset_memory,'r')
                plt.savefig('results/account_value_{}_{}_{}.png'.format(self.mode,self.model_name, self.iteration),index=False)
                plt.close()
                

            # Add outputs to logger interface
            #logger.record("environment/portfolio_value", end_total_asset)
            #logger.record("environment/total_reward", tot_reward)
            #logger.record("environment/total_reward_pct", (tot_reward / (end_total_asset - tot_reward)) * 100)
            #logger.record("environment/total_cost", self.cost)
            #logger.record("environment/total_trades", self.trades)
            #if self.reward<0:
            #    self.reward*=2
            return self.state, self.reward, self.terminal, {}
            #print(4,self.state)
        else:
            #print("start step:",actions,self.openedPos,round(self.state[0],2),round(self.state[1],2),round(self.state[3],2),round(self.state[4],2),round(self.state[5],2),round(self.state[6],2),round(self.state[7],2),round(self.state[8],2),round(self.state[9],2))
            #actions
            #actions = actions * self.hmax #actions initially is scaled between 0 to 1
            #actions=actions*10*abs(self.get_end_asset()//self.state[1],()
            t=actions
            begin_total_asset=self.get_end_asset()
            #if actions[0]==0:
            #    self.closePos()
            stop=actions[1]
            #tp=actions[2]
            #actions=actions[0]
            #self.musor.append(actions)
            #if len(self.musor)%25==0:
            #    print("act::::",self.musor)
            #    self.musor=[]
                
            act=actions[0]#-actions[1]
            if act>0:
              self.long(act)
            elif act<0:
              self.short(abs(act))
            elif act==0:
              pass
                  
            self.stoploss=0.1*(1+stop)/2#0.2 * actions[1]/50
            #self.takeprofit=0.2*(1+tp)/2
            self.actions_memory.append(actions)
            #self.action=argsort_actions
            self.asset_memory.append(begin_total_asset)
            #self.states0_memory.append(self.state[0])
            self.states1_memory.append(self.state[1])
            prev_state1=self.state[1] #необходим что ниже рассчитать стоп по текущей цене(до обновления)
            #print("sss:",self.state[0])
            self.states3_memory.append(self.state[3])
            self.states4_memory.append(self.state[4])
            self.states5_memory.append(self.state[5])
            self.states6_memory.append(self.state[6])
            #print('states:',self.state[3],self.state[4],self.state[5],self.state[6])
            self.states7_memory.append(self.state[7])
            self.states8_memory.append(self.state[8])
            self.states9_memory.append(self.state[9])
            self.states10_memory.append(self.state[10])
            self.states11_memory.append(self.state[11])
            self.states12_memory.append(self.state[12])
            self.states13_memory.append(self.state[13])
            self.states14_memory.append(self.state[14])
            self.states16_memory.append(actions)
            self.states16a_memory.append(self.state[16])
            
            self.day += 1
            self.data = self.df.loc[self.day,:]    
            if self.turbulence_threshold is not None:     
                #self.turbulence = self.data['turbulence'].values[0]
                self.turbulence = self.data['turbulence']
            self.state =  self._update_state()
            
            
            self.state[13]=self.data.low
            self.state[14]=self.data.high
            #if self.openedPos==1:
            #    sl=prev_state1*(1-self.stoploss)
               
            #elif self.openedPos==-1:
            #    sl=prev_state1*(1+self.stoploss)
            #else:
            #    sl=-1
            self.states15_memory.append(self.stoploss)
            
            self.closePos()     
            end_total_asset = self.get_end_asset()
            self.states0_memory.append(self.state[0])

            
            self.date_memory.append(self._get_date())
            self.reward = end_total_asset - begin_total_asset
            #print('end tot ass:',end_total_asset,"step reward:", self.reward, "params: state[0], self.state[3:6]", self.state[0],self.state[3],self.state[4],self.state[5],self.state[6])
            #print("step reward:", self.reward)
            self.rewards_memory.append(self.reward)
            self.reward = self.reward/begin_total_asset*self.reward_scaling
           # print("end step:",actions,self.openedPos,round(self.state[0],2),round(self.state[1],2),round(self.state[3],2),round(self.state[4],2),round(self.state[5],2),round(self.state[6],2),round(self.state[7],2),round(self.state[8],2),round(self.state[9],2))
            if self.reward<0:
                self.reward*=1.3
            #print(end_total_asset,self.state)
        return self.state, self.reward, self.terminal, {}

    def reset(self):  
        #initiate state
        self.state = self._initiate_state()
        if self.state[3]>0:
            self.openedPos=1
        elif self.state[5]>0:
            self.openedPos=-1
        if self.initial:
            self.asset_memory = [self.initial_amount]
            self.states0_memory= [self.state[0]]
            self.states1_memory= [self.state[1]]
            self.states3_memory= [self.state[3]]
            self.states4_memory= [self.state[4]]
            self.states5_memory= [self.state[5]]
            self.states6_memory= [self.state[6]]
            #print('states2:',self.state[3],self.state[4],self.state[5],self.state[6])
            self.states7_memory= [self.state[7]]
            self.states8_memory= [self.state[8]]
            self.states9_memory= [self.state[9]]
            self.states10_memory= [self.state[10]]
            self.states11_memory= [self.state[11]]
            self.states12_memory= [self.state[12]]
            self.states13_memory= [self.state[13]]
            self.states14_memory= [self.state[14]]
            self.states15_memory= [self.state[15]]
            self.states16_memory=[0]
            self.states16a_memory= [self.state[16]]
        else:
            #raise NotImplementedError
            previous_total_asset = self.previous_state[0]+ \
                self.previous_state[4]*self.previous_state[3] + (self.previous_state[1] - self.previous_state[4])*self.previous_state[3] +\
                self.previous_state[6]*self.previous_state[5] + (self.previous_state[6]-self.previous_state[1])*self.previous_state[5]
            previous_total_asset2 = self.state[0]+ \
                self.state[4]*self.state[3] + (self.state[1] - self.state[4])*self.state[3] +\
                self.state[6]*self.state[5] + (self.state[6]-self.state[1])*self.state[5]
           
            #self.previous_state[0]+ \
            #sum(np.array(self.state[1:(self.stock_dim+1)])*np.array(self.previous_state[(self.stock_dim+1):(self.stock_dim*2+1)]))
            self.asset_memory = [previous_total_asset]
            #print('pta:',previous_total_asset)
            #print('current:', self.state[0],self.state[1],self.state[2],self.state[3],self.state[4],self.state[5],self.state[6])
            #print('prev:', self.previous_state[0],self.previous_state[1],self.previous_state[2],self.previous_state[3],self.previous_state[4],self.previous_state[5],self.previous_state[6])
            #print('base:',self.previous_state[0]+ self.state[1:(self.stock_dim+1)]+np.array(self.previous_state[(self.stock_dim+1):(self.stock_dim*2+1)]),self.previous_state[0], self.state[1:(self.stock_dim+1)],np.array(self.previous_state[(self.stock_dim+1):(self.stock_dim*2+1)]))
        self.day = 0
        self.data = self.df.loc[self.day,:]
        self.turbulence = 0
        self.cost = 0
        self.trades = 0
        self.terminal = False 
        # self.iteration=self.iteration
        self.rewards_memory = []
        self.actions_memory=[]
        self.date_memory=[self._get_date()]
        #self.openedPos=0
        self.episode+=1
        #print(6,self.state)
        return self.state
    
    def render(self, mode='human',close=False):
        return self.state

    def _initiate_state(self):
        if self.initial:
            # For Initial State
            if len(self.df.tic.unique())>1:
                # for multiple stock
                state = [self.initial_amount] + \
                         self.data.close.values.tolist() + \
                         [0]*self.stock_dim  + \
                         sum([self.data[tech].values.tolist() for tech in self.tech_indicator_list ], [])
            else:
                # for single stock
                
                state = [self.initial_amount] + \
                        [self.data.close] + \
                        [0]*self.stock_dim  + \
                        [0]+\
                        [0]+\
                        [0]+\
                        [0]+\
                        [0] + [0]+ [0*self.initial_amount] + [0*self.openedPos] +[self.data.pct_ch] +\
                        [self.data.open]+[self.data.low]+[self.data.high]+\
                        [0]+\
                        [0*round(2 * self.initial_amount/self.data.close,self.precision)]+\
                        sum([[self.data[tech]] for tech in self.tech_indicator_list ], [])
                #print(0,'len:',len(state),state)
        else:
            #Using Previous State
            if len(self.df.tic.unique())>1:
                # for multiple stock
                state = [self.previous_state[0]] + \
                         self.data.close.values.tolist() + \
                         self.previous_state[(self.stock_dim+1):(self.stock_dim*2+1)]  + \
                         sum([self.data[tech].values.tolist() for tech in self.tech_indicator_list ], [])
            else:
                # for single stock
                state = [self.previous_state[0]] + \
                        [self.data.close] + \
                        self.previous_state[(self.stock_dim+1):(self.stock_dim*2+1)]  + \
                        [self.previous_state[3]] + [self.previous_state[4]] + [self.previous_state[5]] + [self.previous_state[6]] + \
                        [self.previous_state[7]]+[self.previous_state[8]] + [self.previous_state[9]] +[self.previous_state[10]]+[self.previous_state[11]] +\
                        [self.previous_state[12]]+[self.previous_state[13]]+[self.previous_state[14]]+\
                        [self.previous_state[15]]+\
                        [self.previous_state[16]]+\
                        sum([[self.data[tech]] for tech in self.tech_indicator_list ], [])
                #print(1,state)
        #print("initial:",self.initial)
        return state

    def _update_state(self):
        if len(self.df.tic.unique())>1:
            # for multiple stock
            state =  [self.state[0]] + \
                      self.data.close.values.tolist() + \
                      list(self.state[(self.stock_dim+1):(self.stock_dim*2+1)]) + \
                      sum([self.data[tech].values.tolist() for tech in self.tech_indicator_list ], [])

        else:
            
            self.state[13]=self.data.low
            self.state[14]=self.data.high
            #self.state[1]=self.data.close
            if self.openedPos==1:
                self.stoploss=self.state[1]*(1-self.stoploss)
                if self.state[13]<=self.stoploss and self.stoploss<self.state[1]:
                    #self.states15_memory.append([self.stoploss,self.state[13],self.state[1],self.openedPos])
                    pnl=self.PNL(stopPrice=self.stoploss)
                    self.state[0]+=self.state[3]*self.state[4]+pnl#+self.PNL(stopPrice=self.stoploss)
                    self.state[3]=0
                    self.state[4]=0
                    self.openedPos=0
                    #self.stoploss=0
                    #print('stop from 1:', self.state[13],self.stoploss)
                
                    
            elif self.openedPos==-1:
                self.stoploss=self.state[1]*(1+self.stoploss)
                if self.state[14]>=self.stoploss and self.stoploss>self.state[1]:
                    #self.states15_memory.append([self.stoploss,self.state[14],self.state[1],self.openedPos])
                    pnl=self.PNL(stopPrice=self.stoploss)
                    self.state[0]+=self.state[5]*self.state[6] + pnl#+self.PNL(stopPrice=self.stoploss)
                    self.state[5]=0
                    self.state[6]=0
                    self.openedPos=0
                    #self.stoploss=0

                    
                    #print('stop from -1:', self.state[14],self.stoploss)
            self.state[1]=self.data.close
            
            state =  [self.state[0]] + \
                     [self.data.close] + \
                     list(self.state[(self.stock_dim+1):(self.stock_dim*2+1)]) + \
                     [self.state[3]] + [self.state[4]] + [self.state[5]] + [self.state[6]] + [0*self.avLongQty()/(self.avLongQty()+self.avShortQty()+1e-10)] + [0*self.avShortQty()/(self.avLongQty()+self.avShortQty() +1e-10)]+ [0*self.get_end_asset()] + [0*self.openedPos] +[self.data.pct_ch] +\
                     [self.data.open]+[self.data.low]+[self.data.high]+\
                     [self.stoploss]+\
                     [0*(self.avLongQty()+self.avShortQty())]+\
                     sum([[self.data[tech]] for tech in self.tech_indicator_list ], [])
            #print(5,self.openedPos,self.action,state)
                          
        return state

    def _get_date(self):
        if len(self.df.tic.unique())>1:
            date = self.data.date.unique()[0]
        else:
            date = self.data.date
        return date

    def save_asset_memory(self):
        date_list = self.date_memory
        asset_list = self.asset_memory
        #print(len(date_list))
        #print(len(asset_list))
        df_account_value = pd.DataFrame({'date':date_list,'account_value':asset_list})
        return df_account_value

    def save_action_memory(self):
        if len(self.df.tic.unique())>1:
            # date and close price length must match actions length
            date_list = self.date_memory[:-1]
            df_date = pd.DataFrame(date_list)
            df_date.columns = ['date']
            
            action_list = self.actions_memory
            df_actions = pd.DataFrame(action_list)
            df_actions.columns = self.data.tic.values
            df_actions.index = df_date.date
            #df_actions = pd.DataFrame({'date':date_list,'actions':action_list})
        else:
            date_list = self.date_memory[:-1]
            action_list = self.actions_memory
            df_actions = pd.DataFrame({'date':date_list,'actions':action_list})
        return df_actions

    def _seed(self, seed=None):
        self.np_random, seed = seeding.np_random(seed)
        return [seed]


    def get_sb_env(self):
        e = DummyVecEnv([lambda: self])
        obs = e.reset()
        return e, obs

import numpy as np
import pandas as pd
from gym.utils import seeding
import gym
from gym import spaces
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import pickle
from stable_baselines3.common.vec_env import DummyVecEnv
#from stable_baselines3.common import logger
class StockTradingEnvShortPNL (gym.Env):
    """A stock trading environment for OpenAI gym"""
    metadata = {'render.modes': ['human']}

    def __init__(self, 
                df, 
                stock_dim,
                hmax,                
                initial_amount,
                buy_cost_pct,
                sell_cost_pct,
                reward_scaling,
                state_space,
                action_space,
                tech_indicator_list,
                turbulence_threshold=None,
                make_plots = False, 
                print_verbosity = 10,
                day = 0, 
                initial=True,
                previous_state=[],
                model_name = '',
                mode='',
                iteration=''):
        print("HELLLLOv5!!!")
        self.day = day
        self.df = df
        self.stock_dim = stock_dim
        self.hmax = hmax
        self.initial_amount = initial_amount
        self.buy_cost_pct = buy_cost_pct
        self.sell_cost_pct = sell_cost_pct
        self.reward_scaling = reward_scaling
        self.state_space = state_space
        self.action_space = action_space
        self.tech_indicator_list = tech_indicator_list
        #self.action_space = spaces.Box(low = -1, high = 1,shape = (1,),dtype=np.float32) 
        self.action_space = spaces.Discrete(9) 
        #self.action_space = spaces.MultiDiscrete([2,2,21,51])
        self.observation_space = spaces.Box(low=-np.inf, high=np.inf, shape = (self.state_space,))
        self.data = self.df.loc[self.day,:]
        self.terminal = False
        self.make_plots = make_plots
        self.print_verbosity = print_verbosity
        self.turbulence_threshold = None
        self.initial = initial
        self.previous_state = previous_state
        self.model_name=model_name
        self.mode=mode 
        self.iteration=iteration
        self.openedPos=0
        self.lastPos=0
        self.lastPrice=0
        self.longPosQty=0
        self.longPosBP=0
        self.shortPosQty=0
        self.shortPosBP=0
        self.shortPoses=[]
        # initalize state
        self.precision=10
        self.stoploss=0
        self.takeprofit=0
        self.state = self._initiate_state()
        
        # initialize reward
        self.reward = 0
        self.turbulence = 0
        self.cost = 0
        self.trades = 0
        self.episode = 0
        
        self.musor=[]
        
        # memorize all the total balance change
        self.asset_memory = [self.initial_amount]
        self.states0_memory= [self.state[0]]
        self.states1_memory= [self.state[1]]
        self.states3_memory= [self.state[3]]
        self.states4_memory= [self.state[4]]
        self.states5_memory= [self.state[5]]
        self.states6_memory= [self.state[6]]
        self.states7_memory= [self.state[7]]
        self.states8_memory= [self.state[8]]
        self.states9_memory= [self.state[9]]
        self.states10_memory= [self.state[10]]
        self.states11_memory= [self.state[11]]
        self.states12_memory= [self.state[12]]
        self.states13_memory= [self.state[13]]
        self.states14_memory= [self.state[14]]
        self.states15_memory= [self.state[15]]
        self.states16_memory=[0]
        self.states16a_memory= [self.state[16]]
        self.rewards_memory = []
        self.actions_memory=[]
        self.date_memory=[self._get_date()]
        self.action=0
        #self.reset()
        self._seed()
        

    def PNL(self,stopPrice=None):
        if stopPrice==None:
            if self.openedPos==1:

                return (self.state[1]-self.state[4])*self.state[3]
            elif self.openedPos==-1:
                return (self.state[6]-self.state[1])*self.state[5]
            else:
                #print('0 pos')
                return 0
        else:
            if self.openedPos==1:

                return (stopPrice-self.state[4])*self.state[3]
            elif self.openedPos==-1:
                return (self.state[6]-stopPrice)*self.state[5]
            else:
                #print('0 pos')
                return 0
            
    def _sell_stock(self, action):
        index=0
       # action=self.avShortQty()*abs(action)+self.avLongQty(
        if True:
                short_num_shares = 0
                if self.state[index+1]>0: 
                    # Sell only if the price is > 0 (no missing data in this particular date)
                    # perform sell action based on the sign of the action
                    if self.openedPos==1:
                        #if self.state[index+self.stock_dim+1] > 0:
                            # Sell only if current asset is > 0
                            action=(self.avShortQty()+self.avLongQty())*abs(action)
                            if abs(action)<self.state[3]:
                                #total_contract_value=self.state[3]*self.state[4] - abs(action)*self.state[1]
                                #avg_entry_price=total_contract_value/(self.state[3] - abs(action))
                                pnl_before=self.PNL()
                                self.state[0]+=self.state[4]*abs(action)
                                self.state[3] -= abs(action)
                                #self.state[4] = avg_entry_price
                                pnl_after=self.PNL()
                                self.state[0] += pnl_before - pnl_after
                                
                                self.trades+=1
                                
                            elif abs(action)==self.state[3]:
                                pnl=self.PNL()
                                self.state[0]+=self.state[4]*self.state[3]
                                self.state[3]=0
                                self.state[4]=0
                                self.state[0]+=pnl
                                
                                self.openedPos=0
                                
                                self.trades+=1
                            
                            elif abs(action)>self.state[3]:
                                
                                pnl=self.PNL()
                                add_qty=abs(action)-self.state[3]
                                self.state[0]+=self.state[3]*self.state[4]
                                self.state[3]=0
                                self.state[4]=0
                                self.state[0]+=pnl
                                self.openedPos=0
                                
                                available_qty = max(0,self.state[0]//self.state[1])
                                if available_qty>0:
                                    short_qty = min(add_qty,available_qty)
                                    self.state[5]=short_qty
                                    self.state[6]=self.state[1]
                                    self.state[0]-= short_qty * self.state[1]

                                    self.openedPos=-1

                                self.trades+=1
                               

                    elif self.openedPos==0:
                        action=self.avShortQty()*abs(action)
                        available_qty = max(0,self.state[0] // self.state[1])
                        short_qty = min(available_qty, abs(action))
                        self.state[0] -= short_qty * self.state[1]
                        self.state[5]=short_qty
                        self.state[6]=self.state[1]
                        
                        self.openedPos=-1
                        
                        self.trades+=1
                        
                    elif self.openedPos==-1:
                        action=self.avShortQty()*abs(action)
                        pnl=self.PNL()
                        #self.state[0]+=pnl 
                        available_qty=max(0,(self.state[0]+pnl) // self.state[1])
                        add_qty=min(available_qty,abs(action))
                        if add_qty>0:
                            self.state[0] -= self.state[1] * add_qty ####
                            total_contract_value=self.state[5]*self.state[6] + abs(add_qty) * self.state[1]
                            avg_entry_price=total_contract_value/(self.state[5] + abs(add_qty))
                            self.state[5]+=add_qty
                            self.state[6]=avg_entry_price

                            self.trades+=1
                                                                  
        else:
            short_num_shares = 0
            print("sell action:",action)
        #print('states:',self.state[3],self.state[4],self.state[5],self.state[6])    
        return 0
    
                                                              
         

    
    def _buy_stock(self, action):
        index=0
        #action=self.avLongQty()*abs(action)
        if True:                                                          
            
                if self.state[index+1]>0: 
                    # Sell only if the price is > 0 (no missing data in this particular date)
                    # perform sell action based on the sign of the action
                    if self.openedPos==1:
                        action=self.avLongQty()*abs(action)
                        #if self.state[index+self.stock_dim+1] > 0:
                            # Sell only if current asset is > 0
                        pnl=self.PNL()
                        #self.state[0]+=pnl
                        available_qty=max(0,(self.state[0]+pnl) // self.state[1])
                        add_qty=min(available_qty,abs(action))
                        if add_qty>0:
                            self.state[0] -= self.state[1]*abs(add_qty) ######

                            total_contract_value=self.state[3]*self.state[4] + abs(add_qty) * self.state[1]
                            avg_entry_price=total_contract_value/(self.state[3] + abs(add_qty))
                            self.state[3]+=add_qty
                            self.state[4]=avg_entry_price
                            self.trades+=1
                                

                    elif self.openedPos==0:
                        action=self.avLongQty()*abs(action)
                        available_qty = max(0,self.state[0] // self.state[1])
                        long_qty = min(available_qty, abs(action))
                        self.state[0] -= long_qty * self.state[1]
                        self.state[3]=long_qty
                        self.state[4]=self.state[1]
                        
                        self.openedPos=1
                        
                        self.trades+=1
                        
                    elif self.openedPos==-1:
                        action=(self.avShortQty()+self.avLongQty())*abs(action)
                        if abs(action)<self.state[5]:
                            #total_contract_value=self.state[5]*self.state[6] - abs(action)*self.state[1]
                            #avg_entry_price=total_contract_value/(self.state[5] - abs(action))
                            pnl_before=self.PNL()
                            self.state[0]+=self.state[6]*abs(action)
                            #self.state[0]-=self.state[1]*abs(action) #######
                            self.state[5] -= abs(action)
                            #self.state[6] = avg_entry_price
                            pnl_after=self.PNL()
                            self.state[0] += pnl_before - pnl_after
                            self.trades+=1

                        elif abs(action)==self.state[5]:
                            pnl=self.PNL()
                            self.state[0]+=self.state[6]*abs(action)
                            self.state[5]=0
                            self.state[6]=0
                            self.state[0]+=pnl

                            self.openedPos=0

                            self.trades+=1

                        elif abs(action)>self.state[5]:
                            pnl=self.PNL()
                            add_qty=abs(action)-self.state[5]
                            self.state[0]+=self.state[5]*self.state[6]
                            self.state[5]=0
                            self.state[6]=0
                            self.state[0]+=pnl
                            self.openedPos=0

                            available_qty=max(0,self.state[0]//self.state[1])
                            if available_qty>0:
                                
                                long_qty=min(add_qty,available_qty)
                                self.state[3]=long_qty
                                self.state[4]=self.state[1]
                                self.state[0]-= long_qty * self.state[1]

                                self.openedPos=1

                            self.trades+=1
        
        else:
            short_num_shares = 0
            print("buy action:",action)
        #print('states:',self.state[3],self.state[4],self.state[5],self.state[6])
        return 0
    def avLongQty(self):
        pnl=self.PNL()
        short_value=self.state[5]*self.state[1]
        avQty=round((self.state[0]+pnl + short_value)/self.state[1],self.precision)
        return max(0,avQty)
    
    def avShortQty(self):
        pnl=self.PNL()
        long_value=self.state[3]*self.state[1]
        avQty=round((self.state[0]+pnl + long_value)/self.state[1],self.precision)
        return max(0,avQty)
    
    def shortPos(self,direct,pct_qty):
        
        if self.openedPos==0:
            if direct==0:
                return 0
            elif direct==1:
                shortQty=round(self.avShortQty()*pct_qty,self.precision)
                self.state[5]+=shortQty
                self.state[6]=self.state[1]
                self.state[0]-=shortQty*self.state[1]
                self.openedPos=-1
                self.trades+=1
                
        elif self.openedPos==1:
            avQty=self.avShortQty()
            if avQty>0:
                pnl=self.PNL()
                self.state[0]+=self.state[3]*self.state[4] + pnl
                self.state[3]=0
                self.state[4]=0
                
                self.state[5]=round(avQty * pct_qty,self.precision)
                self.state[6]=self.state[1]
                self.state[0]-=self.state[5]*self.state[6]
                self.openedPos=-1
                self.trades+=1
                
               
                    
        elif self.openedPos==-1:
            if direct==0:
                reduceShortQty=round(self.state[5]*pct_qty,self.precision)
                pnl_before=self.PNL()
                self.state[5]-=reduceShortQty
                self.state[0]+=self.state[6]*reduceShortQty
                pnl_after=self.PNL()
                self.state[0]+=pnl_before-pnl_after
                
                if self.state[5]==0:
                    self.state[6]=0
                    self.openedPos=0
                
                self.trades+=1
                
            elif direct==1:
                avQty=self.avShortQty()
                if avQty>0 and pct_qty>0:
                    shortQty=round(avQty * pct_qty,self.precision)
                    self.state[0] -= self.state[1]*shortQty
                    
                    total_contract_value=self.state[5]*self.state[6] + shortQty * self.state[1]
                    avg_entry_price=round(total_contract_value/(self.state[5] + shortQty),self.precision)
                    self.state[5]+=shortQty
                    self.state[6]=avg_entry_price
                    self.trades+=1
                    
    
    def longPos(self,direct,pct_qty):
        if self.openedPos==0:
            if direct==0:
                return 0
            elif direct==1:
                longQty=round(self.avLongQty()*pct_qty,self.precision)
                self.state[3]+=longQty
                self.state[4]=self.state[1]
                self.state[0]-=longQty*self.state[1]
                self.openedPos=1
                self.trades+=1
        elif self.openedPos==1:
            if direct==0:
                reduceLongQty=round(self.state[3]*pct_qty,self.precision)
                pnl_before=self.PNL()
                self.state[3]-=reduceLongQty
                self.state[0]+=self.state[4]*reduceLongQty
                pnl_after=self.PNL()
                self.state[0]+=pnl_before-pnl_after
                
                if self.state[3]==0:
                    self.state[4]=0
                    self.openedPos=0
                
                self.trades+=1
                
            elif direct==1:
                avQty=self.avLongQty()
                if avQty>0 and pct_qty>0:
                    longQty=round(avQty * pct_qty,self.precision)
                    self.state[0] -= self.state[1]*longQty
                    
                    total_contract_value=self.state[3]*self.state[4] + longQty * self.state[1]
                    avg_entry_price=round(total_contract_value/(self.state[3] + longQty),self.precision)
                    self.state[3]+=longQty
                    self.state[4]=avg_entry_price
                    self.trades+=1
                    
        elif self.openedPos==-1:
            avQty=self.avLongQty()
            if avQty>0:
                pnl=self.PNL()
                self.state[0]+=self.state[5]*self.state[6] + pnl
                self.state[5]=0
                self.state[6]=0
                
                self.state[3]=round(avQty * pct_qty,self.precision)
                self.state[4]=self.state[1]
                self.state[0]-=self.state[3]*self.state[4]
                self.openedPos=1
                self.trades+=1
            
    def closePos(self):
        #pnl=self.PNL()
        self.state[0]=self.get_end_asset()
        self.state[3]=0; self.state[4]=0
        self.state[5]=0; self.state[6]=0
        self.openedPos=0
        
    def long(self,action):
      qty=action*self.state[0]/self.state[1]
      self.state[0]-=qty*self.state[1]
      self.state[3]=qty
      self.state[4]=self.state[1]
      self.openedPos=1
      self.trades+=1
      
      
    def short(self,action):
      qty=action*self.state[0]/self.state[1]
      self.state[0]-=qty*self.state[1]
      self.state[5]=qty
      self.state[6]=self.state[1]
      self.openedPos=-1
      self.trades+=1
                                                          
                    
            
                                                              
    def get_end_asset(self):
        end_total_asset= self.state[0]+ \
                self.state[3]*self.state[4] + (self.state[1] - self.state[4])*self.state[3] +\
                self.state[6]*self.state[5] + (self.state[6]-self.state[1])*self.state[5]
        #print(end_total_asset, self.state[0],self.longPosBP,self.longPosQty,self.shortPosBP,self.shortPosQty)
        return end_total_asset
    def _make_plot(self):
        plt.plot(self.asset_memory,'r')
        plt.savefig('results/account_value_trade_{}.png'.format(self.episode))
        plt.close()

    def step(self, actions):
        self.terminal = self.day >= len(self.df.index.unique())-1
        if self.terminal:
            # print(f"Episode: {self.episode}")
            if self.make_plots:
                self._make_plot()            
            end_total_asset = self.get_end_asset()
            # sum(np.array(self.state[1:(self.stock_dim+1)])*np.array(self.state[(self.stock_dim+1):(self.stock_dim*2+1)]))
            #print(np.array(self.state[1:(self.stock_dim+1)]),np.array(self.state[(self.stock_dim+1):(self.stock_dim*2+1)]),np.array(self.state[1:(self.stock_dim+1)])*np.array(self.state[(self.stock_dim+1):(self.stock_dim*2+1)]))
            df_total_value = pd.DataFrame(self.asset_memory)
            tot_reward =  self.get_end_asset() - self.initial_amount
            #print("tot reward:", tot_reward,"params: state[0], self.state[3:6]", self.state[0],self.state[3],self.state[4],self.state[5],self.state[6])
            df_total_value.columns = ['account_value']
            df_total_value['date'] = self.date_memory
            df_total_value['daily_return']=df_total_value['account_value'].pct_change(1)
            #self.actions_memory.append('nan')
            #df_total_value['actions']=self.actions_memory
            df_total_value['state[0]']=self.states0_memory
            df_total_value['state[1]']=self.states1_memory
            df_total_value['state[3]']=self.states3_memory
            df_total_value['state[4]']=self.states4_memory
            df_total_value['state[5]']=self.states5_memory
            df_total_value['act']=self.states16_memory
            df_total_value['state[6]']=self.states6_memory
            df_total_value['begin_asset']=end_total_asset
            
            df_total_value['avlong']=self.states7_memory
            df_total_value['avshort']=self.states8_memory
            df_total_value['avsum']=self.states16a_memory
            
            df_total_value['gea']=self.states9_memory
            df_total_value['opPos']=self.states10_memory
            df_total_value['pct']=self.states11_memory
            df_total_value['open']=self.states12_memory
            df_total_value['low']=self.states13_memory
            df_total_value['high']=self.states14_memory
            df_total_value['stop']=self.states15_memory

            #df_total_value['actions']
            if df_total_value['daily_return'].std() !=0:
                sharpe = (252**0.5)*df_total_value['daily_return'].mean()/ \
                      df_total_value['daily_return'].std()
            df_rewards = pd.DataFrame(self.rewards_memory)
            df_rewards.columns = ['account_rewards']
            df_rewards['date'] = self.date_memory[:-1]
            if self.episode % self.print_verbosity == 0:
                print(f"day: {self.day}, episode: {self.episode}")
                print(f"begin_total_asset: {self.asset_memory[0]:0.10f}")
                print(f"end_total_asset: {end_total_asset:0.10f}")
                print(f"total_reward: {tot_reward:0.10f}")
                print(f"total_cost: {self.cost:0.10f}")
                print(f"total_trades: {self.trades}")
                if df_total_value['daily_return'].std() != 0:
                    print(f"Sharpe: {sharpe:0.3f}")
                print("=================================")

            if (self.model_name!='') and (self.mode!=''):
                df_actions = self.save_action_memory()
                df_actions.to_csv('results/actions_{}_{}_{}.csv'.format(self.mode,self.model_name, self.iteration))
                df_total_value.to_csv('results/account_value_{}_{}_{}.csv'.format(self.mode,self.model_name, self.iteration),index=False)
                df_rewards.to_csv('results/account_rewards_{}_{}_{}.csv'.format(self.mode,self.model_name, self.iteration),index=False)
                plt.plot(self.asset_memory,'r')
                plt.savefig('results/account_value_{}_{}_{}.png'.format(self.mode,self.model_name, self.iteration),index=False)
                plt.close()
                

            # Add outputs to logger interface
            #logger.record("environment/portfolio_value", end_total_asset)
            #logger.record("environment/total_reward", tot_reward)
            #logger.record("environment/total_reward_pct", (tot_reward / (end_total_asset - tot_reward)) * 100)
            #logger.record("environment/total_cost", self.cost)
            #logger.record("environment/total_trades", self.trades)
            #if self.reward<0:
            #    self.reward*=2
            return self.state, self.reward, self.terminal, {}
            #print(4,self.state)
        else:
            #print("start step:",actions,self.openedPos,round(self.state[0],2),round(self.state[1],2),round(self.state[3],2),round(self.state[4],2),round(self.state[5],2),round(self.state[6],2),round(self.state[7],2),round(self.state[8],2),round(self.state[9],2))
            #actions
            #actions = actions * self.hmax #actions initially is scaled between 0 to 1
            #actions=actions*10*abs(self.get_end_asset()//self.state[1],()
            t=actions
            begin_total_asset=self.get_end_asset()
            #if actions[0]==0:
            #    self.closePos()
            #stop=actions[1]
            #tp=actions[1]
            #actions=actions[0]
            #self.musor.append(actions)
            #if len(self.musor)%25==0:
            #    print("act::::",self.musor)
            #    self.musor=[]
                
            act=actions#[0]#-actions[1]
            if act>0 and act <5:
                act=1/act
                self.long(act)

            elif act>4:
                act=act-4
                act=1/act
                self.short(act)
              #self.short(abs(act))
            elif act==0:
                pass
                  
            #self.stoploss=0.1*(1+stop)/2#0.2 * actions[1]/50
            #self.stoploss=0.07*(1+tp)/2
            self.actions_memory.append(actions)
            #self.action=argsort_actions
            self.asset_memory.append(begin_total_asset)
            #self.states0_memory.append(self.state[0])
            self.states1_memory.append(self.state[1])
            prev_state1=self.state[1] #необходим что ниже рассчитать стоп по текущей цене(до обновления)
            #print("sss:",self.state[0])
            self.states3_memory.append(self.state[3])
            self.states4_memory.append(self.state[4])
            self.states5_memory.append(self.state[5])
            self.states6_memory.append(self.state[6])
            #print('states:',self.state[3],self.state[4],self.state[5],self.state[6])
            self.states7_memory.append(self.state[7])
            self.states8_memory.append(self.state[8])
            self.states9_memory.append(self.state[9])
            self.states10_memory.append(self.state[10])
            self.states11_memory.append(self.state[11])
            self.states12_memory.append(self.state[12])
            self.states13_memory.append(self.state[13])
            self.states14_memory.append(self.state[14])
            self.states16_memory.append(actions)
            self.states16a_memory.append(self.state[16])
            
            self.day += 1
            self.data = self.df.loc[self.day,:]    
            if self.turbulence_threshold is not None:     
                #self.turbulence = self.data['turbulence'].values[0]
                self.turbulence = self.data['turbulence']
            self.state =  self._update_state()
            
            
            self.state[13]=self.data.low
            self.state[14]=self.data.high
            #if self.openedPos==1:
            #    sl=prev_state1*(1-self.stoploss)
               
            #elif self.openedPos==-1:
            #    sl=prev_state1*(1+self.stoploss)
            #else:
            #    sl=-1
            self.states15_memory.append(self.stoploss)
            
            self.closePos()     
            end_total_asset = self.get_end_asset()
            self.states0_memory.append(self.state[0])

            
            self.date_memory.append(self._get_date())
            self.reward = end_total_asset - begin_total_asset
            #print('end tot ass:',end_total_asset,"step reward:", self.reward, "params: state[0], self.state[3:6]", self.state[0],self.state[3],self.state[4],self.state[5],self.state[6])
            #print("step reward:", self.reward)
            self.rewards_memory.append(self.reward)
            self.reward = self.reward/begin_total_asset
           # print("end step:",actions,self.openedPos,round(self.state[0],2),round(self.state[1],2),round(self.state[3],2),round(self.state[4],2),round(self.state[5],2),round(self.state[6],2),round(self.state[7],2),round(self.state[8],2),round(self.state[9],2))
            #if self.reward<0:
            #    self.reward*=1.3
            #print(end_total_asset,self.state)
        return self.state, self.reward*self.reward_scaling, self.terminal, {}

    def reset(self):  
        #initiate state
        self.state = self._initiate_state()
        if self.state[3]>0:
            self.openedPos=1
        elif self.state[5]>0:
            self.openedPos=-1
        if self.initial:
            self.asset_memory = [self.initial_amount]
            self.states0_memory= [self.state[0]]
            self.states1_memory= [self.state[1]]
            self.states3_memory= [self.state[3]]
            self.states4_memory= [self.state[4]]
            self.states5_memory= [self.state[5]]
            self.states6_memory= [self.state[6]]
            #print('states2:',self.state[3],self.state[4],self.state[5],self.state[6])
            self.states7_memory= [self.state[7]]
            self.states8_memory= [self.state[8]]
            self.states9_memory= [self.state[9]]
            self.states10_memory= [self.state[10]]
            self.states11_memory= [self.state[11]]
            self.states12_memory= [self.state[12]]
            self.states13_memory= [self.state[13]]
            self.states14_memory= [self.state[14]]
            self.states15_memory= [self.state[15]]
            self.states16_memory=[0]
            self.states16a_memory= [self.state[16]]
        else:
            #raise NotImplementedError
            previous_total_asset = self.previous_state[0]+ \
                self.previous_state[4]*self.previous_state[3] + (self.previous_state[1] - self.previous_state[4])*self.previous_state[3] +\
                self.previous_state[6]*self.previous_state[5] + (self.previous_state[6]-self.previous_state[1])*self.previous_state[5]
            previous_total_asset2 = self.state[0]+ \
                self.state[4]*self.state[3] + (self.state[1] - self.state[4])*self.state[3] +\
                self.state[6]*self.state[5] + (self.state[6]-self.state[1])*self.state[5]
           
            #self.previous_state[0]+ \
            #sum(np.array(self.state[1:(self.stock_dim+1)])*np.array(self.previous_state[(self.stock_dim+1):(self.stock_dim*2+1)]))
            self.asset_memory = [previous_total_asset]
            #print('pta:',previous_total_asset)
            #print('current:', self.state[0],self.state[1],self.state[2],self.state[3],self.state[4],self.state[5],self.state[6])
            #print('prev:', self.previous_state[0],self.previous_state[1],self.previous_state[2],self.previous_state[3],self.previous_state[4],self.previous_state[5],self.previous_state[6])
            #print('base:',self.previous_state[0]+ self.state[1:(self.stock_dim+1)]+np.array(self.previous_state[(self.stock_dim+1):(self.stock_dim*2+1)]),self.previous_state[0], self.state[1:(self.stock_dim+1)],np.array(self.previous_state[(self.stock_dim+1):(self.stock_dim*2+1)]))
        self.day = 0
        self.data = self.df.loc[self.day,:]
        self.turbulence = 0
        self.cost = 0
        self.trades = 0
        self.terminal = False 
        # self.iteration=self.iteration
        self.rewards_memory = []
        self.actions_memory=[]
        self.date_memory=[self._get_date()]
        #self.openedPos=0
        self.episode+=1
        #print(6,self.state)
        return self.state
    
    def render(self, mode='human',close=False):
        return self.state

    def _initiate_state(self):
        if self.initial:
            # For Initial State
            if len(self.df.tic.unique())>1:
                # for multiple stock
                state = [self.initial_amount] + \
                         self.data.close.values.tolist() + \
                         [0]*self.stock_dim  + \
                         sum([self.data[tech].values.tolist() for tech in self.tech_indicator_list ], [])
            else:
                # for single stock
                
                state = [self.initial_amount] + \
                        [self.data.close] + \
                        [0]*self.stock_dim  + \
                        [0]+\
                        [0]+\
                        [0]+\
                        [0]+\
                        [0] + [0]+ [0*self.initial_amount] + [0*self.openedPos] +[self.data.pct_ch] +\
                        [self.data.open]+[self.data.low]+[self.data.high]+\
                        [0]+\
                        [0*round(2 * self.initial_amount/self.data.close,self.precision)]+\
                        sum([[self.data[tech]] for tech in self.tech_indicator_list ], [])
                #print(0,'len:',len(state),state)
        else:
            #Using Previous State
            if len(self.df.tic.unique())>1:
                # for multiple stock
                state = [self.previous_state[0]] + \
                         self.data.close.values.tolist() + \
                         self.previous_state[(self.stock_dim+1):(self.stock_dim*2+1)]  + \
                         sum([self.data[tech].values.tolist() for tech in self.tech_indicator_list ], [])
            else:
                # for single stock
                state = [self.previous_state[0]] + \
                        [self.data.close] + \
                        self.previous_state[(self.stock_dim+1):(self.stock_dim*2+1)]  + \
                        [self.previous_state[3]] + [self.previous_state[4]] + [self.previous_state[5]] + [self.previous_state[6]] + \
                        [self.previous_state[7]]+[self.previous_state[8]] + [self.previous_state[9]] +[self.previous_state[10]]+[self.previous_state[11]] +\
                        [self.previous_state[12]]+[self.previous_state[13]]+[self.previous_state[14]]+\
                        [self.previous_state[15]]+\
                        [self.previous_state[16]]+\
                        sum([[self.data[tech]] for tech in self.tech_indicator_list ], [])
                #print(1,state)
        #print("initial:",self.initial)
        return state

    def _update_state(self):
        if len(self.df.tic.unique())>1:
            # for multiple stock
            state =  [self.state[0]] + \
                      self.data.close.values.tolist() + \
                      list(self.state[(self.stock_dim+1):(self.stock_dim*2+1)]) + \
                      sum([self.data[tech].values.tolist() for tech in self.tech_indicator_list ], [])

        else:
            
            self.state[13]=self.data.low
            self.state[14]=self.data.high
            #self.state[1]=self.data.close
            if False:#self.openedPos==-1:
                self.stoploss=self.state[1]*(1-self.stoploss)
                if self.state[13]<=self.stoploss and self.stoploss<=self.state[1]:
                    #self.states15_memory.append([self.stoploss,self.state[13],self.state[1],self.openedPos])
                    pnl=self.PNL(stopPrice=self.stoploss)
                    self.state[0]+=self.state[5]*self.state[6]+pnl#+self.PNL(stopPrice=self.stoploss)
                    self.state[5]=0
                    self.state[6]=0
                    self.openedPos=0
                    #self.stoploss=0
                    #print('stop from 1:', self.state[13],self.stoploss)
                
                    
            elif False:#self.openedPos==1:
                self.stoploss=self.state[1]*(1+self.stoploss)
                if self.state[14]>=self.stoploss and self.stoploss>=self.state[1]:
                    #self.states15_memory.append([self.stoploss,self.state[14],self.state[1],self.openedPos])
                    pnl=self.PNL(stopPrice=self.stoploss)
                    self.state[0]+=self.state[3]*self.state[4] + pnl#+self.PNL(stopPrice=self.stoploss)
                    self.state[3]=0
                    self.state[4]=0
                    self.openedPos=0
                    #self.stoploss=0

                    
                    #print('stop from -1:', self.state[14],self.stoploss)
            self.state[1]=self.data.close
            
            state =  [self.state[0]] + \
                     [self.data.close] + \
                     list(self.state[(self.stock_dim+1):(self.stock_dim*2+1)]) + \
                     [self.state[3]] + [self.state[4]] + [self.state[5]] + [self.state[6]] + [0*self.avLongQty()/(self.avLongQty()+self.avShortQty()+1e-10)] + [0*self.avShortQty()/(self.avLongQty()+self.avShortQty() +1e-10)]+ [0*self.get_end_asset()] + [0*self.openedPos] +[self.data.pct_ch] +\
                     [self.data.open]+[self.data.low]+[self.data.high]+\
                     [self.stoploss]+\
                     [0*(self.avLongQty()+self.avShortQty())]+\
                     sum([[self.data[tech]] for tech in self.tech_indicator_list ], [])
            #print(5,self.openedPos,self.action,state)
                          
        return state

    def _get_date(self):
        if len(self.df.tic.unique())>1:
            date = self.data.date.unique()[0]
        else:
            date = self.data.date
        return date

    def save_asset_memory(self):
        date_list = self.date_memory
        asset_list = self.asset_memory
        #print(len(date_list))
        #print(len(asset_list))
        df_account_value = pd.DataFrame({'date':date_list,'account_value':asset_list})
        return df_account_value

    def save_action_memory(self):
        if len(self.df.tic.unique())>1:
            # date and close price length must match actions length
            date_list = self.date_memory[:-1]
            df_date = pd.DataFrame(date_list)
            df_date.columns = ['date']
            
            action_list = self.actions_memory
            df_actions = pd.DataFrame(action_list)
            df_actions.columns = self.data.tic.values
            df_actions.index = df_date.date
            #df_actions = pd.DataFrame({'date':date_list,'actions':action_list})
        else:
            date_list = self.date_memory[:-1]
            action_list = self.actions_memory
            df_actions = pd.DataFrame({'date':date_list,'actions':action_list})
        return df_actions

    def _seed(self, seed=None):
        self.np_random, seed = seeding.np_random(seed)
        return [seed]


    def get_sb_env(self):
        e = DummyVecEnv([lambda: self])
        obs = e.reset()
        return e, obs


class StockTradingEnvShortPNLStop (gym.Env):
    """A stock trading environment for OpenAI gym"""
    metadata = {'render.modes': ['human']}

    def __init__(self, 
                df, 
                stock_dim,
                hmax,                
                initial_amount,
                buy_cost_pct,
                sell_cost_pct,
                reward_scaling,
                state_space,
                action_space,
                tech_indicator_list,
                turbulence_threshold=None,
                make_plots = False, 
                print_verbosity = 10,
                day = 0, 
                initial=True,
                previous_state=[],
                model_name = '',
                mode='',
                iteration=''):
        print("HELLLLOv5!!!")
        self.day = day
        self.df = df
        self.stock_dim = stock_dim
        self.hmax = hmax
        self.initial_amount = initial_amount
        self.buy_cost_pct = buy_cost_pct
        self.sell_cost_pct = sell_cost_pct
        self.reward_scaling = reward_scaling
        self.state_space = state_space
        self.action_space = action_space
        self.tech_indicator_list = tech_indicator_list
        self.action_space = spaces.Box(low = -1, high = 1,shape = (2,),dtype=np.float32) 
        #self.action_space = spaces.MultiDiscrete([2,2,21,51])
        self.observation_space = spaces.Box(low=-np.inf, high=np.inf, shape = (self.state_space,))
        self.data = self.df.loc[self.day,:]
        self.terminal = False
        self.make_plots = make_plots
        self.print_verbosity = print_verbosity
        self.turbulence_threshold = None
        self.initial = initial
        self.previous_state = previous_state
        self.model_name=model_name
        self.mode=mode 
        self.iteration=iteration
        self.openedPos=0
        self.lastPos=0
        self.lastPrice=0
        self.longPosQty=0
        self.longPosBP=0
        self.shortPosQty=0
        self.shortPosBP=0
        self.shortPoses=[]
        # initalize state
        self.precision=10
        self.stoploss=0
        self.takeprofit=0
        self.state = self._initiate_state()
        
        # initialize reward
        self.reward = 0
        self.turbulence = 0
        self.cost = 0
        self.trades = 0
        self.episode = 0
        
        self.musor=[]
        
        # memorize all the total balance change
        self.asset_memory = [self.initial_amount]
        self.states0_memory= [self.state[0]]
        self.states1_memory= [self.state[1]]
        self.states3_memory= [self.state[3]]
        self.states4_memory= [self.state[4]]
        self.states5_memory= [self.state[5]]
        self.states6_memory= [self.state[6]]
        self.states7_memory= [self.state[7]]
        self.states8_memory= [self.state[8]]
        self.states9_memory= [self.state[9]]
        self.states10_memory= [self.state[10]]
        self.states11_memory= [self.state[11]]
        self.states12_memory= [self.state[12]]
        self.states13_memory= [self.state[13]]
        self.states14_memory= [self.state[14]]
        self.states15_memory= [self.state[15]]
        self.states16_memory=[0]
        self.states16a_memory= [self.state[16]]
        self.rewards_memory = []
        self.actions_memory=[]
        self.date_memory=[self._get_date()]
        self.action=0
        #self.reset()
        self._seed()
        

    def PNL(self,stopPrice=None):
        if stopPrice==None:
            if self.openedPos==1:

                return (self.state[1]-self.state[4])*self.state[3]
            elif self.openedPos==-1:
                return (self.state[6]-self.state[1])*self.state[5]
            else:
                #print('0 pos')
                return 0
        else:
            if self.openedPos==1:

                return (stopPrice-self.state[4])*self.state[3]
            elif self.openedPos==-1:
                return (self.state[6]-stopPrice)*self.state[5]
            else:
                #print('0 pos')
                return 0
            
    def _sell_stock(self, action):
        index=0
       # action=self.avShortQty()*abs(action)+self.avLongQty(
        if True:
                short_num_shares = 0
                if self.state[index+1]>0: 
                    # Sell only if the price is > 0 (no missing data in this particular date)
                    # perform sell action based on the sign of the action
                    if self.openedPos==1:
                        #if self.state[index+self.stock_dim+1] > 0:
                            # Sell only if current asset is > 0
                            action=(self.avShortQty()+self.avLongQty())*abs(action)
                            if abs(action)<self.state[3]:
                                #total_contract_value=self.state[3]*self.state[4] - abs(action)*self.state[1]
                                #avg_entry_price=total_contract_value/(self.state[3] - abs(action))
                                pnl_before=self.PNL()
                                self.state[0]+=self.state[4]*abs(action)
                                self.state[3] -= abs(action)
                                #self.state[4] = avg_entry_price
                                pnl_after=self.PNL()
                                self.state[0] += pnl_before - pnl_after
                                
                                self.trades+=1
                                
                            elif abs(action)==self.state[3]:
                                pnl=self.PNL()
                                self.state[0]+=self.state[4]*self.state[3]
                                self.state[3]=0
                                self.state[4]=0
                                self.state[0]+=pnl
                                
                                self.openedPos=0
                                
                                self.trades+=1
                            
                            elif abs(action)>self.state[3]:
                                
                                pnl=self.PNL()
                                add_qty=abs(action)-self.state[3]
                                self.state[0]+=self.state[3]*self.state[4]
                                self.state[3]=0
                                self.state[4]=0
                                self.state[0]+=pnl
                                self.openedPos=0
                                
                                available_qty = max(0,self.state[0]//self.state[1])
                                if available_qty>0:
                                    short_qty = min(add_qty,available_qty)
                                    self.state[5]=short_qty
                                    self.state[6]=self.state[1]
                                    self.state[0]-= short_qty * self.state[1]

                                    self.openedPos=-1

                                self.trades+=1
                               

                    elif self.openedPos==0:
                        action=self.avShortQty()*abs(action)
                        available_qty = max(0,self.state[0] // self.state[1])
                        short_qty = min(available_qty, abs(action))
                        self.state[0] -= short_qty * self.state[1]
                        self.state[5]=short_qty
                        self.state[6]=self.state[1]
                        
                        self.openedPos=-1
                        
                        self.trades+=1
                        
                    elif self.openedPos==-1:
                        action=self.avShortQty()*abs(action)
                        pnl=self.PNL()
                        #self.state[0]+=pnl 
                        available_qty=max(0,(self.state[0]+pnl) // self.state[1])
                        add_qty=min(available_qty,abs(action))
                        if add_qty>0:
                            self.state[0] -= self.state[1] * add_qty ####
                            total_contract_value=self.state[5]*self.state[6] + abs(add_qty) * self.state[1]
                            avg_entry_price=total_contract_value/(self.state[5] + abs(add_qty))
                            self.state[5]+=add_qty
                            self.state[6]=avg_entry_price

                            self.trades+=1
                                                                  
        else:
            short_num_shares = 0
            print("sell action:",action)
        #print('states:',self.state[3],self.state[4],self.state[5],self.state[6])    
        return 0
    
                                                              
         

    
    def _buy_stock(self, action):
        index=0
        #action=self.avLongQty()*abs(action)
        if True:                                                          
            
                if self.state[index+1]>0: 
                    # Sell only if the price is > 0 (no missing data in this particular date)
                    # perform sell action based on the sign of the action
                    if self.openedPos==1:
                        action=self.avLongQty()*abs(action)
                        #if self.state[index+self.stock_dim+1] > 0:
                            # Sell only if current asset is > 0
                        pnl=self.PNL()
                        #self.state[0]+=pnl
                        available_qty=max(0,(self.state[0]+pnl) // self.state[1])
                        add_qty=min(available_qty,abs(action))
                        if add_qty>0:
                            self.state[0] -= self.state[1]*abs(add_qty) ######

                            total_contract_value=self.state[3]*self.state[4] + abs(add_qty) * self.state[1]
                            avg_entry_price=total_contract_value/(self.state[3] + abs(add_qty))
                            self.state[3]+=add_qty
                            self.state[4]=avg_entry_price
                            self.trades+=1
                                

                    elif self.openedPos==0:
                        action=self.avLongQty()*abs(action)
                        available_qty = max(0,self.state[0] // self.state[1])
                        long_qty = min(available_qty, abs(action))
                        self.state[0] -= long_qty * self.state[1]
                        self.state[3]=long_qty
                        self.state[4]=self.state[1]
                        
                        self.openedPos=1
                        
                        self.trades+=1
                        
                    elif self.openedPos==-1:
                        action=(self.avShortQty()+self.avLongQty())*abs(action)
                        if abs(action)<self.state[5]:
                            #total_contract_value=self.state[5]*self.state[6] - abs(action)*self.state[1]
                            #avg_entry_price=total_contract_value/(self.state[5] - abs(action))
                            pnl_before=self.PNL()
                            self.state[0]+=self.state[6]*abs(action)
                            #self.state[0]-=self.state[1]*abs(action) #######
                            self.state[5] -= abs(action)
                            #self.state[6] = avg_entry_price
                            pnl_after=self.PNL()
                            self.state[0] += pnl_before - pnl_after
                            self.trades+=1

                        elif abs(action)==self.state[5]:
                            pnl=self.PNL()
                            self.state[0]+=self.state[6]*abs(action)
                            self.state[5]=0
                            self.state[6]=0
                            self.state[0]+=pnl

                            self.openedPos=0

                            self.trades+=1

                        elif abs(action)>self.state[5]:
                            pnl=self.PNL()
                            add_qty=abs(action)-self.state[5]
                            self.state[0]+=self.state[5]*self.state[6]
                            self.state[5]=0
                            self.state[6]=0
                            self.state[0]+=pnl
                            self.openedPos=0

                            available_qty=max(0,self.state[0]//self.state[1])
                            if available_qty>0:
                                
                                long_qty=min(add_qty,available_qty)
                                self.state[3]=long_qty
                                self.state[4]=self.state[1]
                                self.state[0]-= long_qty * self.state[1]

                                self.openedPos=1

                            self.trades+=1
        
        else:
            short_num_shares = 0
            print("buy action:",action)
        #print('states:',self.state[3],self.state[4],self.state[5],self.state[6])
        return 0
    def avLongQty(self):
        pnl=self.PNL()
        short_value=self.state[5]*self.state[1]
        avQty=round((self.state[0]+pnl + short_value)/self.state[1],self.precision)
        return max(0,avQty)
    
    def avShortQty(self):
        pnl=self.PNL()
        long_value=self.state[3]*self.state[1]
        avQty=round((self.state[0]+pnl + long_value)/self.state[1],self.precision)
        return max(0,avQty)
    
    def shortPos(self,direct,pct_qty):
        
        if self.openedPos==0:
            if direct==0:
                return 0
            elif direct==1:
                shortQty=round(self.avShortQty()*pct_qty,self.precision)
                self.state[5]+=shortQty
                self.state[6]=self.state[1]
                self.state[0]-=shortQty*self.state[1]
                self.openedPos=-1
                self.trades+=1
                
        elif self.openedPos==1:
            avQty=self.avShortQty()
            if avQty>0:
                pnl=self.PNL()
                self.state[0]+=self.state[3]*self.state[4] + pnl
                self.state[3]=0
                self.state[4]=0
                
                self.state[5]=round(avQty * pct_qty,self.precision)
                self.state[6]=self.state[1]
                self.state[0]-=self.state[5]*self.state[6]
                self.openedPos=-1
                self.trades+=1
                
               
                    
        elif self.openedPos==-1:
            if direct==0:
                reduceShortQty=round(self.state[5]*pct_qty,self.precision)
                pnl_before=self.PNL()
                self.state[5]-=reduceShortQty
                self.state[0]+=self.state[6]*reduceShortQty
                pnl_after=self.PNL()
                self.state[0]+=pnl_before-pnl_after
                
                if self.state[5]==0:
                    self.state[6]=0
                    self.openedPos=0
                
                self.trades+=1
                
            elif direct==1:
                avQty=self.avShortQty()
                if avQty>0 and pct_qty>0:
                    shortQty=round(avQty * pct_qty,self.precision)
                    self.state[0] -= self.state[1]*shortQty
                    
                    total_contract_value=self.state[5]*self.state[6] + shortQty * self.state[1]
                    avg_entry_price=round(total_contract_value/(self.state[5] + shortQty),self.precision)
                    self.state[5]+=shortQty
                    self.state[6]=avg_entry_price
                    self.trades+=1
                    
    
    def longPos(self,direct,pct_qty):
        if self.openedPos==0:
            if direct==0:
                return 0
            elif direct==1:
                longQty=round(self.avLongQty()*pct_qty,self.precision)
                self.state[3]+=longQty
                self.state[4]=self.state[1]
                self.state[0]-=longQty*self.state[1]
                self.openedPos=1
                self.trades+=1
        elif self.openedPos==1:
            if direct==0:
                reduceLongQty=round(self.state[3]*pct_qty,self.precision)
                pnl_before=self.PNL()
                self.state[3]-=reduceLongQty
                self.state[0]+=self.state[4]*reduceLongQty
                pnl_after=self.PNL()
                self.state[0]+=pnl_before-pnl_after
                
                if self.state[3]==0:
                    self.state[4]=0
                    self.openedPos=0
                
                self.trades+=1
                
            elif direct==1:
                avQty=self.avLongQty()
                if avQty>0 and pct_qty>0:
                    longQty=round(avQty * pct_qty,self.precision)
                    self.state[0] -= self.state[1]*longQty
                    
                    total_contract_value=self.state[3]*self.state[4] + longQty * self.state[1]
                    avg_entry_price=round(total_contract_value/(self.state[3] + longQty),self.precision)
                    self.state[3]+=longQty
                    self.state[4]=avg_entry_price
                    self.trades+=1
                    
        elif self.openedPos==-1:
            avQty=self.avLongQty()
            if avQty>0:
                pnl=self.PNL()
                self.state[0]+=self.state[5]*self.state[6] + pnl
                self.state[5]=0
                self.state[6]=0
                
                self.state[3]=round(avQty * pct_qty,self.precision)
                self.state[4]=self.state[1]
                self.state[0]-=self.state[3]*self.state[4]
                self.openedPos=1
                self.trades+=1
            
    def closePos(self):
        #pnl=self.PNL()
        self.state[0]=self.get_end_asset()
        self.state[3]=0; self.state[4]=0
        self.state[5]=0; self.state[6]=0
        self.openedPos=0
        
    def long(self,action):
      qty=action*self.state[0]/self.state[1]
      self.state[0]-=qty*self.state[1]
      self.state[3]=qty
      self.state[4]=self.state[1]
      self.openedPos=1
      self.trades+=1
      
      
    def short(self,action):
      qty=action*self.state[0]/self.state[1]
      self.state[0]-=qty*self.state[1]
      self.state[5]=qty
      self.state[6]=self.state[1]
      self.openedPos=-1
      self.trades+=1
                                                          
                    
            
                                                              
    def get_end_asset(self):
        end_total_asset= self.state[0]+ \
                self.state[3]*self.state[4] + (self.state[1] - self.state[4])*self.state[3] +\
                self.state[6]*self.state[5] + (self.state[6]-self.state[1])*self.state[5]
        #print(end_total_asset, self.state[0],self.longPosBP,self.longPosQty,self.shortPosBP,self.shortPosQty)
        return end_total_asset
    def _make_plot(self):
        plt.plot(self.asset_memory,'r')
        plt.savefig('results/account_value_trade_{}.png'.format(self.episode))
        plt.close()

    def step(self, actions):
        self.terminal = self.day >= len(self.df.index.unique())-1
        if self.terminal:
            # print(f"Episode: {self.episode}")
            if self.make_plots:
                self._make_plot()            
            end_total_asset = self.get_end_asset()
            # sum(np.array(self.state[1:(self.stock_dim+1)])*np.array(self.state[(self.stock_dim+1):(self.stock_dim*2+1)]))
            #print(np.array(self.state[1:(self.stock_dim+1)]),np.array(self.state[(self.stock_dim+1):(self.stock_dim*2+1)]),np.array(self.state[1:(self.stock_dim+1)])*np.array(self.state[(self.stock_dim+1):(self.stock_dim*2+1)]))
            df_total_value = pd.DataFrame(self.asset_memory)
            tot_reward =  self.get_end_asset() - self.initial_amount
            #print("tot reward:", tot_reward,"params: state[0], self.state[3:6]", self.state[0],self.state[3],self.state[4],self.state[5],self.state[6])
            df_total_value.columns = ['account_value']
            df_total_value['date'] = self.date_memory
            df_total_value['daily_return']=df_total_value['account_value'].pct_change(1)
            #self.actions_memory.append('nan')
            #df_total_value['actions']=self.actions_memory
            df_total_value['state[0]']=self.states0_memory
            df_total_value['state[1]']=self.states1_memory
            df_total_value['state[3]']=self.states3_memory
            df_total_value['state[4]']=self.states4_memory
            df_total_value['state[5]']=self.states5_memory
            df_total_value['act']=self.states16_memory
            df_total_value['state[6]']=self.states6_memory
            df_total_value['begin_asset']=end_total_asset
            
            df_total_value['avlong']=self.states7_memory
            df_total_value['avshort']=self.states8_memory
            df_total_value['avsum']=self.states16a_memory
            
            df_total_value['gea']=self.states9_memory
            df_total_value['opPos']=self.states10_memory
            df_total_value['pct']=self.states11_memory
            df_total_value['open']=self.states12_memory
            df_total_value['low']=self.states13_memory
            df_total_value['high']=self.states14_memory
            df_total_value['stop']=self.states15_memory

            #df_total_value['actions']
            if df_total_value['daily_return'].std() !=0:
                sharpe = (252**0.5)*df_total_value['daily_return'].mean()/ \
                      df_total_value['daily_return'].std()
            df_rewards = pd.DataFrame(self.rewards_memory)
            df_rewards.columns = ['account_rewards']
            df_rewards['date'] = self.date_memory[:-1]
            if self.episode % self.print_verbosity == 0:
                print(f"day: {self.day}, episode: {self.episode}")
                print(f"begin_total_asset: {self.asset_memory[0]:0.10f}")
                print(f"end_total_asset: {end_total_asset:0.10f}")
                print(f"total_reward: {tot_reward:0.10f}")
                print(f"total_cost: {self.cost:0.10f}")
                print(f"total_trades: {self.trades}")
                if df_total_value['daily_return'].std() != 0:
                    print(f"Sharpe: {sharpe:0.3f}")
                print("=================================")

            if (self.model_name!='') and (self.mode!=''):
                df_actions = self.save_action_memory()
                df_actions.to_csv('results/actions_{}_{}_{}.csv'.format(self.mode,self.model_name, self.iteration))
                df_total_value.to_csv('results/account_value_{}_{}_{}.csv'.format(self.mode,self.model_name, self.iteration),index=False)
                df_rewards.to_csv('results/account_rewards_{}_{}_{}.csv'.format(self.mode,self.model_name, self.iteration),index=False)
                plt.plot(self.asset_memory,'r')
                plt.savefig('results/account_value_{}_{}_{}.png'.format(self.mode,self.model_name, self.iteration),index=False)
                plt.close()
                

            # Add outputs to logger interface
            #logger.record("environment/portfolio_value", end_total_asset)
            #logger.record("environment/total_reward", tot_reward)
            #logger.record("environment/total_reward_pct", (tot_reward / (end_total_asset - tot_reward)) * 100)
            #logger.record("environment/total_cost", self.cost)
            #logger.record("environment/total_trades", self.trades)
            #if self.reward<0:
            #    self.reward*=2
            return self.state, self.reward, self.terminal, {}
            #print(4,self.state)
        else:
            #print("start step:",actions,self.openedPos,round(self.state[0],2),round(self.state[1],2),round(self.state[3],2),round(self.state[4],2),round(self.state[5],2),round(self.state[6],2),round(self.state[7],2),round(self.state[8],2),round(self.state[9],2))
            #actions
            #actions = actions * self.hmax #actions initially is scaled between 0 to 1
            #actions=actions*10*abs(self.get_end_asset()//self.state[1],()
            t=actions
            begin_total_asset=self.get_end_asset()
            #if actions[0]==0:
            #    self.closePos()
            stop=actions[1]
            #tp=actions[2]
            #actions=actions[0]
            #self.musor.append(actions)
            #if len(self.musor)%25==0:
            #    print("act::::",self.musor)
            #    self.musor=[]
                
            act=actions[0]#-actions[1]
            if act>0:
              self.long(act)
            elif act<0:
              self.short(abs(act))
            elif act==0:
              pass
                  
            self.stoploss=0.1*(1+stop)/2#0.2 * actions[1]/50
            #self.takeprofit=0.2*(1+tp)/2
            self.actions_memory.append(actions)
            #self.action=argsort_actions
            self.asset_memory.append(begin_total_asset)
            #self.states0_memory.append(self.state[0])
            self.states1_memory.append(self.state[1])
            prev_state1=self.state[1] #необходим что ниже рассчитать стоп по текущей цене(до обновления)
            #print("sss:",self.state[0])
            self.states3_memory.append(self.state[3])
            self.states4_memory.append(self.state[4])
            self.states5_memory.append(self.state[5])
            self.states6_memory.append(self.state[6])
            #print('states:',self.state[3],self.state[4],self.state[5],self.state[6])
            self.states7_memory.append(self.state[7])
            self.states8_memory.append(self.state[8])
            self.states9_memory.append(self.state[9])
            self.states10_memory.append(self.state[10])
            self.states11_memory.append(self.state[11])
            self.states12_memory.append(self.state[12])
            self.states13_memory.append(self.state[13])
            self.states14_memory.append(self.state[14])
            self.states16_memory.append(actions)
            self.states16a_memory.append(self.state[16])
            
            self.day += 1
            self.data = self.df.loc[self.day,:]    
            if self.turbulence_threshold is not None:     
                #self.turbulence = self.data['turbulence'].values[0]
                self.turbulence = self.data['turbulence']
            self.state =  self._update_state()
            
            
            self.state[13]=self.data.low
            self.state[14]=self.data.high
            #if self.openedPos==1:
            #    sl=prev_state1*(1-self.stoploss)
               
            #elif self.openedPos==-1:
            #    sl=prev_state1*(1+self.stoploss)
            #else:
            #    sl=-1
            self.states15_memory.append(self.stoploss)
            
            self.closePos()     
            end_total_asset = self.get_end_asset()
            self.states0_memory.append(self.state[0])

            
            self.date_memory.append(self._get_date())
            self.reward = end_total_asset - begin_total_asset
            #print('end tot ass:',end_total_asset,"step reward:", self.reward, "params: state[0], self.state[3:6]", self.state[0],self.state[3],self.state[4],self.state[5],self.state[6])
            #print("step reward:", self.reward)
            self.rewards_memory.append(self.reward)
            self.reward = self.reward/begin_total_asset*self.reward_scaling
           # print("end step:",actions,self.openedPos,round(self.state[0],2),round(self.state[1],2),round(self.state[3],2),round(self.state[4],2),round(self.state[5],2),round(self.state[6],2),round(self.state[7],2),round(self.state[8],2),round(self.state[9],2))
            if self.reward<0:
                self.reward*=1.3
            #print(end_total_asset,self.state)
        return self.state, self.reward, self.terminal, {}

    def reset(self):  
        #initiate state
        self.state = self._initiate_state()
        if self.state[3]>0:
            self.openedPos=1
        elif self.state[5]>0:
            self.openedPos=-1
        if self.initial:
            self.asset_memory = [self.initial_amount]
            self.states0_memory= [self.state[0]]
            self.states1_memory= [self.state[1]]
            self.states3_memory= [self.state[3]]
            self.states4_memory= [self.state[4]]
            self.states5_memory= [self.state[5]]
            self.states6_memory= [self.state[6]]
            #print('states2:',self.state[3],self.state[4],self.state[5],self.state[6])
            self.states7_memory= [self.state[7]]
            self.states8_memory= [self.state[8]]
            self.states9_memory= [self.state[9]]
            self.states10_memory= [self.state[10]]
            self.states11_memory= [self.state[11]]
            self.states12_memory= [self.state[12]]
            self.states13_memory= [self.state[13]]
            self.states14_memory= [self.state[14]]
            self.states15_memory= [self.state[15]]
            self.states16_memory=[0]
            self.states16a_memory= [self.state[16]]
        else:
            #raise NotImplementedError
            previous_total_asset = self.previous_state[0]+ \
                self.previous_state[4]*self.previous_state[3] + (self.previous_state[1] - self.previous_state[4])*self.previous_state[3] +\
                self.previous_state[6]*self.previous_state[5] + (self.previous_state[6]-self.previous_state[1])*self.previous_state[5]
            previous_total_asset2 = self.state[0]+ \
                self.state[4]*self.state[3] + (self.state[1] - self.state[4])*self.state[3] +\
                self.state[6]*self.state[5] + (self.state[6]-self.state[1])*self.state[5]
           
            #self.previous_state[0]+ \
            #sum(np.array(self.state[1:(self.stock_dim+1)])*np.array(self.previous_state[(self.stock_dim+1):(self.stock_dim*2+1)]))
            self.asset_memory = [previous_total_asset]
            #print('pta:',previous_total_asset)
            #print('current:', self.state[0],self.state[1],self.state[2],self.state[3],self.state[4],self.state[5],self.state[6])
            #print('prev:', self.previous_state[0],self.previous_state[1],self.previous_state[2],self.previous_state[3],self.previous_state[4],self.previous_state[5],self.previous_state[6])
            #print('base:',self.previous_state[0]+ self.state[1:(self.stock_dim+1)]+np.array(self.previous_state[(self.stock_dim+1):(self.stock_dim*2+1)]),self.previous_state[0], self.state[1:(self.stock_dim+1)],np.array(self.previous_state[(self.stock_dim+1):(self.stock_dim*2+1)]))
        self.day = 0
        self.data = self.df.loc[self.day,:]
        self.turbulence = 0
        self.cost = 0
        self.trades = 0
        self.terminal = False 
        # self.iteration=self.iteration
        self.rewards_memory = []
        self.actions_memory=[]
        self.date_memory=[self._get_date()]
        #self.openedPos=0
        self.episode+=1
        #print(6,self.state)
        return self.state
    
    def render(self, mode='human',close=False):
        return self.state

    def _initiate_state(self):
        if self.initial:
            # For Initial State
            if len(self.df.tic.unique())>1:
                # for multiple stock
                state = [self.initial_amount] + \
                         self.data.close.values.tolist() + \
                         [0]*self.stock_dim  + \
                         sum([self.data[tech].values.tolist() for tech in self.tech_indicator_list ], [])
            else:
                # for single stock
                
                state = [self.initial_amount] + \
                        [self.data.close] + \
                        [0]*self.stock_dim  + \
                        [0]+\
                        [0]+\
                        [0]+\
                        [0]+\
                        [0] + [0]+ [0*self.initial_amount] + [0*self.openedPos] +[self.data.pct_ch] +\
                        [self.data.open]+[self.data.low]+[self.data.high]+\
                        [0]+\
                        [0*round(2 * self.initial_amount/self.data.close,self.precision)]+\
                        sum([[self.data[tech]] for tech in self.tech_indicator_list ], [])
                #print(0,'len:',len(state),state)
        else:
            #Using Previous State
            if len(self.df.tic.unique())>1:
                # for multiple stock
                state = [self.previous_state[0]] + \
                         self.data.close.values.tolist() + \
                         self.previous_state[(self.stock_dim+1):(self.stock_dim*2+1)]  + \
                         sum([self.data[tech].values.tolist() for tech in self.tech_indicator_list ], [])
            else:
                # for single stock
                state = [self.previous_state[0]] + \
                        [self.data.close] + \
                        self.previous_state[(self.stock_dim+1):(self.stock_dim*2+1)]  + \
                        [self.previous_state[3]] + [self.previous_state[4]] + [self.previous_state[5]] + [self.previous_state[6]] + \
                        [self.previous_state[7]]+[self.previous_state[8]] + [self.previous_state[9]] +[self.previous_state[10]]+[self.previous_state[11]] +\
                        [self.previous_state[12]]+[self.previous_state[13]]+[self.previous_state[14]]+\
                        [self.previous_state[15]]+\
                        [self.previous_state[16]]+\
                        sum([[self.data[tech]] for tech in self.tech_indicator_list ], [])
                #print(1,state)
        #print("initial:",self.initial)
        return state

    def _update_state(self):
        if len(self.df.tic.unique())>1:
            # for multiple stock
            state =  [self.state[0]] + \
                      self.data.close.values.tolist() + \
                      list(self.state[(self.stock_dim+1):(self.stock_dim*2+1)]) + \
                      sum([self.data[tech].values.tolist() for tech in self.tech_indicator_list ], [])

        else:
            
            self.state[13]=self.data.low
            self.state[14]=self.data.high
            #self.state[1]=self.data.close
            if self.openedPos==1:
                self.stoploss=self.state[1]*(1-self.stoploss)
                if self.state[13]<=self.stoploss and self.stoploss<self.state[1]:
                    #self.states15_memory.append([self.stoploss,self.state[13],self.state[1],self.openedPos])
                    pnl=self.PNL(stopPrice=self.stoploss)
                    self.state[0]+=self.state[3]*self.state[4]+pnl#+self.PNL(stopPrice=self.stoploss)
                    self.state[3]=0
                    self.state[4]=0
                    self.openedPos=0
                    #self.stoploss=0
                    #print('stop from 1:', self.state[13],self.stoploss)
                
                    
            elif self.openedPos==-1:
                self.stoploss=self.state[1]*(1+self.stoploss)
                if self.state[14]>=self.stoploss and self.stoploss>self.state[1]:
                    #self.states15_memory.append([self.stoploss,self.state[14],self.state[1],self.openedPos])
                    pnl=self.PNL(stopPrice=self.stoploss)
                    self.state[0]+=self.state[5]*self.state[6] + pnl#+self.PNL(stopPrice=self.stoploss)
                    self.state[5]=0
                    self.state[6]=0
                    self.openedPos=0
                    #self.stoploss=0

                    
                    #print('stop from -1:', self.state[14],self.stoploss)
            self.state[1]=self.data.close
            
            state =  [self.state[0]] + \
                     [self.data.close] + \
                     list(self.state[(self.stock_dim+1):(self.stock_dim*2+1)]) + \
                     [self.state[3]] + [self.state[4]] + [self.state[5]] + [self.state[6]] + [0*self.avLongQty()/(self.avLongQty()+self.avShortQty()+1e-10)] + [0*self.avShortQty()/(self.avLongQty()+self.avShortQty() +1e-10)]+ [0*self.get_end_asset()] + [0*self.openedPos] +[self.data.pct_ch] +\
                     [self.data.open]+[self.data.low]+[self.data.high]+\
                     [self.stoploss]+\
                     [0*(self.avLongQty()+self.avShortQty())]+\
                     sum([[self.data[tech]] for tech in self.tech_indicator_list ], [])
            #print(5,self.openedPos,self.action,state)
                          
        return state

    def _get_date(self):
        if len(self.df.tic.unique())>1:
            date = self.data.date.unique()[0]
        else:
            date = self.data.date
        return date

    def save_asset_memory(self):
        date_list = self.date_memory
        asset_list = self.asset_memory
        #print(len(date_list))
        #print(len(asset_list))
        df_account_value = pd.DataFrame({'date':date_list,'account_value':asset_list})
        return df_account_value

    def save_action_memory(self):
        if len(self.df.tic.unique())>1:
            # date and close price length must match actions length
            date_list = self.date_memory[:-1]
            df_date = pd.DataFrame(date_list)
            df_date.columns = ['date']
            
            action_list = self.actions_memory
            df_actions = pd.DataFrame(action_list)
            df_actions.columns = self.data.tic.values
            df_actions.index = df_date.date
            #df_actions = pd.DataFrame({'date':date_list,'actions':action_list})
        else:
            date_list = self.date_memory[:-1]
            action_list = self.actions_memory
            df_actions = pd.DataFrame({'date':date_list,'actions':action_list})
        return df_actions

    def _seed(self, seed=None):
        self.np_random, seed = seeding.np_random(seed)
        return [seed]


    def get_sb_env(self):
        e = DummyVecEnv([lambda: self])
        obs = e.reset()
        return e, obs
#from stable_baselines3.common import logger A stock trading environment for OpenAI gym #self.action_space = spaces.Box(low = -1, high = 1,shape = (1,),dtype=np.float32) #self.action_space = spaces.MultiDiscrete([2,2,21,51]) # initalize state # initialize reward # memorize all the total balance change #self.reset() #print('0 pos') #print('0 pos') # action=self.avShortQty()*abs(action)+self.avLongQty( # Sell only if the price is > 0 (no missing data in this particular date) # perform sell action based on the sign of the action #if self.state[index+self.stock_dim+1] > 0: # Sell only if current asset is > 0 #total_contract_value=self.state[3]*self.state[4] - abs(action)*self.state[1] #avg_entry_price=total_contract_value/(self.state[3] - abs(action)) #self.state[4] = avg_entry_price #self.state[0]+=pnl #### #print('states:',self.state[3],self.state[4],self.state[5],self.state[6]) #action=self.avLongQty()*abs(action) # Sell only if the price is > 0 (no missing data in this particular date) # perform sell action based on the sign of the action #if self.state[index+self.stock_dim+1] > 0: # Sell only if current asset is > 0 #self.state[0]+=pnl ###### #total_contract_value=self.state[5]*self.state[6] - abs(action)*self.state[1] #avg_entry_price=total_contract_value/(self.state[5] - abs(action)) #self.state[0]-=self.state[1]*abs(action) ####### #self.state[6] = avg_entry_price #print('states:',self.state[3],self.state[4],self.state[5],self.state[6]) #pnl=self.PNL() #print(end_total_asset, self.state[0],self.longPosBP,self.longPosQty,self.shortPosBP,self.shortPosQty) # print(f"Episode: {self.episode}") # sum(np.array(self.state[1:(self.stock_dim+1)])*np.array(self.state[(self.stock_dim+1):(self.stock_dim*2+1)])) #print(np.array(self.state[1:(self.stock_dim+1)]),np.array(self.state[(self.stock_dim+1):(self.stock_dim*2+1)]),np.array(self.state[1:(self.stock_dim+1)])*np.array(self.state[(self.stock_dim+1):(self.stock_dim*2+1)])) #print("tot reward:", tot_reward,"params: state[0], self.state[3:6]", self.state[0],self.state[3],self.state[4],self.state[5],self.state[6]) #self.actions_memory.append('nan') #df_total_value['actions']=self.actions_memory #df_total_value['actions'] # Add outputs to logger interface #logger.record("environment/portfolio_value", end_total_asset) #logger.record("environment/total_reward", tot_reward) #logger.record("environment/total_reward_pct", (tot_reward / (end_total_asset - tot_reward)) * 100) #logger.record("environment/total_cost", self.cost) #logger.record("environment/total_trades", self.trades) #if self.reward<0: #    self.reward*=2 #print(4,self.state) #print("start step:",actions,self.openedPos,round(self.state[0],2),round(self.state[1],2),round(self.state[3],2),round(self.state[4],2),round(self.state[5],2),round(self.state[6],2),round(self.state[7],2),round(self.state[8],2),round(self.state[9],2)) #actions #actions = actions * self.hmax #actions initially is scaled between 0 to 1 #actions=actions*10*abs(self.get_end_asset()//self.state[1],() #if actions[0]==0: #    self.closePos() #stop=actions[1] #tp=actions[1] #actions=actions[0] #self.musor.append(actions) #if len(self.musor)%25==0: #    print("act::::",self.musor) #    self.musor=[] #[0]#-actions[1] #self.short(abs(act)) #self.stoploss=0.1*(1+stop)/2#0.2 * actions[1]/50 #self.stoploss=0.07*(1+tp)/2 #self.action=argsort_actions #self.states0_memory.append(self.state[0]) #необходим что ниже рассчитать стоп по текущей цене(до обновления) #print("sss:",self.state[0]) #print('states:',self.state[3],self.state[4],self.state[5],self.state[6]) #self.turbulence = self.data['turbulence'].values[0] #if self.openedPos==1: #    sl=prev_state1*(1-self.stoploss) #elif self.openedPos==-1: #    sl=prev_state1*(1+self.stoploss) #else: #    sl=-1 #print('end tot ass:',end_total_asset,"step reward:", self.reward, "params: state[0], self.state[3:6]", self.state[0],self.state[3],self.state[4],self.state[5],self.state[6]) #print("step reward:", self.reward) # print("end step:",actions,self.openedPos,round(self.state[0],2),round(self.state[1],2),round(self.state[3],2),round(self.state[4],2),round(self.state[5],2),round(self.state[6],2),round(self.state[7],2),round(self.state[8],2),round(self.state[9],2)) #if self.reward<0: #    self.reward*=1.3 #print(end_total_asset,self.state) #initiate state #print('states2:',self.state[3],self.state[4],self.state[5],self.state[6]) #raise NotImplementedError #self.previous_state[0]+ \ #sum(np.array(self.state[1:(self.stock_dim+1)])*np.array(self.previous_state[(self.stock_dim+1):(self.stock_dim*2+1)])) #print('pta:',previous_total_asset) #print('current:', self.state[0],self.state[1],self.state[2],self.state[3],self.state[4],self.state[5],self.state[6]) #print('prev:', self.previous_state[0],self.previous_state[1],self.previous_state[2],self.previous_state[3],self.previous_state[4],self.previous_state[5],self.previous_state[6]) #print('base:',self.previous_state[0]+ self.state[1:(self.stock_dim+1)]+np.array(self.previous_state[(self.stock_dim+1):(self.stock_dim*2+1)]),self.previous_state[0], self.state[1:(self.stock_dim+1)],np.array(self.previous_state[(self.stock_dim+1):(self.stock_dim*2+1)])) # self.iteration=self.iteration #self.openedPos=0 #print(6,self.state) # For Initial State # for multiple stock # for single stock #print(0,'len:',len(state),state) #Using Previous State # for multiple stock # for single stock #print(1,state) #print("initial:",self.initial) # for multiple stock #self.state[1]=self.data.close #self.openedPos==-1: #self.states15_memory.append([self.stoploss,self.state[13],self.state[1],self.openedPos]) #+self.PNL(stopPrice=self.stoploss) #self.stoploss=0 #print('stop from 1:', self.state[13],self.stoploss) #self.openedPos==1: #self.states15_memory.append([self.stoploss,self.state[14],self.state[1],self.openedPos]) #+self.PNL(stopPrice=self.stoploss) #self.stoploss=0 #print('stop from -1:', self.state[14],self.stoploss) #print(5,self.openedPos,self.action,state) #print(len(date_list)) #print(len(asset_list)) # date and close price length must match actions length #df_actions = pd.DataFrame({'date':date_list,'actions':action_list}) A stock trading environment for OpenAI gym #self.action_space = spaces.MultiDiscrete([2,2,21,51]) # initalize state # initialize reward # memorize all the total balance change #self.reset() #print('0 pos') #print('0 pos') # action=self.avShortQty()*abs(action)+self.avLongQty( # Sell only if the price is > 0 (no missing data in this particular date) # perform sell action based on the sign of the action #if self.state[index+self.stock_dim+1] > 0: # Sell only if current asset is > 0 #total_contract_value=self.state[3]*self.state[4] - abs(action)*self.state[1] #avg_entry_price=total_contract_value/(self.state[3] - abs(action)) #self.state[4] = avg_entry_price #self.state[0]+=pnl #### #print('states:',self.state[3],self.state[4],self.state[5],self.state[6]) #action=self.avLongQty()*abs(action) # Sell only if the price is > 0 (no missing data in this particular date) # perform sell action based on the sign of the action #if self.state[index+self.stock_dim+1] > 0: # Sell only if current asset is > 0 #self.state[0]+=pnl ###### #total_contract_value=self.state[5]*self.state[6] - abs(action)*self.state[1] #avg_entry_price=total_contract_value/(self.state[5] - abs(action)) #self.state[0]-=self.state[1]*abs(action) ####### #self.state[6] = avg_entry_price #print('states:',self.state[3],self.state[4],self.state[5],self.state[6]) #pnl=self.PNL() #print(end_total_asset, self.state[0],self.longPosBP,self.longPosQty,self.shortPosBP,self.shortPosQty) # print(f"Episode: {self.episode}") # sum(np.array(self.state[1:(self.stock_dim+1)])*np.array(self.state[(self.stock_dim+1):(self.stock_dim*2+1)])) #print(np.array(self.state[1:(self.stock_dim+1)]),np.array(self.state[(self.stock_dim+1):(self.stock_dim*2+1)]),np.array(self.state[1:(self.stock_dim+1)])*np.array(self.state[(self.stock_dim+1):(self.stock_dim*2+1)])) #print("tot reward:", tot_reward,"params: state[0], self.state[3:6]", self.state[0],self.state[3],self.state[4],self.state[5],self.state[6]) #self.actions_memory.append('nan') #df_total_value['actions']=self.actions_memory #df_total_value['actions'] # Add outputs to logger interface #logger.record("environment/portfolio_value", end_total_asset) #logger.record("environment/total_reward", tot_reward) #logger.record("environment/total_reward_pct", (tot_reward / (end_total_asset - tot_reward)) * 100) #logger.record("environment/total_cost", self.cost) #logger.record("environment/total_trades", self.trades) #if self.reward<0: #    self.reward*=2 #print(4,self.state) #print("start step:",actions,self.openedPos,round(self.state[0],2),round(self.state[1],2),round(self.state[3],2),round(self.state[4],2),round(self.state[5],2),round(self.state[6],2),round(self.state[7],2),round(self.state[8],2),round(self.state[9],2)) #actions #actions = actions * self.hmax #actions initially is scaled between 0 to 1 #actions=actions*10*abs(self.get_end_asset()//self.state[1],() #if actions[0]==0: #    self.closePos() #tp=actions[2] #actions=actions[0] #self.musor.append(actions) #if len(self.musor)%25==0: #    print("act::::",self.musor) #    self.musor=[] #-actions[1] #0.2 * actions[1]/50 #self.takeprofit=0.2*(1+tp)/2 #self.action=argsort_actions #self.states0_memory.append(self.state[0]) #необходим что ниже рассчитать стоп по текущей цене(до обновления) #print("sss:",self.state[0]) #print('states:',self.state[3],self.state[4],self.state[5],self.state[6]) #self.turbulence = self.data['turbulence'].values[0] #if self.openedPos==1: #    sl=prev_state1*(1-self.stoploss) #elif self.openedPos==-1: #    sl=prev_state1*(1+self.stoploss) #else: #    sl=-1 #print('end tot ass:',end_total_asset,"step reward:", self.reward, "params: state[0], self.state[3:6]", self.state[0],self.state[3],self.state[4],self.state[5],self.state[6]) #print("step reward:", self.reward) # print("end step:",actions,self.openedPos,round(self.state[0],2),round(self.state[1],2),round(self.state[3],2),round(self.state[4],2),round(self.state[5],2),round(self.state[6],2),round(self.state[7],2),round(self.state[8],2),round(self.state[9],2)) #print(end_total_asset,self.state) #initiate state #print('states2:',self.state[3],self.state[4],self.state[5],self.state[6]) #raise NotImplementedError #self.previous_state[0]+ \ #sum(np.array(self.state[1:(self.stock_dim+1)])*np.array(self.previous_state[(self.stock_dim+1):(self.stock_dim*2+1)])) #print('pta:',previous_total_asset) #print('current:', self.state[0],self.state[1],self.state[2],self.state[3],self.state[4],self.state[5],self.state[6]) #print('prev:', self.previous_state[0],self.previous_state[1],self.previous_state[2],self.previous_state[3],self.previous_state[4],self.previous_state[5],self.previous_state[6]) #print('base:',self.previous_state[0]+ self.state[1:(self.stock_dim+1)]+np.array(self.previous_state[(self.stock_dim+1):(self.stock_dim*2+1)]),self.previous_state[0], self.state[1:(self.stock_dim+1)],np.array(self.previous_state[(self.stock_dim+1):(self.stock_dim*2+1)])) # self.iteration=self.iteration #self.openedPos=0 #print(6,self.state) # For Initial State # for multiple stock # for single stock #print(0,'len:',len(state),state) #Using Previous State # for multiple stock # for single stock #print(1,state) #print("initial:",self.initial) # for multiple stock #self.state[1]=self.data.close #self.states15_memory.append([self.stoploss,self.state[13],self.state[1],self.openedPos]) #+self.PNL(stopPrice=self.stoploss) #self.stoploss=0 #print('stop from 1:', self.state[13],self.stoploss) #self.states15_memory.append([self.stoploss,self.state[14],self.state[1],self.openedPos]) #+self.PNL(stopPrice=self.stoploss) #self.stoploss=0 #print('stop from -1:', self.state[14],self.stoploss) #print(5,self.openedPos,self.action,state) #print(len(date_list)) #print(len(asset_list)) # date and close price length must match actions length #df_actions = pd.DataFrame({'date':date_list,'actions':action_list})