.NAN)
    #param = {'eta': 0.05,'max_depth': 8,'eval_metric': 'auc', 'objective': 'binary:logistic','silent': 1}

    param={
        'objective':"reg:linear",
        'booster':"gbtree",
        'eval_metric':"rmse",
        'eta':0.1,
       'max_depth':6,
        #'colsample_bytree':0.7,
        #'min_child_weight':1,
        #'num_boost_round':20,
       'silent': 1
        }

    evallist = [(dtrain,'dtrain'),(dval, 'eval')]
    num_round = 200
    bst = xgb.train(param, dTrain, num_round, evals=evallist)
    ypred = bst.predict(dtest, ntree_limit=bst.best_ntree_limit)
    ypred = pd.DataFrame(ypred)
    return ypred


# ## --------------------------------------------

# ## Starting with the dataset

data = pd.read_csv("dataset.csv")
meta_data = pd.read_csv("ML_Bond_metadata.csv")

data.head()


# ## **
#

# First, we have clean the given dataset
# Second, we have converted the given dataset to include the timeofday when on given dates trade are not done.
# This we have generated and stored in train_data_gen.csv. 
# So, for this code, we already had the generated file and so we are just mentioning the function by which we generated.
# ##### Making this train_data_gen (i.e., making data for the timeofday on given dates when no trade is done)

# +
def generate_train_data_gen(data):
    m_new=data.groupby(["isin","date","side","timeofday"]).volume.sum().reset_index()
    m_backup = m_new.copy()
    m_new["date"] = convert_epoch(m_new["date"]).values
    data_cl = data_clean(m_new)
    print(data_cl.head())
    tiger = list(data_cl.groupby(["isin","date"]))
    poss_combo = [(0,1), (0,0), (1,0), (1,1)]
    len_tiger = len(tiger)

    main_isin_list = []
    print("start")
    for i in range(len_tiger):
        if i%10000 == 0:
            print(i)
        side_l = list(tiger[i][1]["side"])
        timeofday_l = list(tiger[i][1]["timeofday"])
        combo_list = list(zip(side_l, timeofday_l))
        new_side_l = []
        new_timeofday_l = []
        for combo in poss_combo:
            if combo not in combo_list:
                new_side_l.append(combo[0])
                new_timeofday_l.append(combo[1])

        len_this = len(new_side_l)
        list_df = []
        weekday_l = tiger[i][1]["weekday"].iloc[0]
        if len_this!=0:
            list_df.append(tiger[i][1])
            tmp_df = pd.DataFrame(index = np.arange(len_this))    
            tmp_df["isin"] = tiger[i][0][0]
            tmp_df["date"] = tiger[i][0][1]
            tmp_df["side"] = new_side_l
            tmp_df["timeofday"] = new_timeofday_l
            tmp_df["volume"] = 0
            #print(tiger[i][1]["weekday"])
            tmp_df["weekday"] = weekday_l
            list_df.append(tmp_df)
            main_kl = pd.concat(list_df)
            main_isin_list.append(main_kl)
            #print(main_kl)
        else:
            main_isin_list.append(tiger[i][1])


    main_isin_df = pd.concat(main_isin_list)
    print(main_isin_df.shape)
    print(main_isin_df.head())
    main_isin_df.to_csv("train_data_gen.csv")
    
# Generating train_data_gen.csv
# Here we have already generated files in our previous codes, so just putting the function here and the way to generate it
generate_train_data_gen(data)
# -

main_isin_df = pd.read_csv("train_data_gen.csv")
print(main_isin_df.shape)
main_isin_df.head()

# +
main_isin_df = pd.read_csv("train_data_gen.csv")
print(main_isin_df.shape)
main_isin_df.head()

main_isin_df = main_isin_df.drop("Unnamed: 0", axis = 1)
# -

main_isin_df_group = list(main_isin_df.groupby("isin"))

# +
#try_12 = main_isin_df_group[0][1].copy()
#try_12.sort_values(by=['date', 'timeofday'], ascending=[True, False])
# -

# ## ********************

# Now, from the made train_data, we now added the dates which were not present.
# #### We included the days when no trade was done for a given bond. But assuring that on those days, one of the bond was traded. This improved the accuracy.
# #### This clearly shows that we have to take into account for each bond when they were not traded, as there is a high possibility for some bonds that they will not be traded on 10th, 13th and 14th June. 

mn = main_isin_df["date"].max()
ep_to_day(mn)
#datetime.fromtimestamp(mn)

print(main_isin_df["date"].nunique())
poss_dates = list(main_isin_df["date"].unique())
#print(poss_dates)

# +
len_tiger = len(main_isin_df_group)
main_isin_list = []
print("start")
side_list = [0, 0, 1, 1]
time_list = [0, 1, 0, 1]
full_list_bond_df = []
for i in range(len_tiger):
    if i%10000 == 0:
        print(i)
    main_per_bondname = main_isin_df_group[i][0]
    main_per_bond = main_isin_df_group[i][1]
    weekday_l = main_per_bond["weekday"].iloc[0]
    date_list = list(main_per_bond["date"].unique())
    list_for_each_bond = []
    list_for_each_bond.append(main_per_bond)
    for x in poss_dates:
        if x not in date_list:
            tmp_df = pd.DataFrame(index = np.arange(4))
            tmp_df["isin"] = main_per_bondname
            tmp_df["date"] = x
            tmp_df["side"] = side_list
            tmp_df["timeofday"] = time_list
            tmp_df["volume"] = 0
            tmp_df["weekday"] = weekday_l
            list_for_each_bond.append(tmp_df)
    for_each_bond_df = pd.concat(list_for_each_bond)
    full_list_bond_df.append(for_each_bond_df)


# -

MAIN_df_now = pd.concat(full_list_bond_df)

MAIN_df_now.to_csv("MAIN_DF.csv")

MAIN_df_now.shape

MAIN_df_now.head()

MAIN_df_now.shape

MAIN_df_now.describe()

MAIN_df_now["date"].max() - MAIN_df_now["date"].min()

min_date = MAIN_df_now["date"].min()
again_poss_dates = list(MAIN_df_now["date"].unique())

print(min_date)
MAIN_df_now["date