# -*- coding: utf-8 -*-
"""
Created on Fri May  1 19:29:28 2020

@author: Mike
"""

##############################################################################
configfilename = "configfile.csv"
##############################################################################

import numpy as np
import pandas as pd
from datetime import datetime

from tensorflow.keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation, Flatten
from tensorflow.keras.layers import Bidirectional, GlobalMaxPool1D, BatchNormalization, Conv1D, GlobalMaxPooling1D
from tensorflow.keras.models import Model, Sequential
from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau, CSVLogger
from tensorflow.keras.models import model_from_json
from tensorflow.keras import metrics
#from tensorflow.keras.utils import multi_gpu_model
from tensorflow.keras.optimizers import Nadam, Adam

from sklearn.metrics import mean_squared_error, mean_absolute_error
from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score
from sklearn.model_selection import train_test_split as split

import tensorflow.keras.backend as K
import tensorflow.keras.callbacks as Kc

# TF2: Mixed precision

#from tensorflow.keras.mixed_precision import experimental as mixed_precision
#mixed_precision.set_policy('mixed_float16')

# TF2: Disable eager execution
import tensorflow as tf
tf.compat.v1.disable_eager_execution()
tf.get_logger().setLevel('INFO')


import time




def AE(y, y_hat):
    AE = np.abs(y - y_hat)
    return AE


# Aggregated metrics:
    
# MAE = np.mean(AE)
# MEP = np.mean(EP)

def ear(table, alpha=2):    
    #absolute error
    table["AE"] = AE(table["y"], table["y_pred"])
    
    #temporal decay
    table["AE_td"] = table["AE"]/alpha*table["prefix_number"]
    
    # Calculate metrics:
        
    # MAE:
    MAE_td = np.mean(table["AE_td"])
     
    #Aggregated metrics:
    table["MAE_td"] = MAE_td
    
    return table


def acc(table):
    """
    Evaluates earliness alone
    
    Input: 
        Inference table with relevant variables
        
            T = "num_events"
            t = "event_number"
            
            case = "caseid"
            y = "y"
            y_pred = "y_pred"
    
    Output:
        EP, AE, MEP, MAE, MAEPE
    """
    
    # Calculate metrics:
        
    #Absolute error
    table["AE"] = AE(table["y"], table["y_pred"])

    # MAE:
    MAE = np.mean(table["AE"])
     
    #Aggregated metrics:
    table["MAE"] = MAE
    
    return table



def Earliness(Inference_test, parallel=False, EAR=True, TS=True):
    
    start_time = time.time()
 
    """
    Earliness
    """
    print("Earliness...")
    Inference_test = acc(Inference_test)
    Inference_test = ear(Inference_test, alpha=1)
    
    end_time = time.time()
    Time_sec = end_time - start_time
    print(Time_sec)

    return Inference_test
    
"""
Evaluation pipeline
"""

def evaluate_model(data_objects, transform="log"):
    print("Initial-model evaluation..")
    
    # Load config-file
    configfile = pd.read_csv(configfilename)
    Project_dir = configfile.Project_dir.values[0]    
    RUN = configfile["RUN"][0]
    F_dataset = configfile["F_dataset"][0]
    

    # Load inference tables
    #Inference_train = data_objects["Inference_train"]
    Inference_test = data_objects["Inference_test"]   
    
    # Load the model
    model = data_objects["model"]
  
    
    # Load the training data
    x_test, y_test = data_objects["x_test"], data_objects["y_test"]
    
    # Predict on inference table
    Inference_test["y_pred"] = model.predict(x_test, verbose=1, batch_size=2048)
    
    # Save information about experiment
    Inference_test["RUN"] = [RUN]*len(Inference_test)
    Inference_test["F_dataset"] = [F_dataset]*len(Inference_test)
      
    # Time information
    time_taken = 0
    now = datetime.now() # current date and time
    timestamp = now.strftime("%Y/%m/%d, %H:%M:%S")
    
    if "train_time" in data_objects:
        time_taken = data_objects["train_time"]
    
    # Get model training info
    model_params = data_objects["model_params"]
    epochs = data_objects["epochs"]
    
    ###################################################################
    #Log transform inverse  
    if transform == "log":    
        Inference_test["y_pred"] = np.exp(Inference_test["y_pred"])-1
        #Inference_test["y"] = np.exp(Inference_test["y"])-1
        
    #Range transform inverse  
    if transform == "range":
        Inference_test["y_pred"] = (Inference_test["y_pred"] * (data_objects["y_test_max"] - data_objects["y_test_min"])) + data_objects["y_test_min"]
        
    ###################################################################
    
    # Calculate earliness    
    Inference_test = Earliness(Inference_test, EAR=True)
    
    ########## ACCURACY #######################
    
    mae_test = np.mean(Inference_test["MAE"])/(24.0*3600)
    maetd_test = np.mean(Inference_test["MAE_td"])/(24.0*3600)
        
    # Print status:
    print('_'*60)
    print('Test MAE:     ', mae_test, ' (days)')
    print("================================"*3)
        
    
    ########### Generate a report on individual level ########

    # Dump the inference tables for debugging ## REMOVED 26/09
    Inference_test.to_csv(Project_dir+"/experiments/"+str(RUN)+"/inference_tables/Inf_test.csv",index=False)
        
    # Generate report with parameters of interest
    results = {"RUN":RUN,
               "Time":timestamp,
               "Traintime":time_taken,
               "MAE":mae_test,
               "MAE_td":maetd_test}

    report = dict(results, **model_params)
    
    report = pd.DataFrame(report, index=[0])
    
    # Storing individual
    report.to_csv(Project_dir+"/experiments/"+str(RUN)+"/"+str(RUN)+".csv",index=False)
    
    """
    Generate prefix results:
    """
    
    """
    #flexible
    max_events = np.max(Inference_test.event_number)

    if max_events < 10:
        events = 5
    else:
        events = 10
    """  
    #fixed
    events = 5
        
    inf_results = Inference_test.loc[Inference_test.event_number < events+1]
    inf_results.AE = inf_results.AE /(24.0*3600)
    
    pivottable = pd.pivot_table(inf_results, 
                           values='AE',
                        columns=['event_number'], aggfunc=np.mean)
    
    newcols = []
    for colno in range(0,len(pivottable.columns)):
        colno = colno + 1
        name = "AE_"+str(colno)+""
        newcols.append(name)
    
    pivottable.columns = newcols
    pivottable.index = [0]
    
    return pivottable


# -*- coding: utf-8 -*-
"""
Created on Fri May  1 19:29:28 2020

@author: Mike
"""

##############################################################################
configfilename = "configfile.csv"
##############################################################################

import numpy as np
import pandas as pd
from datetime import datetime

from tensorflow.keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation, Flatten
from tensorflow.keras.layers import Bidirectional, GlobalMaxPool1D, BatchNormalization, Conv1D, GlobalMaxPooling1D
from tensorflow.keras.models import Model, Sequential
from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau, CSVLogger
from tensorflow.keras.models import model_from_json
from tensorflow.keras import metrics
#from tensorflow.keras.utils import multi_gpu_model
from tensorflow.keras.optimizers import Nadam, Adam

from sklearn.metrics import mean_squared_error, mean_absolute_error
from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score
from sklearn.model_selection import train_test_split as split

import tensorflow.keras.backend as K
import tensorflow.keras.callbacks as Kc

# TF2: Mixed precision

#from tensorflow.keras.mixed_precision import experimental as mixed_precision
#mixed_precision.set_policy('mixed_float16')

# TF2: Disable eager execution
import tensorflow as tf
tf.compat.v1.disable_eager_execution()
tf.get_logger().setLevel('INFO')


import time




def AE(y, y_hat):
    AE = np.abs(y - y_hat)
    return AE


# Aggregated metrics:
    
# MAE = np.mean(AE)
# MEP = np.mean(EP)

def ear(table, alpha=2):    
    #absolute error
    table["AE"] = AE(table["y"], table["y_pred"])
    
    #temporal decay
    table["AE_td"] = table["AE"]/alpha*table["prefix_number"]
    
    # Calculate metrics:
        
    # MAE:
    MAE_td = np.mean(table["AE_td"])
     
    #Aggregated metrics:
    table["MAE_td"] = MAE_td
    
    return table


def acc(table):
    """
    Evaluates earliness alone
    
    Input: 
        Inference table with relevant variables
        
            T = "num_events"
            t = "event_number"
            
            case = "caseid"
            y = "y"
            y_pred = "y_pred"
    
    Output:
        EP, AE, MEP, MAE, MAEPE
    """
    
    # Calculate metrics:
        
    #Absolute error
    table["AE"] = AE(table["y"], table["y_pred"])

    # MAE:
    MAE = np.mean(table["AE"])
     
    #Aggregated metrics:
    table["MAE"] = MAE
    
    return table



def Earliness(Inference_test, parallel=False, EAR=True, TS=True):
    
    start_time = time.time()
 
    """
    Earliness
    """
    print("Earliness...")
    Inference_test = acc(Inference_test)
    Inference_test = ear(Inference_test, alpha=1)
    
    end_time = time.time()
    Time_sec = end_time - start_time
    print(Time_sec)

    return Inference_test
    
"""
Evaluation pipeline
"""

def evaluate_model(data_objects, transform="log"):
    print("Initial-model evaluation..")
    
    # Load config-file
    configfile = pd.read_csv(configfilename)
    Project_dir = configfile.Project_dir.values[0]    
    RUN = configfile["RUN"][0]
    F_dataset = configfile["F_dataset"][0]
    

    # Load inference tables
    #Inference_train = data_objects["Inference_train"]
    Inference_test = data_objects["Inference_test"]   
    
    # Load the model
    model = data_objects["model"]
  
    
    # Load the training data
    x_test, y_test = data_objects["x_test"], data_objects["y_test"]
    
    # Predict on inference table
    Inference_test["y_pred"] = model.predict(x_test, verbose=1, batch_size=2048)
    
    # Save information about experiment
    Inference_test["RUN"] = [RUN]*len(Inference_test)
    Inference_test["F_dataset"] = [F_dataset]*len(Inference_test)
      
    # Time information
    time_taken = 0
    now = datetime.now() # current date and time
    timestamp = now.strftime("%Y/%m/%d, %H:%M:%S")
    
    if "train_time" in data_objects:
        time_taken = data_objects["train_time"]
    
    # Get model training info
    model_params = data_objects["model_params"]
    epochs = data_objects["epochs"]
    
    ###################################################################
    #Log transform inverse  
    if transform == "log":    
        Inference_test["y_pred"] = np.exp(Inference_test["y_pred"])-1
        #Inference_test["y"] = np.exp(Inference_test["y"])-1
        
    #Range transform inverse  
    if transform == "range":
        Inference_test["y_pred"] = (Inference_test["y_pred"] * (data_objects["y_test_max"] - data_objects["y_test_min"])) + data_objects["y_test_min"]
        
    ###################################################################
    
    # Calculate earliness    
    Inference_test = Earliness(Inference_test, EAR=True)
    
    ########## ACCURACY #######################
    
    mae_test = np.mean(Inference_test["MAE"])/(24.0*3600)
    maetd_test = np.mean(Inference_test["MAE_td"])/(24.0*3600)
        
    # Print status:
    print('_'*60)
    print('Test MAE:     ', mae_test, ' (days)')
    print("================================"*3)
        
    
    ########### Generate a report on individual level ########

    # Dump the inference tables for debugging ## REMOVED 26/09
    Inference_test.to_csv(Project_dir+"/experiments/"+str(RUN)+"/inference_tables/Inf_test.csv",index=False)
        
    # Generate report with parameters of interest
    results = {"RUN":RUN,
               "Time":timestamp,
               "Traintime":time_taken,
               "MAE":mae_test,
               "MAE_td":maetd_test}

    report = dict(results, **model_params)
    
    report = pd.DataFrame(report, index=[0])
    
    # Storing individual
    report.to_csv(Project_dir+"/experiments/"+str(RUN)+"/"+str(RUN)+".csv",index=False)
    
    """
    Generate prefix results:
    """
    
    """
    #flexible
    max_events = np.max(Inference_test.event_number)

    if max_events < 10:
        events = 5
    else:
        events = 10
    """  
    #fixed
    events = 5
        
    inf_results = Inference_test.loc[Inference_test.event_number < events+1]
    inf_results.AE = inf_results.AE /(24.0*3600)
    
    pivottable = pd.pivot_table(inf_results, 
                           values='AE',
                        columns=['event_number'], aggfunc=np.mean)
    
    newcols = []
    for colno in range(0,len(pivottable.columns)):
        colno = colno + 1
        name = "AE_"+str(colno)+""
        newcols.append(name)
    
    pivottable.columns = newcols
    pivottable.index = [0]
    
    return pivottable


# -*- coding: utf-8 -*- Created on Fri May  1 19:29:28 2020  @author: Mike ############################################################################## ############################################################################## #from tensorflow.keras.utils import multi_gpu_model # TF2: Mixed precision #from tensorflow.keras.mixed_precision import experimental as mixed_precision #mixed_precision.set_policy('mixed_float16') # TF2: Disable eager execution # Aggregated metrics: # MAE = np.mean(AE) # MEP = np.mean(EP) #absolute error #temporal decay # Calculate metrics: # MAE: #Aggregated metrics: Evaluates earliness alone          Input:          Inference table with relevant variables                      T = "num_events"             t = "event_number"                          case = "caseid"             y = "y"             y_pred = "y_pred"          Output:         EP, AE, MEP, MAE, MAEPE # Calculate metrics: #Absolute error # MAE: #Aggregated metrics: Earliness Evaluation pipeline # Load config-file # Load inference tables #Inference_train = data_objects["Inference_train"] # Load the model # Load the training data # Predict on inference table # Save information about experiment # Time information # current date and time # Get model training info ################################################################### #Log transform inverse #Inference_test["y"] = np.exp(Inference_test["y"])-1 #Range transform inverse ################################################################### # Calculate earliness ########## ACCURACY ####################### # Print status: ########### Generate a report on individual level ######## # Dump the inference tables for debugging ## REMOVED 26/09 # Generate report with parameters of interest # Storing individual Generate prefix results: #flexible     max_events = np.max(Inference_test.event_number)      if max_events < 10:         events = 5     else:         events = 10 #fixed