# -*- coding: utf-8 -*-
"""
Spyder Editor

This is a temporary script file.
"""

# %% Import packages
from sklearn import tree, feature_selection
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import RandomizedSearchCV, GridSearchCV, RepeatedStratifiedKFold
from sklearn.feature_selection import SelectFromModel
from sklearn import metrics
from sklearn.metrics import classification_report
import numpy as np
from numpy import random
import pickle
import joblib

# %% Fit the model
def fit(X_train, Y_train, X_test, Y_test,
        use_local_parameters = False):
    
    # build a decision tree model
    
    # train the decision tree model
    classifier = tree.DecisionTreeClassifier()
    tree_model = classifier.fit(X_train, Y_train)
    
    # Evaluate the performance of the model
    tree_score = tree_model.score(X_test,Y_test)
    
    # Build a random forests algorithm
    if use_local_parameters:
        try:
            # Load the local file containing the hyperparameter settings
            directory = "Hyperparameters/RandomForests.pkl"
            RandomForests_params_file = open(directory, "rb")
            params = pickle.load(RandomForests_params_file)
            RandomForests_params_file.close()
            
            # Intiliase random forests classifier with local hyperparameters
            classifier = RandomForestClassifier(
                bootstrap = params['bootstrap'],
                max_features = params['max_features'],
                min_samples_leaf = params['min_samples_leaf'],
                min_samples_split = params['min_samples_split'],
                n_estimators = params['n_estimators'])
            
        except Exception as e:
            print('Could not load local hyperparameters!')
            print('Error: ' + str(e))
            print('Will now continue with default paramters.')
            # Intiliase random forests classifier with deafault hyperparameters
            classifier = RandomForestClassifier()
    else:
        # Intiliase random forests classifier with deafault hyperparameters
        classifier = RandomForestClassifier()
    
    # Assign a random seed    
    classifier.random_state  = random.randint(1,10000)
    # Tell the classifier to use all CPU cores when fitting
    classifier.n_jobs = -1
    # Tell the classifier to 
    classifier.warm_start = False
    # Tell it to balance the class weights
    classifier.class_weight = 'balanced'
    
    # Train the Random Forests model on top of the previous built model
    forest_model = classifier.fit(X_train, Y_train)
        
    
    # report performance
    
    # Evaluate the performance of the model
    forest_score = forest_model.score(X_test,Y_test)
    forest_prediction = forest_model.predict(X_test)
    report = classification_report(Y_test, forest_prediction)
    print(report)
    
    # %% print final results
    print(F"Decision tree accuracy: {tree_score}\n")
    print(F"Random forests accuracy: {forest_score}")
        
        
    return (tree_model, forest_model)

# %% Function to find the optimal parameters

# Some possible scorers:
    # average_precision
    # f1
    # roc_auc

def hyperparameter_tuning(X_train, Y_train, X_test, Y_test,
                          score = 'balanced_accuracy', save_name = 'forest_model'):
    classifier =  RandomForestClassifier()
    
    #===Set the range of possible values for all parameters===#
        
    # Number of trees in random forest
    n_estimators = [50, 100, 200, 400, 800, 1000, 1400, 2000]
    
    # Number of features to consider at every split
    max_features = ['auto', None]
    
    # Maximum number of levels in tree
    max_depth = [10, 20, 30, 40, 60, 80, 110]
    
    # Grow trees with max_leaf_nodes in best-first fashion
    max_leaf_nodes = [10, 100]
    
    # Minimum number of samples required to split a node
    min_samples_split = [2, 10, 20]
    
    # Minimum number of samples required at each leaf node
    min_samples_leaf = [1, 4, 8]
    
    # Wrap the parameter space in random_grid
    random_grid = {'n_estimators': n_estimators,
                   'max_features': max_features,
                   'max_depth': max_depth,
                   'max_leaf_nodes' : max_leaf_nodes,
                   'min_samples_split': min_samples_split,
                   'min_samples_leaf': min_samples_leaf}
    
    
    #===Run a randomised search for the optimal parameter setting===#
    classifier_random = RandomizedSearchCV(estimator = classifier,
                                           param_distributions = random_grid,
                                           n_iter = 100,
                                           cv = 5,
                                           verbose=2,
                                           random_state=42,
                                           n_jobs = -1,
                                           scoring = score)
    
    # Fit the random search model
    param_opt_rand = classifier_random.fit(X_train, Y_train)
    
    #=== Narrow down random optimal solutions to the best hyperparamets ===#
    # Extrapolate random optimal parameter values
    max_depth = param_opt_rand.best_estimator_.max_depth
    max_features = param_opt_rand.best_estimator_.max_features
    max_leaf_nodes = param_opt_rand.best_estimator_.max_leaf_nodes
    min_samples_leaf = param_opt_rand.best_estimator_.min_samples_leaf
    min_samples_split = param_opt_rand.best_estimator_.min_samples_split
    n_estimators = param_opt_rand.best_estimator_.n_estimators
    
    # IDEA: implement a precision variable. Divide the step size and the offset
    # used in the np.arange() by the precision which should by default be 1.
    param_grid = {
        'max_depth': [np.arange(max_depth-10, max_depth + 21, 10), None],
        'max_features': ['auto', None],
        'min_samples_leaf': np.arange(min_samples_leaf - 1 ,
                                      min_samples_leaf + 2),
        'max_leaf_nodes' : [np.arange(max_leaf_nodes - 10,
                                      max_leaf_nodes + 10, 2), None],
        'min_samples_split': np.arange(min_samples_split - 1,
                                       min_samples_split + 2),
        'n_estimators': np.arange(n_estimators - 200,
                                  n_estimators + 401,
                                  200)
    }
    
    # Find the optimal hyperparameters using gridsearch
    # initialise the grid search with cross validation
    classifier_gridsearch = GridSearchCV(estimator = classifier,
                                         param_grid = param_grid, 
                                         cv = 5, n_jobs = -1, verbose = 2,
                                         scoring = score)
    # Run the grid search to find the model with the optimal hyperparamterers
    classifier_gridsearch.fit(X_train, Y_train)
    # Extrapolate the optimal hyperparamters
    RandomForests_params_opt = classifier_gridsearch.best_params_
    
    
    #=== print and save the results ===#
    
    # Locally save results
    RandomForests_params_file = open("Hyperparameters/RandomForests.pkl", "wb")
    pickle.dump(RandomForests_params_opt, RandomForests_params_file)
    RandomForests_params_file.close()
    
    # Locally save the entire model
    joblib.dump(classifier_gridsearch.best_estimator_,
                f'Models/RandomForests/{save_name}.pkl')
    
    # Evaluate the optimal model
    final_accuracy = classifier_gridsearch.best_estimator_.score(X_test,
                                                                 Y_test)
    final_roc_auc_score = metrics.roc_auc_score(Y_test,
                                                classifier_gridsearch.best_estimator_.predict(X_test))

    # print the results
    print("Random Forests hyperparamter optimisation completed succesfully.")
    print(f"Best random forests model accuracy: {final_accuracy}")
    print(f"Best random forests model roc_auc: {final_roc_auc_score}")
    
    return classifier_gridsearch

# %% Find the features that satisfy a given importance threshold
def select_features(X_train, Y_train, threshold = 'mean',
                    return_importance = False):
    # IDEA: implement a check for the availability of a local hyperparameter
    # fil such that that the feature selection can be performed using a
    # hyperparameter optimised random forest opposed to a default one.
    
    # Initialise the forst model with feature selection
    classifier =  SelectFromModel(RandomForestClassifier(),
                                  threshold = 'mean')
    # Setup the repeated Kfold cross validation
    cv = RepeatedStratifiedKFold(n_splits=10, n_repeats=3)

    # Fit the model on the entire training data and initialise the matrices
    # of the cummulative feature importances and counts
    classifier.fit(X_train, Y_train)
    feat_importances = classifier.estimator_.feature_importances_
    # Feat_counts is currently redundant as it counts how often a feature is 
    # selected based on the 'SelectFromModel' threshold which does not support
    # a top n features. Feat_counts is therefore not used at the moment.
    feat_counts = classifier.get_support().astype(int)

    # Run through the cross validation loops and record the feature selection
    count = 0
    for train, test in cv.split(X_train, Y_train):
        X = X_train[train]
        Y = Y_train.iloc[train]
        classifier.fit(X, Y)
        feat_importances += classifier.estimator_.feature_importances_
        feat_counts += classifier.get_support().astype(int)
        count += 1
    feat_importances = feat_importances/count
    
    # use_top is by default false such that a numeric threshold is used
    use_top = False
    
    # Select the important features
    if threshold == 'mean':
        threshold = np.mean(feat_importances)
    elif threshold == 'median':
        threshold = np.median(feat_importances)
    elif type(threshold) == str and threshold[0:3] == 'top':
        # If a top n features is specified, use_top will be set to True so
        top = int(threshold[3:len(threshold)])
        use_top = True
    else:
        try:
            threshold = float(threshold)
        except:
            raise Exception('Invalid theshold!')
    
    if use_top:
        # Set threshold to the lowest importance value in the top n features
        threshold = np.sort(feat_importances, axis = -1)[::-1][top-1]
    # Create an index array of which features satisfie the importance threshold
    important_feat_index = feat_importances >= threshold
    
    if return_importance == True:
        return (important_feat_index, feat_importances)
    else:
        return important_feat_index
# -*- coding: utf-8 -*-
"""
Spyder Editor

This is a temporary script file.
"""

# %% Import packages
from sklearn import tree, feature_selection
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import RandomizedSearchCV, GridSearchCV, RepeatedStratifiedKFold
from sklearn.feature_selection import SelectFromModel
from sklearn import metrics
from sklearn.metrics import classification_report
import numpy as np
from numpy import random
import pickle
import joblib

# %% Fit the model
def fit(X_train, Y_train, X_test, Y_test,
        use_local_parameters = False):
    
    # build a decision tree model
    
    # train the decision tree model
    classifier = tree.DecisionTreeClassifier()
    tree_model = classifier.fit(X_train, Y_train)
    
    # Evaluate the performance of the model
    tree_score = tree_model.score(X_test,Y_test)
    
    # Build a random forests algorithm
    if use_local_parameters:
        try:
            # Load the local file containing the hyperparameter settings
            directory = "Hyperparameters/RandomForests.pkl"
            RandomForests_params_file = open(directory, "rb")
            params = pickle.load(RandomForests_params_file)
            RandomForests_params_file.close()
            
            # Intiliase random forests classifier with local hyperparameters
            classifier = RandomForestClassifier(
                bootstrap = params['bootstrap'],
                max_features = params['max_features'],
                min_samples_leaf = params['min_samples_leaf'],
                min_samples_split = params['min_samples_split'],
                n_estimators = params['n_estimators'])
            
        except Exception as e:
            print('Could not load local hyperparameters!')
            print('Error: ' + str(e))
            print('Will now continue with default paramters.')
            # Intiliase random forests classifier with deafault hyperparameters
            classifier = RandomForestClassifier()
    else:
        # Intiliase random forests classifier with deafault hyperparameters
        classifier = RandomForestClassifier()
    
    # Assign a random seed    
    classifier.random_state  = random.randint(1,10000)
    # Tell the classifier to use all CPU cores when fitting
    classifier.n_jobs = -1
    # Tell the classifier to 
    classifier.warm_start = False
    # Tell it to balance the class weights
    classifier.class_weight = 'balanced'
    
    # Train the Random Forests model on top of the previous built model
    forest_model = classifier.fit(X_train, Y_train)
        
    
    # report performance
    
    # Evaluate the performance of the model
    forest_score = forest_model.score(X_test,Y_test)
    forest_prediction = forest_model.predict(X_test)
    report = classification_report(Y_test, forest_prediction)
    print(report)
    
    # %% print final results
    print(F"Decision tree accuracy: {tree_score}\n")
    print(F"Random forests accuracy: {forest_score}")
        
        
    return (tree_model, forest_model)

# %% Function to find the optimal parameters

# Some possible scorers:
    # average_precision
    # f1
    # roc_auc

def hyperparameter_tuning(X_train, Y_train, X_test, Y_test,
                          score = 'balanced_accuracy', save_name = 'forest_model'):
    classifier =  RandomForestClassifier()
    
    #===Set the range of possible values for all parameters===#
        
    # Number of trees in random forest
    n_estimators = [50, 100, 200, 400, 800, 1000, 1400, 2000]
    
    # Number of features to consider at every split
    max_features = ['auto', None]
    
    # Maximum number of levels in tree
    max_depth = [10, 20, 30, 40, 60, 80, 110]
    
    # Grow trees with max_leaf_nodes in best-first fashion
    max_leaf_nodes = [10, 100]
    
    # Minimum number of samples required to split a node
    min_samples_split = [2, 10, 20]
    
    # Minimum number of samples required at each leaf node
    min_samples_leaf = [1, 4, 8]
    
    # Wrap the parameter space in random_grid
    random_grid = {'n_estimators': n_estimators,
                   'max_features': max_features,
                   'max_depth': max_depth,
                   'max_leaf_nodes' : max_leaf_nodes,
                   'min_samples_split': min_samples_split,
                   'min_samples_leaf': min_samples_leaf}
    
    
    #===Run a randomised search for the optimal parameter setting===#
    classifier_random = RandomizedSearchCV(estimator = classifier,
                                           param_distributions = random_grid,
                                           n_iter = 100,
                                           cv = 5,
                                           verbose=2,
                                           random_state=42,
                                           n_jobs = -1,
                                           scoring = score)
    
    # Fit the random search model
    param_opt_rand = classifier_random.fit(X_train, Y_train)
    
    #=== Narrow down random optimal solutions to the best hyperparamets ===#
    # Extrapolate random optimal parameter values
    max_depth = param_opt_rand.best_estimator_.max_depth
    max_features = param_opt_rand.best_estimator_.max_features
    max_leaf_nodes = param_opt_rand.best_estimator_.max_leaf_nodes
    min_samples_leaf = param_opt_rand.best_estimator_.min_samples_leaf
    min_samples_split = param_opt_rand.best_estimator_.min_samples_split
    n_estimators = param_opt_rand.best_estimator_.n_estimators
    
    # IDEA: implement a precision variable. Divide the step size and the offset
    # used in the np.arange() by the precision which should by default be 1.
    param_grid = {
        'max_depth': [np.arange(max_depth-10, max_depth + 21, 10), None],
        'max_features': ['auto', None],
        'min_samples_leaf': np.arange(min_samples_leaf - 1 ,
                                      min_samples_leaf + 2),
        'max_leaf_nodes' : [np.arange(max_leaf_nodes - 10,
                                      max_leaf_nodes + 10, 2), None],
        'min_samples_split': np.arange(min_samples_split - 1,
                                       min_samples_split + 2),
        'n_estimators': np.arange(n_estimators - 200,
                                  n_estimators + 401,
                                  200)
    }
    
    # Find the optimal hyperparameters using gridsearch
    # initialise the grid search with cross validation
    classifier_gridsearch = GridSearchCV(estimator = classifier,
                                         param_grid = param_grid, 
                                         cv = 5, n_jobs = -1, verbose = 2,
                                         scoring = score)
    # Run the grid search to find the model with the optimal hyperparamterers
    classifier_gridsearch.fit(X_train, Y_train)
    # Extrapolate the optimal hyperparamters
    RandomForests_params_opt = classifier_gridsearch.best_params_
    
    
    #=== print and save the results ===#
    
    # Locally save results
    RandomForests_params_file = open("Hyperparameters/RandomForests.pkl", "wb")
    pickle.dump(RandomForests_params_opt, RandomForests_params_file)
    RandomForests_params_file.close()
    
    # Locally save the entire model
    joblib.dump(classifier_gridsearch.best_estimator_,
                f'Models/RandomForests/{save_name}.pkl')
    
    # Evaluate the optimal model
    final_accuracy = classifier_gridsearch.best_estimator_.score(X_test,
                                                                 Y_test)
    final_roc_auc_score = metrics.roc_auc_score(Y_test,
                                                classifier_gridsearch.best_estimator_.predict(X_test))

    # print the results
    print("Random Forests hyperparamter optimisation completed succesfully.")
    print(f"Best random forests model accuracy: {final_accuracy}")
    print(f"Best random forests model roc_auc: {final_roc_auc_score}")
    
    return classifier_gridsearch

# %% Find the features that satisfy a given importance threshold
def select_features(X_train, Y_train, threshold = 'mean',
                    return_importance = False):
    # IDEA: implement a check for the availability of a local hyperparameter
    # fil such that that the feature selection can be performed using a
    # hyperparameter optimised random forest opposed to a default one.
    
    # Initialise the forst model with feature selection
    classifier =  SelectFromModel(RandomForestClassifier(),
                                  threshold = 'mean')
    # Setup the repeated Kfold cross validation
    cv = RepeatedStratifiedKFold(n_splits=10, n_repeats=3)

    # Fit the model on the entire training data and initialise the matrices
    # of the cummulative feature importances and counts
    classifier.fit(X_train, Y_train)
    feat_importances = classifier.estimator_.feature_importances_
    # Feat_counts is currently redundant as it counts how often a feature is 
    # selected based on the 'SelectFromModel' threshold which does not support
    # a top n features. Feat_counts is therefore not used at the moment.
    feat_counts = classifier.get_support().astype(int)

    # Run through the cross validation loops and record the feature selection
    count = 0
    for train, test in cv.split(X_train, Y_train):
        X = X_train[train]
        Y = Y_train.iloc[train]
        classifier.fit(X, Y)
        feat_importances += classifier.estimator_.feature_importances_
        feat_counts += classifier.get_support().astype(int)
        count += 1
    feat_importances = feat_importances/count
    
    # use_top is by default false such that a numeric threshold is used
    use_top = False
    
    # Select the important features
    if threshold == 'mean':
        threshold = np.mean(feat_importances)
    elif threshold == 'median':
        threshold = np.median(feat_importances)
    elif type(threshold) == str and threshold[0:3] == 'top':
        # If a top n features is specified, use_top will be set to True so
        top = int(threshold[3:len(threshold)])
        use_top = True
    else:
        try:
            threshold = float(threshold)
        except:
            raise Exception('Invalid theshold!')
    
    if use_top:
        # Set threshold to the lowest importance value in the top n features
        threshold = np.sort(feat_importances, axis = -1)[::-1][top-1]
    # Create an index array of which features satisfie the importance threshold
    important_feat_index = feat_importances >= threshold
    
    if return_importance == True:
        return (important_feat_index, feat_importances)
    else:
        return important_feat_index
# -*- coding: utf-8 -*- Spyder Editor  This is a temporary script file. # %% Import packages # %% Fit the model # build a decision tree model # train the decision tree model # Evaluate the performance of the model # Build a random forests algorithm # Load the local file containing the hyperparameter settings # Intiliase random forests classifier with local hyperparameters # Intiliase random forests classifier with deafault hyperparameters # Intiliase random forests classifier with deafault hyperparameters # Assign a random seed # Tell the classifier to use all CPU cores when fitting # Tell the classifier to # Tell it to balance the class weights # Train the Random Forests model on top of the previous built model # report performance # Evaluate the performance of the model # %% print final results # %% Function to find the optimal parameters # Some possible scorers: # average_precision # f1 # roc_auc #===Set the range of possible values for all parameters===# # Number of trees in random forest # Number of features to consider at every split # Maximum number of levels in tree # Grow trees with max_leaf_nodes in best-first fashion # Minimum number of samples required to split a node # Minimum number of samples required at each leaf node # Wrap the parameter space in random_grid #===Run a randomised search for the optimal parameter setting===# # Fit the random search model #=== Narrow down random optimal solutions to the best hyperparamets ===# # Extrapolate random optimal parameter values # IDEA: implement a precision variable. Divide the step size and the offset # used in the np.arange() by the precision which should by default be 1. # Find the optimal hyperparameters using gridsearch # initialise the grid search with cross validation # Run the grid search to find the model with the optimal hyperparamterers # Extrapolate the optimal hyperparamters #=== print and save the results ===# # Locally save results # Locally save the entire model # Evaluate the optimal model # print the results # %% Find the features that satisfy a given importance threshold # IDEA: implement a check for the availability of a local hyperparameter # fil such that that the feature selection can be performed using a # hyperparameter optimised random forest opposed to a default one. # Initialise the forst model with feature selection # Setup the repeated Kfold cross validation # Fit the model on the entire training data and initialise the matrices # of the cummulative feature importances and counts # Feat_counts is currently redundant as it counts how often a feature is # selected based on the 'SelectFromModel' threshold which does not support # a top n features. Feat_counts is therefore not used at the moment. # Run through the cross validation loops and record the feature selection # use_top is by default false such that a numeric threshold is used # Select the important features # If a top n features is specified, use_top will be set to True so # Set threshold to the lowest importance value in the top n features # Create an index array of which features satisfie the importance threshold