<reponame>dmacko232/AndroidMalwareLabeling
"""
File contains functions that can be used to evaluate the experiments.

Author: <NAME>
"""

from typing import Union, List, Any, Dict, Optional, Dict

import numpy as np
import pandas as pd
import seaborn as sns
from imblearn.metrics import classification_report_imbalanced
from sklearn.metrics import cohen_kappa_score, confusion_matrix, accuracy_score
import matplotlib.pyplot as plt

def evaluate_binary_prediction(true_y: Union[np.array, pd.Series, List[Any]],
                               pred_y: Union[np.array, pd.Series, List[Any]],
                               positive_class: Any=True,
                               name: str="score",
                               rename: Dict[str, str]={
                                   "pre": "precision",
                                   "rec": "recall",
                                   "spe": "specificity",
                                   "geo": "geometric mean"
                               },
                               drop=["sup", "iba"]
                              ) -> pd.DataFrame:
    """Evaluates a single binary prediction.
    
    true_y - true labels
    pred_y - predicted labels
    positive_class - which class to consider positive
    name - how to name the row
    rename - columns and how to rename them
    drop - columns to drop
    
    returns a dataframe
    """

    report = classification_report_imbalanced(true_y, pred_y, output_dict=True, zero_division=0)[positive_class]
    report["macro averaged accuracy"] = (report["rec"] + report["spe"]) / 2
    report["cohen kappa"] = cohen_kappa_score(true_y, pred_y)
    report["accuracy"] = accuracy_score(true_y, pred_y)
    return pd.DataFrame.from_dict({name: report}, orient="index").drop(columns=drop).rename(columns=rename)

def evaluate_binary_predictions(
    true_y: Union[np.array, pd.Series, List[Any]],
    predictions: Dict[str, pd.Series],
    positive_class: Any=True,
    rename: Dict[str, str]={
                "pre": "precision",
                "rec": "recall",
                "spe": "specificity",
                "geo": "geometric mean",
        },
    drop=["sup", "iba"]
    ) -> pd.DataFrame:
    """Evaluates binary predictions.
    
    true_y - true values
    predictions - dictionary of predictions per model (model name is key and value is prediction)
    positive_class - what to consider in predictions as positive class
    rename - columns and how to rename them
    drop - specifies columns to drop
    
    returns a dataframe
    """

    df = None
    for name, pred_y in predictions.items():
        curr = evaluate_binary_prediction(
            true_y,
            pred_y,
            positive_class=positive_class,
            rename=rename,
            drop=drop,
            name=name
        )
        if df is None:
            df = curr
        else:
            df = df.append(curr)
    return df

def evaluate_imbalanced_multiclass_prediction(
    true_y: Union[np.array, pd.Series, List[Any]],
    pred_y: Union[np.array, pd.Series, List[Any]], 
    rename: Dict[str, str]={
        "pre": "precision",
        "rec": "recall",
        "spe": "specificity",
        "geo": "geometric mean",
    },
    drop: List[str]=["sup", "iba"],
    total_row_name: str="TOTAL"
    ) -> pd.DataFrame:
    """Evaluates prediction using imbalanced metrics class-wise and average total.
    
    true_y - true labels
    pred_y - predicted labels
    rename - columns and how to rename them
    total_row_name - name of row to be used for the renamed columns

    returns dataframe with metrics per class and total
    """

    report = classification_report_imbalanced(
        true_y,
        pred_y,
        output_dict=True,
        zero_division=0
    )

    # filter out total scores
    report = { k: v for k, v in report.items() if k in set(true_y).union(set(pred_y)) }

    df = pd.DataFrame.from_dict(report, orient="index")
    df["macro averaged accuracy"] = (df["rec"] + df["spe"]) / 2
    df.loc[total_row_name] = df.mean()
    df.loc[total_row_name, "cohen kappa"] = cohen_kappa_score(true_y, pred_y)
    return df.drop(columns=drop).rename(columns=rename)

def evaluate_imbalanced_multiclass_predictions(true_y: Union[np.array, pd.Series, List[Any]],
                                               predictions: Dict[str, pd.Series],
                                               rename: Dict[str, str]={
                                                   "pre": "precision",
                                                   "rec": "recall",
                                                   "spe": "specificity",
                                                   "geo": "geometric mean"
                                               },
                                               drop: List[str]=["sup", "iba"],
                                              ) -> pd.DataFrame:
    """Evalutes imbalanced predictions comparing them among themselves using average metrics.
    
    true_y - specifies true labels
    predictions - specifies dictionary with name/model as key and prediction as value
    rename - specifies which columns to take and how to rename them
    drop - specifies columns that are to be dropped

    returns dataframe with comparison of metrics
    """
    
    df = None
    for name, pred_y in predictions.items():
        curr = evaluate_imbalanced_multiclass_prediction(
            true_y,
            pred_y,
            rename=rename,
            drop=drop,
            total_row_name=name
        ).loc[[name]]
        if df is None:
            df = curr
        else:
            df = df.append(curr)
    return df

def plot_conf_matrix(true_y: Union[np.array, pd.Series, List[Any]],
                     pred_y: Union[np.array, pd.Series, List[Any]],
                     label_mapping: Optional[Dict[Any, str]]=None,
                     output_path: Optional[str]=None,
                     cmap: str="Blues",
                     dpi: int=300
                    ) -> None: 
    """Plots confusion matrix for given prediction and true labels
    
    true_y - true values
    pred_y - predictions of a model
    label_mapping - mapping how to rename the labels in true_y and pred_y
    output_path - optional path where to store the matrix
    """

    if label_mapping:
        true_y = [label_mapping[label] for label in true_y]
        pred_y = [label_mapping[label] for label in pred_y]

    labels = sorted([*set(true_y)])
    if len(labels) == 2: # binary problem
        if label_mapping and True in label_mapping and False in label_mapping: # get label mappings in order True False
            labels = [label_mapping[True], label_mapping[False]]
        else:
            labels = sorted(labels, reverse=True) # sort in reverse to get True, False
    
    matrix = confusion_matrix(true_y, pred_y, labels=labels, normalize="true")
    df_cm = pd.DataFrame(matrix, columns=labels, index = labels)
    df_cm.index.name = "True label"
    df_cm.columns.name = "Predicted label"
    plt.figure(figsize = (8, 6))
    sns_plot = sns.heatmap(df_cm, cmap=cmap, annot=True, fmt=".2f").get_figure()
    if output_path is not None:
        sns_plot.savefig(output_path, dpi=dpi)



"""
File contains functions that can be used to evaluate the experiments.

Author: <NAME>
"""

from typing import Union, List, Any, Dict, Optional, Dict

import numpy as np
import pandas as pd
import seaborn as sns
from imblearn.metrics import classification_report_imbalanced
from sklearn.metrics import cohen_kappa_score, confusion_matrix, accuracy_score
import matplotlib.pyplot as plt

def evaluate_binary_prediction(true_y: Union[np.array, pd.Series, List[Any]],
                               pred_y: Union[np.array, pd.Series, List[Any]],
                               positive_class: Any=True,
                               name: str="score",
                               rename: Dict[str, str]={
                                   "pre": "precision",
                                   "rec": "recall",
                                   "spe": "specificity",
                                   "geo": "geometric mean"
                               },
                               drop=["sup", "iba"]
                              ) -> pd.DataFrame:
    """Evaluates a single binary prediction.
    
    true_y - true labels
    pred_y - predicted labels
    positive_class - which class to consider positive
    name - how to name the row
    rename - columns and how to rename them
    drop - columns to drop
    
    returns a dataframe
    """

    report = classification_report_imbalanced(true_y, pred_y, output_dict=True, zero_division=0)[positive_class]
    report["macro averaged accuracy"] = (report["rec"] + report["spe"]) / 2
    report["cohen kappa"] = cohen_kappa_score(true_y, pred_y)
    report["accuracy"] = accuracy_score(true_y, pred_y)
    return pd.DataFrame.from_dict({name: report}, orient="index").drop(columns=drop).rename(columns=rename)

def evaluate_binary_predictions(
    true_y: Union[np.array, pd.Series, List[Any]],
    predictions: Dict[str, pd.Series],
    positive_class: Any=True,
    rename: Dict[str, str]={
                "pre": "precision",
                "rec": "recall",
                "spe": "specificity",
                "geo": "geometric mean",
        },
    drop=["sup", "iba"]
    ) -> pd.DataFrame:
    """Evaluates binary predictions.
    
    true_y - true values
    predictions - dictionary of predictions per model (model name is key and value is prediction)
    positive_class - what to consider in predictions as positive class
    rename - columns and how to rename them
    drop - specifies columns to drop
    
    returns a dataframe
    """

    df = None
    for name, pred_y in predictions.items():
        curr = evaluate_binary_prediction(
            true_y,
            pred_y,
            positive_class=positive_class,
            rename=rename,
            drop=drop,
            name=name
        )
        if df is None:
            df = curr
        else:
            df = df.append(curr)
    return df

def evaluate_imbalanced_multiclass_prediction(
    true_y: Union[np.array, pd.Series, List[Any]],
    pred_y: Union[np.array, pd.Series, List[Any]], 
    rename: Dict[str, str]={
        "pre": "precision",
        "rec": "recall",
        "spe": "specificity",
        "geo": "geometric mean",
    },
    drop: List[str]=["sup", "iba"],
    total_row_name: str="TOTAL"
    ) -> pd.DataFrame:
    """Evaluates prediction using imbalanced metrics class-wise and average total.
    
    true_y - true labels
    pred_y - predicted labels
    rename - columns and how to rename them
    total_row_name - name of row to be used for the renamed columns

    returns dataframe with metrics per class and total
    """

    report = classification_report_imbalanced(
        true_y,
        pred_y,
        output_dict=True,
        zero_division=0
    )

    # filter out total scores
    report = { k: v for k, v in report.items() if k in set(true_y).union(set(pred_y)) }

    df = pd.DataFrame.from_dict(report, orient="index")
    df["macro averaged accuracy"] = (df["rec"] + df["spe"]) / 2
    df.loc[total_row_name] = df.mean()
    df.loc[total_row_name, "cohen kappa"] = cohen_kappa_score(true_y, pred_y)
    return df.drop(columns=drop).rename(columns=rename)

def evaluate_imbalanced_multiclass_predictions(true_y: Union[np.array, pd.Series, List[Any]],
                                               predictions: Dict[str, pd.Series],
                                               rename: Dict[str, str]={
                                                   "pre": "precision",
                                                   "rec": "recall",
                                                   "spe": "specificity",
                                                   "geo": "geometric mean"
                                               },
                                               drop: List[str]=["sup", "iba"],
                                              ) -> pd.DataFrame:
    """Evalutes imbalanced predictions comparing them among themselves using average metrics.
    
    true_y - specifies true labels
    predictions - specifies dictionary with name/model as key and prediction as value
    rename - specifies which columns to take and how to rename them
    drop - specifies columns that are to be dropped

    returns dataframe with comparison of metrics
    """
    
    df = None
    for name, pred_y in predictions.items():
        curr = evaluate_imbalanced_multiclass_prediction(
            true_y,
            pred_y,
            rename=rename,
            drop=drop,
            total_row_name=name
        ).loc[[name]]
        if df is None:
            df = curr
        else:
            df = df.append(curr)
    return df

def plot_conf_matrix(true_y: Union[np.array, pd.Series, List[Any]],
                     pred_y: Union[np.array, pd.Series, List[Any]],
                     label_mapping: Optional[Dict[Any, str]]=None,
                     output_path: Optional[str]=None,
                     cmap: str="Blues",
                     dpi: int=300
                    ) -> None: 
    """Plots confusion matrix for given prediction and true labels
    
    true_y - true values
    pred_y - predictions of a model
    label_mapping - mapping how to rename the labels in true_y and pred_y
    output_path - optional path where to store the matrix
    """

    if label_mapping:
        true_y = [label_mapping[label] for label in true_y]
        pred_y = [label_mapping[label] for label in pred_y]

    labels = sorted([*set(true_y)])
    if len(labels) == 2: # binary problem
        if label_mapping and True in label_mapping and False in label_mapping: # get label mappings in order True False
            labels = [label_mapping[True], label_mapping[False]]
        else:
            labels = sorted(labels, reverse=True) # sort in reverse to get True, False
    
    matrix = confusion_matrix(true_y, pred_y, labels=labels, normalize="true")
    df_cm = pd.DataFrame(matrix, columns=labels, index = labels)
    df_cm.index.name = "True label"
    df_cm.columns.name = "Predicted label"
    plt.figure(figsize = (8, 6))
    sns_plot = sns.heatmap(df_cm, cmap=cmap, annot=True, fmt=".2f").get_figure()
    if output_path is not None:
        sns_plot.savefig(output_path, dpi=dpi)
File contains functions that can be used to evaluate the experiments.  Author: <NAME> Evaluates a single binary prediction.          true_y - true labels     pred_y - predicted labels     positive_class - which class to consider positive     name - how to name the row     rename - columns and how to rename them     drop - columns to drop          returns a dataframe Evaluates binary predictions.          true_y - true values     predictions - dictionary of predictions per model (model name is key and value is prediction)     positive_class - what to consider in predictions as positive class     rename - columns and how to rename them     drop - specifies columns to drop          returns a dataframe Evaluates prediction using imbalanced metrics class-wise and average total.          true_y - true labels     pred_y - predicted labels     rename - columns and how to rename them     total_row_name - name of row to be used for the renamed columns      returns dataframe with metrics per class and total # filter out total scores Evalutes imbalanced predictions comparing them among themselves using average metrics.          true_y - specifies true labels     predictions - specifies dictionary with name/model as key and prediction as value     rename - specifies which columns to take and how to rename them     drop - specifies columns that are to be dropped      returns dataframe with comparison of metrics Plots confusion matrix for given prediction and true labels          true_y - true values     pred_y - predictions of a model     label_mapping - mapping how to rename the labels in true_y and pred_y     output_path - optional path where to store the matrix # binary problem # get label mappings in order True False # sort in reverse to get True, False