<gh_stars>1-10
# AUTOGENERATED! DO NOT EDIT! File to edit: 24_machine_learning.ipynb (unless otherwise specified).

__all__ = ['k_means', 'k_means_range', 'k_means_ensemble', 'agglomerative_cluster', 'describe_clusters',
           'neural_network_classification', 'neural_network_regression', 'logistic_regression',
           'lasso_logistic_regression', 'svm_eps_regression', 'svm_c_classification', 'svm_one_classification',
           'rf_regression', 'rf_classification', 'compute_performance', 'plot_cluster_sizes',
           'plot_agglomeration_dendrogram']

# Cell
from sklearn.cluster import KMeans
import numpy as np
#export
def k_means(measures_table, clusters=4, data_only=False, plot=False, loud=False):
    "Applies the k-means clustering algorithm to a measures table. The resulting clustering is then reported in terms of its inertia (sum of squared distances of samples to their closest cluster center) and its silhouette score (how distinct clusters are within the sample [see the skikit learn docs for details]). The measures passed as the first parameter can be returned with an added column reporting the cluster each player belongs to using the data_only parameter."

    # get variable names from the behavioural measures
    variables = list(measures_table.columns)[1:]

    Kmean = KMeans(n_clusters=clusters)

    data = np.array(measures_table[variables].values)
    Kmean.fit(data)

    silhouette = metrics.silhouette_score(data, Kmean.labels_, metric="euclidean")

    cluster_centers = Kmean.cluster_centers_

    clustered_data = measures_table.copy()
    clustered_data["cluster"] = Kmean.labels_

    if loud:
        print("variables:", variables)
        print("centers:", Kmean.cluster_centers_)
        print("inertia:", Kmean.inertia_)
        print("silhouette:", silhouette)

    if plot:
        bars = []
        heights = []
        for label in set(sorted(Kmean.labels_)):
            bars.append(label)
            heights.append(list(Kmean.labels_).count(label))

        colors = ["C0", "C1", "C2", "C3", "C4", "C5", "C6", "C7", "C8", "C9"]
        plt.bar(bars, heights, color=colors[: len(bars)])
        plt.title(
            "\nClusters: "
            + str(len(bars))
            + "\nInertia: "
            + str(round(Kmean.inertia_))
            + "\nIterations: "  # Kmean.inertia_ is the sum of squared distances of samples to their closest cluster center
            + str(Kmean.n_iter_),
            x=1.01,
            y=0.5,
            ha="left",
        )
        plt.xlabel("Cluster ID")
        plt.ylabel("Number of Members")
        plt.show()

    if data_only:
        return clustered_data

    return clustered_data, Kmean.inertia_, silhouette

# Cell
def k_means_range(measures_table, min_clusters=2, max_clusters=13):
    "Computes the k_means calculation above across a range of cluster counts, returning their goodness of fit measures (inertia and silhouette)."

    # print('calculating k means in range', min_clusters, max_clusters)

    inertias = []
    silhouettes = []

    for x in range(min_clusters, max_clusters + 1):
        k_means_result = k_means(measures_table, clusters=x)
        inertias.append(k_means_result[1])
        silhouettes.append(k_means_result[2])

    return inertias, silhouettes

# Cell
def k_means_ensemble(measures_table, ensemble_size=100, min_clusters=2, max_clusters=13):
    "Computes the k_means clustering algorithm across a range of cluster counts, a number of times. This is useful for determining clusters in a more robust way but can be slow on large data sets."

    all_inertias = []
    all_silhouettes = []

    # call the k_means_range function n times, storing scores in the above arrays
    for x in range(ensemble_size):
        k_means_range_result = k_means_range(
            measures_table, min_clusters=min_clusters, max_clusters=max_clusters
        )
        all_inertias.append(k_means_range_result[0])
        all_silhouettes.append(k_means_range_result[1])

    # now average each of the elements in the score lists
    ensemble_inertias = []
    ensemble_silhouettes = []
    for cluster_num in range(len(all_inertias[0])):
        inertia_scores = [all_inertias[x][cluster_num] for x in range(ensemble_size)]
        ensemble_inertias.append(statistics.mean(inertia_scores))

        silhouette_scores = [
            all_silhouettes[x][cluster_num] for x in range(ensemble_size)
        ]
        ensemble_silhouettes.append(statistics.mean(silhouette_scores))

    return ensemble_inertias, ensemble_silhouettes

# Cell
from sklearn.cluster import AgglomerativeClustering
#export
def agglomerative_cluster(measures_table, distance_threshold=0, n_clusters=None):
    "Performs sklearn's agglomerative clustering algorithm on a dataframe of behavioural measures. See their documentation for details. Note: Either the distance threshold or the n_cluster parameter must be None."

    variables = list(measures_table.columns)[1:]
    X = measures_table[variables].values

    model = AgglomerativeClustering(
        distance_threshold=distance_threshold, n_clusters=n_clusters
    )
    model = model.fit(X)
    return model

# Cell
import pandas as pd
def describe_clusters(clustered_measures_table, cluster_col="cluster"):
    "Describes cluster centroids (mean values of each measure) for each cluster in a clustered measures table."

    unique_clusters = list(set(clustered_measures_table["cluster"].values))

    descriptive_table = pd.DataFrame()
    descriptive_table["cluster_centroid"] = clustered_measures_table.columns[1:]

    for value in unique_clusters:
        members = clustered_measures_table[
            clustered_measures_table[cluster_col] == value
        ]
        centroid = [members[col].mean() for col in members.columns[1:]]
        descriptive_table["n=" + str(len(members))] = centroid

    descriptive_table.set_index("cluster_centroid", inplace=True)
    return descriptive_table

# Cell
from keras.layers import Dense, Activation
from keras.models import Sequential
import numpy as np
import gamba.measures as measures

def neural_network_classification(measures_table, label, train_test_split=0.7):
    "A basic example of a classification network using a single sigmoid-activated output node. The output is the probability of the player belonging to the value of the label. As with the `neural_network_regression` this network closely follows examples in [Deep Learning with Python](http://faculty.neu.edu.cn/yury/AAI/Textbook/Deep%20Learning%20with%20Python.pdf) which is a great resource for extending this example."

    train_measures, test_measures = measures.split_measures_table(measures_table, frac=train_test_split)
    test_labels = test_measures[label]

    train_data = train_measures.drop(['player_id', label], axis=1)
    train_labels = train_measures[label]
    test_data = test_measures.drop(['player_id',label], axis=1)
    test_labels = test_measures[label]

    model = Sequential()
    model.add(Dense(17, activation = 'relu', input_dim = 17))
    model.add(Dense(50, activation = 'relu'))
    model.add(Dense(units = 1, activation='sigmoid')) # this is the 'classification' part

    model.compile(optimizer = 'adam',
                  loss = 'mean_squared_error',
                  metrics=['accuracy'])

    history = model.fit(train_data, train_labels,
                        batch_size = 20, epochs = 100,
                        validation_data=(test_data, test_labels),
                        verbose=False)

    # now make a prediction and clip the values to 0 or 1 as in the original code
    raw_prediction = model.predict(test_data)
    predicted_labels = [value[0] for value in np.where(raw_prediction >= 0.5, 1, 0)]

    return test_labels, predicted_labels

# Cell
import gamba.measures as measures
def neural_network_regression(measures_table, label, train_test_split=0.7):
    "A very simple neural network for regression using two hidden layers and one output node with a linear activation function. This method is very similar to the example in [Deep Learning with Python](http://faculty.neu.edu.cn/yury/AAI/Textbook/Deep%20Learning%20with%20Python.pdf), and can be used as a template for more complex network topologies."

    # note that the metric of accuracy doesn't apply for regression problems (mean absolute error is better)

    train_measures, test_measures = measures.split_measures_table(measures_table, frac=train_test_split)
    test_labels = test_measures[label]

    train_data = train_measures.drop(['player_id', label], axis=1)
    train_labels = train_measures[label]
    test_data = test_measures.drop(['player_id',label], axis=1)
    test_labels = test_measures[label]

    model = Sequential()
    model.add(Dense(17, activation = 'relu', input_dim = 17))
    model.add(Dense(50, activation = 'relu'))
    model.add(Dense(units = 1)) # see page 86 of Deep Learning with Python by Chollet for an explanation

    model.compile(optimizer = 'adam',
                  loss = 'mean_squared_error',
                  metrics=['accuracy'])

    history = model.fit(train_data, train_labels,
                        batch_size = 20, epochs = 100,
                        validation_data=(test_data, test_labels),
                        verbose=False)

    # now make a prediction and clip the values to 0 or 1 as in the original code
    raw_prediction = model.predict(test_data)
    predicted_labels = [value[0] for value in np.where(raw_prediction >= 0.5, 1, 0)]

    return test_labels, predicted_labels

# Cell
import statsmodels.api as sm
from sklearn.linear_model import LogisticRegression
from sklearn import svm
import gamba.measures as measures
#export
def logistic_regression(measures_table, label, train_test_split=0.7):
    "Performs a logistic regression using the [statsmodels library](https://www.statsmodels.org/stable/index.html), returning the predicted labels rounded to the nearest integer. Note: this method is currently hard-coded to only function on Philander 2014's data set. Abstracted logistic regression function coming soon."

    train_measures, test_measures = measures.split_measures_table(measures_table, frac=train_test_split)
    test_labels = test_measures[label]


    # defines the R style formula to fit
    formula = str(label) + " ~ gender+age+total_wagered+num_bets+frequency+duration+bets_per_day+net_loss+intensity+variability+frequency_1m+trajectory+z_intensity+z_variability+z_frequency+z_trajectory"
    model = sm.formula.glm(formula=formula, family=sm.families.Binomial(), data=train_measures)

    # this is where the stepwise bit could happen - see original code
    fit_model = model.fit()

    raw_prediction = fit_model.predict(test_measures)
    predicted_labels = [value for value in np.where(raw_prediction >= 0.5, 1, 0)]

    return test_labels, predicted_labels

# Cell
import gamba.measures as measures
def lasso_logistic_regression(measures_table, label, train_test_split=0.7):
    "Performs a 'lasso' (optimizes a least-square problem with L1 penalty) logistic regression using [sklearn's linear_model](https://scikit-learn.org/stable/modules/classes.html#module-sklearn.linear_model). This [stackoverflow post](https://stackoverflow.com/questions/41639557/how-to-perform-logistic-lasso-in-python>) contains a useful discussion on this type of function-estimation regression pair."

    train_measures, test_measures = measures.split_measures_table(measures_table, frac=train_test_split)
    test_labels = test_measures[label]

    train_data = train_measures.drop(['player_id', label], axis=1)
    train_labels = train_measures[label]
    test_data = test_measures.drop(['player_id', label], axis=1)

    model = LogisticRegression(penalty='l1', solver='liblinear')

    model.fit(train_data, train_labels)

    predicted_labels = model.predict(test_data)

    return test_labels, predicted_labels

# Cell
import gamba.measures as measures
def svm_eps_regression(measures_table, label, train_test_split=0.7):
    "Creates and trains a support vector machine for epsilon-support vector regression using the sklearn library's implementation."

    train_measures, test_measures = measures.split_measures_table(measures_table, frac=train_test_split)
    test_labels = test_measures[label]

    train_data = train_measures.drop(['player_id', label], axis=1)
    train_labels = train_measures[label]
    test_data = test_measures.drop(['player_id', label], axis=1)

    model = svm.SVR(kernel='rbf')

    model.fit(train_data, train_labels)

    predicted_labels = model.predict(test_data)

    # convert probabilities to binary labels for comparison
    regression_cutoff = 0.5
    predicted_labels = np.where(predicted_labels < regression_cutoff, 0, 1)

    return test_labels, predicted_labels

# Cell
import gamba.measures as measures
def svm_c_classification(measures_table, label, train_test_split=0.7):
    "Creates and trains a support vector machine for classification using the sklearn library's implementation."

    train_measures, test_measures = measures.split_measures_table(measures_table, frac=train_test_split)
    test_labels = test_measures[label]

    train_data = train_measures.drop(['player_id', label], axis=1)
    train_labels = train_measures[label]
    test_data = test_measures.drop(['player_id', label], axis=1)

    model = svm.SVC(kernel='rbf')

    model.fit(train_data, train_labels)

    predicted_labels = model.predict(test_data)

    return test_labels, predicted_labels

# Cell
import gamba.measures as measures
def svm_one_classification(measures_table, label, train_test_split=0.7):
    "Creates and trains a support vector machine for one-class classification using the sklearn library's implementation."

    train_measures, test_measures = measures.split_measures_table(measures_table, frac=train_test_split)
    test_labels = test_measures[label]

    train_data = train_measures.drop(['player_id', label], axis=1)
    train_labels = train_measures[label]
    test_data = test_measures.drop(['player_id', label], axis=1)

    model = svm.OneClassSVM(kernel='rbf')

    model.fit(train_data, train_labels)

    predicted_labels = model.predict(test_data)

    # need to add a correction step for the labels here as OneClassSVM returns -1 for outliers and 1 for inliers
    predicted_labels = np.where(predicted_labels < 0, 1, 0)

    return test_labels, predicted_labels

# Cell
from sklearn.ensemble import RandomForestRegressor
from sklearn.ensemble import RandomForestClassifier
import gamba.measures as measures
#export
def rf_regression(measures_table, label, train_test_split=0.7):
    "Creates and fits a random forest regressor using [sklearn's ensemble module](https://scikit-learn.org/stable/modules/ensemble.html#forest), returning the predicted labels rounded to the nearest integer."

    train_measures, test_measures = measures.split_measures_table(measures_table, frac=train_test_split)
    test_labels = test_measures[label]

    train_data = train_measures.drop(['player_id', label], axis=1)
    train_labels = train_measures[label]
    test_data = test_measures.drop(['player_id', label], axis=1)

    model = RandomForestRegressor()

    model.fit(train_data, train_labels)

    predicted_labels = model.predict(test_data)

    # convert probabilities to binary labels for comparison
    regression_cutoff = 0.5
    predicted_labels = np.where(predicted_labels < regression_cutoff, 0, 1)

    return test_labels, predicted_labels

# Cell
import gamba.measures as measures
def rf_classification(measures_table, label, train_test_split=0.7):
    "Creates a random forest for classification also using the sklearn library's implementation."

    train_measures, test_measures = measures.split_measures_table(measures_table, frac=train_test_split)
    test_labels = test_measures[label]

    train_data = train_measures.drop(['player_id', label], axis=1)
    train_labels = train_measures[label]
    test_data = test_measures.drop(['player_id', label], axis=1)

    model = RandomForestClassifier(n_estimators=100)

    model.fit(train_data, train_labels)

    predicted_labels = model.predict(test_data)

    return test_labels, predicted_labels

# Cell
from sklearn import metrics
#export
def compute_performance(method_name, actual, predicted):
    "Computes performance metrics including sensitivity, specificity, accuracy, confusion matrix values, odds ratio, and area under curve, for a given classification/regression using its actual and predicted values."

    # resources:
    # describes odds ratio and precision equations
    # https://cran.r-project.org/web/packages/ROCR/ROCR.pdf
    # describes sklearn's confusion matrix
    # https://scikit-learn.org/stable/modules/generated/sklearn.metrics.confusion_matrix.html#sklearn-metrics-confusion-matrix

    result = metrics.classification_report(actual, y_pred=predicted, output_dict=True)
    sensitivity = result['1']['recall']
    specificity = result['0']['recall']
    accuracy = result['accuracy']
    fpr, tpr, thresholds = metrics.roc_curve(actual, predicted)
    auc = metrics.auc(fpr, tpr)
    confusion_matrix = metrics.confusion_matrix(actual, predicted)
    tn, fp, fn, tp = confusion_matrix.ravel()

    # odds ratio is (tp x tn)/(fp x fn)
    odds_ratio = 0
    if fp != 0 and fn != 0:
        odds_ratio = (tp * tn)/(fp * fn)

    # precision is tp / (tp + fp)
    precision = tp / (tp + fp)

    metrics_df = pd.DataFrame()
    metrics_df['sensitivity'] = [round(sensitivity, 3)]
    metrics_df['specificity'] = [round(specificity, 3)]
    metrics_df['accuracy'] = [round(accuracy, 3)]
    metrics_df['precision'] = [round(precision, 3)]
    metrics_df['auc'] = [round(auc, 3)]
    metrics_df['odds_ratio'] = [round(odds_ratio, 3)]
    metrics_df.index = [method_name]
    return metrics_df

# Cell
def plot_cluster_sizes(model):
    "Create a bar chart using a previously computed clustering model. Each bar represents a single cluster, with the height of the bars representing the number of members (players) in each cluster."

    plt.figure()
    cluster_ids = list(set(list(model.labels_)))
    cluster_sizes = [list(model.labels_).count(x) for x in cluster_ids]
    plt.bar(
        cluster_ids,
        cluster_sizes,
        color=plt.rcParams["axes.prop_cycle"].by_key()["color"],
    )
    locs, labels = plt.xticks()
    plt.xticks(range(len(cluster_ids)), cluster_ids)
    plt.xlabel("Cluster ID")
    plt.ylabel("Number of Players per Cluster")
    plt.grid(axis="x")
    return plt

# Cell
def plot_agglomeration_dendrogram(model, dt_cutoff=None, **kwargs):
    "Create a dendrogram visualising a heirarchical clustering method (agglomerative clustering). A horisontal line can be added using the dt_cutoff parameter to visualise the number of clusters at a given distance threshold."

    # Create linkage matrix and then plot the sch.dendrogram
    # create the counts of samples under each node
    counts = np.zeros(model.children_.shape[0])
    n_samples = len(model.labels_)
    for i, merge in enumerate(model.children_):
        current_count = 0
        for child_idx in merge:
            if child_idx < n_samples:
                current_count += 1  # leaf node
            else:
                current_count += counts[child_idx - n_samples]
        counts[i] = current_count

    linkage_matrix = np.column_stack(
        [model.children_, model.distances_, counts]
    ).astype(float)

    # Plot the corresponding dendrogram
    plt.figure(figsize=(12, 4))
    plt.title("Hierarchical Clustering dendrogram")
    sch.dendrogram(linkage_matrix, truncate_mode="level", p=3)
    if dt_cutoff != None:
        plt.plot(list(plt.xlim()), [dt_cutoff, dt_cutoff], linestyle="--", color="grey")
    plt.xlabel("Number of points in node (or index of point if no parenthesis).")
    plt.ylabel("Distance threshold")
    plt.grid(False)
    return plt
# AUTOGENERATED! DO NOT EDIT! File to edit: 24_machine_learning.ipynb (unless otherwise specified).

__all__ = ['k_means', 'k_means_range', 'k_means_ensemble', 'agglomerative_cluster', 'describe_clusters',
           'neural_network_classification', 'neural_network_regression', 'logistic_regression',
           'lasso_logistic_regression', 'svm_eps_regression', 'svm_c_classification', 'svm_one_classification',
           'rf_regression', 'rf_classification', 'compute_performance', 'plot_cluster_sizes',
           'plot_agglomeration_dendrogram']

# Cell
from sklearn.cluster import KMeans
import numpy as np
#export
def k_means(measures_table, clusters=4, data_only=False, plot=False, loud=False):
    "Applies the k-means clustering algorithm to a measures table. The resulting clustering is then reported in terms of its inertia (sum of squared distances of samples to their closest cluster center) and its silhouette score (how distinct clusters are within the sample [see the skikit learn docs for details]). The measures passed as the first parameter can be returned with an added column reporting the cluster each player belongs to using the data_only parameter."

    # get variable names from the behavioural measures
    variables = list(measures_table.columns)[1:]

    Kmean = KMeans(n_clusters=clusters)

    data = np.array(measures_table[variables].values)
    Kmean.fit(data)

    silhouette = metrics.silhouette_score(data, Kmean.labels_, metric="euclidean")

    cluster_centers = Kmean.cluster_centers_

    clustered_data = measures_table.copy()
    clustered_data["cluster"] = Kmean.labels_

    if loud:
        print("variables:", variables)
        print("centers:", Kmean.cluster_centers_)
        print("inertia:", Kmean.inertia_)
        print("silhouette:", silhouette)

    if plot:
        bars = []
        heights = []
        for label in set(sorted(Kmean.labels_)):
            bars.append(label)
            heights.append(list(Kmean.labels_).count(label))

        colors = ["C0", "C1", "C2", "C3", "C4", "C5", "C6", "C7", "C8", "C9"]
        plt.bar(bars, heights, color=colors[: len(bars)])
        plt.title(
            "\nClusters: "
            + str(len(bars))
            + "\nInertia: "
            + str(round(Kmean.inertia_))
            + "\nIterations: "  # Kmean.inertia_ is the sum of squared distances of samples to their closest cluster center
            + str(Kmean.n_iter_),
            x=1.01,
            y=0.5,
            ha="left",
        )
        plt.xlabel("Cluster ID")
        plt.ylabel("Number of Members")
        plt.show()

    if data_only:
        return clustered_data

    return clustered_data, Kmean.inertia_, silhouette

# Cell
def k_means_range(measures_table, min_clusters=2, max_clusters=13):
    "Computes the k_means calculation above across a range of cluster counts, returning their goodness of fit measures (inertia and silhouette)."

    # print('calculating k means in range', min_clusters, max_clusters)

    inertias = []
    silhouettes = []

    for x in range(min_clusters, max_clusters + 1):
        k_means_result = k_means(measures_table, clusters=x)
        inertias.append(k_means_result[1])
        silhouettes.append(k_means_result[2])

    return inertias, silhouettes

# Cell
def k_means_ensemble(measures_table, ensemble_size=100, min_clusters=2, max_clusters=13):
    "Computes the k_means clustering algorithm across a range of cluster counts, a number of times. This is useful for determining clusters in a more robust way but can be slow on large data sets."

    all_inertias = []
    all_silhouettes = []

    # call the k_means_range function n times, storing scores in the above arrays
    for x in range(ensemble_size):
        k_means_range_result = k_means_range(
            measures_table, min_clusters=min_clusters, max_clusters=max_clusters
        )
        all_inertias.append(k_means_range_result[0])
        all_silhouettes.append(k_means_range_result[1])

    # now average each of the elements in the score lists
    ensemble_inertias = []
    ensemble_silhouettes = []
    for cluster_num in range(len(all_inertias[0])):
        inertia_scores = [all_inertias[x][cluster_num] for x in range(ensemble_size)]
        ensemble_inertias.append(statistics.mean(inertia_scores))

        silhouette_scores = [
            all_silhouettes[x][cluster_num] for x in range(ensemble_size)
        ]
        ensemble_silhouettes.append(statistics.mean(silhouette_scores))

    return ensemble_inertias, ensemble_silhouettes

# Cell
from sklearn.cluster import AgglomerativeClustering
#export
def agglomerative_cluster(measures_table, distance_threshold=0, n_clusters=None):
    "Performs sklearn's agglomerative clustering algorithm on a dataframe of behavioural measures. See their documentation for details. Note: Either the distance threshold or the n_cluster parameter must be None."

    variables = list(measures_table.columns)[1:]
    X = measures_table[variables].values

    model = AgglomerativeClustering(
        distance_threshold=distance_threshold, n_clusters=n_clusters
    )
    model = model.fit(X)
    return model

# Cell
import pandas as pd
def describe_clusters(clustered_measures_table, cluster_col="cluster"):
    "Describes cluster centroids (mean values of each measure) for each cluster in a clustered measures table."

    unique_clusters = list(set(clustered_measures_table["cluster"].values))

    descriptive_table = pd.DataFrame()
    descriptive_table["cluster_centroid"] = clustered_measures_table.columns[1:]

    for value in unique_clusters:
        members = clustered_measures_table[
            clustered_measures_table[cluster_col] == value
        ]
        centroid = [members[col].mean() for col in members.columns[1:]]
        descriptive_table["n=" + str(len(members))] = centroid

    descriptive_table.set_index("cluster_centroid", inplace=True)
    return descriptive_table

# Cell
from keras.layers import Dense, Activation
from keras.models import Sequential
import numpy as np
import gamba.measures as measures

def neural_network_classification(measures_table, label, train_test_split=0.7):
    "A basic example of a classification network using a single sigmoid-activated output node. The output is the probability of the player belonging to the value of the label. As with the `neural_network_regression` this network closely follows examples in [Deep Learning with Python](http://faculty.neu.edu.cn/yury/AAI/Textbook/Deep%20Learning%20with%20Python.pdf) which is a great resource for extending this example."

    train_measures, test_measures = measures.split_measures_table(measures_table, frac=train_test_split)
    test_labels = test_measures[label]

    train_data = train_measures.drop(['player_id', label], axis=1)
    train_labels = train_measures[label]
    test_data = test_measures.drop(['player_id',label], axis=1)
    test_labels = test_measures[label]

    model = Sequential()
    model.add(Dense(17, activation = 'relu', input_dim = 17))
    model.add(Dense(50, activation = 'relu'))
    model.add(Dense(units = 1, activation='sigmoid')) # this is the 'classification' part

    model.compile(optimizer = 'adam',
                  loss = 'mean_squared_error',
                  metrics=['accuracy'])

    history = model.fit(train_data, train_labels,
                        batch_size = 20, epochs = 100,
                        validation_data=(test_data, test_labels),
                        verbose=False)

    # now make a prediction and clip the values to 0 or 1 as in the original code
    raw_prediction = model.predict(test_data)
    predicted_labels = [value[0] for value in np.where(raw_prediction >= 0.5, 1, 0)]

    return test_labels, predicted_labels

# Cell
import gamba.measures as measures
def neural_network_regression(measures_table, label, train_test_split=0.7):
    "A very simple neural network for regression using two hidden layers and one output node with a linear activation function. This method is very similar to the example in [Deep Learning with Python](http://faculty.neu.edu.cn/yury/AAI/Textbook/Deep%20Learning%20with%20Python.pdf), and can be used as a template for more complex network topologies."

    # note that the metric of accuracy doesn't apply for regression problems (mean absolute error is better)

    train_measures, test_measures = measures.split_measures_table(measures_table, frac=train_test_split)
    test_labels = test_measures[label]

    train_data = train_measures.drop(['player_id', label], axis=1)
    train_labels = train_measures[label]
    test_data = test_measures.drop(['player_id',label], axis=1)
    test_labels = test_measures[label]

    model = Sequential()
    model.add(Dense(17, activation = 'relu', input_dim = 17))
    model.add(Dense(50, activation = 'relu'))
    model.add(Dense(units = 1)) # see page 86 of Deep Learning with Python by Chollet for an explanation

    model.compile(optimizer = 'adam',
                  loss = 'mean_squared_error',
                  metrics=['accuracy'])

    history = model.fit(train_data, train_labels,
                        batch_size = 20, epochs = 100,
                        validation_data=(test_data, test_labels),
                        verbose=False)

    # now make a prediction and clip the values to 0 or 1 as in the original code
    raw_prediction = model.predict(test_data)
    predicted_labels = [value[0] for value in np.where(raw_prediction >= 0.5, 1, 0)]

    return test_labels, predicted_labels

# Cell
import statsmodels.api as sm
from sklearn.linear_model import LogisticRegression
from sklearn import svm
import gamba.measures as measures
#export
def logistic_regression(measures_table, label, train_test_split=0.7):
    "Performs a logistic regression using the [statsmodels library](https://www.statsmodels.org/stable/index.html), returning the predicted labels rounded to the nearest integer. Note: this method is currently hard-coded to only function on Philander 2014's data set. Abstracted logistic regression function coming soon."

    train_measures, test_measures = measures.split_measures_table(measures_table, frac=train_test_split)
    test_labels = test_measures[label]


    # defines the R style formula to fit
    formula = str(label) + " ~ gender+age+total_wagered+num_bets+frequency+duration+bets_per_day+net_loss+intensity+variability+frequency_1m+trajectory+z_intensity+z_variability+z_frequency+z_trajectory"
    model = sm.formula.glm(formula=formula, family=sm.families.Binomial(), data=train_measures)

    # this is where the stepwise bit could happen - see original code
    fit_model = model.fit()

    raw_prediction = fit_model.predict(test_measures)
    predicted_labels = [value for value in np.where(raw_prediction >= 0.5, 1, 0)]

    return test_labels, predicted_labels

# Cell
import gamba.measures as measures
def lasso_logistic_regression(measures_table, label, train_test_split=0.7):
    "Performs a 'lasso' (optimizes a least-square problem with L1 penalty) logistic regression using [sklearn's linear_model](https://scikit-learn.org/stable/modules/classes.html#module-sklearn.linear_model). This [stackoverflow post](https://stackoverflow.com/questions/41639557/how-to-perform-logistic-lasso-in-python>) contains a useful discussion on this type of function-estimation regression pair."

    train_measures, test_measures = measures.split_measures_table(measures_table, frac=train_test_split)
    test_labels = test_measures[label]

    train_data = train_measures.drop(['player_id', label], axis=1)
    train_labels = train_measures[label]
    test_data = test_measures.drop(['player_id', label], axis=1)

    model = LogisticRegression(penalty='l1', solver='liblinear')

    model.fit(train_data, train_labels)

    predicted_labels = model.predict(test_data)

    return test_labels, predicted_labels

# Cell
import gamba.measures as measures
def svm_eps_regression(measures_table, label, train_test_split=0.7):
    "Creates and trains a support vector machine for epsilon-support vector regression using the sklearn library's implementation."

    train_measures, test_measures = measures.split_measures_table(measures_table, frac=train_test_split)
    test_labels = test_measures[label]

    train_data = train_measures.drop(['player_id', label], axis=1)
    train_labels = train_measures[label]
    test_data = test_measures.drop(['player_id', label], axis=1)

    model = svm.SVR(kernel='rbf')

    model.fit(train_data, train_labels)

    predicted_labels = model.predict(test_data)

    # convert probabilities to binary labels for comparison
    regression_cutoff = 0.5
    predicted_labels = np.where(predicted_labels < regression_cutoff, 0, 1)

    return test_labels, predicted_labels

# Cell
import gamba.measures as measures
def svm_c_classification(measures_table, label, train_test_split=0.7):
    "Creates and trains a support vector machine for classification using the sklearn library's implementation."

    train_measures, test_measures = measures.split_measures_table(measures_table, frac=train_test_split)
    test_labels = test_measures[label]

    train_data = train_measures.drop(['player_id', label], axis=1)
    train_labels = train_measures[label]
    test_data = test_measures.drop(['player_id', label], axis=1)

    model = svm.SVC(kernel='rbf')

    model.fit(train_data, train_labels)

    predicted_labels = model.predict(test_data)

    return test_labels, predicted_labels

# Cell
import gamba.measures as measures
def svm_one_classification(measures_table, label, train_test_split=0.7):
    "Creates and trains a support vector machine for one-class classification using the sklearn library's implementation."

    train_measures, test_measures = measures.split_measures_table(measures_table, frac=train_test_split)
    test_labels = test_measures[label]

    train_data = train_measures.drop(['player_id', label], axis=1)
    train_labels = train_measures[label]
    test_data = test_measures.drop(['player_id', label], axis=1)

    model = svm.OneClassSVM(kernel='rbf')

    model.fit(train_data, train_labels)

    predicted_labels = model.predict(test_data)

    # need to add a correction step for the labels here as OneClassSVM returns -1 for outliers and 1 for inliers
    predicted_labels = np.where(predicted_labels < 0, 1, 0)

    return test_labels, predicted_labels

# Cell
from sklearn.ensemble import RandomForestRegressor
from sklearn.ensemble import RandomForestClassifier
import gamba.measures as measures
#export
def rf_regression(measures_table, label, train_test_split=0.7):
    "Creates and fits a random forest regressor using [sklearn's ensemble module](https://scikit-learn.org/stable/modules/ensemble.html#forest), returning the predicted labels rounded to the nearest integer."

    train_measures, test_measures = measures.split_measures_table(measures_table, frac=train_test_split)
    test_labels = test_measures[label]

    train_data = train_measures.drop(['player_id', label], axis=1)
    train_labels = train_measures[label]
    test_data = test_measures.drop(['player_id', label], axis=1)

    model = RandomForestRegressor()

    model.fit(train_data, train_labels)

    predicted_labels = model.predict(test_data)

    # convert probabilities to binary labels for comparison
    regression_cutoff = 0.5
    predicted_labels = np.where(predicted_labels < regression_cutoff, 0, 1)

    return test_labels, predicted_labels

# Cell
import gamba.measures as measures
def rf_classification(measures_table, label, train_test_split=0.7):
    "Creates a random forest for classification also using the sklearn library's implementation."

    train_measures, test_measures = measures.split_measures_table(measures_table, frac=train_test_split)
    test_labels = test_measures[label]

    train_data = train_measures.drop(['player_id', label], axis=1)
    train_labels = train_measures[label]
    test_data = test_measures.drop(['player_id', label], axis=1)

    model = RandomForestClassifier(n_estimators=100)

    model.fit(train_data, train_labels)

    predicted_labels = model.predict(test_data)

    return test_labels, predicted_labels

# Cell
from sklearn import metrics
#export
def compute_performance(method_name, actual, predicted):
    "Computes performance metrics including sensitivity, specificity, accuracy, confusion matrix values, odds ratio, and area under curve, for a given classification/regression using its actual and predicted values."

    # resources:
    # describes odds ratio and precision equations
    # https://cran.r-project.org/web/packages/ROCR/ROCR.pdf
    # describes sklearn's confusion matrix
    # https://scikit-learn.org/stable/modules/generated/sklearn.metrics.confusion_matrix.html#sklearn-metrics-confusion-matrix

    result = metrics.classification_report(actual, y_pred=predicted, output_dict=True)
    sensitivity = result['1']['recall']
    specificity = result['0']['recall']
    accuracy = result['accuracy']
    fpr, tpr, thresholds = metrics.roc_curve(actual, predicted)
    auc = metrics.auc(fpr, tpr)
    confusion_matrix = metrics.confusion_matrix(actual, predicted)
    tn, fp, fn, tp = confusion_matrix.ravel()

    # odds ratio is (tp x tn)/(fp x fn)
    odds_ratio = 0
    if fp != 0 and fn != 0:
        odds_ratio = (tp * tn)/(fp * fn)

    # precision is tp / (tp + fp)
    precision = tp / (tp + fp)

    metrics_df = pd.DataFrame()
    metrics_df['sensitivity'] = [round(sensitivity, 3)]
    metrics_df['specificity'] = [round(specificity, 3)]
    metrics_df['accuracy'] = [round(accuracy, 3)]
    metrics_df['precision'] = [round(precision, 3)]
    metrics_df['auc'] = [round(auc, 3)]
    metrics_df['odds_ratio'] = [round(odds_ratio, 3)]
    metrics_df.index = [method_name]
    return metrics_df

# Cell
def plot_cluster_sizes(model):
    "Create a bar chart using a previously computed clustering model. Each bar represents a single cluster, with the height of the bars representing the number of members (players) in each cluster."

    plt.figure()
    cluster_ids = list(set(list(model.labels_)))
    cluster_sizes = [list(model.labels_).count(x) for x in cluster_ids]
    plt.bar(
        cluster_ids,
        cluster_sizes,
        color=plt.rcParams["axes.prop_cycle"].by_key()["color"],
    )
    locs, labels = plt.xticks()
    plt.xticks(range(len(cluster_ids)), cluster_ids)
    plt.xlabel("Cluster ID")
    plt.ylabel("Number of Players per Cluster")
    plt.grid(axis="x")
    return plt

# Cell
def plot_agglomeration_dendrogram(model, dt_cutoff=None, **kwargs):
    "Create a dendrogram visualising a heirarchical clustering method (agglomerative clustering). A horisontal line can be added using the dt_cutoff parameter to visualise the number of clusters at a given distance threshold."

    # Create linkage matrix and then plot the sch.dendrogram
    # create the counts of samples under each node
    counts = np.zeros(model.children_.shape[0])
    n_samples = len(model.labels_)
    for i, merge in enumerate(model.children_):
        current_count = 0
        for child_idx in merge:
            if child_idx < n_samples:
                current_count += 1  # leaf node
            else:
                current_count += counts[child_idx - n_samples]
        counts[i] = current_count

    linkage_matrix = np.column_stack(
        [model.children_, model.distances_, counts]
    ).astype(float)

    # Plot the corresponding dendrogram
    plt.figure(figsize=(12, 4))
    plt.title("Hierarchical Clustering dendrogram")
    sch.dendrogram(linkage_matrix, truncate_mode="level", p=3)
    if dt_cutoff != None:
        plt.plot(list(plt.xlim()), [dt_cutoff, dt_cutoff], linestyle="--", color="grey")
    plt.xlabel("Number of points in node (or index of point if no parenthesis).")
    plt.ylabel("Distance threshold")
    plt.grid(False)
    return plt
# AUTOGENERATED! DO NOT EDIT! File to edit: 24_machine_learning.ipynb (unless otherwise specified). # Cell #export # get variable names from the behavioural measures # Kmean.inertia_ is the sum of squared distances of samples to their closest cluster center # Cell # print('calculating k means in range', min_clusters, max_clusters) # Cell # call the k_means_range function n times, storing scores in the above arrays # now average each of the elements in the score lists # Cell #export # Cell # Cell # this is the 'classification' part # now make a prediction and clip the values to 0 or 1 as in the original code # Cell # note that the metric of accuracy doesn't apply for regression problems (mean absolute error is better) # see page 86 of Deep Learning with Python by Chollet for an explanation # now make a prediction and clip the values to 0 or 1 as in the original code # Cell #export # defines the R style formula to fit # this is where the stepwise bit could happen - see original code # Cell #module-sklearn.linear_model). This [stackoverflow post](https://stackoverflow.com/questions/41639557/how-to-perform-logistic-lasso-in-python>) contains a useful discussion on this type of function-estimation regression pair." # Cell # convert probabilities to binary labels for comparison # Cell # Cell # need to add a correction step for the labels here as OneClassSVM returns -1 for outliers and 1 for inliers # Cell #export #forest), returning the predicted labels rounded to the nearest integer." # convert probabilities to binary labels for comparison # Cell # Cell #export # resources: # describes odds ratio and precision equations # https://cran.r-project.org/web/packages/ROCR/ROCR.pdf # describes sklearn's confusion matrix # https://scikit-learn.org/stable/modules/generated/sklearn.metrics.confusion_matrix.html#sklearn-metrics-confusion-matrix # odds ratio is (tp x tn)/(fp x fn) # precision is tp / (tp + fp) # Cell # Cell # Create linkage matrix and then plot the sch.dendrogram # create the counts of samples under each node # leaf node # Plot the corresponding dendrogram