class DefaultColors(object):

    """Default colors for plotting."""

    def __init__(self):
        self.ecfp_color = (0, 0, 0)
        self.ecfp_chiral_color = (.39, .49, .39)
        self.e2fp_color = (.12, .47, .71)
        self.e2fp_stereo_color = (.2, .63, .17)
        self.e3fp_nostereo_color = (.94, .35, .35)
        self.e3fp_color = (1, .6, .18)
        self.rocs_shape_color = (.41, .34, .52)
        self.rocs_combo_color = (0, .18, .31)
        self.mol_colors = {'Alphaprodine': (0.216, 0.495, 0.720),
                           'Anpirtoline': (0.894, 0.102, 0.110),
                           'Cypenamine': (0.304, 0.683, 0.293)} class DefaultFonts(object):

    """Default fonts/sizes for plotting."""

    def __init__(self):
        self.panel_label_fontsize = 12
        self.title_fontsize = 8
        self.ax_label_fontsize = 8
        self.ax_ticks_fontsize = 7
        self.legend_fontsize = 7 def add_panel_labels(fig=None, axes=None, xoffset=.17, yoffset=.03,
                     label_offset=0):
    """Add labels to all axes."""
    panel_labels = string.ascii_uppercase[label_offset:]

    if axes is None or len(axes) == 0:
        axes = fig.get_axes()

    try:
        x = [-v for v in xoffset]
    except TypeError:
        x = [-xoffset for v in axes]

    try:
        y = [1 + v for v in yoffset]
    except TypeError:
        y = [1 + yoffset for v in axes]

    for i, ax in enumerate(axes):
        label = panel_labels[i]
        ax.text(x[i], y[i], s=label, fontsize=fonts.panel_label_fontsize,
                fontweight='bold', va='bottom', ha='right',
                transform=ax.transAxes) def plot_roc_curves(roc_lists, ax, y_min=0., names=None, colors=None,
                    linestyles=None, ref_line=True, show_legend=True,
                    only_best=True, title="", alpha=1., show_inset=False,
                    legend_fontsize=fonts.legend_fontsize):
    """Plot ROC curves

    Parameters
    ----------
    roc_lists : list of ndarray or list of list of ndarray
        If list of ndarray, each array corresponds to a single ROC curve,
        where the first and second indices of the array correspond to the FPR
        and TPR, respectively. If list of list of ndarray, then each list of
        ndarray is considered to be a set of ROC curves that should be
        grouped.
    ax : axis
        matplotlib axis
    y_min : float, optional
        Minimum value of y, used to visually discriminate between well-
        performing curves.
    names : list of str or None, optional
        Names of ROC sets in `roc_lists`
    colors : list or None, optional
        Colors to be used for ROC sets in `roc_lists`
    linestyles : list or None, optional
        Line styles to be used for ROC sets in `roc_lists`
    ref_line : bool, optional
        Plot expected ROC curve of random classifier.
    show_legend : bool, optional
        Show legend.
    only_best : bool, optional
        Only plot best (highest AUC) ROC curve in a set.
    title : str, optional
        Title of plot
    alpha : float, optional
        Alpha of curves.
    show_inset : bool, optional
        Show inset of plot in lower right corner.
    legend_fontsize : int, optional
        Font size of legend
    """
    legend_lines = []
    legend_names = []

    if isinstance(roc_lists[0], np.ndarray):
        roc_list = [[x] for x in roc_lists]

    if show_inset:
        if show_legend:
            ax_inset = inset_axes(ax, "25%", "25%", loc=4, borderpad=.7)
        else:
            ax_inset = inset_axes(ax, "33%", "33%", loc=4, borderpad=.7)

    for i, roc_list in enumerate(roc_lists):
        color = get_from_list(colors, i)
        name = get_from_list(names, i)
        linestyle = get_from_list(linestyles, i)
        aucs = [get_auc(x[0], x[1]) for x in roc_list]
        auc_rocs = sorted(zip(aucs, roc_list), reverse=True)
        try:
            auc = auc_rocs[0][0]
        except IndexError:
            continue
        if only_best:
            line, = ax.plot(auc_rocs[0][1][0], auc_rocs[0][1][1], linewidth=2,
                            linestyle=linestyle, zorder=i + 2, color=color,
                            alpha=alpha)
            if show_inset:
                ax_inset.plot(auc_rocs[0][1][0], auc_rocs[0][1][1],
                              linewidth=1, linestyle=linestyle, zorder=i + 2,
                              color=color, alpha=alpha)
            name += " ({:.4f})".format(auc)
        else:
            for j, (auc, roc) in enumerate(auc_rocs):
                line, = ax.plot(roc[0], roc[1], linewidth=1,
                                linestyle=linestyle, zorder=i + 2, color=color,
                                alpha=alpha)
                if show_inset:
                    ax_inset.plot(roc[0], roc[1], linewidth=1,
                                  linestyle=linestyle, zorder=i + 2,
                                  color=color, alpha=alpha)

            name += " ({:.4f} +/- {:.4f})".format(np.mean(zip(*auc_rocs)[0]),
                                                  np.std(zip(*auc_rocs)[0]))
        legend_lines.append(line)
        legend_names.append(name)

    legend_lines = legend_lines[::-1]
    legend_names = legend_names[::-1]
    if ref_line:
        line, = ax.plot([0, 1], [0, 1], linewidth=1, color="lightgrey",
                        linestyle="--", label="Random", zorder=1)
        if show_inset:
            ax_inset.plot([0, 1], [0, 1], linewidth=1, color="lightgrey",
                          linestyle="--", label="Random", zorder=1)
        legend_lines.append(line)
        legend_names.append("Random (0.5)")

    ax.set_xlim(0., 1.01)
    ax.set_ylim(y_min, 1.001)
    ax.set_aspect(1. / ax.get_data_ratio())
    if show_inset:
        ax_inset.set_xlim(-.02, 1.02)
        ax_inset.set_ylim(-.02, 1.02)
        ax_inset.set_aspect(1.)
        ax_inset.set_xticks([0., 1.])
        ax_inset.set_yticks([0., 1.])
    ax.set_xlabel("False Positive Rate", fontsize=fonts.ax_label_fontsize)
    ax.set_ylabel("True Positive Rate", fontsize=fonts.ax_label_fontsize)
    ax.set_title(title, fontsize=fonts.title_fontsize)
    if show_legend:
        if show_inset:
            legend_loc = (.125, 0)
        else:
            legend_loc = 'lower center'
        legend = ax.legend(legend_lines, legend_names, loc=legend_loc,
                           fontsize=legend_fontsize, frameon=True)
        legend.get_frame().set_linewidth(0) def plot_prc_curves(prc_lists, ax, names=None, colors=None, linestyles=None,
                    ref_val=None, show_legend=True, only_best=True, title="",
                    alpha=1., legend_fontsize=fonts.legend_fontsize):
    """Plot precision-recall (PRC) curves

    Parameters
    ----------
    prc_lists : list of ndarray or list of list of ndarray
        If list of ndarray, each array corresponds to a single PRC curve,
        where the first and second indices of the array correspond to the FPR
        and TPR, respectively. If list of list of ndarray, then each list of
        ndarray is considered to be a set of PRC curves that should be
        grouped.
    ax : axis
        matplotlib axis
    names : list of str or None, optional
        Names of PRC sets in `prc_lists`
    colors : list or None, optional
        Colors to be used for PRC sets in `prc_lists`
    linestyles : list or None, optional
        Line styles to be used for ROC sets in `prc_lists`
    ref_val : float or None, optional
        If float is provided, value corresponds to the PRC expected from a
        random classifier, which is a horizontal line corresponding to percent
        of pairs which are positives.
    show_legend : bool, optional
        Show legend.
    only_best : bool, optional
        Only plot best (highest AUC) PRC curve in a set.
    title : str, optional
        Title of plot
    alpha : float, optional
        Alpha of curves.
    legend_fontsize : int, optional
        Font size of legend

    Deleted Parameters
    ------------------
    y_min : float, optional
        Minimum value of y, used to visually discriminate between well-
        performing curves.
    """
    legend_lines = []
    legend_names = []

    if isinstance(prc_lists[0], np.ndarray):
        prc_list = [[x] for x in prc_lists]

    for i, prc_list in enumerate(prc_lists):
        color = get_from_list(colors, i)
        name = get_from_list(names, i)
        linestyle = get_from_list(linestyles, i)
        aucs = [get_auc(x[0], x[1]) for x in prc_list]
        auc_prcs = sorted(zip(aucs, prc_list), reverse=True)
        try:
            auc = auc_prcs[0][0]
        except IndexError:
            continue
        if only_best:
            line, = ax.plot(auc_prcs[0][1][0], auc_prcs[0][1][1], linewidth=2,
                            linestyle=linestyle, zorder=i + 2, color=color,
                            alpha=alpha)
            name += " ({:.4f})".format(auc)
        else:
            for j, (auc, prc) in enumerate(auc_prcs):
                line, = ax.plot(prc[0], prc[1], linewidth=1,
                                linestyle=linestyle, zorder=i + 2,
                                color=color, alpha=alpha)
            name += " ({:.4f} +/- {:.4f})".format(np.mean(zip(*auc_prcs)[0]),
                                                  np.std(zip(*auc_prcs)[0]))
        legend_lines.append(line)
        legend_names.append(name)

    legend_lines = legend_lines[::-1]
    legend_names = legend_names[::-1]
    if ref_val is not None:
        line, = ax.plot([0, 1], [ref_val, ref_val], linewidth=1,
                        color="lightgrey", linestyle="--", label="Random",
                        zorder=1)
        legend_lines.append(line)
        legend_names.append("Random ({:.4f})".format(ref_val))

    ax.set_xlim(0., 1.01)
    ax.set_ylim(0, 1.001)
    ax.set_aspect(1. / ax.get_data_ratio())
    ax.set_xlabel("Recall", fontsize=fonts.ax_label_fontsize)
    ax.set_ylabel("Precision", fontsize=fonts.ax_label_fontsize)
    ax.set_title(title, fontsize=fonts.title_fontsize)
    if show_legend:
        legend = ax.legend(legend_lines, legend_names, loc='lower left',
                           fontsize=legend_fontsize, frameon=True)
        legend.get_frame().set_linewidth(0) def plot_auc_stats(repeat_aucs_list, ax, names=None, colors=None,
                   show_legend=True, show_inset=False, xlabel=""):
    """Plot errorbars of AUCs.

    Parameters
    ----------
    repeat_aucs_list : list of list of float
        List of list of fold AUCs from bootstrapped ROC curves.
    ax : axis
        matplotlib axis
    names : list of str or None, optional
        Names of PRC sets in `prc_lists`
    colors : list or None, optional
        Colors to be used for PRC sets in `prc_lists`
    show_legend : bool, optional
        Show legend
    show_inset : bool, optional
        Show insert over full range
    xlabel : str, optional
        X-axis label, indicating type of AUC
    """
    legend_lines = []
    if show_inset:
        xmax = .52
        ax_inset = inset_axes(ax, .3, 1.2, loc=4, borderpad=.2)
    else:
        xmax = .85

    ticks = np.linspace(.04, xmax, len(repeat_aucs_list))

    legend_names = []
    for i, aucs in enumerate(repeat_aucs_list):
        color = get_from_list(colors, i)
        name = get_from_list(names, i)
        mean_auc, std_auc = np.mean(aucs), np.std(aucs)
        _, caps, _ = ax.errorbar(ticks[i], mean_auc, yerr=std_auc,
                                 zorder=i + 2, color=color, fmt='o',
                                 capsize=2, ms=6, linewidth=1)
        legend_names.append(name)
        legend_lines.append(caps)

        for cap in caps:
            cap.set_markeredgewidth(1)

        if show_inset:
            _, caps, _ = ax_inset.errorbar(ticks[i], mean_auc, yerr=std_auc,
                                           ms=4, zorder=i + 2, color=color,
                                           fmt='o', capsize=1.5, linewidth=1)
            for cap in caps:
                cap.set_markeredgewidth(1)

    ax.set_xlim(-.05, 1.25)
    ax.set_xlabel(xlabel)
    if names is not None:
        ax.set_xticks(ticks)
        ax.set_xticklabels(legend_names, rotation=45)
    else:
        ax.set_xticks([])

    if show_inset:
        ax_inset.set_xlim(0., xmax + .1)
        ax_inset.set_ylim(-0.1, 1.1)
        ax_inset.set_xticks([])
        ax_inset.set_yticks([0, 1])

    if show_legend and names is not None:
        ax.legend(legend_lines, names, loc=3, fontsize=fonts.legend_fontsize) def plot_auc_scatter(aucs_dictx, aucs_dicty, ax, colors=None, ref_line=True,
                     xlabel="X AUCs", ylabel="Y AUCs", title=""):
    """Make scatter plot of AUCs.

    Parameters
    ----------
    aucs_dictx : dict
        dict matching key to AUC Only keys common to `aucs_dicty` are used.
    aucs_dicty : dict
        dict matching key to AUC. Only keys common to `aucs_dictx` are used.
    ax : axis
        Matplotlib axis
    colors : list or None, optional
        Colors to be used for ROC sets in `roc_lists`
    ref_line : bool, optional
        Plot expected ROC curve of random classifier.
    xlabel : str, optional
        x-axis label
    ylabel : str, optional
        y-axis label
    title : str, optional
        Title of plot
    """
    data = np.array([(v, aucs_dicty[k]) for k, v in aucs_dictx.iteritems()
                     if k in aucs_dicty], dtype=np.double)

    if ref_line:
        ax.plot([0, 1], [0, 1], linewidth=1, color="lightgrey",
                linestyle="--", label="Random", zorder=1)

    above = np.where(data[:, 1] > data[:, 0])
    below = np.where(data[:, 1] <= data[:, 0])
    x, y = data[above].T
    ax.scatter(x, y, s=20, marker="o", facecolors=colors[0],
               edgecolors='none', alpha=0.3, zorder=3)
    x, y = data[below].T
    ax.scatter(x, y, s=20, marker="o", facecolors=colors[1],
               edgecolors='none', alpha=0.3, zorder=2)

    min_val = np.amin(data)
    ax_min = int(min_val * 20) / 20.0
    ax.set_xlim(ax_min, 1.005)
    ax.set_ylim(ax_min, 1.005)
    ax.set_aspect('equal')
    ax.set_xlabel(xlabel, fontsize=fonts.ax_label_fontsize)
    ax.set_ylabel(ylabel, fontsize=fonts.ax_label_fontsize)
    ax.set_title(title, fontsize=fonts.title_fontsize) def plot_tc_scatter(outlier_df, ax):
    ul = outlier_df.loc["Upper left"]
    ur = outlier_df.loc["Upper"]
    lr = outlier_df.loc["Lower Right"]
    fr = outlier_df.loc["Far Right"]
    for df, ms in zip([ul, ur, lr, fr], ['X', 'P', 's', 'D']):
        ax.scatter(df["ECFP4 TC"], df["E3FP TC"], s=20, linewidths=1,
                   marker=ms, facecolors='r', edgecolors='none', alpha=.8,
                   zorder=100) def plot_tc_hists(counts_df, ax, cols=[], title="", colors=[], thresholds=[],
                  logscale=True, show_legend=True,
                  legend_fontsize=fonts.legend_fontsize):
    if logscale:
        ax.set_yscale("log")
    count = counts_df["Count"]
    ref_line_colors = ["royalblue", "forestgreen"]
    for i, col in enumerate(cols):
        name = col.split(" ")[0]
        tcs = counts_df[col]
        alpha = 1.
        if i > 0:
            alpha = .9
        else:
            alpha = 1.
        try:
            color = colors[i]
        except IndexError:
            color = None
        sns.distplot(tcs, label=name, kde=False, ax=ax, norm_hist=True,
                     color=color, hist_kws={"linewidth": 0,
                                            "histtype": "stepfilled",
                                            "alpha": alpha,
                                            "zorder": 2 * i + 1,
                                            "weights": count})
        if len(thresholds) > 0:
            ax.axvline(thresholds[i], linewidth=1, color=ref_line_colors[i],
                       alpha=.75, linestyle='--', zorder=2 * i + 2)

    if logscale:
        ylabel = "Log Frequency"
    else:
        ylabel = "Frequency"
    ax.set_xlabel("TC", fontsize=fonts.ax_label_fontsize)
    ax.set_ylabel(ylabel, fontsize=fonts.ax_label_fontsize)
    ax.set_xlim(0, counts_df[cols].values.max())
    ax.set_title(title, fontsize=fonts.title_fontsize)
    if show_legend:
        legend = ax.legend(loc=1, fontsize=legend_fontsize, frameon=True)
        legend.get_frame().set_linewidth(0) def plot_tc_heatmap(counts_df, ax, outliers_df=None, cols=[], title="",
                    ref_line=True, fit_line=True, thresholds=[], cmap="bone_r",
                    limit=.5, set_auto_limits=False, logscale=True):
    x, y = counts_df[cols[0]], counts_df[cols[1]]
    non_nan_inds = ~(np.isnan(x) | np.isnan(y))
    x, y = x[non_nan_inds], y[non_nan_inds]
    try:
        count = counts_df["Count"]
        count = count[non_nan_inds]
    except:
        count = None

    if ref_line:
        ax.plot([0, 1], [0, 1], linewidth=1, color="gray", alpha=.5,
                linestyle="--", label="Equal", zorder=2)
        if len(thresholds) > 0:
            xthresh, ythresh = thresholds
            ax.axvline(xthresh, linewidth=1, color="royalblue", alpha=.75,
                       linestyle='--', zorder=2)
            ax.axhline(ythresh, linewidth=1, color="forestgreen",
                       alpha=.75, linestyle='--', zorder=2)

    xmax, ymax = max(max(x), limit), max(max(y), limit)
    if xmax <= 1 and ymax <= 1:
        xmax = ymax = 1
    xmin = ymin = 0
    if set_auto_limits:
        xmin, ymin = round(min(x)), round(min(y))
        xmax, ymax = round(xmax), round(ymax)
    extent = (xmin, xmax, ymin, ymax)

    norm = None
    if logscale:
        norm = matplotlib.colors.LogNorm()
    ax.hexbin(x, y, C=count, cmap=cmap, norm=norm, gridsize=50, extent=extent,
              edgecolors='none', zorder=1)

    if outliers_df is not None:
        x, y = outliers_df[cols[0]], outliers_df[cols[1]]
        if len(x) > 0:
            ax.scatter(x, y, s=15, marker="x", facecolors="r",
                       edgecolors='none', alpha=.3, zorder=3)

    if fit_line:
        slope, intercept, r2 = get_weighted_leastsq(x, y, weights=count)
        ax.plot([xmin, xmax],
                [slope * xmin + intercept, slope * xmax + intercept],
                linewidth=1, color="r", alpha=.75, linestyle="--",
                label="Trend", zorder=2)
        print("Fit with slope {:.4f}, intercept {:.4f}, and R^2 {:.4f}".format(
            slope, intercept, r2))

    ax.set_xlim(xmin, xmax + .02)
    ax.set_ylim(ymin, ymax + .02)
    if xmax <= 1 and ymax <= 1:
        ax.set_aspect('equal')
    xticks = np.linspace(xmin, xmax, 5)
    ax.set_xticks(xticks)
    yticks = np.linspace(ymin, ymax, 5)
    ax.set_yticks(yticks)
    ax.set_xlabel(cols[0], fontsize=fonts.ax_label_fontsize)
    ax.set_ylabel(cols[1], fontsize=fonts.ax_label_fontsize)
    ax.set_title(title, fontsize=fonts.title_fontsize) def rocs_from_cv_dir(cv_dir, fold=None, basename="combined_roc"):
    return results_from_cv_dir(cv_dir, fold=fold, basename=basename) def prcs_from_cv_dir(cv_dir, fold=None, basename="combined_prc"):
    return results_from_cv_dir(cv_dir, fold=fold, basename=basename) def prc_roc_aucs_from_cv_dirs(cv_dirs):
    aucs_list = []
    for cv_dir in cv_dirs:
        log_file = glob.glob(os.path.join(cv_dir, "log.txt"))[0]
        with smart_open(log_file, "r") as f:
            for line in f:
                try:
                    m = re.search(
                        'Fold.*AUROC of (0\.\d+).*AUPRC of (0\.\d+)', line)
                    aucs = float(m.group(1)), float(m.group(2))
                    aucs_list.append(aucs)
                except AttributeError:
                    continue
    aurocs, auprcs = zip(*aucs_list)
    return aurocs, auprcs def target_aucs_from_cv_dirs(cv_dirs):
    target_aurocs_dict = {}
    target_auprcs_dict = {}
    if isinstance(cv_dirs, str):
        cv_dirs = [cv_dirs]
    for cv_dir in cv_dirs:
        log_file = glob.glob(os.path.join(cv_dir, "log.txt"))[0]
        with smart_open(log_file, "r") as f:
            for line in f:
                try:
                    m = re.search(
                        'Target ([\w\d]+) .*AUROC of (0\.\d+).*AUPRC of (0\.\d+)', line)
                    tid = m.group(1)
                    aucs = float(m.group(2)), float(m.group(3))
                    target_aurocs_dict.setdefault(tid, [])
                    target_auprcs_dict.setdefault(tid, [])
                    target_aurocs_dict[tid].append(aucs[0])
                    target_auprcs_dict[tid].append(aucs[1])
                except AttributeError:
                    continue
    return target_aurocs_dict, target_auprcs_dict
import os
import glob
import logging
import numpy as np
import pandas as pd
import seaborn as sns
from matplotlib import pyplot as plt
from matplotlib import gridspec
from python_utilities.scripting import setup_logging
from e3fp_paper.plotting.defaults import DefaultColors, DefaultFonts
from e3fp_paper.plotting.util import add_panel_labels
from e3fp_paper.plotting.validation import plot_roc_curves, plot_prc_curves, \
                                           plot_auc_stats, plot_auc_scatter
from e3fp_paper.plotting.comparison import plot_tc_scatter, plot_tc_hists, \
                                           plot_tc_heatmap
from e3fp_paper.crossvalidation.util import rocs_from_cv_dir, \
                                            prcs_from_cv_dir, \
                                            prc_roc_aucs_from_cv_dirs, \
                                            target_aucs_from_cv_dirs

Author: Seth Axen
E-mail: seth.axen@gmail.com
"""



fonts = DefaultFonts()
def_colors = DefaultColors()

PROJECT_DIR = os.environ['E3FP_PROJECT']
CV_BASEDIR = os.path.join(PROJECT_DIR, 'crossvalidation', 'sea')
E3FP_CV_DIR = os.path.join(CV_BASEDIR, "e3fp")
ECFP_CV_DIR = os.path.join(CV_BASEDIR, "ecfp4")

E3FP_NOSTEREO_REPEAT_DIRS = glob.glob(os.path.join(CV_BASEDIR,
                                                   "e3fp-nostereo*"))
E3FP_REPEAT_DIRS = [x for x in glob.glob(os.path.join(CV_BASEDIR, "e3fp*"))
                    if x not in E3FP_NOSTEREO_REPEAT_DIRS]
E2FP_STEREO_REPEAT_DIRS = glob.glob(os.path.join(CV_BASEDIR, "e2fp-stereo*"))
E2FP_REPEAT_DIRS = [x for x in glob.glob(os.path.join(CV_BASEDIR, "e2fp*"))
                    if x not in E2FP_STEREO_REPEAT_DIRS]
ECFP_CHIRAL_REPEAT_DIRS = glob.glob(os.path.join(CV_BASEDIR, "ecfp4-chiral*"))
ECFP_REPEAT_DIRS = [x for x in glob.glob(os.path.join(CV_BASEDIR, "ecfp4*"))
                    if x not in ECFP_CHIRAL_REPEAT_DIRS]

TC_COUNTS_FILE = os.path.join(PROJECT_DIR, "fingerprint_comparison",
                              "ecfp_e3fp_tcs_counts.csv.bz2")
EXAMPLES_FILE = os.path.join(PROJECT_DIR, "data", "examples.txt")
OUT_IMAGE_BASENAME = "fig_2"
ROC_YMIN = .969
FRAC_POS = .0051
ECFP_TC_THRESH = .3
E3FP_TC_THRESH = .2
setup_logging(verbose=True)


def get_best_fold_by_prc(cv_dir):
    log_file = os.path.join(cv_dir, "log.txt")
    logging.debug("Opening log file: {}".format(log_file))
    fold_results = []
    with open(log_file, "r") as f:
        for line in f:
            if "|Fold " in line:
                fold = int(line.split('Fold ')[1].split()[0])
                auprc = float(line.split('AUPRC of ')[1].split()[0][:-1])
                fold_results.append((auprc, fold))
    return sorted(fold_results)[-1][1]


def get_best_fold_by_roc(cv_dir):
    log_file = os.path.join(cv_dir, "log.txt")
    logging.debug("Opening log file: {}".format(log_file))
    fold_results = []
    with open(log_file, "r") as f:
        for line in f:
            if "|Fold " in line:
                fold = int(line.split('Fold ')[1].split()[0])
                auroc = float(line.split('AUROC of ')[1].split()[0])
                fold_results.append((auroc, fold))
    return sorted(fold_results)[-1][1]


if __name__ == "__main__":
    panel_axes = []
    panel_xoffsets = []
    fig = plt.figure(figsize=(6.4, 8))
    gs = gridspec.GridSpec(3, 4)
    tc_counts_df = pd.DataFrame.from_csv(TC_COUNTS_FILE, sep="\t",
                                         index_col=None)
    tc_examples_df = pd.DataFrame.from_csv(EXAMPLES_FILE, sep="\t")
    ax = plt.subplot(gs[0, :2])
    plot_tc_heatmap(tc_counts_df, ax=ax, outliers_df=None,
                    cols=["ECFP4 TC", "E3FP Max TC"],
                    thresholds=[ECFP_TC_THRESH, E3FP_TC_THRESH])
    plot_tc_scatter(tc_examples_df, ax)
    sns.despine(ax=ax, offset=10)
    panel_axes.append(ax)
    panel_xoffsets.append(.23)

    ax = plt.subplot(gs[0, 2:])
    plot_tc_hists(tc_counts_df, cols=["ECFP4 TC", "E3FP Max TC"], ax=ax,
                  colors=[def_colors.ecfp_color, def_colors.e3fp_color],
                  thresholds=[ECFP_TC_THRESH, E3FP_TC_THRESH])
    sns.despine(ax=ax, offset=10)
    panel_axes.append(ax)
    panel_xoffsets.append(.20)

    cv_repeat_dirs = [ECFP_REPEAT_DIRS, ECFP_CHIRAL_REPEAT_DIRS,
                      E2FP_REPEAT_DIRS, E2FP_STEREO_REPEAT_DIRS,
                      E3FP_NOSTEREO_REPEAT_DIRS,
                      E3FP_REPEAT_DIRS]
    cv_repeat_names = ["ECFP4", "ECFP4-Chiral", "E2FP", "E2FP-Stereo",
                       "E3FP-NoStereo", "E3FP"]
    all_colors = [def_colors.ecfp_color, def_colors.ecfp_chiral_color,
                  def_colors.e2fp_color, def_colors.e2fp_stereo_color,
                  def_colors.e3fp_nostereo_color, def_colors.e3fp_color]
    ax = plt.subplot(gs[1, :2])
    prc_lists = [[z for y in x for z in prcs_from_cv_dir(y)]
                 for x in cv_repeat_dirs]
    plot_prc_curves(prc_lists, ref_val=FRAC_POS, ax=ax, names=cv_repeat_names,
                    colors=all_colors, alpha=.75, only_best=True)
    del prc_lists
    sns.despine(ax=ax, offset=10)
    panel_axes.append(ax)
    panel_xoffsets.append(.20)

    ax = plt.subplot(gs[1, 2:])
    roc_lists = [[z for y in x for z in rocs_from_cv_dir(y)]
                 for x in cv_repeat_dirs]
    plot_roc_curves(roc_lists, ax=ax, names=cv_repeat_names,
                    colors=all_colors, y_min=ROC_YMIN, alpha=.75,
                    only_best=True)
    del roc_lists
    sns.despine(ax=ax, offset=10)
    panel_axes.append(ax)
    panel_xoffsets.append(.28)

    repeat_aucs = [prc_roc_aucs_from_cv_dirs(x) for x in cv_repeat_dirs]
    ax1 = plt.subplot(gs[2, 0])

    plot_auc_stats(