class KoverDataset(object):
	def __init__(self, file):
		self.path = file
		self.dataset_open = partial(_hdf5_open_no_chunk_cache, file)

	@property
	def classification_type(self):
		dataset = self.dataset_open()
		# Backwards compatibilty with pre-2.0.0 Kover datasets
		try:
			classification_type = dataset.attrs["classification_type"]
		except:
			classification_type = "binary"
		return classification_type

	@property
	def compression(self):
		dataset = self.dataset_open()
		return dataset.attrs["compression"]

	@property
	def kmer_filter(self):
		dataset = self.dataset_open()
		return dataset.attrs["filter"]

	@property
	def genome_count(self):
		dataset = self.dataset_open()
		return dataset["genome_identifiers"].shape[0]

	@property
	def genome_identifiers(self):
		dataset = self.dataset_open()
		return dataset["genome_identifiers"]

	@property
	def genome_source(self):
		dataset = self.dataset_open()
		return dataset.attrs["genomic_data"]

	@property
	def genome_source_type(self):
		dataset = self.dataset_open()
		return dataset.attrs["genome_source_type"]

	@property
	def kmer_by_matrix_column(self):
		dataset = self.dataset_open()
		return dataset["kmer_by_matrix_column"]

	@property
	def kmer_count(self):
		dataset = self.dataset_open()
		return dataset["kmer_sequences"].shape[0]

	@property
	def kmer_length(self):
		dataset = self.dataset_open()
		return len(dataset["kmer_sequences"][0])

	@property
	def kmer_matrix(self):
		dataset = self.dataset_open()
		return dataset["kmer_matrix"]

	@property
	def kmer_sequences(self):
		dataset = self.dataset_open()
		return dataset["kmer_sequences"]

	@property
	def phenotype(self):
		dataset = self.dataset_open()
		# Backwards compatibilty with pre-2.0.0 Kover datasets
		try:
			description = dataset.attrs["phenotype_description"]
		except:
			description = dataset.attrs["phenotype_name"]

		try:
			tags = dataset["phenotype_tags"]
		except:
			tags = np.array(['0', '1'])

		return KoverDatasetPhenotype(description=description,
									 tags=tags,
									 metadata=dataset["phenotype"],
									 metadata_source=dataset.attrs["phenotype_metadata_source"])

	@property
	def splits(self):
		dataset = self.dataset_open()
		if "splits" in dataset:
			return [self.get_split(split_name) for split_name in dataset["splits"]]
		else:
			return []

	@property
	def uuid(self):
		dataset = self.dataset_open()
		return dataset.attrs["uuid"]

	def get_split(self, name):
		dataset = self.dataset_open()
		split = dataset["splits"][name]
		return KoverDatasetSplit(name,
								 split.attrs["train_proportion"],
								 # Backwards compatibility with datasets without test_proportion
								 split.attrs["test_proportion"] if "test_proportion" in split.attrs else
																1.0 - split.attrs["train_proportion"],
								 split["train_genome_idx"],
								 split["test_genome_idx"],
								 split["unique_risks"],
								 split["unique_risk_by_kmer"],
								 split["unique_risk_by_anti_kmer"],
								 [] if not "folds" in split
								 else [KoverDatasetFold(fold_name,
														fold["train_genome_idx"],
														fold["test_genome_idx"],
														fold["unique_risks"],
														fold["unique_risk_by_kmer"],
														fold["unique_risk_by_anti_kmer"]) for fold_name, fold in split["folds"].iteritems()],
								 split.attrs["random_seed"]) class ConjunctionModel(SCMModel):
    def predict_proba(self, X):
        predictions = np.ones(X.shape[0], np.float32)
        for a in self.rules:
            predictions *= a.classify(X)
        return predictions

    @property
    def type(self):
        return conjunction

    def __str__(self):
        return self._to_string(separator=" and ") class DisjunctionModel(SCMModel):
    def predict_proba(self, X):
        predictions = np.ones(X.shape[0], dtype=np.float32)
        for a in self.rules:
            predictions *= 1.0 - a.classify(X) # Proportion of the voters that predict False in a
        return 1.0 - predictions

    @property
    def type(self):
        return disjunction

    def __str__(self):
        return self._to_string(separator=" or ") class LazyKmerRuleList(object):
    """
    By convention, the first half of the list contains presence rules and the second half contains the absence rules in
    the same order.
    """
    def __init__(self, kmer_sequences, kmer_by_rule):
        self.n_rules = kmer_by_rule.shape[0] * 2
        self.kmer_sequences = kmer_sequences
        self.kmer_by_rule = kmer_by_rule

    def __getitem__(self, idx):
        if idx >= self.n_rules:
            raise ValueError("Index %d is out of range for list of size %d" % (idx, self.n_rules))
        if idx >= len(self.kmer_sequences):
            type = "absence"
            kmer_idx = self.kmer_by_rule[idx % len(self.kmer_sequences)]
        else:
            type = "presence"
            kmer_idx = self.kmer_by_rule[idx]
        return KmerRule(idx % len(self.kmer_sequences), self.kmer_sequences[kmer_idx], type)

    def __len__(self):
        return self.n_rules class KmerRuleClassifications(BaseRuleClassifications):
    """
    Methods involving columns account for presence and absence rules
    """
    # TODO: Clean up. Get rid of the code to handle deleted rows. We don't need this.
    def __init__(self, dataset, n_rows, block_size=None):
        self.dataset = dataset
        self.dataset_initial_n_rows = n_rows
        self.dataset_n_rows = n_rows
        self.dataset_removed_rows = []
        self.dataset_removed_rows_mask = np.zeros(self.dataset_initial_n_rows, dtype=np.bool)
        self.block_size = (None, None)

        if block_size is None:
            if self.dataset.chunks is None:
                self.block_size = (1, self.dataset.shape[1])
            else:
                self.block_size = self.dataset.chunks
        else:
            if len(block_size) != 2 or not isinstance(block_size[0], int) or not isinstance(block_size[1], int):
                raise ValueError("The block size must be a tuple of 2 integers.")
            self.block_size = block_size

        # Get the size of the ints used to store the data
        if self.dataset.dtype == np.uint32:
            self.dataset_pack_size = 32
            self.inplace_popcount = inplace_popcount_32
        elif self.dataset.dtype == np.uint64:
            self.dataset_pack_size = 64
            self.inplace_popcount = inplace_popcount_64
        else:
            raise ValueError("Unsupported data type for packed attribute classifications array. The supported data" +
                             " types are np.uint32 and np.uint64.")

        super(BaseRuleClassifications, self).__init__()

    def get_columns(self, columns):
        """
        Columns can be an integer (or any object that implements __index__) or a sorted list/ndarray.
        """
        #TODO: Support slicing, make this more efficient than getting the columns individually.
        columns_is_int = False
        if hasattr(columns, "__index__"):  # All int types implement the __index__ method (PEP 357)
            columns = [columns.__index__()]
            columns_is_int = True
        elif isinstance(columns, np.ndarray):
            columns = columns.tolist()
        elif isinstance(columns, list):
            pass
        else:
            columns = list(columns)

        # Detect where an inversion is needed (columns corresponding to absence rules)
        columns, invert_result = zip(* (((column if column < self.dataset.shape[1] else column % self.dataset.shape[1]),
                                         (True if column > self.dataset.shape[1] else False)) for column in columns))
        columns = list(columns)
        invert_result = np.array(invert_result)

        # Don't return rows that have been deleted
        row_mask = np.ones(self.dataset.shape[0] * self.dataset_pack_size, dtype=np.bool)
        row_mask[self.dataset_initial_n_rows:] = False
        row_mask[self.dataset_removed_rows] = False

        # h5py requires that the column indices are sorted
        unique, inverse = np.unique(columns, return_inverse=True)
        result = _unpack_binary_bytes_from_ints(self.dataset[:, unique.tolist()])[row_mask]
        result = result[:, inverse]
        result[:, invert_result] = 1 - result[:, invert_result]

        if columns_is_int:
            return result.reshape(-1)
        else:
            return result

    def remove_rows(self, rows):
        # Find in which dataset the rows must be removed
        dataset_removed_rows = []
        # TODO: This is inefficient!
        for row_idx in rows:
            current_idx = -1
            n_active_elements_seen = 0
            while n_active_elements_seen <= row_idx:
                current_idx += 1
                if not self.dataset_removed_rows_mask[current_idx]:
                    n_active_elements_seen += 1
            dataset_removed_rows.append(current_idx)
        # Update the dataset removed row lists
        # Update the start and stop indexes
        # Adjust the shape
        # Adjust the number of rows in each dataset
        # Store the sorted relative removed row indexes by dataset
        if len(dataset_removed_rows) > 0:
            self.dataset_removed_rows = sorted(set(self.dataset_removed_rows + dataset_removed_rows))
            self.dataset_removed_rows_mask = np.zeros(self.dataset_initial_n_rows, dtype=np.bool)
            self.dataset_removed_rows_mask[self.dataset_removed_rows] = True
            self.dataset_n_rows = self.dataset_initial_n_rows - len(self.dataset_removed_rows)

    @property
    def shape(self):
        return self.dataset_n_rows, self.dataset.shape[1] * 2

    # TODO: allow summing over multiple lists of rows at a time (saves i/o operations)
    def sum_rows(self, rows):
        """
        Note: Assumes that the rows argument does not contain duplicate elements. Rows will not be considered more than once.
        """
        rows = np.asarray(rows)
        result_dtype = _minimum_uint_size(rows.shape[0])
        result = np.zeros(self.dataset.shape[1] * 2, dtype=result_dtype)

        # Builds a mask to turn off the bits of the rows we do not want to count in the sum.
        def build_row_mask(example_idx, n_examples, mask_n_bits):
            if mask_n_bits not in [8, 16, 32, 64, 128]:
                raise ValueError("Unsupported mask format. Use 8, 16, 32, 64 or 128 bits.")

            n_masks = int(ceil(float(n_examples) / mask_n_bits))
            masks = [0] * n_masks

            for idx in example_idx:
                example_mask = idx / mask_n_bits
                example_mask_idx = mask_n_bits - (idx - mask_n_bits * example_mask) - 1
                masks[example_mask] |= 1 << example_mask_idx

            return np.array(masks, dtype="u" + str(mask_n_bits / 8))

        # Find the rows that occur in each dataset and their relative index
        rows = np.sort(rows)
        dataset_relative_rows = []
        for row_idx in rows:
            # Find which row in the dataset corresponds to the requested row
            # TODO: This is inefficient! Could exploit the fact that rows is sorted to reuse previous iterations.
            current_idx = -1
            n_active_elements_seen = 0
            while n_active_elements_seen <= row_idx:
                current_idx += 1
                if not self.dataset_removed_rows_mask[current_idx]:
                    n_active_elements_seen += 1
            dataset_relative_rows.append(current_idx)

        # Create a row mask for each dataset
        row_mask = build_row_mask(dataset_relative_rows, self.dataset_initial_n_rows, self.dataset_pack_size)
        del dataset_relative_rows

        # For each dataset load the rows for which the mask is not 0. Support column slicing aswell
        n_col_blocks = int(ceil(1.0 * self.dataset.shape[1] / self.block_size[1]))
        rows_to_load = np.where(row_mask != 0)[0]
        n_row_blocks = int(ceil(1.0 * len(rows_to_load) / self.block_size[0]))

        for row_block in xrange(n_row_blocks):
            block_row_mask = row_mask[rows_to_load[row_block * self.block_size[0]:(row_block + 1) * self.block_size[0]]]

            for col_block in xrange(n_col_blocks):

                # Load the appropriate rows/columns based on the block sizes
                block = self.dataset[rows_to_load[row_block * self.block_size[0]:(row_block + 1) * self.block_size[0]],
                                     col_block * self.block_size[1]:(col_block + 1) * self.block_size[1]]

                # Popcount
                if len(block.shape) == 1:
                    block = block.reshape(1, -1)
                self.inplace_popcount(block, block_row_mask)

                # Increment the sum
                result[col_block * self.block_size[1]:min((col_block + 1) * self.block_size[1], self.dataset.shape[1])] += np.sum(block, axis=0)

        # Compute the sum for absence rules
        result[self.dataset.shape[1] : ] = len(rows) - result[: self.dataset.shape[1]]

        return result class SetCoveringMachine(BaseSetCoveringMachine):
    """
    The Set Covering Machine (SCM).
    Marchand, M., & Taylor, J. S. (2003). The set covering machine. Journal of Machine Learning Research, 3, 723–746.
    Parameters:
    -----------
    model_type: pyscm.model.conjunction or pyscm.model.disjunction, default=pyscm.model.conjunction
        The type of model to be built.
    p: float, default=1.0
        A parameter to control the importance of making prediction errors on positive examples in the utility function.
    max_rules: int, default=10
        The maximum number of binary rules to include in the model.
    verbose: bool, default=False
        Sets verbose mode on/off.
    """

    def __init__(self, model_type=conjunction, p=1.0, max_rules=10):
        super(SetCoveringMachine, self).__init__(model_type=model_type, max_rules=max_rules)

        if model_type == conjunction:
            self.model = ConjunctionModel()
        elif model_type == disjunction:
            self.model = DisjunctionModel()
        else:
            raise ValueError("Unsupported model type.")

        self.p = p

    def _get_best_utility_rules(self, rule_classifications, positive_example_idx, negative_example_idx,
                                rule_blacklist=[]):
        assert isinstance(rule_blacklist, list) or isinstance(rule_blacklist, np.ndarray)
        rule_is_blacklisted = np.zeros(rule_classifications.shape[1], dtype=np.bool)
        rule_is_blacklisted[rule_blacklist] = True

        logging.debug("Counting covered negative examples")
        negative_cover_counts = negative_example_idx.shape[0] - rule_classifications.sum_rows(negative_example_idx)

        logging.debug("Counting errors on positive examples")
        # It is possible that there are no more positive examples to be considered. This is not possible for negative
        # examples because of the SCM's stopping criterion.
        if positive_example_idx.shape[0] > 0:
            positive_error_counts = positive_example_idx.shape[0] - rule_classifications.sum_rows(positive_example_idx)
        else:
            positive_error_counts = np.zeros(rule_classifications.shape[1], dtype=negative_cover_counts.dtype)

        logging.debug("Computing rule utilities")
        # We compute the rule utilities in blocks. This limits the effect of the conversion of integers to floats,
        # which is very memory intensive.
        best_utility = -np.infty
        best_utility_idx = np.array([])
        best_utility_pos_error_count = np.array([])
        best_utility_neg_cover_count = np.array([])
        for block in xrange(int(ceil(1.0 * rule_classifications.shape[1] / UTIL_BLOCK_SIZE))):
            block_utilities = negative_cover_counts[block * UTIL_BLOCK_SIZE : (block + 1) * UTIL_BLOCK_SIZE] - \
                              float(self.p) * positive_error_counts[block * UTIL_BLOCK_SIZE : (block + 1) * UTIL_BLOCK_SIZE]

            # Discard blacklisted rules
            block_utilities[rule_is_blacklisted[block * UTIL_BLOCK_SIZE : (block + 1) * UTIL_BLOCK_SIZE]] = -np.infty

            # Check if there is a better rule or equal in this block
            block_max_utility = np.max(block_utilities)
            if block_max_utility > best_utility or np.allclose(best_utility, block_max_utility):
                # Find the indices of the better rules that are not blacklisted
                block_utility_argmax = np.where(np.isclose(block_utilities, block_max_utility))[0] + block * UTIL_BLOCK_SIZE

                # Update the best utility value and other infos
                if np.allclose(block_max_utility, best_utility):
                    best_utility_idx = np.hstack((best_utility_idx, block_utility_argmax))
                    best_utility_pos_error_count = np.hstack((best_utility_pos_error_count,
                                                              positive_error_counts[block_utility_argmax]))
                    best_utility_neg_cover_count = np.hstack((best_utility_neg_cover_count,
                                                              negative_cover_counts[block_utility_argmax]))
                else:
                    best_utility = block_max_utility
                    best_utility_idx = block_utility_argmax
                    best_utility_pos_error_count = positive_error_counts[block_utility_argmax]
                    best_utility_neg_cover_count = negative_cover_counts[block_utility_argmax]

        return best_utility, best_utility_idx, best_utility_pos_error_count, best_utility_neg_cover_count def _duplicate_last_element(l, length):
    """
    Duplicates the last element of a list until a given length is reached. (In-place)
    """
    l += [l[-1]] * (length - len(l))
    return l def _unpack_binary_bytes_from_ints(a):
    """
    Unpacks binary values stored in bytes into ints
    """
    type = a.dtype

    if type == np.uint32:
        pack_size = 32
    elif type == np.uint64:
        pack_size = 64
    else:
        raise ValueError("Supported data types are 32-bit and 64-bit integers.")

    unpacked_n_rows = a.shape[0] * pack_size
    unpacked_n_columns = a.shape[1] if len(a.shape) > 1 else 1
    b = np.zeros((unpacked_n_rows, a.shape[1]) if len(a.shape) > 1 else unpacked_n_rows, dtype=np.uint8)

    packed_rows = 0
    packing_row = 0
    for i in xrange(b.shape[0]):
        if packed_rows == pack_size:
            packed_rows = 0
            packing_row += 1
        tmp = np.left_shift(np.ones(unpacked_n_columns, dtype=type), pack_size - (i - pack_size * packing_row)-1)
        np.bitwise_and(a[packing_row], tmp, tmp)
        b[i] = tmp > 0
        packed_rows += 1

    return b def _parse_kmer_blacklist(blacklist_path, expected_kmer_len):
    data = []
    
    # Fasta file format
    fasta_extensions = [".fasta", ".fa", ".fas", ".fna"]
    if any(blacklist_path.endswith(extension) for extension in fasta_extensions):
        data = _fasta_to_sequences(blacklist_path)
            
    # Other file format (one kmer per line)
    else:
        # Loading data and splitting on every line
        data = [l.rstrip('\n') for l in open(blacklist_path, "r")]
        
        # Filtering for empty strings (text file with one kmer per line)
        data = [x for x in data if x]
        
    def is_valid_kmer(k):
        return len(set(k).difference(["A", "C", "G", "T", "a", "c", "g", "t"])) == 0
    for kmer in data:
        if not is_valid_kmer(kmer):
            raise ValueError("{} is not a valid DNA sequence".format(kmer))
        
    if not(all(len(kmer) == expected_kmer_len for kmer in data)):
        raise ValueError("Extracted k-mers to blacklist do not have all the same length as the dataset k-mers")
    return data def _get_binary_metrics(predictions, answers):
    """
    Compute metrics for binary predictions.
    Parameters:
    -----------
    predictions: numpy_array, shape=(n_predictions, n_examples)
	    The predictions array.
    answers: numpy_array, shape=(n_examples,)
	    The labels vector associated to the examples.
    Returns:
    --------
    metrics: dictionary
	    A dictionary containing for each metric key,
	    a vector of values associated to the predictions.
    """
    if len(predictions.shape) == 1:
	    predictions = predictions.reshape(1, -1)
    metrics = defaultdict(list)
    for i in xrange(predictions.shape[0]):
	p = predictions[i]
	risk = 1.0 * len(p[p != answers]) / len(answers)
	tp = len(np.where(p[answers == 1] == 1)[0])
	fp = len(np.where(p[answers == 0] == 1)[0])
	tn = len(np.where(p[answers == 0] == 0)[0])
	fn = len(np.where(p[answers == 1] == 0)[0])
	precision = 1.0 * tp / (tp + fp) if (tp + fp) != 0 else -np.infty
	sensitivity = recall = 1.0 * tp / (tp + fn) if (tp + fn) != 0 else -np.infty
	specificity = 1.0 * tn / (fp + tn) if (fp + tn) != 0 else -np.infty
	f1_score = 2.0 * precision * recall / (precision + recall) if (precision + recall) > 0.0 else -np.infty
	metrics["risk"].append(risk)
	metrics["tp"].append(tp)
	metrics["fp"].append(fp)
	metrics["tn"].append(tn)
	metrics["fn"].append(fn)
	metrics["precision"].append(precision)
	metrics["sensitivity"].append(sensitivity)
	metrics["recall"].append(recall)
	metrics["specificity"].append(specificity)
	metrics["f1_score"].append(f1_score)
    return metrics
import logging
import numpy as np
from collections import defaultdict
from copy import deepcopy
from functools import partial
from itertools import product
from math import exp, log as ln, pi
from multiprocessing import Pool, cpu_count
from scipy.misc import comb
from ...dataset.ds import KoverDataset
from ..common.models import ConjunctionModel, DisjunctionModel
from ..common.rules import LazyKmerRuleList, KmerRuleClassifications
from ..learners.scm import SetCoveringMachine
from ...utils import _duplicate_last_element, _unpack_binary_bytes_from_ints, _parse_kmer_blacklist
from ..experiments.metrics import _get_binary_metrics
                k = k.upper()
                try:
                    presence_rule_idx = kmer_by_matrix_column.index(kmer_sequences.index(k))
                    absence_rule_idx = presence_rule_idx + n_kmers
                    rule_blacklist += [presence_rule_idx, absence_rule_idx]
                except ValueError:
                    kmers_not_found.append(k)

            if(len(kmers_not_found) > 0):
                warning_callback("The following kmers could not be found in the dataset: " + ", ".join(kmers_not_found))

    return rule_blacklist

def learn_SCM(dataset_file, split_name, model_type, p, kmer_blacklist_file,max_rules, max_equiv_rules,
              parameter_selection, n_cpu, random_seed, authorized_rules, bound_delta=None, bound_max_genome_size=None,
			  progress_callback=None, warning_callback=None, error_callback=None):
    """
    parameter_selection: bound, cv, none (use first value of each if multiple)
    """
    # Execution callback functions
    if warning_callback is None:
        warning_callback = lambda w: logging.warning(w)
    if error_callback is None:
        def normal_raise(exception):
            raise exception
        error_callback = normal_raise
    if progress_callback is None:
        progress_callback = lambda t, p: None

    if n_cpu is None:
        n_cpu = cpu_count()

    random_generator = np.random.RandomState(random_seed)

    model_type = np.unique(model_type)
    p = np.unique(p)

    logging.debug("Searching for blacklisted rules.")
    rule_blacklist = _find_rule_blacklist(dataset_file=dataset_file,
                                          kmer_blacklist_file=kmer_blacklist_file,
                                          warning_callback=warning_callback)

    dataset = KoverDataset(dataset_file)

    # Score the hyperparameter combinations
    # ------------------------------------------------------------------------------------------------------------------
    if parameter_selection == "bound":
        if bound_delta is None or bound_max_genome_size is None:
            error_callback(Exception("Bound selection cannot be performed without delta and the maximum genome length."))

        # For bound selection, there is no need to retrain the algorithm after selecting the best hyperparameters.
        # The model is already obtained from all the training data. This is why we save the model here.
        best_hp_score, \
        best_hp, \
        best_model, \
        best_rule_importances, \
        best_predictor_equiv_rules = _bound_selection(dataset_file=dataset_file,
                                                      split_name=split_name,
                                                      model_types=model_type,
                                                      p_values=p,
                                                      max_rules=max_rules,
                                                      rule_blacklist=rule_blacklist,
                                                      max_equiv_rules=max_equiv_rules,
                                                      bound_delta=bound_delta,
                                                      bound_max_genome_size=bound_max_genome_size,
                                                      n_cpu=n_cpu,
                                                      random_generator=random_generator,
                                                      progress_callback=progress_callback,
                                                      warning_callback=warning_callback,
                                                      error_callback =error_callback)

    elif parameter_selection == "cv":
        n_folds = len(dataset.get_split(split_name).folds)
        if n_folds < 1:
            error_callback(Exception("Cross-validation cannot be performed on a split with no folds."))
        best_hp_score, best_hp = _cross_validation(dataset_file=dataset_file,
                                                   split_name=split_name,
                                                   model_types=model_type,
                                                   p_values=p,
                                                   max_rules=max_rules,
												   rule_blacklist=rule_blacklist,
                                                   n_cpu=n_cpu,
                                                   progress_callback=progress_callback,
                                                   warning_callback=warning_callback,
                                                   error_callback=error_callback)

    else:
        # Use the first value provided for each parameter
        best_hp = {"model_type": model_type[0], "p": p[0], "max_rules": max_rules}
        best_hp_score = None

    # Use the best hyperparameters to train/test on the split
    # ------------------------------------------------------------------------------------------------------------------
    if parameter_selection == "bound":
        model = best_model
        equivalent_rules = best_predictor_equiv_rules
        rule_importances = best_rule_importances
    else:
        model, rule_importances, \
        equivalent_rules = _full_train(dataset=dataset,
                                       split_name=split_name,
                                       model_type=best_hp["model_type"],
                                       p=best_hp["p"],
                                       max_rules=best_hp["max_rules"],
                                       max_equiv_rules=max_equiv_rules,
									   rule_blacklist=rule_blacklist,
                                       random_generator=random_generator,
                                       progress_callback=progress_callback)

    split = dataset.get_split(split_name)
    train_example_idx = split.train_genome_idx
    test_example_idx = split.test_genome_idx

    train_predictions, test_predictions = _predictions(model=model,
                                                       kmer_matrix=dataset.kmer_matrix,
                                                       train_example_idx=train_example_idx,
                                                       test_example_idx=test_example_idx,
                                                       progress_callback=progress_callback)

    train_answers = dataset.phenotype.metadata[train_example_idx]

    train_metrics = _get_binary_metrics(train_predictions, train_answers)