#!/usr/bin/env python
# -*- encoding: utf-8 -*-
#
# Copyright 2016 H2O.ai;  Apache License Version 2.0 (see LICENSE for details)
#
"""Shared utilities used by various classes, all placed here to avoid circular imports.

This file INTENTIONALLY has NO module dependencies!
"""
from __future__ import absolute_import, division, print_function, unicode_literals

import imp
import itertools
import os
import re
import sys
import zipfile
import io
import string
import subprocess
import csv
import shutil
import tempfile

from h2o.exceptions import H2OValueError
from h2o.utils.compatibility import *  # NOQA
from h2o.utils.typechecks import assert_is_type, is_type, numeric
from h2o.backend.server import H2OLocalServer

_id_ctr = 0

# The set of characters allowed in frame IDs. Since frame ids are used within REST API urls, they may
# only contain characters allowed within the "segment" part of the URL (see RFC 3986). Additionally, we
# forbid all characters that are declared as "illegal" in Key.java.
_id_allowed_characters = set("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-._~")

__all__ = ('mojo_predict_csv', 'mojo_predict_pandas')

def _py_tmp_key(append):
    global _id_ctr
    _id_ctr += 1
    return "py_" + str(_id_ctr) + append

def check_frame_id(frame_id):
    """Check that the provided frame id is valid in Rapids language."""
    if frame_id is None:
        return
    if frame_id.strip() == "":
        raise H2OValueError("Frame id cannot be an empty string: %r" % frame_id)
    for i, ch in enumerate(frame_id):
        # '$' character has special meaning at the beginning of the string; and prohibited anywhere else
        if ch == "$" and i == 0: continue
        if ch not in _id_allowed_characters:
            raise H2OValueError("Character '%s' is illegal in frame id: %s" % (ch, frame_id))
    if re.match(r"-?[0-9]", frame_id):
        raise H2OValueError("Frame id cannot start with a number: %s" % frame_id)

def temp_ctr():
    return _id_ctr

def can_use_pandas():
    try:
        imp.find_module('pandas')
        return True
    except ImportError:
        return False

def can_use_numpy():
    try:
        imp.find_module('numpy')
        return True
    except ImportError:
        return False

_url_safe_chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_.~"
_url_chars_map = [chr(i) if chr(i) in _url_safe_chars else "%%%02X" % i for i in range(256)]

def url_encode(s):
    # Note: type cast str(s) will not be needed once all code is made compatible
    return "".join(_url_chars_map[c] for c in bytes_iterator(s))

def quote(s):
    return url_encode(s)

def urlopen():
    if PY3:
        from urllib import request
        return request.urlopen
    else:
        import urllib2
        return urllib2.urlopen

def clamp(x, xmin, xmax):
    """Return the value of x, clamped from below by `xmin` and from above by `xmax`."""
    return max(xmin, min(x, xmax))

def _gen_header(cols):
    return ["C" + str(c) for c in range(1, cols + 1, 1)]

def _check_lists_of_lists(python_obj):
    # check we have a lists of flat lists
    # returns longest length of sublist
    most_cols = 1
    for l in python_obj:
        # All items in the list must be a list!
        if not isinstance(l, (tuple, list)):
            raise ValueError("`python_obj` is a mixture of nested lists and other types.")
        most_cols = max(most_cols, len(l))
        for ll in l:
            # in fact, we must have a list of flat lists!
            if isinstance(ll, (tuple, list)):
                raise ValueError("`python_obj` is not a list of flat lists!")
    return most_cols

def _handle_python_lists(python_obj, check_header):
    # convert all inputs to lol
    if _is_list_of_lists(python_obj):  # do we have a list of lists: [[...], ..., [...]] ?
        ncols = _check_lists_of_lists(python_obj)  # must be a list of flat lists, raise ValueError if not
    elif isinstance(python_obj, (list, tuple)):  # single list
        ncols = 1
        python_obj = [[e] for e in python_obj]
    else:  # scalar
        python_obj = [[python_obj]]
        ncols = 1
    # create the header
    if check_header == 1:
        header = python_obj[0]
        python_obj = python_obj[1:]
    else:
        header = _gen_header(ncols)
    # shape up the data for csv.DictWriter
    # data_to_write = [dict(list(zip(header, row))) for row in python_obj]
    return header, python_obj

def stringify_dict(d):
    return stringify_list(["{'key': %s, 'value': %s}" % (_quoted(k), v) for k, v in d.items()])

def stringify_list(arr):
    return "[%s]" % ",".join(stringify_list(item) if isinstance(item, list) else _str(item)
                             for item in arr)

def _str(item):
    return _str_tuple(item) if isinstance(item, tuple) else str(item)

def _str_tuple(t):
    return "{%s}" % ",".join(["%s: %s" % (ti[0], str(ti[1])) for ti in zip(list(string.ascii_lowercase), t)])

def _is_list(l):
    return isinstance(l, (tuple, list))

def _is_str_list(l):
    return is_type(l, [str])

def _is_num_list(l):
    return is_type(l, [numeric])

def _is_list_of_lists(o):
    return any(isinstance(l, (tuple, list)) for l in o)

def _handle_numpy_array(python_obj, header):
    return _handle_python_lists(python_obj.tolist(), header)

def _handle_pandas_data_frame(python_obj, header):
    data = _handle_python_lists(python_obj.values.tolist(), -1)[1]
    return list(python_obj.columns), data

def _handle_python_dicts(python_obj, check_header):
    header = list(python_obj.keys()) if python_obj else _gen_header(1)
    is_valid = all(re.match(r"^[a-zA-Z_][a-zA-Z0-9_.]*$", col) for col in header)  # is this a valid header?
    if not is_valid:
        raise ValueError(
            "Did not get a valid set of column names! Must match the regular expression: ^[a-zA-Z_][a-zA-Z0-9_.]*$ ")
    for k in python_obj:  # check that each value entry is a flat list/tuple or single int, float, or string
        v = python_obj[k]
        if isinstance(v, (tuple, list)):  # if value is a tuple/list, then it must be flat
            if _is_list_of_lists(v):
                raise ValueError("Values in the dictionary must be flattened!")
        elif is_type(v, str, numeric):
            python_obj[k] = [v]
        else:
            raise ValueError("Encountered invalid dictionary value when constructing H2OFrame. Got: {0}".format(v))

    zipper = getattr(itertools, "zip_longest", None) or getattr(itertools, "izip_longest", None) or zip
    rows = list(map(list, zipper(*list(python_obj.values()))))
    data_to_write = [dict(list(zip(header, row))) for row in rows]
    return header, data_to_write

def _is_fr(o):
    return o.__class__.__name__ == "H2OFrame"  # hack to avoid circular imports

def _quoted(key):
    if key is None: return "\"\""
    # mimic behavior in R to replace "%" and "&" characters, which break the call to /Parse, with "."
    # key = key.replace("%", ".")
    # key = key.replace("&", ".")
    is_quoted = len(re.findall(r'\"(.+?)\"', key)) != 0
    key = key if is_quoted else '"' + key + '"'
    return key

def _locate(path):
    """Search for a relative path and turn it into an absolute path.
    This is handy when hunting for data files to be passed into h2o and used by import file.
    Note: This function is for unit testing purposes only.

    Parameters
    ----------
    path : str
      Path to search for

    :return: Absolute path if it is found.  None otherwise.
    """

    tmp_dir = os.path.realpath(os.getcwd())
    possible_result = os.path.join(tmp_dir, path)
    while True:
        if os.path.exists(possible_result):
            return possible_result

        next_tmp_dir = os.path.dirname(tmp_dir)
        if next_tmp_dir == tmp_dir:
            raise ValueError("File not found: " + path)

        tmp_dir = next_tmp_dir
        possible_result = os.path.join(tmp_dir, path)

def _colmean(column):
    """Return the mean of a single-column frame."""
    assert column.ncols == 1
    return column.mean(return_frame=True).flatten()

def get_human_readable_bytes(size):
    """
    Convert given number of bytes into a human readable representation, i.e. add prefix such as kb, Mb, Gb,
    etc. The `size` argument must be a non-negative integer.

    :param size: integer representing byte size of something
    :return: string representation of the size, in human-readable form
    """
    if size == 0: return "0"
    if size is None: return ""
    assert_is_type(size, int)
    assert size >= 0, "`size` cannot be negative, got %d" % size
    suffixes = "PTGMk"
    maxl = len(suffixes)
    for i in range(maxl + 1):
        shift = (maxl - i) * 10
        if size >> shift == 0: continue
        ndigits = 0
        for nd in [3, 2, 1]:
            if size >> (shift + 12 - nd * 3) == 0:
                ndigits = nd
                break
        if ndigits == 0 or size == (size >> shift) << shift:
            rounded_val = str(size >> shift)
        else:
            rounded_val = "%.*f" % (ndigits, size / (1 << shift))
        return "%s %sb" % (rounded_val, suffixes[i] if i < maxl else "")

def get_human_readable_time(time_ms):
    """
    Convert given duration in milliseconds into a human-readable representation, i.e. hours, minutes, seconds,
    etc. More specifically, the returned string may look like following:
        1 day 3 hours 12 mins
        3 days 0 hours 0 mins
        8 hours 12 mins
        34 mins 02 secs
        13 secs
        541 ms
    In particular, the following rules are applied:
        * milliseconds are printed only if the duration is less than a second;
        * seconds are printed only if the duration is less than an hour;
        * for durations greater than 1 hour we print days, hours and minutes keeping zeros in the middle (i.e. we
          return "4 days 0 hours 12 mins" instead of "4 days 12 mins").

    :param time_ms: duration, as a number of elapsed milliseconds.
    :return: human-readable string representation of the provided duration.
    """
    millis = time_ms % 1000
    secs = (time_ms // 1000) % 60
    mins = (time_ms // 60000) % 60
    hours = (time_ms // 3600000) % 24
    days = (time_ms // 86400000)

    res = ""
    if days > 1:
        res += "%d days" % days
    elif days == 1:
        res += "1 day"

    if hours > 1 or (hours == 0 and res):
        res += " %d hours" % hours
    elif hours == 1:
        res += " 1 hour"

    if mins > 1 or (mins == 0 and res):
        res += " %d mins" % mins
    elif mins == 1:
        res += " 1 min"

    if days == 0 and hours == 0:
        res += " %02d secs" % secs
    if not res:
        res = " %d ms" % millis

    return res.strip()

def print2(msg, flush=False, end="\n"):
    """
    This function exists here ONLY because Sphinx.ext.autodoc gets into a bad state when seeing the print()
    function. When in that state, autodoc doesn't display any errors or warnings, but instead completely
    ignores the "bysource" member-order option.
    """
    print(msg, end=end)
    if flush: sys.stdout.flush()

def normalize_slice(s, total):
    """
    Return a "canonical" version of slice ``s``.

    :param slice s: the original slice expression
    :param total int: total number of elements in the collection sliced by ``s``
    :return slice: a slice equivalent to ``s`` but not containing any negative indices or Nones.
    """
    newstart = 0 if s.start is None else max(0, s.start + total) if s.start < 0 else min(s.start, total)
    newstop = total if s.stop is None else max(0, s.stop + total) if s.stop < 0 else min(s.stop, total)
    newstep = 1 if s.step is None else s.step
    return slice(newstart, newstop, newstep)

def slice_is_normalized(s):
    """Return True if slice ``s`` in "normalized" form."""
    return (s.start is not None and s.stop is not None and s.step is not None and s.start <= s.stop)

gen_header = _gen_header
py_tmp_key = _py_tmp_key
locate = _locate
quoted = _quoted
is_list = _is_list
is_fr = _is_fr
handle_python_dicts = _handle_python_dicts
handle_pandas_data_frame = _handle_pandas_data_frame
handle_numpy_array = _handle_numpy_array
is_list_of_lists = _is_list_of_lists
is_num_list = _is_num_list
is_str_list = _is_str_list
handle_python_lists = _handle_python_lists
check_lists_of_lists = _check_lists_of_lists

gen_model_file_name = "h2o-genmodel.jar"
h2o_predictor_class = "hex.genmodel.tools.PredictCsv"

def mojo_predict_pandas(dataframe, mojo_zip_path, genmodel_jar_path=None, classpath=None, java_options=None, verbose=False):
    """
    MOJO scoring function to take a Pandas frame and use MOJO model as zip file to score.

    :param dataframe: Pandas frame to score.
    :param mojo_zip_path: Path to MOJO zip downloaded from H2O.
    :param genmodel_jar_path: Optional, path to genmodel jar file. If None (default) then the h2o-genmodel.jar in the same
        folder as the MOJO zip will be used.
    :param classpath: Optional, specifies custom user defined classpath which will be used when scoring. If None
        (default) then the default classpath for this MOJO model will be used.
    :param java_options: Optional, custom user defined options for Java. By default ``-Xmx4g`` is used.
    :param verbose: Optional, if True, then additional debug information will be printed. False by default.
    :return: Pandas frame with predictions
    """
    tmp_dir = tempfile.mkdtemp()
    try:
        if not can_use_pandas():
            raise RuntimeException('Cannot import pandas')
        import pandas
        assert_is_type(dataframe, pandas.DataFrame)
        input_csv_path = os.path.join(tmp_dir, 'input.csv')
        prediction_csv_path = os.path.join(tmp_dir, 'prediction.csv')
        dataframe.to_csv(input_csv_path)
        mojo_predict_csv(input_csv_path=input_csv_path, mojo_zip_path=mojo_zip_path,
                         output_csv_path=prediction_csv_path, genmodel_jar_path=genmodel_jar_path,
                         classpath=classpath, java_options=java_options, verbose=verbose)
        return pandas.read_csv(prediction_csv_path)
    finally:
        shutil.rmtree(tmp_dir)

def mojo_predict_csv(input_csv_path, mojo_zip_path, output_csv_path=None, genmodel_jar_path=None, classpath=None, java_options=None, verbose=False):
    """
    MOJO scoring function to take a CSV file and use MOJO model as zip file to score.

    :param input_csv_path: Path to input CSV file.
    :param mojo_zip_path: Path to MOJO zip downloaded from H2O.
    :param output_csv_path: Optional, name of the output CSV file with computed predictions. If None (default), then
        predictions will be saved as prediction.csv in the same folder as the MOJO zip.
    :param genmodel_jar_path: Optional, path to genmodel jar file. If None (default) then the h2o-genmodel.jar in the same
        folder as the MOJO zip will be used.
    :param classpath: Optional, specifies custom user defined classpath which will be used when scoring. If None
        (default) then the default classpath for this MOJO model will be used.
    :param java_options: Optional, custom user defined options for Java. By default ``-Xmx4g -XX:ReservedCodeCacheSize=256m`` is used.
    :param verbose: Optional, if True, then additional debug information will be printed. False by default.
    :return: List of computed predictions
    """
    default_java_options = '-Xmx4g -XX:ReservedCodeCacheSize=256m'
    prediction_output_file = 'prediction.csv'

    # Checking java
    java = H2OLocalServer._find_java()
    H2OLocalServer._check_java(java=java, verbose=verbose)

    # Ensure input_csv exists
    if verbose:
        print("input_csv:\t%s" % input_csv_path)
    if not os.path.isfile(input_csv_path):
        raise RuntimeError("Input csv cannot be found at %s" % input_csv_path)

    # Ensure mojo_zip exists
    mojo_zip_path = os.path.abspath(mojo_zip_path)
    if verbose:
        print("mojo_zip:\t%s" % mojo_zip_path)
    if not os.path.isfile(mojo_zip_path):
        raise RuntimeError("MOJO zip cannot be found at %s" % mojo_zip_path)

    parent_dir = os.path.dirname(mojo_zip_path)

    # Set output_csv if necessary
    if output_csv_path is None:
        output_csv_path = os.path.join(parent_dir, prediction_output_file)

    # Set path to h2o-genmodel.jar if necessary and check it's valid
    if genmodel_jar_path is None:
        genmodel_jar_path = os.path.join(parent_dir, gen_model_file_name)
    if verbose:
        print("genmodel_jar:\t%s" % genmodel_jar_path)
    if not os.path.isfile(genmodel_jar_path):
        raise RuntimeError("Genmodel jar cannot be found at %s" % genmodel_jar_path)

    if verbose and output_csv_path is not None:
        print("output_csv:\t%s" % output_csv_path)

    # Set classpath if necessary
    if classpath is None:
        classpath = genmodel_jar_path
    if verbose:
        print("classpath:\t%s" % classpath)

    # Set java_options if necessary
    if java_options is None:
        java_options = default_java_options
    if verbose:
        print("java_options:\t%s" % java_options)

    # Construct command to invoke java
    cmd = [java]
    for option in java_options.split(' '):
        cmd += [option]
    cmd += ["-cp", classpath, h2o_predictor_class, "--mojo", mojo_zip_path, "--input", input_csv_path,
            '--output', output_csv_path, '--decimal']
    if verbose:
        cmd_str = " ".join(cmd)
        print("java cmd:\t%s" % cmd_str)

    # invoke the command
    subprocess.check_call(cmd, shell=False)

    # load predictions in form of a dict
    with open(output_csv_path) as csv_file:
        result = list(csv.DictReader(csv_file))
    return result

def deprecated(message):
    """The decorator to mark deprecated functions."""
    from traceback import extract_stack
    assert message, "`message` argument in @deprecated is required."

    def deprecated_decorator(fun):
        def decorator_invisible(*args, **kwargs):
            stack = extract_stack()
            assert len(stack) >= 2 and stack[-1][2] == "decorator_invisible", "Got confusing stack... %r" % stack
            print("[WARNING] in %s line %d:" % (stack[-2][0], stack[-2][1]))
            print("    >>> %s" % (stack[-2][3] or "????"))
            print("        ^^^^ %s" % message)
            return fun(*args, **kwargs)

        decorator_invisible.__doc__ = message
        decorator_invisible.__name__ = fun.__name__
        decorator_invisible.__module__ = fun.__module__
        decorator_invisible.__deprecated__ = True
        return decorator_invisible

    return deprecated_decorator

class InMemoryZipArch(object):
    def __init__(self, file_name = None, compression = zipfile.ZIP_DEFLATED):
        self._data = io.BytesIO()
        self._arch = zipfile.ZipFile(self._data, "w", compression, False)
        self._file_name = file_name

    def append(self, filename_in_zip, file_contents):
        self._arch.writestr(filename_in_zip, file_contents)
        return self

    def write_to_file(self, filename):
        # Mark the files as having been created on Windows so that
        # Unix permissions are not inferred as 0000
        for zfile in self._arch.filelist:
            zfile.create_system = 0
        self._arch.close()
        with open(filename, 'wb') as f:
            f.write(self._data.getvalue())

    def __enter__(self):
            return self

    def __exit__(self, exc_type, exc_value, traceback):
        if self._file_name is None:
            return
        self.write_to_file(self._file_name)


# -*- encoding: utf-8 -*-
from __future__ import division, print_function, absolute_import, unicode_literals

import itertools

import h2o
from h2o.job import H2OJob
from h2o.frame import H2OFrame
from h2o.exceptions import H2OValueError
from h2o.estimators.estimator_base import H2OEstimator
from h2o.two_dim_table import H2OTwoDimTable
from h2o.display import H2ODisplay
from h2o.grid.metrics import *  # NOQA
from h2o.utils.backward_compatibility import backwards_compatible
from h2o.utils.shared_utils import deprecated, quoted
from h2o.utils.compatibility import *  # NOQA
from h2o.utils.typechecks import assert_is_type, is_type

class H2OGridSearch(backwards_compatible()):
    """
    Grid Search of a Hyper-Parameter Space for a Model

    :param model: The type of model to be explored initialized with optional parameters that will be
        unchanged across explored models.
    :param hyper_params: A dictionary of string parameters (keys) and a list of values to be explored by grid
        search (values).
    :param str grid_id: The unique id assigned to the resulting grid object. If none is given, an id will
        automatically be generated.
    :param search_criteria:  A dictionary of directives which control the search of the hyperparameter space.
        The default strategy "Cartesian" covers the entire space of hyperparameter combinations. Specify the
        "RandomDiscrete" strategy to get random search of all the combinations of your hyperparameters.
        RandomDiscrete should usually be combined with at least one early stopping criterion: max_models
        and/or max_runtime_secs, e.g::

            >>> criteria = {"strategy": "RandomDiscrete", "max_models": 42,
            ...             "max_runtime_secs": 28800, "seed": 1234}
            >>> criteria = {"strategy": "RandomDiscrete", "stopping_metric": "AUTO",
            ...             "stopping_tolerance": 0.001, "stopping_rounds": 10}
            >>> criteria = {"strategy": "RandomDiscrete", "stopping_rounds": 5,
            ...             "stopping_metric": "misclassification",
            ...             "stopping_tolerance": 0.00001}
    :returns: a new H2OGridSearch instance

    Examples
    --------
        >>> from h2o.grid.grid_search import H2OGridSearch
        >>> from h2o.estimators.glm import H2OGeneralizedLinearEstimator
        >>> hyper_parameters = {'alpha': [0.01,0.5], 'lambda': [1e-5,1e-6]}
        >>> gs = H2OGridSearch(H2OGeneralizedLinearEstimator(family='binomial'), hyper_parameters)
        >>> training_data = h2o.import_file("smalldata/logreg/benign.csv")
        >>> gs.train(x=range(3) + range(4,11),y=3, training_frame=training_data)
        >>> gs.show()
    """

    def __init__(self, model, hyper_params, grid_id=None, search_criteria=None):
        super(H2OGridSearch, self).__init__()
        assert_is_type(model, None, H2OEstimator, lambda mdl: issubclass(mdl, H2OEstimator))
        assert_is_type(hyper_params, dict)
        assert_is_type(grid_id, None, str)
        assert_is_type(search_criteria, None, dict)
        if not (model is None or is_type(model, H2OEstimator)): model = model()
        self._id = grid_id
        self.model = model
        self.hyper_params = dict(hyper_params)
        self.search_criteria = None if search_criteria is None else dict(search_criteria)
        self._grid_json = None
        self.models = None  # list of H2O Estimator instances
        self._parms = {}  # internal, for object recycle #
        self.parms = {}  # external#
        self._future = False  # used by __repr__/show to query job state#
        self._job = None  # used when _future is True#

    @property
    def grid_id(self):
        """A key that identifies this grid search object in H2O."""
        return self._id

    @grid_id.setter
    def grid_id(self, value):
        oldname = self.grid_id
        self._id = value
        h2o.rapids('(rename "{}" "{}")'.format(oldname, value))

    @property
    def model_ids(self):
        return [i['name'] for i in self._grid_json["model_ids"]]

    @property
    def hyper_names(self):
        return self._grid_json["hyper_names"]

    @property
    def failed_params(self):
        return self._grid_json.get("failed_params", None)

    @property
    def failure_details(self):
        return self._grid_json.get("failure_details", None)

    @property
    def failure_stack_traces(self):
        return self._grid_json.get("failure_stack_traces", None)

    @property
    def failed_raw_params(self):
        return self._grid_json.get("failed_raw_params", None)

    def start(self, x, y=None, training_frame=None, offset_column=None, fold_column=None, weights_column=None,
              validation_frame=None, **params):
        """
        Asynchronous model build by specifying the predictor columns, response column, and any
        additional frame-specific values.

        To block for results, call :meth:`join`.

        :param x: A list of column names or indices indicating the predictor columns.
        :param y: An index or a column name indicating the response column.
        :param training_frame: The H2OFrame having the columns indicated by x and y (as well as any
            additional columns specified by fold, offset, and weights).
        :param offset_column: The name or index of the column in training_frame that holds the offsets.
        :param fold_column: The name or index of the column in training_frame that holds the per-row fold
            assignments.
        :param weights_column: The name or index of the column in training_frame that holds the per-row weights.
        :param validation_frame: H2OFrame with validation data to be scored on while training.
        """
        self._future = True
        self.train(x=x,
                   y=y,
                   training_frame=training_frame,
                   offset_column=offset_column,
                   fold_column=fold_column,
                   weights_column=weights_column,
                   validation_frame=validation_frame,
                   **params)

    def join(self):
        """Wait until grid finishes computing."""
        self._future = False
        self._job.poll()
        self._job = None

    def train(self, x=None, y=None, training_frame=None, offset_column=None, fold_column=None, weights_column=None,
              validation_frame=None, **params):
        """
        Train the model synchronously (i.e. do not return until the model finishes training).

        To train asynchronously call :meth:`start`.

        :param x: A list of column names or indices indicating the predictor columns.
        :param y: An index or a column name indicating the response column.
        :param training_frame: The H2OFrame having the columns indicated by x and y (as well as any
            additional columns specified by fold, offset, and weights).
        :param offset_column: The name or index of the column in training_frame that holds the offsets.
        :param fold_column: The name or index of the column in training_frame that holds the per-row fold
            assignments.
        :param weights_column: The name or index of the column in training_frame that holds the per-row weights.
        :param validation_frame: H2OFrame with validation data to be scored on while training.
        """
        algo_params = locals()
        parms = self._parms.copy()
        parms.update({k: v for k, v in algo_params.items() if k not in ["self", "params", "algo_params", "parms"]})
        # dictionaries have special handling in grid search, avoid the implicit conversion
        parms["search_criteria"] = None if self.search_criteria is None else str(self.search_criteria)
        parms["hyper_parameters"] = None if self.hyper_params  is None else str(self.hyper_params) # unique to grid search
        parms.update({k: v for k, v in list(self.model._parms.items()) if v is not None})  # unique to grid search
        parms.update(params)
        if '__class__' in parms:  # FIXME: hackt for PY3
            del parms['__class__']
        y = algo_params["y"]
        tframe = algo_params["training_frame"]
        if tframe is None: raise ValueError("Missing training_frame")
        if y is not None:
            if is_type(y, list, tuple):
                if len(y) == 1:
                    parms["y"] = y[0]
                else:
                    raise ValueError('y must be a single column reference')
        if x is None:
            if(isinstance(y, int)):
                xset = set(range(training_frame.ncols)) - {y}
            else:
                xset = set(training_frame.names) - {y}
        else:
            xset = set()
            if is_type(x, int, str): x = [x]
            for xi in x:
                if is_type(xi, int):
                    if not (-training_frame.ncols <= xi < training_frame.ncols):
                        raise H2OValueError("Column %d does not exist in the training frame" % xi)
                    xset.add(training_frame.names[xi])
                else:
                    if xi not in training_frame.names:
                        raise H2OValueError("Column %s not in the training frame" % xi)
                    xset.add(xi)
        x = list(xset)
        parms["x"] = x
        self.build_model(parms)

    def build_model(self, algo_params):
        """(internal)"""
        if algo_params["training_frame"] is None: raise ValueError("Missing training_frame")
        x = algo_params.pop("x")
        y = algo_params.pop("y", None)
        training_frame = algo_params.pop("training_frame")
        validation_frame = algo_params.pop("validation_frame", None)
        is_auto_encoder = (algo_params is not None) and ("autoencoder" in algo_params and algo_params["autoencoder"])
        algo = self.model._compute_algo()  # unique to grid search
        is_unsupervised = is_auto_encoder or algo == "pca" or algo == "svd" or algo == "kmeans" or algo == "glrm"
        if is_auto_encoder and y is not None: raise ValueError("y should not be specified for autoencoder.")
        if not is_unsupervised and y is None: raise ValueError("Missing response")
        if not is_unsupervised:
            y = y if y in training_frame.names else training_frame.names[y]
            self.model._estimator_type = "classifier" if training_frame.types[y] == "enum" else "regressor"
        self._model_build(x, y, training_frame, validation_frame, algo_params)

    def _model_build(self, x, y, tframe, vframe, kwargs):
        kwargs['training_frame'] = tframe
        if vframe is not None: kwargs["validation_frame"] = vframe
        if is_type(y, int): y = tframe.names[y]
        if y is not None: kwargs['response_column'] = y
        if not is_type(x, list, tuple): x = [x]
        if is_type(x[0], int):
            x = [tframe.names[i] for i in x]
        offset = kwargs["offset_column"]
        folds = kwargs["fold_column"]
        weights = kwargs["weights_column"]
        ignored_columns = list(set(tframe.names) - set(x + [y, offset, folds, weights]))
        kwargs["ignored_columns"] = None if not ignored_columns else [quoted(col) for col in ignored_columns]
        kwargs = dict([(k, kwargs[k].frame_id if isinstance(kwargs[k], H2OFrame) else kwargs[k]) for k in kwargs if
                       kwargs[k] is not None])  # gruesome one-liner
        algo = self.model._compute_algo()  # unique to grid search
        if self.grid_id is not None: kwargs["grid_id"] = self.grid_id
        rest_ver = kwargs.pop("_rest_version") if "_rest_version" in kwargs else None

        grid = H2OJob(h2o.api("POST /99/Grid/%s" % algo, data=kwargs), job_type=(algo + " Grid Build"))

        if self._future:
            self._job = grid
            return

        grid.poll()

        grid_json = h2o.api("GET /99/Grids/%s" % (grid.dest_key))
        failure_messages_stacks = ""
        error_index = 0
        if len(grid_json["failure_details"]) > 0:
            print("Errors/Warnings building gridsearch model\n")
# will raise error if no grid model is returned, store error messages here

            for error_message in grid_json["failure_details"]:
                if isinstance(grid_json["failed_params"][error_index], dict):
                    for h_name in grid_json['hyper_names']:
                        print("Hyper-parameter: {0}, {1}".format(h_name,
                                                                 grid_json['failed_params'][error_index][h_name]))

                if len(grid_json["failure_stack_traces"]) > error_index:
                    print("failure_details: {0}\nfailure_stack_traces: "
                          "{1}\n".format(error_message, grid_json['failure_stack_traces'][error_index]))
                    failure_messages_stacks += error_message+'\n'
                error_index += 1

        self.models = [h2o.get_model(key['name']) for key in grid_json['model_ids']]
        for model in self.models:
            model._estimator_type = self.model._estimator_type

        # get first model returned in list of models from grid search to get model class (binomial, multinomial, etc)
        # sometimes no model is returned due to bad parameter values provided by the user.
        if len(grid_json['model_ids']) > 0:
            first_model_json = h2o.api("GET /%d/Models/%s" %
                                       (rest_ver or 3, grid_json['model_ids'][0]['name']))['models'][0]
            self._resolve_grid(grid.dest_key, grid_json, first_model_json)
        else:
            if len(failure_messages_stacks)>0:
                raise ValueError(failure_messages_stacks)
            else:
                raise ValueError("Gridsearch returns no model due to bad parameter values or other reasons....")

    def _resolve_grid(self, grid_id, grid_json, first_model_json):
        model_class = H2OGridSearch._metrics_class(first_model_json)
        m = model_class()
        m._id = grid_id
        m._grid_json = grid_json
        # m._metrics_class = metrics_class
        m._parms = self._parms
        H2OEstimator.mixin(self, model_class)
        self.__dict__.update(m.__dict__.copy())

    def __getitem__(self, item):
        return self.models[item]

    def __iter__(self):
        nmodels = len(self.models)
        return (self[i] for i in range(nmodels))

    def __len__(self):
        return len(self.models)

    def __repr__(self):
        self.show()
        return ""

    def predict(self, test_data):
        """
        Predict on a dataset.

        :param H2OFrame test_data: Data to be predicted on.
        :returns: H2OFrame filled with predictions.
        """
        return {model.model_id: model.predict(test_data) for model in self.models}

    def is_cross_validated(self):
        """Return True if the model was cross-validated."""
        return {model.model_id: model.is_cross_validated() for model in self.models}

    def xval_keys(self):
        """Model keys for the cross-validated model."""
        return {model.model_id: model.xval_keys() for model in self.models}

    def get_xval_models(self, key=None):
        """
        Return a Model object.

        :param str key: If None, return all cross-validated models; otherwise return the model
            specified by the key.
        :returns: A model or a list of models.
        """
        return {model.model_id: model.get_xval_models(key) for model in self.models}

    def xvals(self):
        """Return the list of cross-validated models."""
        return {model.model_id: model.xvals for model in self.models}

    def deepfeatures(self, test_data, layer):
        """
        Obtain a hidden layer's details on a dataset.

        :param test_data: Data to create a feature space on.
        :param int layer: Index of the hidden layer.
        :returns: A dictionary of hidden layer details for each model.
        """
        return {model.model_id: model.deepfeatures(test_data, layer) for model in self.models}

    def weights(self, matrix_id=0):
        """
        Return the frame for the respective weight matrix.

        :param: matrix_id: an integer, ranging from 0 to number of layers, that specifies the weight matrix to return.
        :returns: an H2OFrame which represents the weight matrix identified by matrix_id
        """
        return {model.model_id: model.weights(matrix_id) for model in self.models}

    def biases(self, vector_id=0):
        """
        Return the frame for the respective bias vector.

        :param: vector_id: an integer, ranging from 0 to number of layers, that specifies the bias vector to return.
        :returns: an H2OFrame which represents the bias vector identified by vector_id
        """
        return {model.model_id: model.biases(vector_id) for model in self.models}

    def normmul(self):
        """Normalization/Standardization multipliers for numeric predictors."""
        return {model.model_id: model.normmul() for model in self.models}

    def normsub(self):
        """Normalization/Standardization offsets for numeric predictors."""
        return {model.model_id: model.normsub() for model in self.models}

    def respmul(self):
        """Normalization/Standardization multipliers for numeric response."""
        return {model.model_id: model.respmul() for model in self.models}

    def respsub(self):
        """Normalization/Standardization offsets for numeric response."""
        return {model.model_id: model.respsub() for model in self.models}

    def catoffsets(self):
        """
        Categorical offsets for one-hot encoding
        """
        return {model.model_id: model.catoffsets() for model in self.models}

    def model_performance(self, test_data=None, train=False, valid=False, xval=False):
        """
        Generate model metrics for this model on test_data.

        :param test_data: Data set for which model metrics shall be computed against. All three of train, valid
            and xval arguments are ignored if test_data is not None.
        :param train: Report the training metrics for the model.
        :param valid: Report the validation metrics for the model.
        :param xval: Report the validation metrics for the model.
        :return: An object of class H2OModelMetrics.
        """
        return {model.model_id: model.model_performance(test_data, train, valid, xval) for model in self.models}

    def scoring_history(self):
        """
        Retrieve model scoring history.

        :returns: Score history (H2OTwoDimTable)
        """
        return {model.model_id: model.scoring_history() for model in self.models}

    def summary(self, header=True):
        """Print a detailed summary of the explored models."""
        table = []
        for model in self.models:
            model_summary = model._model_json["output"]["model_summary"]
            r_values = list(model_summary.cell_values[0])
            r_values[0] = model.model_id
            table.append(r_values)

        # if h2o.can_use_pandas():
        #  import pandas
        #  pandas.options.display.max_rows = 20
        #  print pandas.DataFrame(table,columns=self.col_header)
        #  return
        print()
        if header:
            print('Grid Summary:')
        print()
        H2ODisplay(table, ['Model Id'] + model_summary.col_header[1:], numalign="left", stralign="left")

    def show(self):
        """Print models sorted by metric."""
        hyper_combos = itertools.product(*list(self.hyper_params.values()))
        if not self.models:
            c_values = [[idx + 1, list(val)] for idx, val in enumerate(hyper_combos)]
            print(H2OTwoDimTable(
                col_header=['Model', 'Hyperparameters: [' + ', '.join(list(self.hyper_params.keys())) + ']'],
                table_header='Grid Search of Model ' + self.model.__class__.__name__, cell_values=c_values))
        else:
            print(self.sorted_metric_table())

    def varimp(self, use_pandas=False):
        """
        Pretty print the variable importances, or return them in a list/pandas DataFrame.

        :param bool use_pandas: If True, then the variable importances will be returned as a pandas data frame.

        :returns: A dictionary of lists or Pandas DataFrame instances.
        """
        return {model.model_id: model.varimp(use_pandas) for model in self.models}

    def residual_deviance(self, train=False, valid=False, xval=False):
        """
        Retreive the residual deviance if this model has the attribute, or None otherwise.

        :param bool train: Get the residual deviance for the training set. If both train and valid are False,
            then train is selected by default.
        :param bool valid: Get the residual deviance for the validation set. If both train and valid are True,
            then train is selected by default.
        :param bool xval: Get the residual deviance for the cross-validated models.

        :returns: the residual deviance, or None if it is not present.
        """
        return {model.model_id: model.residual_deviance(train, valid, xval) for model in self.models}

    def residual_degrees_of_freedom(self, train=False, valid=False, xval=False):
        """
        Retreive the residual degress of freedom if this model has the attribute, or None otherwise.

        :param bool train: Get the residual dof for the training set. If both train and valid are False, then
            train is selected by default.
        :param bool valid: Get the residual dof for the validation set. If both train and valid are True, then
            train is selected by default.
        :param bool xval: Get the residual dof for the cross-validated models.

        :returns: the residual degrees of freedom, or None if they are not present.
        """
        return {model.model_id: model.residual_degrees_of_freedom(train, valid, xval) for model in self.models}

    def null_deviance(self, train=False, valid=False, xval=False):
        """
        Retreive the null deviance if this model has the attribute, or None otherwise.

        :param bool train: Get the null deviance for the training set. If both train and valid are False, then
            train is selected by default.
        :param bool valid: Get the null deviance for the validation set. If both train and valid are True, then
            train is selected by default.
        :param bool xval: Get the null deviance for the cross-validated models.

        :returns: the null deviance, or None if it is not present.
        """
        return {model.model_id: model.null_deviance(train, valid, xval) for model in self.models}

    def null_degrees_of_freedom(self, train=False, valid=False, xval=False):
        """
        Retreive the null degress of freedom if this model has the attribute, or None otherwise.

        :param bool train: Get the null dof for the training set. If both train and valid are False, then train is
            selected by default.
        :param bool valid: Get the null dof for the validation set. If both train and valid are True, then train is
            selected by default.
        :param bool xval: Get the null dof for the cross-validated models.

        :returns: the null dof, or None if it is not present.
        """
        return {model.model_id: model.null_degrees_of_freedom(train, valid, xval) for model in self.models}

    def pprint_coef(self):
        """Pretty print the coefficents table (includes normalized coefficients)."""
        for i, model in enumerate(self.models):
            print('Model', i)
            model.pprint_coef()
            print()

    def coef(self):
        """Return the coefficients that can be applied to the non-standardized data.

        Note: standardize = True by default. If set to False, then coef() returns the coefficients that are fit directly.

        """
        return {model.model_id: model.coef() for model in self.models}

    def coef_norm(self):
        """Return coefficients fitted on the standardized data (requires standardize = True, which is on by default). These coefficients can be used to evaluate variable importance.

        """
        return {model.model_id: model.coef_norm() for model in self.models}

    def r2(self, train=False, valid=False, xval=False):
        """
        Return the R^2 for this regression model.

        The R^2 value is defined to be ``1 - MSE/var``, where ``var`` is computed as ``sigma^2``.

        If all are False (default), then return the training metric value.
        If more than one options is set to True, then return a dictionary of metrics where the keys are "train",
        "valid", and "xval".

        :param bool train: If train is True, then return the R^2 value for the training data.
        :param bool valid: If valid is True, then return the R^2 value for the validation data.
        :param bool xval:  If xval is True, then return the R^2 value for the cross validation data.

        :returns: The R^2 for this regression model.
        """
        return {model.model_id: model.r2(train, valid, xval) for model in self.models}

    def mse(self, train=False, valid=False, xval=False):
        """
        Get the MSE(s).

        If all are False (default), then return the training metric value.
        If more than one options is set to True, then return a dictionary of metrics where the keys are "train",
        "valid", and "xval".

        :param bool train: If train is True, then return the MSE value for the training data.
        :param bool valid: If valid is True, then return the MSE value for the validation data.
        :param bool xval:  If xval is True, then return the MSE value for the cross validation data.
        :returns: The MSE for this regression model.
        """
        return {model.model_id: model.mse(train, valid, xval) for model in self.models}

    def logloss(self, train=False, valid=False, xval=False):
        """
        Get the Log Loss(s).

        If all are False (default), then return the training metric value.
        If more than one options is set to True, then return a dictionary of metrics where the keys are "train",
        "valid", and "xval".

        :param bool train: If train is True, then return the Log Loss value for the training data.
        :param bool valid: If valid is True, then return the Log Loss value for the validation data.
        :param bool xval:  If xval is True, then return the Log Loss value for the cross validation data.

        :returns: The Log Loss for this binomial model.
        """
        return {model.model_id: model.logloss(train, valid, xval) for model in self.models}

    def mean_residual_deviance(self, train=False, valid=False, xval=False):
        """
        Get the Mean Residual Deviances(s).

        If all are False (default), then return the training metric value.
        If more than one options is set to True, then return a dictionary of metrics where the keys are "train",
        "valid", and "xval".

        :param bool train: If train is True, then return the Mean Residual Deviance value for the training data.
        :param bool valid: If valid is True, then return the Mean Residual Deviance value for the validation data.
        :param bool xval:  If xval is True, then return the Mean Residual Deviance value for the cross validation data.
        :returns: The Mean Residual Deviance for this regression model.
        """
        return {model.model_id: model.mean_residual_deviance(train, valid, xval) for model in self.models}

    def auc(self, train=False, valid=False, xval=False):
        """
        Get the AUC(s).

        If all are False (default), then return the training metric value.
        If more than one options is set to True, then return a dictionary of metrics where the keys are "train",
        "valid", and "xval".

        :param bool train: If train is True, then return the AUC value for the training data.
        :param bool valid: If valid is True, then return the AUC value for the validation data.
        :param bool xval:  If xval is True, then return the AUC value for the validation data.

        :returns: The AUC.
        """
        return {model.model_id: model.auc(train, valid, xval) for model in self.models}

    def aic(self, train=False, valid=False, xval=False):
        """
        Get the AIC(s).

        If all are False (default), then return the training metric value.
        If more than one options is set to True, then return a dictionary of metrics where the keys are "train",
        "valid", and "xval".

        :param bool train: If train is True, then return the AIC value for the training data.
        :param bool valid: If valid is True, then return the AIC value for the validation data.
        :param bool xval:  If xval is True, then return the AIC value for the validation data.

        :returns: The AIC.
        """
        return {model.model_id: model.aic(train, valid, xval) for model in self.models}

    def gini(self, train=False, valid=False, xval=False):
        """
        Get the Gini Coefficient(s).

        If all are False (default), then return the training metric value.
        If more than one options is set to True, then return a dictionary of metrics where the keys are "train",
        "valid", and "xval".

        :param bool train: If train is True, then return the Gini Coefficient value for the training data.
        :param bool valid: If valid is True, then return the Gini Coefficient value for the validation data.
        :param bool xval:  If xval is True, then return the Gini Coefficient value for the cross validation data.

        :returns: The Gini Coefficient for this binomial model.
        """
        return {model.model_id: model.gini(train, valid, xval) for model in self.models}

    def get_hyperparams(self, id, display=True):
        """
        Get the hyperparameters of a model explored by grid search.

        :param str id: The model id of the model with hyperparameters of interest.
        :param bool display: Flag to indicate whether to display the hyperparameter names.

        :returns: A list of the hyperparameters for the specified model.
        """
        idx = id if is_type(id, int) else self.model_ids.index(id)
        model = self[idx]

        # if cross-validation is turned on, parameters in one of the fold model actuall contains the max_runtime_secs
        # parameter and not the main model that is returned.
        if model._is_xvalidated:
            model = h2o.get_model(model._xval_keys[0])

        res = [model.params[h]['actual'][0] if isinstance(model.params[h]['actual'], list)
               else model.params[h]['actual']
               for h in self.hyper_params]
        if display: print('Hyperparameters: [' + ', '.join(list(self.hyper_params.keys())) + ']')
        return res

    def get_hyperparams_dict(self, id, display=True):
        """
        Derived and returned the model parameters used to train the particular grid search model.

        :param str id: The model id of the model with hyperparameters of interest.
        :param bool display: Flag to indicate whether to display the hyperparameter names.

        :returns: A dict of model pararmeters derived from the hyper-parameters used to train this particular model.
        """
        idx = id if is_type(id, int) else self.model_ids.index(id)
        model = self[idx]

        model_params = dict()

        # if cross-validation is turned on, parameters in one of the fold model actual contains the max_runtime_secs
        # parameter and not the main model that is returned.
        if model._is_xvalidated:
            model = h2o.get_model(model._xval_keys[0])

        for param_name in self.hyper_names:
            model_params[param_name] = model.params[param_name]['actual'][0] if \
                isinstance(model.params[param_name]['actual'], list) else model.params[param_name]['actual']

        if display: print('Hyperparameters: [' + ', '.join(list(self.hyper_params.keys())) + ']')
        return model_params

    def sorted_metric_table(self):
        """
        Retrieve summary table of an H2O Grid Search.

        :returns: The summary table as an H2OTwoDimTable or a Pandas DataFrame.
        """
        summary = self._grid_json["summary_table"]
        if summary is not None: return summary.as_data_frame()
        print("No sorted metric table for this grid search")

    @staticmethod
    def _metrics_class(model_json):
        model_type = model_json["output"]["model_category"]
        if model_type == "Binomial":
            model_class = H2OBinomialGridSearch
        elif model_type == "Clustering":
            model_class = H2OClusteringGridSearch
        elif model_type == "Regression":
            model_class = H2ORegressionGridSearch
        elif model_type == "Multinomial":
            model_class = H2OMultinomialGridSearch
        elif model_type == "Ordinal":
            model_class = H2OOrdinalGridSearch
        elif model_type == "AutoEncoder":
            model_class = H2OAutoEncoderGridSearch
        elif model_type == "DimReduction":
            model_class = H2ODimReductionGridSearch
        else:
            raise NotImplementedError(model_type)
        return model_class

    def get_grid(self, sort_by=None, decreasing=None):
        """
        Retrieve an H2OGridSearch instance.

        Optionally specify a metric by which to sort models and a sort order.
        Note that if neither cross-validation nor a validation frame is used in the grid search, then the
        training metrics will display in the "get grid" output. If a validation frame is passed to the grid, and
        ``nfolds = 0``, then the validation metrics will display. However, if ``nfolds`` > 1, then cross-validation
        metrics will display even if a validation frame is provided.

        :param str sort_by: A metric by which to sort the models in the grid space. Choices are: ``"logloss"``,
            ``"residual_deviance"``, ``"mse"``, ``"auc"``, ``"r2"``, ``"accuracy"``, ``"precision"``, ``"recall"``,
            ``"f1"``, etc.
        :param bool decreasing: Sort the models in decreasing order of metric if true, otherwise sort in increasing
            order (default).

        :returns: A new H2OGridSearch instance optionally sorted on the specified metric.
        """
        if sort_by is None and decreasing is None: return self

        grid_json = h2o.api("GET /99/Grids/%s" % self._id, data={"sort_by": sort_by, "decreasing": decreasing})
        grid = H2OGridSearch(self.model, self.hyper_params, self._id)
        grid.models = [h2o.get_model(key['name']) for key in grid_json['model_ids']]  # reordered
        first_model_json = h2o.api("GET /99/Models/%s" % grid_json['model_ids'][0]['name'])['models'][0]
        model_class = H2OGridSearch._metrics_class(first_model_json)
        m = model_class()
        m._id = self._id
        m._grid_json = grid_json
        # m._metrics_class = metrics_class
        m._parms = grid._parms
        H2OEstimator.mixin(grid, model_class)
        grid.__dict__.update(m.__dict__.copy())
        return grid

    # Deprecated functions; left here for backward compatibility
    _bcim = {
        "giniCoef": lambda self, *args, **kwargs: self.gini(*args, **kwargs)
    }

    @deprecated("grid.sort_by() is deprecated; use grid.get_grid() instead")
    def sort_by(self, metric, increasing=True):
        """Deprecated since 2016-12-12, use grid.get_grid() instead."""

        if metric[-1] != ')': metric += '()'
        c_values = [list(x) for x in zip(*sorted(eval('self.' + metric + '.items()'), key=lambda k_v: k_v[1]))]
        c_values.insert(1, [self.get_hyperparams(model_id, display=False) for model_id in c_values[0]])
        if not increasing:
            for col in c_values: col.reverse()
        if metric[-2] == '(': metric = metric[:-2]
        return H2OTwoDimTable(
            col_header=['Model Id', 'Hyperparameters: [' + ', '.join(list(self.hyper_params.keys())) + ']', metric],
            table_header='Grid Search Results for ' + self.model.__class__.__name__,
            cell_values=[list(x) for x in zip(*c_values)])


# -*- encoding: utf-8 -*-
from __future__ import absolute_import, division, print_function, unicode_literals

#-----------------------------------------------------------------------------------------------------------------------
# AutoEncoder Grid Search
#-----------------------------------------------------------------------------------------------------------------------

class H2OAutoEncoderGridSearch(object):

    def anomaly(self, test_data, per_feature=False):
        """
        Obtain the reconstruction error for the input test_data.

        :param H2OFrame test_data: The dataset upon which the reconstruction error is computed.
        :param bool per_feature: Whether to return the square reconstruction error per feature. Otherwise, return
            the mean square error.
        :returns: the reconstruction error.
        """
        return {model.model_id: model.anomaly(test_data, per_feature) for model in self.models}

#-----------------------------------------------------------------------------------------------------------------------
# Binomial Grid Search
#-----------------------------------------------------------------------------------------------------------------------

class H2OBinomialGridSearch(object):

    def F1(self, thresholds=None, train=False, valid=False, xval=False):
        """
        Get the F1 values for a set of thresholds for the models explored.

        If all are False (default), then return the training metric value.
        If more than one options is set to True, then return a dictionary of metrics where
        the keys are "train", "valid", and "xval".

        :param List thresholds: If None, then the thresholds in this set of metrics will be used.
        :param bool train: If True, return the F1 value for the training data.
        :param bool valid: If True, return the F1 value for the validation data.
        :param bool xval: If True, return the F1 value for each of the cross-validated splits.
        :returns: Dictionary of model keys to F1 values
        """
        return {model.model_id: model.F1(thresholds, train, valid, xval) for model in
                self.models}  # dict model key -> F1 score

    def F2(self, thresholds=None, train=False, valid=False, xval=False):
        """
        Get the F2 for a set of thresholds.

        If all are False (default), then return the training metric value.
        If more than one options is set to True, then return a dictionary of metrics where the keys are "train",
        "valid", and "xval".

        :param thresholds: thresholds parameter must be a list (i.e. [0.01, 0.5, 0.99]). If None, then the thresholds
            in this set of metrics will be used.
        :param bool train: If train is True, then return the F2 value for the training data.
        :param bool valid: If valid is True, then return the F2 value for the validation data.
        :param bool xval:  If xval is True, then return the F2 value for the cross validation data.
        :returns: Dictionary of model keys to F2 values.
        """
        return {model.model_id: model.F2(thresholds, train, valid, xval) for model in self.models}

    def F0point5(self, thresholds=None, train=False, valid=False, xval=False):
        """
        Get the F0.5 for a set of thresholds.

        If all are False (default), then return the training metric value.
        If more than one options is set to True, then return a dictionary of metrics where the keys are "train",
        "valid", and "xval".

        :param thresholds: thresholds parameter must be a list (i.e. [0.01, 0.5, 0.99]). If None, then the thresholds
            in this set of metrics will be used.
        :param bool train: If train is True, then return the F0point5 value for the training data.
        :param bool valid: If valid is True, then return the F0point5 value for the validation data.
        :param bool xval:  If xval is True, then return the F0point5 value for the cross validation data.
        :returns: The F0point5 for this binomial model.
        """
        return {model.model_id: model.F0point5(thresholds, train, valid, xval) for model in self.models}

    def accuracy(self, thresholds=None, train=False, valid=False, xval=False):
        """
        Get the accuracy for a set of thresholds.

        If all are False (default), then return the training metric value.
        If more than one options is set to True, then return a dictionary of metrics where the keys are "train",
        "valid", and "xval".

        :param thresholds: thresholds parameter must be a list (i.e. [0.01, 0.5, 0.99]). If None, then the thresholds
            in this set of metrics will be used.
        :param bool train: If train is True, then return the accuracy value for the training data.
        :param bool valid: If valid is True, then return the accuracy value for the validation data.
        :param bool xval:  If xval is True, then return the accuracy value for the cross validation data.
        :returns: The accuracy for this binomial model.
        """
        return {model.model_id: model.accuracy(thresholds, train, valid, xval) for model in self.models}

    def error(self, thresholds=None, train=False, valid=False, xval=False):
        """
        Get the error for a set of thresholds.

        If all are False (default), then return the training metric value.
        If more than one options is set to True, then return a dictionary of metrics where the keys are "train",
        "valid", and "xval".

        :param thresholds: thresholds parameter must be a list (i.e. [0.01, 0.5, 0.99]). If None, then the thresholds
            in this set of metrics will be used.
        :param bool train: If train is True, then return the error value for the training data.
        :param bool valid: If valid is True, then return the error value for the validation data.
        :param bool xval:  If xval is True, then return the error value for the cross validation data.
        :returns: The error for this binomial model.
        """
        return {model.model_id: model.error(thresholds, train, valid, xval) for model in self.models}

    def precision(self, thresholds=None, train=False, valid=False, xval=False):
        """
        Get the precision for a set of thresholds.

        If all are False (default), then return the training metric value.
        If more than one options is set to True, then return a dictionary of metrics where the keys are "train",
        "valid", and "xval".

        :param thresholds: thresholds parameter must be a list (i.e. [0.01, 0.5, 0.99]). If None, then the thresholds
            in this set of metrics will be used.
        :param bool train: If train is True, then return the precision value for the training data.
        :param bool valid: If valid is True, then return the precision value for the validation data.
        :param bool xval:  If xval is True, then return the precision value for the cross validation data.
        :returns: The precision for this binomial model.
        """
        return {model.model_id: model.precision(thresholds, train, valid, xval) for model in self.models}

    def tpr(self, thresholds=None, train=False, valid=False, xval=False):
        """
        Get the True Positive Rate for a set of thresholds.

        If all are False (default), then return the training metric value.
        If more than one options is set to True, then return a dictionary of metrics where the keys are "train",
        "valid", and "xval".

        :param thresholds: thresholds parameter must be a list (i.e. [0.01, 0.5, 0.99]). If None, then the thresholds
            in this set of metrics will be used.
        :param bool train: If train is True, then return the TPR value for the training data.
        :param bool valid: If valid is True, then return the TPR value for the validation data.
        :param bool xval:  If xval is True, then return the TPR value for the cross validation data.
        :returns: The TPR for this binomial model.
        """
        return {model.model_id: model.tpr(thresholds, train, valid, xval) for model in self.models}

    def tnr(self, thresholds=None, train=False, valid=False, xval=False):
        """
        Get the True Negative Rate for a set of thresholds.

        If all are False (default), then return the training metric value.
        If more than one options is set to True, then return a dictionary of metrics where the keys are "train",
        "valid", and "xval".

        :param thresholds: thresholds parameter must be a list (i.e. [0.01, 0.5, 0.99]). If None, then the thresholds
            in this set of metrics will be used.
        :param bool train: If train is True, then return the TNR value for the training data.
        :param bool valid: If valid is True, then return the TNR value for the validation data.
        :param bool xval:  If xval is True, then return the TNR value for the cross validation data.
        :returns: The TNR for this binomial model.
        """
        return {model.model_id: model.tnr(thresholds, train, valid, xval) for model in self.models}

    def fnr(self, thresholds=None, train=False, valid=False, xval=False):
        """
        Get the False Negative Rates for a set of thresholds.
        If all are False (default), then return the training metric value.
        If more than one options is set to True, then return a dictionary of metrics where the keys are "train",
        "valid", and "xval".

        :param thresholds: thresholds parameter must be a list (i.e. [0.01, 0.5, 0.99]). If None, then the thresholds
            in this set of metrics will be used.
        :param bool train: If train is True, then return the FNR value for the training data.
        :param bool valid: If valid is True, then return the FNR value for the validation data.
        :param bool xval:  If xval is True, then return the FNR value for the cross validation data.
        :returns: The FNR for this binomial model.
        """
        return {model.model_id: model.fnr(thresholds, train, valid, xval) for model in self.models}

    def fpr(self, thresholds=None, train=False, valid=False, xval=False):
        """
        Get the False Positive Rates for a set of thresholds.

        If all are False (default), then return the training metric value.
        If more than one options is set to True, then return a dictionary of metrics where the keys are "train",
        "valid", and "xval".

        :param thresholds: thresholds parameter must be a list (i.e. [0.01, 0.5, 0.99]). If None, then the thresholds
            in this set of metrics will be used.
        :param bool train: If train is True, then return the FPR value for the training data.
        :param bool valid: If valid is True, then return the FPR value for the validation data.
        :param bool xval:  If xval is True, then return the FPR value for the cross validation data.
        :returns: The FPR for this binomial model.
        """
        return {model.model_id: model.fpr(thresholds, train, valid, xval) for model in self.models}

    def recall(self, thresholds=None, train=False, valid=False, xval=False):
        """
        Get the Recall (AKA True Positive Rate) for a set of thresholds.

        If all are False (default), then return the training metric value.
        If more than one options is set to True, then return a dictionary of metrics where the keys are "train",
        "valid", and "xval".

        :param thresholds: thresholds parameter must be a list (i.e. [0.01, 0.5, 0.99]). If None, then the thresholds
            in this set of metrics will be used.
        :param bool train: If train is True, then return the recall value for the training data.
        :param bool valid: If valid is True, then return the recall value for the validation data.
        :param bool xval:  If xval is True, then return the recall value for the cross validation data.
        :returns: The recall for this binomial model.
        """
        return {model.model_id: model.recall(thresholds, train, valid, xval) for model in self.models}

    def sensitivity(self, thresholds=None, train=False, valid=False, xval=False):
        """
        Get the sensitivity (AKA True Positive Rate or Recall) for a set of thresholds.

        If all are False (default), then return the training metric value.
        If more than one options is set to True, then return a dictionary of metrics where the keys are "train",
        "valid", and "xval".

        :param thresholds: thresholds parameter must be a list (i.e. [0.01, 0.5, 0.99]). If None, then the thresholds
            in this set of metrics will be used.
        :param bool train: If train is True, then return the sensitivity value for the training data.
        :param bool valid: If valid is True, then return the sensitivity value for the validation data.
        :param bool xval:  If xval is True, then return the sensitivity value for the cross validation data.
        :returns: The sensitivity for this binomial model.
        """
        return {model.model_id: model.sensitivity(thresholds, train, valid, xval) for model in self.models}

    def fallout(self, thresholds=None, train=False, valid=False, xval=False):
        """
        Get the Fallout (AKA False Positive Rate) for a set of thresholds.

        If all are False (default), then return the training metric value.
        If more than one options is set to True, then return a dictionary of metrics where the keys are "train",
        "valid", and "xval".

        :param thresholds: thresholds parameter must be a list (i.e. [0.01, 0.5, 0.99]). If None, then the thresholds
            in this set of metrics will be used.
        :param bool train: If train is True, then return the fallout value for the training data.
        :param bool valid: If valid is True, then return the fallout value for the validation data.
        :param bool xval:  If xval is True, then return the fallout value for the cross validation data.
        :returns: The fallout for this binomial model.
        """
        return {model.model_id: model.fallout(thresholds, train, valid, xval) for model in self.models}

    def missrate(self, thresholds=None, train=False, valid=False, xval=False):
        """
        Get the miss rate (AKA False Negative Rate) for a set of thresholds.

        If all are False (default), then return the training metric value.
        If more than one options is set to True, then return a dictionary of metrics where the keys are "train",
        "valid", and "xval".

        :param thresholds: thresholds parameter must be a list (i.e. [0.01, 0.5, 0.99]). If None, then the thresholds
            in this set of metrics will be used.
        :param bool train: If train is True, then return the missrate value for the training data.
        :param bool valid: If valid is True, then return the missrate value for the validation data.
        :param bool xval:  If xval is True, then return the missrate value for the cross validation data.
        :returns: The missrate for this binomial model.
        """
        return {model.model_id: model.missrate(thresholds, train, valid, xval) for model in self.models}

    def specificity(self, thresholds=None, train=False, valid=False, xval=False):
        """
        Get the specificity (AKA True Negative Rate) for a set of thresholds.

        If all are False (default), then return the training metric value.
        If more than one options is set to True, then return a dictionary of metrics where the keys are "train",
        "valid", and "xval".

        :param thresholds: thresholds parameter must be a list (i.e. [0.01, 0.5, 0.99]). If None, then the thresholds
            in this set of metrics will be used.
        :param bool train: If train is True, then return the specificity value for the training data.
        :param bool valid: If valid is True, then return the specificity value for the validation data.
        :param bool xval:  If xval is True, then return the specificity value for the cross validation data.
        :returns: The specificity for this binomial model.
        """
        return {model.model_id: model.specificity(thresholds, train, valid, xval) for model in self.models}

    def mcc(self, thresholds=None, train=False, valid=False, xval=False):
        """
        Get the MCC for a set of thresholds.

        If all are False (default), then return the training metric value.
        If more than one options is set to True, then return a dictionary of metrics where the keys are "train",
        "valid", and "xval".

        :param thresholds: thresholds parameter must be a list (i.e. [0.01, 0.5, 0.99]). If None, then the thresholds
            in this set of metrics will be used.
        :param bool train: If train is True, then return the mcc value for the training data.
        :param bool valid: If valid is True, then return the mcc value for the validation data.
        :param bool xval:  If xval is True, then return the mcc value for the cross validation data.
        :returns: The MCC for this binomial model.
        """
        return {model.model_id: model.mcc(thresholds, train, valid, xval) for model in self.models}

    def max_per_class_error(self, thresholds=None, train=False, valid=False, xval=False):
        """
        Get the max per class error for a set of thresholds.

        If all are False (default), then return the training metric value.
        If more than one options is set to True, then return a dictionary of metrics where the keys are "train",
        "valid", and "xval".

        :param thresholds: thresholds parameter must be a list (i.e. [0.01, 0.5, 0.99]). If None, then the thresholds
            in this set of metrics will be used.
        :param bool train: If train is True, then return the max_per_class_error value for the training data.
        :param bool valid: If valid is True, then return the max_per_class_error value for the validation data.
        :param bool xval:  If xval is True, then return the max_per_class_error value for the cross validation data.
        :returns: The max per class error for this binomial model.
        """
        return {model.model_id: model.max_per_class_error(thresholds, train, valid, xval) for model in self.models}

    def mean_per_class_error(self, thresholds=None, train=False, valid=False, xval=False):
        """
        Get the mean per class error for a set of thresholds.

        If all are False (default), then return the training metric value.
        If more than one options is set to True, then return a dictionary of metrics where the keys are "train",
        "valid", and "xval".

        :param thresholds: thresholds parameter must be a list (i.e. [0.01, 0.5, 0.99]). If None, then the thresholds
            in this set of metrics will be used.
        :param bool train: If train is True, then return the mean_per_class_error value for the training data.
        :param bool valid: If valid is True, then return the mean_per_class_error value for the validation data.
        :param bool xval:  If xval is True, then return the mean_per_class_error value for the cross validation data.
        :returns: The mean per class error for this binomial model.
        """
        return {model.model_id: model.mean_per_class_error(thresholds, train, valid, xval) for model in self.models}

    def metric(self, metric, thresholds=None, train=False, valid=False, xval=False):
        """
        Get the metric value for a set of thresholds.

        If all are False (default), then return the training metric value.
        If more than one options is set to True, then return a dictionary of metrics where the keys are "train",
        "valid", and "xval".

        :param metric: name of the metric to compute.
        :param thresholds: thresholds parameter must be a list (i.e. [0.01, 0.5, 0.99]). If None, then the thresholds
            in this set of metrics will be used.
        :param bool train: If train is True, then return the metrics for the training data.
        :param bool valid: If valid is True, then return the metrics for the validation data.
        :param bool xval:  If xval is True, then return the metrics for the cross validation data.
        :returns: The metrics for this binomial model.
        """
        return {model.model_id: model.metric(metric, thresholds, train, valid, xval) for model in self.models}

    def roc(self, train=False, valid=False, xval=False):
        """
        Return the coordinates of the ROC curve for a given set of data, as a two-tuple containing the false positive
        rates as a list and true positive rates as a list.

        If all are False (default), then return  the training data.
        If more than one ROC curve is requested, the data is returned as a dictionary of two-tuples.

        :param bool train: If train is true, then return the ROC coordinates for the training data.
        :param bool valid: If valid is true, then return the ROC coordinates for the validation data.
        :param bool xval: If xval is true, then return the ROC coordinates for the cross validation data.
        :returns: the true cooridinates of the roc curve.
        """
        return {model.model_id: model.roc(train, valid, xval) for model in self.models}

    def confusion_matrix(self, metrics=None, thresholds=None, train=False, valid=False, xval=False):
        """
        Get the confusion matrix for the specified metrics/thresholds.

        If all are False (default), then return the training metric value.
        If more than one options is set to True, then return a dictionary of metrics where the keys are "train",
        "valid", and "xval".

        :param metrics: A list of metrics or a single metric among ``"min_per_class_accuracy"``, ``"absolute_mcc"``,
            ``"tnr"``, ``"fnr"``, ``"fpr"``, ``"tpr"``, ``"precision"``, ``"accuracy"``, ``"f0point5"``, ``"f2"``,
            ``"f1"``.
        :param thresholds: thresholds parameter must be a list (i.e. [0.01, 0.5, 0.99]). If None, then the thresholds
            in this set of metrics will be used.
        :param bool train: If train is True, then return the confusion matrix value for the training data.
        :param bool valid: If valid is True, then return the confusion matrix value for the validation data.
        :param bool xval:  If xval is True, then return the confusion matrix value for the cross validation data.
        :returns: The confusion matrix for this binomial model.
        """
        return {model.model_id: model.confusion_matrix(metrics, thresholds, train, valid, xval) for model in
                self.models}

    def find_threshold_by_max_metric(self, metric, train=False, valid=False, xval=False):
        """
        If all are False (default), then return the training metric value.
        If more than one options is set to True, then return a dictionary of metrics where the keys are "train",
        "valid", and "xval".

        :param str metric: The name of the metric to search for.
        :param bool train: If train is True, then return the threshold_by_max_metric value for the training data.
        :param bool valid: If valid is True, then return the threshold_by_max_metric value for the validation data.
        :param bool xval:  If xval is True, then return the threshold_by_max_metric value for the cross validation data.
        :returns: The threshold_by_max_metric for this binomial model.
        """
        return {model.model_id: model.find_threshold_by_max_metric(metric, train, valid, xval) for model in self.models}

    def find_idx_by_threshold(self, threshold, train=False, valid=False, xval=False):
        """
        Retrieve the index in this metric's threshold list at which the given threshold is located.

        If all are False (default), then return the training metric value.
        If more than one options is set to True, then return a dictionary of metrics where the keys are "train",
        "valid", and "xval".

        :param float threshold: The threshold value to search for.
        :param bool train: If train is True, then return the idx_by_threshold for the training data.
        :param bool valid: If valid is True, then return the idx_by_threshold for the validation data.
        :param bool xval:  If xval is True, then return the idx_by_threshold for the cross validation data.
        :returns: The idx_by_threshold for this binomial model.
        """
        return {model.model_id: model.find_idx_by_threshold(threshold, train, valid, xval) for model in self.models}

#-----------------------------------------------------------------------------------------------------------------------
# Clustering Grid Search
#-----------------------------------------------------------------------------------------------------------------------

class H2OClusteringGridSearch(object):

    def size(self, train=False, valid=False, xval=False):
        """
        Get the sizes of each cluster.

        If all are False (default), then return the training metric value.
        If more than one options is set to True, then return a dictionary of metrics where
        the keys are "train", "valid", and "xval".

        :param bool train: If True, then return the cluster sizes for the training data.
        :param bool valid: If True, then return the cluster sizes for the validation data.
        :param bool xval: If True, then return the cluster sizes for each of the cross-validated splits.
        :returns: the cluster sizes for the specified key(s).
        """
        return {model.model_id: model.size(train, valid, xval) for model in self.models}

    def num_iterations(self):
        """Get the number of iterations that it took to converge or reach max iterations."""
        return {model.model_id: model.num_iterations() for model in self.models}

    def betweenss(self, train=False, valid=False, xval=False):
        """
        Get the between cluster sum of squares.

        If all are False (default), then return the training metric value.
        If more than one options is set to True, then return a dictionary of metrics where
        the keys are "train", "valid", and "xval".

        :param bool train: If True, then return the between cluster sum of squares value for the training data.
        :param bool valid: If True, then return the between cluster sum of squares value for the validation data.
        :param bool xval: If True, then return the between cluster sum of squares value for each of the
            cross-validated splits.
        :returns: the between cluster sum of squares values for the specified key(s).
        """
        return {model.model_id: model.betweenss(train, valid, xval) for model in self.models}

    def totss(self, train=False, valid=False, xval=False):
        """
        Get the total sum of squares.

        If all are False (default), then return the training metric value.
        If more than one options is set to True, then return a dictionary of metrics where
        the keys are "train", "valid", and "xval".

        :param bool train: If True, then return total sum of squares for the training data.
        :param bool valid: If True, then return the total sum of squares for the validation data.
        :param bool xval: If True, then return the total sum of squares for each of the cross-validated splits.
        :returns: the total sum of squares values for the specified key(s).
        """
        return {model.model_id: model.totss(train, valid, xval) for model in self.models}

    def tot_withinss(self, train=False, valid=False, xval=False):
        """
        Get the total within cluster sum of squares.

        If all are False (default), then return the training metric value.
        If more than one options is set to True, then return a dictionary of metrics where
        the keys are "train", "valid", and "xval".

        :param bool train: If True, then return the total within cluster sum of squares for the training data.
        :param bool valid: If True, then return the total within cluster sum of squares for the validation data.
        :param bool xval: If True, then return the total within cluster sum of squares for each of the
            cross-validated splits.
        :returns: the total within cluster sum of squares values for the specified key(s).
        """
        return {model.model_id: model.tot_withinss(train, valid, xval) for model in self.models}

    def withinss(self, train=False, valid=False, xval=False):
        """
        Get the within cluster sum of squares for each cluster.

        If all are False (default), then return the training metric value.
        If more than one options is set to True, then return a dictionary of metrics where
        the keys are "train", "valid", and "xval".

        :param bool train: If True, then return within cluster sum of squares for the training data.
        :param bool valid: If True, then return the within cluster sum of squares for the validation data.
        :param bool xval: If True, then return the within cluster sum of squares for each of the
            cross-validated splits.
        :returns: the within cluster sum of squares values for the specified key(s).
        """
        return {model.model_id: model.withinss(train, valid, xval) for model in self.models}

    def centroid_stats(self, train=False, valid=False, xval=False):
        """
        Get the centroid statistics for each cluster.

        If all are False (default), then return the training metric value.
        If more than one options is set to True, then return a dictionary of metrics where
        the keys are "train", "valid", and "xval".

        :param bool train: If True, then return the centroid statistics for the training data.
        :param bool valid: If True, then return the centroid statistics for the validation data.
        :param bool xval: If True, then return the centroid statistics for each of the cross-validated splits.
        :returns: the centroid statistics for the specified key(s).
        """
        return {model.model_id: model.centroid_stats(train, valid, xval) for model in self.models}

    def centers(self):
        """Returns the centers for the KMeans model."""
        return {model.model_id: model.centers() for model in self.models}

    def centers_std(self):
        """Returns the standardized centers for the kmeans model."""
        return {model.model_id: model.centers_std() for model in self.models}

#-----------------------------------------------------------------------------------------------------------------------
# Dimensionality Reduction Grid Search
#-----------------------------------------------------------------------------------------------------------------------

class H2ODimReductionGridSearch(object):
    def num_iterations(self):
        """
        Get the number of iterations that it took to converge or reach max iterations.

        :returns: number of iterations (integer)
        """
        return {model.model_id: model.num_iterations for model in self.models}

    def objective(self):
        """
        Get the final value of the objective function from the GLRM model.

        :returns: final objective value (double)
        """
        return {model.model_id: model.objective for model in self.models}

    def final_step(self):
        """
        Get the final step size from the GLRM model.

        :returns: final step size (double)
        """
        return {model.model_id: model.final_step for model in self.models}

    def archetypes(self):
        """
        :returns: the archetypes (Y) of the GLRM model.
        """
        return {model.model_id: model.archetypes for model in self.models}

#-----------------------------------------------------------------------------------------------------------------------
# Multinomial Grid Search
#-----------------------------------------------------------------------------------------------------------------------

class H2OMultinomialGridSearch(object):
    def confusion_matrix(self, data):
        """
        Returns a confusion matrix based of H2O's default prediction threshold for a dataset.

        :param data: metric for which the confusion matrix will be calculated.
        """
        return {model.model_id: model.confusion_matrix(data) for model in self.models}

    def hit_ratio_table(self, train=False, valid=False, xval=False):
        """
        Retrieve the Hit Ratios.

        If all are False (default), then return the training metric value.
        If more than one option is set to True, then return a dictionary of metrics where the keys are "train",
        "valid", and "xval".

        :param bool train: If train is True, then return the hit ratio value for the training data.
        :param bool valid: If valid is True, then return the hit ratio value for the validation data.
        :param bool xval:  If xval is True, then return the hit ratio value for the cross validation data.
        :returns: The hit ratio for this multinomial model.
        """
        return {model.model_id: model.hit_ratio_table(train, valid, xval) for model in self.models}

    def mean_per_class_error(self, train=False, valid=False, xval=False):
        """
        Get the mean per class error.

        If all are False (default), then return the training metric value.
        If more than one options is set to True, then return a dictionary of metrics where the keys are "train",
        "valid", and "xval".

        :param bool train: If train is True, then return the mean per class error value for the training data.
        :param bool valid: If valid is True, then return the mean per class error value for the validation data.
        :param bool xval:  If xval is True, then return the mean per class error value for the cross validation data.
        :returns: The mean per class error for this multinomial model.
        """
        return {model.model_id: model.mean_per_class_error(train, valid, xval) for model in self.models}

#-----------------------------------------------------------------------------------------------------------------------
# Ordinal Grid Search
#-----------------------------------------------------------------------------------------------------------------------

class H2OOrdinalGridSearch(object):
    def confusion_matrix(self, data):
        """
        Returns a confusion matrix based of H2O's default prediction threshold for a dataset.

        :param data: metric for which the confusion matrix will be calculated.
        """
        return {model.model_id: model.confusion_matrix(data) for model in self.models}

    def hit_ratio_table(self, train=False, valid=False, xval=False):
        """
        Retrieve the Hit Ratios.

        If all are False (default), then return the training metric value.
        If more than one option is set to True, then return a dictionary of metrics where the keys are "train",
        "valid", and "xval".

        :param bool train: If train is True, then return the hit ratio value for the training data.
        :param bool valid: If valid is True, then return the hit ratio value for the validation data.
        :param bool xval:  If xval is True, then return the hit ratio value for the cross validation data.
        :returns: The hit ratio for this ordinal model.
        """
        return {model.model_id: model.hit_ratio_table(train, valid, xval) for model in self.models}

    def mean_per_class_error(self, train=False, valid=False, xval=False):
        """
        Get the mean per class error.

        If all are False (default), then return the training metric value.
        If more than one options is set to True, then return a dictionary of metrics where the keys are "train",
        "valid", and "xval".

        :param bool train: If train is True, then return the mean per class error value for the training data.
        :param bool valid: If valid is True, then return the mean per class error value for the validation data.
        :param bool xval:  If xval is True, then return the mean per class error value for the cross validation data.
        :returns: The mean per class error for this ordinal model.
        """
        return {model.model_id: model.mean_per_class_error(train, valid, xval) for model in self.models}

#-----------------------------------------------------------------------------------------------------------------------
# Regression Grid Search
#-----------------------------------------------------------------------------------------------------------------------

class H2ORegressionGridSearch(object):
    pass


# -*- encoding: utf-8 -*-
from __future__ import absolute_import, division, print_function, unicode_literals
from h2o.utils.shared_utils import can_use_pandas
from h2o.utils.compatibility import *  # NOQA

from .model_base import ModelBase
from .metrics_base import *  # NOQA
import h2o

class H2ODimReductionModel(ModelBase):
    """
    Dimension reduction model, such as PCA or GLRM.
    """

    def varimp(self, use_pandas=False):
        """
        Return the Importance of components associcated with a pca model.

        use_pandas: ``bool``  (default: ``False``).
        """
        model = self._model_json["output"]
        if "importance" in list(model.keys()) and model["importance"]:
            vals = model["importance"].cell_values
            header = model["importance"].col_header
            if use_pandas and can_use_pandas():
                import pandas
                return pandas.DataFrame(vals, columns=header)
            else:
                return vals
        else:
            print("Warning: This model doesn't have importances of components.")

    def num_iterations(self):
        """Get the number of iterations that it took to converge or reach max iterations."""
        o = self._model_json["output"]
        return o["model_summary"]["number_of_iterations"][0]

    def objective(self):
        """Get the final value of the objective function."""
        o = self._model_json["output"]
        return o["model_summary"]["final_objective_value"][0]

    def final_step(self):
        """Get the final step size for the model."""
        o = self._model_json["output"]
        return o["model_summary"]["final_step_size"][0]

    def archetypes(self):
        """The archetypes (Y) of the GLRM model."""
        o = self._model_json["output"]
        yvals = o["archetypes"].cell_values
        archetypes = []
        for yidx, yval in enumerate(yvals):
            archetypes.append(list(yvals[yidx])[1:])
        return archetypes

    def reconstruct(self, test_data, reverse_transform=False):
        """
        Reconstruct the training data from the model and impute all missing values.

        :param H2OFrame test_data: The dataset upon which the model was trained.
        :param bool reverse_transform: Whether the transformation of the training data during model-building
            should be reversed on the reconstructed frame.

        :returns: the approximate reconstruction of the training data.
        """
        if test_data is None or test_data.nrow == 0: raise ValueError("Must specify test data")
        j = h2o.api("POST /3/Predictions/models/%s/frames/%s" % (self.model_id, test_data.frame_id),
                    data={"reconstruct_train": True, "reverse_transform": reverse_transform})
        return h2o.get_frame(j["model_metrics"][0]["predictions"]["frame_id"]["name"])

    def proj_archetypes(self, test_data, reverse_transform=False):
        """
        Convert archetypes of the model into original feature space.

        :param H2OFrame test_data: The dataset upon which the model was trained.
        :param bool reverse_transform: Whether the transformation of the training data during model-building
            should be reversed on the projected archetypes.

        :returns: model archetypes projected back into the original training data's feature space.
        """
        if test_data is None or test_data.nrow == 0: raise ValueError("Must specify test data")
        j = h2o.api("POST /3/Predictions/models/%s/frames/%s" % (self.model_id, test_data.frame_id),
                    data={"project_archetypes": True, "reverse_transform": reverse_transform})
        return h2o.get_frame(j["model_metrics"][0]["predictions"]["frame_id"]["name"])

    def screeplot(self, type="barplot", **kwargs):
        """
        Produce the scree plot.

        Library ``matplotlib`` is required for this function.

        :param str type: either ``"barplot"`` or ``"lines"``.
        """
        # check for matplotlib. exit if absent.
        is_server = kwargs.pop("server")
        if kwargs:
            raise ValueError("Unknown arguments %s to screeplot()" % ", ".join(kwargs.keys()))
        try:
            import matplotlib
            if is_server: matplotlib.use('Agg', warn=False)
            import matplotlib.pyplot as plt
        except ImportError:
            print("matplotlib is required for this function!")
            return

        variances = [s ** 2 for s in self._model_json['output']['importance'].cell_values[0][1:]]
        plt.xlabel('Components')
        plt.ylabel('Variances')
        plt.title('Scree Plot')
        plt.xticks(list(range(1, len(variances) + 1)))
        if type == "barplot":
            plt.bar(list(range(1, len(variances) + 1)), variances)
        elif type == "lines":
            plt.plot(list(range(1, len(variances) + 1)), variances, 'b--')
        if not is_server: plt.show()