# Copyright 2017 Neural Networks and Deep Learning lab, MIPT
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from logging import getLogger

import tensorflow as tf
from tensorflow.contrib.layers import xavier_initializer as xav

from deeppavlov.core.layers import tf_csoftmax_attention as csoftmax_attention

log = getLogger(__name__)

def general_attention(key, context, hidden_size, projected_align=False):
    """ It is a implementation of the Luong et al. attention mechanism with general score. Based on the paper:
        https://arxiv.org/abs/1508.04025 "Effective Approaches to Attention-based Neural Machine Translation"
    Args:
        key: A tensorflow tensor with dimensionality [None, None, key_size]
        context: A tensorflow tensor with dimensionality [None, None, max_num_tokens, token_size]
        hidden_size: Number of units in hidden representation
        projected_align: Using bidirectional lstm for hidden representation of context.
        If true, beetween input and attention mechanism insert layer of bidirectional lstm with dimensionality [hidden_size].
        If false, bidirectional lstm is not used.
    Returns:
        output: Tensor at the output with dimensionality [None, None, hidden_size]
    """

    if hidden_size % 2 != 0:
        raise ValueError("hidden size must be dividable by two")
    batch_size = tf.shape(context)[0]
    max_num_tokens, token_size = context.get_shape().as_list()[-2:]
    r_context = tf.reshape(context, shape=[-1, max_num_tokens, token_size])

    # projected_key: [None, None, hidden_size]
    projected_key = \
        tf.layers.dense(key, hidden_size, kernel_initializer=xav())
    r_projected_key = tf.reshape(projected_key, shape=[-1, hidden_size, 1])

    lstm_fw_cell = tf.nn.rnn_cell.LSTMCell(hidden_size//2)
    lstm_bw_cell = tf.nn.rnn_cell.LSTMCell(hidden_size//2)
    (output_fw, output_bw), states = \
        tf.nn.bidirectional_dynamic_rnn(cell_fw=lstm_fw_cell,
                                        cell_bw=lstm_bw_cell,
                                        inputs=r_context,
                                        dtype=tf.float32)
    # bilstm_output: [-1, max_num_tokens, hidden_size]
    bilstm_output = tf.concat([output_fw, output_bw], -1)

    attn = tf.nn.softmax(tf.matmul(bilstm_output, r_projected_key), dim=1)

    if projected_align:
        log.info("Using projected attention alignment")
        t_context = tf.transpose(bilstm_output, [0, 2, 1])
        output = tf.reshape(tf.matmul(t_context, attn),
                            shape=[batch_size, -1, hidden_size])
    else:
        log.info("Using without projected attention alignment")
        t_context = tf.transpose(r_context, [0, 2, 1])
        output = tf.reshape(tf.matmul(t_context, attn),
                            shape=[batch_size, -1, token_size])
    return output

def light_general_attention(key, context, hidden_size, projected_align=False):
    """ It is a implementation of the Luong et al. attention mechanism with general score. Based on the paper:
        https://arxiv.org/abs/1508.04025 "Effective Approaches to Attention-based Neural Machine Translation"
    Args:
        key: A tensorflow tensor with dimensionality [None, None, key_size]
        context: A tensorflow tensor with dimensionality [None, None, max_num_tokens, token_size]
        hidden_size: Number of units in hidden representation
        projected_align: Using dense layer for hidden representation of context.
        If true, between input and attention mechanism insert a dense layer with dimensionality [hidden_size].
        If false, a dense layer is not used.
    Returns:
        output: Tensor at the output with dimensionality [None, None, hidden_size]
    """
    batch_size = tf.shape(context)[0]
    max_num_tokens, token_size = context.get_shape().as_list()[-2:]
    r_context = tf.reshape(context, shape=[-1, max_num_tokens, token_size])

    # projected_key: [None, None, hidden_size]
    projected_key = tf.layers.dense(key, hidden_size, kernel_initializer=xav())
    r_projected_key = tf.reshape(projected_key, shape=[-1, hidden_size, 1])

    # projected context: [None, None, hidden_size]
    projected_context = \
        tf.layers.dense(r_context, hidden_size, kernel_initializer=xav())

    attn = tf.nn.softmax(tf.matmul(projected_context, r_projected_key), dim=1)

    if projected_align:
        log.info("Using projected attention alignment")
        t_context = tf.transpose(projected_context, [0, 2, 1])
        output = tf.reshape(tf.matmul(t_context, attn),
                            shape=[batch_size, -1, hidden_size])
    else:
        log.info("Using without projected attention alignment")
        t_context = tf.transpose(r_context, [0, 2, 1])
        output = tf.reshape(tf.matmul(t_context, attn),
                            shape=[batch_size, -1, token_size])
    return output

def cs_general_attention(key, context, hidden_size, depth, projected_align=False):
    """ It is a implementation of the Luong et al. attention mechanism with general score and the constrained softmax (csoftmax).
        Based on the papers:
        https://arxiv.org/abs/1508.04025 "Effective Approaches to Attention-based Neural Machine Translation"
        https://andre-martins.github.io/docs/emnlp2017_final.pdf "Learning What's Easy: Fully Differentiable Neural Easy-First Taggers"
    Args:
        key: A tensorflow tensor with dimensionality [None, None, key_size]
        context: A tensorflow tensor with dimensionality [None, None, max_num_tokens, token_size]
        hidden_size: Number of units in hidden representation
        depth: Number of csoftmax usages
        projected_align: Using bidirectional lstm for hidden representation of context.
        If true, beetween input and attention mechanism insert layer of bidirectional lstm with dimensionality [hidden_size].
        If false, bidirectional lstm is not used.
    Returns:
        output: Tensor at the output with dimensionality [None, None, depth * hidden_size]
    """
    if hidden_size % 2 != 0:
        raise ValueError("hidden size must be dividable by two")
    key_size = tf.shape(key)[-1]
    batch_size = tf.shape(context)[0]
    max_num_tokens, token_size = context.get_shape().as_list()[-2:]
    r_context = tf.reshape(context, shape=[-1, max_num_tokens, token_size])
    # projected_context: [None, max_num_tokens, token_size]
    projected_context = tf.layers.dense(r_context, token_size,
                                        kernel_initializer=xav(),
                                        name='projected_context')

    lstm_fw_cell = tf.nn.rnn_cell.LSTMCell(hidden_size//2)
    lstm_bw_cell = tf.nn.rnn_cell.LSTMCell(hidden_size//2)
    (output_fw, output_bw), states = \
        tf.nn.bidirectional_dynamic_rnn(cell_fw=lstm_fw_cell,
                                        cell_bw=lstm_bw_cell,
                                        inputs=projected_context,
                                        dtype=tf.float32)
    # bilstm_output: [-1, max_num_tokens, hidden_size]
    bilstm_output = tf.concat([output_fw, output_bw], -1)
    h_state_for_sketch = bilstm_output

    if projected_align:
        log.info("Using projected attention alignment")
        h_state_for_attn_alignment = bilstm_output
        aligned_h_state = csoftmax_attention.attention_gen_block(
            h_state_for_sketch, h_state_for_attn_alignment, key, depth)
        output = \
            tf.reshape(aligned_h_state, shape=[batch_size, -1, depth * hidden_size])
    else:
        log.info("Using without projected attention alignment")
        h_state_for_attn_alignment = projected_context
        aligned_h_state = csoftmax_attention.attention_gen_block(
            h_state_for_sketch, h_state_for_attn_alignment, key, depth)
        output = \
            tf.reshape(aligned_h_state, shape=[batch_size, -1, depth * token_size])
    return output

def bahdanau_attention(key, context, hidden_size, projected_align=False):
    """ It is a implementation of the Bahdanau et al. attention mechanism. Based on the paper:
        https://arxiv.org/abs/1409.0473 "Neural Machine Translation by Jointly Learning to Align and Translate"
    Args:
        key: A tensorflow tensor with dimensionality [None, None, key_size]
        context: A tensorflow tensor with dimensionality [None, None, max_num_tokens, token_size]
        hidden_size: Number of units in hidden representation
        projected_align: Using bidirectional lstm for hidden representation of context.
        If true, beetween input and attention mechanism insert layer of bidirectional lstm with dimensionality [hidden_size].
        If false, bidirectional lstm is not used.
    Returns:
        output: Tensor at the output with dimensionality [None, None, hidden_size]
    """
    if hidden_size % 2 != 0:
        raise ValueError("hidden size must be dividable by two")
    batch_size = tf.shape(context)[0]
    max_num_tokens, token_size = context.get_shape().as_list()[-2:]
    r_context = tf.reshape(context, shape=[-1, max_num_tokens, token_size])

    # projected_key: [None, None, hidden_size]
    projected_key = tf.layers.dense(key, hidden_size, kernel_initializer=xav())
    r_projected_key = \
        tf.tile(tf.reshape(projected_key, shape=[-1, 1, hidden_size]),
                [1, max_num_tokens, 1])

    lstm_fw_cell = tf.nn.rnn_cell.LSTMCell(hidden_size//2)
    lstm_bw_cell = tf.nn.rnn_cell.LSTMCell(hidden_size//2)
    (output_fw, output_bw), states = \
        tf.nn.bidirectional_dynamic_rnn(cell_fw=lstm_fw_cell,
                                        cell_bw=lstm_bw_cell,
                                        inputs=r_context,
                                        dtype=tf.float32)

    # bilstm_output: [-1,self.max_num_tokens,_n_hidden]
    bilstm_output = tf.concat([output_fw, output_bw], -1)
    concat_h_state = tf.concat([r_projected_key, output_fw, output_bw], -1)
    projected_state = \
        tf.layers.dense(concat_h_state, hidden_size, use_bias=False,
                        kernel_initializer=xav())
    score = \
        tf.layers.dense(tf.tanh(projected_state), units=1, use_bias=False,
                        kernel_initializer=xav())

    attn = tf.nn.softmax(score, dim=1)

    if projected_align:
        log.info("Using projected attention alignment")
        t_context = tf.transpose(bilstm_output, [0, 2, 1])
        output = tf.reshape(tf.matmul(t_context, attn),
                            shape=[batch_size, -1, hidden_size])
    else:
        log.info("Using without projected attention alignment")
        t_context = tf.transpose(r_context, [0, 2, 1])
        output = tf.reshape(tf.matmul(t_context, attn),
                            shape=[batch_size, -1, token_size])
    return output

def light_bahdanau_attention(key, context, hidden_size, projected_align=False):
    """ It is a implementation of the Bahdanau et al. attention mechanism. Based on the paper:
        https://arxiv.org/abs/1409.0473 "Neural Machine Translation by Jointly Learning to Align and Translate"
    Args:
        key: A tensorflow tensor with dimensionality [None, None, key_size]
        context: A tensorflow tensor with dimensionality [None, None, max_num_tokens, token_size]
        hidden_size: Number of units in hidden representation
        projected_align: Using dense layer for hidden representation of context.
        If true, between input and attention mechanism insert a dense layer with dimensionality [hidden_size].
        If false, a dense layer is not used.
    Returns:
        output: Tensor at the output with dimensionality [None, None, hidden_size]
    """
    batch_size = tf.shape(context)[0]
    max_num_tokens, token_size = context.get_shape().as_list()[-2:]
    r_context = tf.reshape(context, shape=[-1, max_num_tokens, token_size])

    # projected_key: [None, None, hidden_size]
    projected_key = tf.layers.dense(key, hidden_size, kernel_initializer=xav())
    r_projected_key = \
        tf.tile(tf.reshape(projected_key, shape=[-1, 1, hidden_size]),
                [1, max_num_tokens, 1])

    # projected_context: [None, max_num_tokens, hidden_size]
    projected_context = \
        tf.layers.dense(r_context, hidden_size, kernel_initializer=xav())
    concat_h_state = tf.concat([projected_context, r_projected_key], -1)

    projected_state = \
        tf.layers.dense(concat_h_state, hidden_size, use_bias=False,
                        kernel_initializer=xav())
    score = \
        tf.layers.dense(tf.tanh(projected_state), units=1, use_bias=False,
                        kernel_initializer=xav())

    attn = tf.nn.softmax(score, dim=1)

    if projected_align:
        log.info("Using projected attention alignment")
        t_context = tf.transpose(projected_context, [0, 2, 1])
        output = tf.reshape(tf.matmul(t_context, attn),
                            shape=[batch_size, -1, hidden_size])
    else:
        log.info("Using without projected attention alignment")
        t_context = tf.transpose(r_context, [0, 2, 1])
        output = tf.reshape(tf.matmul(t_context, attn),
                            shape=[batch_size, -1, token_size])
    return output

def cs_bahdanau_attention(key, context, hidden_size, depth, projected_align=False):
    """ It is a implementation of the Bahdanau et al. attention mechanism. Based on the papers:
        https://arxiv.org/abs/1409.0473 "Neural Machine Translation by Jointly Learning to Align and Translate"
        https://andre-martins.github.io/docs/emnlp2017_final.pdf "Learning What's Easy: Fully Differentiable Neural Easy-First Taggers"
    Args:
        key: A tensorflow tensor with dimensionality [None, None, key_size]
        context: A tensorflow tensor with dimensionality [None, None, max_num_tokens, token_size]
        hidden_size: Number of units in hidden representation
        depth: Number of csoftmax usages
        projected_align: Using bidirectional lstm for hidden representation of context.
        If true, beetween input and attention mechanism insert layer of bidirectional lstm with dimensionality [hidden_size].
        If false, bidirectional lstm is not used.
    Returns:
        output: Tensor at the output with dimensionality [None, None, depth * hidden_size]
    """
    if hidden_size % 2 != 0:
        raise ValueError("hidden size must be dividable by two")
    batch_size = tf.shape(context)[0]
    max_num_tokens, token_size = context.get_shape().as_list()[-2:]

    r_context = tf.reshape(context, shape=[-1, max_num_tokens, token_size])
    # projected context: [None, max_num_tokens, token_size]
    projected_context = tf.layers.dense(r_context, token_size,
                                        kernel_initializer=xav(),
                                        name='projected_context')

    # projected_key: [None, None, hidden_size]
    projected_key = tf.layers.dense(key, hidden_size, kernel_initializer=xav(),
                                    name='projected_key')
    r_projected_key = \
        tf.tile(tf.reshape(projected_key, shape=[-1, 1, hidden_size]),
                [1, max_num_tokens, 1])

    lstm_fw_cell = tf.nn.rnn_cell.LSTMCell(hidden_size//2)
    lstm_bw_cell = tf.nn.rnn_cell.LSTMCell(hidden_size//2)
    (output_fw, output_bw), states = \
        tf.nn.bidirectional_dynamic_rnn(cell_fw=lstm_fw_cell,
                                        cell_bw=lstm_bw_cell,
                                        inputs=projected_context,
                                        dtype=tf.float32)

    # bilstm_output: [-1, max_num_tokens, hidden_size]
    bilstm_output = tf.concat([output_fw, output_bw], -1)
    concat_h_state = tf.concat([r_projected_key, output_fw, output_bw], -1)

    if projected_align:
        log.info("Using projected attention alignment")
        h_state_for_attn_alignment = bilstm_output
        aligned_h_state = csoftmax_attention.attention_bah_block(
            concat_h_state, h_state_for_attn_alignment, depth)
        output = \
            tf.reshape(aligned_h_state, shape=[batch_size, -1, depth * hidden_size])
    else:
        log.info("Using without projected attention alignment")
        h_state_for_attn_alignment = projected_context
        aligned_h_state = csoftmax_attention.attention_bah_block(
            concat_h_state, h_state_for_attn_alignment, depth)
        output = \
            tf.reshape(aligned_h_state, shape=[batch_size, -1, depth * token_size])
    return output


#!/usr/bin/env python
# -*- encoding: utf-8 -*-
#
# This file is auto-generated by h2o-3/h2o-bindings/bin/gen_python.py
# Copyright 2016 H2O.ai;  Apache License Version 2.0 (see LICENSE for details)
#
from __future__ import absolute_import, division, print_function, unicode_literals

from h2o.estimators.estimator_base import H2OEstimator
from h2o.exceptions import H2OValueError
from h2o.frame import H2OFrame
from h2o.utils.typechecks import assert_is_type, Enum, numeric

class H2OGenericEstimator(H2OEstimator):
    """
    Generic Model

    """

    algo = "generic"

    def __init__(self, **kwargs):
        super(H2OGenericEstimator, self).__init__()
        self._parms = {}
        names_list = {"model_id", "model_key"}
        if "Lambda" in kwargs: kwargs["lambda_"] = kwargs.pop("Lambda")
        for pname, pvalue in kwargs.items():
            if pname == 'model_id':
                self._id = pvalue
                self._parms["model_id"] = pvalue
            elif pname in names_list:
                # Using setattr(...) will invoke type-checking of the arguments
                setattr(self, pname, pvalue)
            else:
                raise H2OValueError("Unknown parameter %s = %r" % (pname, pvalue))

    @property
    def model_key(self):
        """
        Key to the self-contained model archive already uploaded to H2O.

        Type: ``H2OFrame``.
        """
        return self._parms.get("model_key")

    @model_key.setter
    def model_key(self, model_key):
        assert_is_type(model_key, None, H2OFrame)
        self._parms["model_key"] = model_key

    def _requires_training_frame(self):
        """
        Determines if Generic model requires a training frame.
        :return: False.
        """
        return False

    @staticmethod
    def from_file(file=str):
        """
        Creates new Generic model by loading existing embedded model into library, e.g. from H2O MOJO.
        The imported model must be supported by H2O.
        :param file: A string containing path to the file to create the model from
        :return: H2OGenericEstimator instance representing the generic model
        """
        from h2o import lazy_import, get_frame
        model_key = lazy_import(file)
        model_bytes_frame = get_frame(model_key[0])
        model = H2OGenericEstimator(model_key = model_bytes_frame)
        model.train()

        return model


#!/usr/bin/env python
# -*- encoding: utf-8 -*-
#
# This file is auto-generated by h2o-3/h2o-bindings/bin/gen_python.py
# Copyright 2016 H2O.ai;  Apache License Version 2.0 (see LICENSE for details)
#
from __future__ import absolute_import, division, print_function, unicode_literals

from h2o.estimators.estimator_base import H2OEstimator
from h2o.exceptions import H2OValueError
from h2o.frame import H2OFrame
from h2o.utils.typechecks import assert_is_type, Enum, numeric
import h2o

class H2OGeneralizedLinearEstimator(H2OEstimator):
    """
    Generalized Linear Modeling

    Fits a generalized linear model, specified by a response variable, a set of predictors, and a
    description of the error distribution.

    A subclass of :class:`ModelBase` is returned. The specific subclass depends on the machine learning task
    at hand (if it's binomial classification, then an H2OBinomialModel is returned, if it's regression then a
    H2ORegressionModel is returned). The default print-out of the models is shown, but further GLM-specific
    information can be queried out of the object. Upon completion of the GLM, the resulting object has
    coefficients, normalized coefficients, residual/null deviance, aic, and a host of model metrics including
    MSE, AUC (for logistic regression), degrees of freedom, and confusion matrices.
    """

    algo = "glm"

    def __init__(self, **kwargs):
        super(H2OGeneralizedLinearEstimator, self).__init__()
        self._parms = {}
        names_list = {"model_id", "training_frame", "validation_frame", "nfolds", "seed",
                      "keep_cross_validation_models", "keep_cross_validation_predictions",
                      "keep_cross_validation_fold_assignment", "fold_assignment", "fold_column", "response_column",
                      "ignored_columns", "ignore_const_cols", "score_each_iteration", "offset_column", "weights_column",
                      "family", "tweedie_variance_power", "tweedie_link_power", "theta", "solver", "alpha", "lambda_",
                      "lambda_search", "early_stopping", "nlambdas", "standardize", "missing_values_handling",
                      "compute_p_values", "remove_collinear_columns", "intercept", "non_negative", "max_iterations",
                      "objective_epsilon", "beta_epsilon", "gradient_epsilon", "link", "prior", "lambda_min_ratio",
                      "beta_constraints", "max_active_predictors", "interactions", "interaction_pairs", "obj_reg",
                      "export_checkpoints_dir", "balance_classes", "class_sampling_factors", "max_after_balance_size",
                      "max_confusion_matrix_size", "max_hit_ratio_k", "max_runtime_secs", "custom_metric_func"}
        if "Lambda" in kwargs: kwargs["lambda_"] = kwargs.pop("Lambda")
        for pname, pvalue in kwargs.items():
            if pname == 'model_id':
                self._id = pvalue
                self._parms["model_id"] = pvalue
            elif pname in names_list:
                # Using setattr(...) will invoke type-checking of the arguments
                setattr(self, pname, pvalue)
            else:
                raise H2OValueError("Unknown parameter %s = %r" % (pname, pvalue))

    @property
    def training_frame(self):
        """
        Id of the training data frame.

        Type: ``H2OFrame``.
        """
        return self._parms.get("training_frame")

    @training_frame.setter
    def training_frame(self, training_frame):
        assert_is_type(training_frame, None, H2OFrame)
        self._parms["training_frame"] = training_frame

    @property
    def validation_frame(self):
        """
        Id of the validation data frame.

        Type: ``H2OFrame``.
        """
        return self._parms.get("validation_frame")

    @validation_frame.setter
    def validation_frame(self, validation_frame):
        assert_is_type(validation_frame, None, H2OFrame)
        self._parms["validation_frame"] = validation_frame

    @property
    def nfolds(self):
        """
        Number of folds for K-fold cross-validation (0 to disable or >= 2).

        Type: ``int``  (default: ``0``).
        """
        return self._parms.get("nfolds")

    @nfolds.setter
    def nfolds(self, nfolds):
        assert_is_type(nfolds, None, int)
        self._parms["nfolds"] = nfolds

    @property
    def seed(self):
        """
        Seed for pseudo random number generator (if applicable)

        Type: ``int``  (default: ``-1``).
        """
        return self._parms.get("seed")

    @seed.setter
    def seed(self, seed):
        assert_is_type(seed, None, int)
        self._parms["seed"] = seed

    @property
    def keep_cross_validation_models(self):
        """
        Whether to keep the cross-validation models.

        Type: ``bool``  (default: ``True``).
        """
        return self._parms.get("keep_cross_validation_models")

    @keep_cross_validation_models.setter
    def keep_cross_validation_models(self, keep_cross_validation_models):
        assert_is_type(keep_cross_validation_models, None, bool)
        self._parms["keep_cross_validation_models"] = keep_cross_validation_models

    @property
    def keep_cross_validation_predictions(self):
        """
        Whether to keep the predictions of the cross-validation models.

        Type: ``bool``  (default: ``False``).
        """
        return self._parms.get("keep_cross_validation_predictions")

    @keep_cross_validation_predictions.setter
    def keep_cross_validation_predictions(self, keep_cross_validation_predictions):
        assert_is_type(keep_cross_validation_predictions, None, bool)
        self._parms["keep_cross_validation_predictions"] = keep_cross_validation_predictions

    @property
    def keep_cross_validation_fold_assignment(self):
        """
        Whether to keep the cross-validation fold assignment.

        Type: ``bool``  (default: ``False``).
        """
        return self._parms.get("keep_cross_validation_fold_assignment")

    @keep_cross_validation_fold_assignment.setter
    def keep_cross_validation_fold_assignment(self, keep_cross_validation_fold_assignment):
        assert_is_type(keep_cross_validation_fold_assignment, None, bool)
        self._parms["keep_cross_validation_fold_assignment"] = keep_cross_validation_fold_assignment

    @property
    def fold_assignment(self):
        """
        Cross-validation fold assignment scheme, if fold_column is not specified. The 'Stratified' option will stratify
        the folds based on the response variable, for classification problems.

        One of: ``"auto"``, ``"random"``, ``"modulo"``, ``"stratified"``  (default: ``"auto"``).
        """
        return self._parms.get("fold_assignment")

    @fold_assignment.setter
    def fold_assignment(self, fold_assignment):
        assert_is_type(fold_assignment, None, Enum("auto", "random", "modulo", "stratified"))
        self._parms["fold_assignment"] = fold_assignment

    @property
    def fold_column(self):
        """
        Column with cross-validation fold index assignment per observation.

        Type: ``str``.
        """
        return self._parms.get("fold_column")

    @fold_column.setter
    def fold_column(self, fold_column):
        assert_is_type(fold_column, None, str)
        self._parms["fold_column"] = fold_column

    @property
    def response_column(self):
        """
        Response variable column.

        Type: ``str``.
        """
        return self._parms.get("response_column")

    @response_column.setter
    def response_column(self, response_column):
        assert_is_type(response_column, None, str)
        self._parms["response_column"] = response_column

    @property
    def ignored_columns(self):
        """
        Names of columns to ignore for training.

        Type: ``List[str]``.
        """
        return self._parms.get("ignored_columns")

    @ignored_columns.setter
    def ignored_columns(self, ignored_columns):
        assert_is_type(ignored_columns, None, [str])
        self._parms["ignored_columns"] = ignored_columns

    @property
    def ignore_const_cols(self):
        """
        Ignore constant columns.

        Type: ``bool``  (default: ``True``).
        """
        return self._parms.get("ignore_const_cols")

    @ignore_const_cols.setter
    def ignore_const_cols(self, ignore_const_cols):
        assert_is_type(ignore_const_cols, None, bool)
        self._parms["ignore_const_cols"] = ignore_const_cols

    @property
    def score_each_iteration(self):
        """
        Whether to score during each iteration of model training.

        Type: ``bool``  (default: ``False``).
        """
        return self._parms.get("score_each_iteration")

    @score_each_iteration.setter
    def score_each_iteration(self, score_each_iteration):
        assert_is_type(score_each_iteration, None, bool)
        self._parms["score_each_iteration"] = score_each_iteration

    @property
    def offset_column(self):
        """
        Offset column. This will be added to the combination of columns before applying the link function.

        Type: ``str``.
        """
        return self._parms.get("offset_column")

    @offset_column.setter
    def offset_column(self, offset_column):
        assert_is_type(offset_column, None, str)
        self._parms["offset_column"] = offset_column

    @property
    def weights_column(self):
        """
        Column with observation weights. Giving some observation a weight of zero is equivalent to excluding it from the
        dataset; giving an observation a relative weight of 2 is equivalent to repeating that row twice. Negative
        weights are not allowed. Note: Weights are per-row observation weights and do not increase the size of the data
        frame. This is typically the number of times a row is repeated, but non-integer values are supported as well.
        During training, rows with higher weights matter more, due to the larger loss function pre-factor.

        Type: ``str``.
        """
        return self._parms.get("weights_column")

    @weights_column.setter
    def weights_column(self, weights_column):
        assert_is_type(weights_column, None, str)
        self._parms["weights_column"] = weights_column

    @property
    def family(self):
        """
        Family. Use binomial for classification with logistic regression, others are for regression problems.

        One of: ``"gaussian"``, ``"binomial"``, ``"quasibinomial"``, ``"ordinal"``, ``"multinomial"``, ``"poisson"``,
        ``"gamma"``, ``"tweedie"``, ``"negativebinomial"``  (default: ``"gaussian"``).
        """
        return self._parms.get("family")

    @family.setter
    def family(self, family):
        assert_is_type(family, None, Enum("gaussian", "binomial", "quasibinomial", "ordinal", "multinomial", "poisson", "gamma", "tweedie", "negativebinomial"))
        self._parms["family"] = family

    @property
    def tweedie_variance_power(self):
        """
        Tweedie variance power

        Type: ``float``  (default: ``0``).
        """
        return self._parms.get("tweedie_variance_power")

    @tweedie_variance_power.setter
    def tweedie_variance_power(self, tweedie_variance_power):
        assert_is_type(tweedie_variance_power, None, numeric)
        self._parms["tweedie_variance_power"] = tweedie_variance_power

    @property
    def tweedie_link_power(self):
        """
        Tweedie link power

        Type: ``float``  (default: ``1``).
        """
        return self._parms.get("tweedie_link_power")

    @tweedie_link_power.setter
    def tweedie_link_power(self, tweedie_link_power):
        assert_is_type(tweedie_link_power, None, numeric)
        self._parms["tweedie_link_power"] = tweedie_link_power

    @property
    def theta(self):
        """
        Theta

        Type: ``float``  (default: ``1e-10``).
        """
        return self._parms.get("theta")

    @theta.setter
    def theta(self, theta):
        assert_is_type(theta, None, numeric)
        self._parms["theta"] = theta

    @property
    def solver(self):
        """
        AUTO will set the solver based on given data and the other parameters. IRLSM is fast on on problems with small
        number of predictors and for lambda-search with L1 penalty, L_BFGS scales better for datasets with many columns.

        One of: ``"auto"``, ``"irlsm"``, ``"l_bfgs"``, ``"coordinate_descent_naive"``, ``"coordinate_descent"``,
        ``"gradient_descent_lh"``, ``"gradient_descent_sqerr"``  (default: ``"auto"``).
        """
        return self._parms.get("solver")

    @solver.setter
    def solver(self, solver):
        assert_is_type(solver, None, Enum("auto", "irlsm", "l_bfgs", "coordinate_descent_naive", "coordinate_descent", "gradient_descent_lh", "gradient_descent_sqerr"))
        self._parms["solver"] = solver

    @property
    def alpha(self):
        """
        Distribution of regularization between the L1 (Lasso) and L2 (Ridge) penalties. A value of 1 for alpha
        represents Lasso regression, a value of 0 produces Ridge regression, and anything in between specifies the
        amount of mixing between the two. Default value of alpha is 0 when SOLVER = 'L-BFGS'; 0.5 otherwise.

        Type: ``List[float]``.
        """
        return self._parms.get("alpha")

    @alpha.setter
    def alpha(self, alpha):
        assert_is_type(alpha, None, numeric, [numeric])
        self._parms["alpha"] = alpha

    @property
    def lambda_(self):
        """
        Regularization strength

        Type: ``List[float]``.
        """
        return self._parms.get("lambda")

    @lambda_.setter
    def lambda_(self, lambda_):
        assert_is_type(lambda_, None, numeric, [numeric])
        self._parms["lambda"] = lambda_

    @property
    def lambda_search(self):
        """
        Use lambda search starting at lambda max, given lambda is then interpreted as lambda min

        Type: ``bool``  (default: ``False``).
        """
        return self._parms.get("lambda_search")

    @lambda_search.setter
    def lambda_search(self, lambda_search):
        assert_is_type(lambda_search, None, bool)
        self._parms["lambda_search"] = lambda_search

    @property
    def early_stopping(self):
        """
        Stop early when there is no more relative improvement on train or validation (if provided)

        Type: ``bool``  (default: ``True``).
        """
        return self._parms.get("early_stopping")

    @early_stopping.setter
    def early_stopping(self, early_stopping):
        assert_is_type(early_stopping, None, bool)
        self._parms["early_stopping"] = early_stopping

    @property
    def nlambdas(self):
        """
        Number of lambdas to be used in a search. Default indicates: If alpha is zero, with lambda search set to True,
        the value of nlamdas is set to 30 (fewer lambdas are needed for ridge regression) otherwise it is set to 100.

        Type: ``int``  (default: ``-1``).
        """
        return self._parms.get("nlambdas")

    @nlambdas.setter
    def nlambdas(self, nlambdas):
        assert_is_type(nlambdas, None, int)
        self._parms["nlambdas"] = nlambdas

    @property
    def standardize(self):
        """
        Standardize numeric columns to have zero mean and unit variance

        Type: ``bool``  (default: ``True``).
        """
        return self._parms.get("standardize")

    @standardize.setter
    def standardize(self, standardize):
        assert_is_type(standardize, None, bool)
        self._parms["standardize"] = standardize

    @property
    def missing_values_handling(self):
        """
        Handling of missing values. Either MeanImputation or Skip.

        One of: ``"mean_imputation"``, ``"skip"``  (default: ``"mean_imputation"``).
        """
        return self._parms.get("missing_values_handling")

    @missing_values_handling.setter
    def missing_values_handling(self, missing_values_handling):
        assert_is_type(missing_values_handling, None, Enum("mean_imputation", "skip"))
        self._parms["missing_values_handling"] = missing_values_handling

    @property
    def compute_p_values(self):
        """
        Request p-values computation, p-values work only with IRLSM solver and no regularization

        Type: ``bool``  (default: ``False``).
        """
        return self._parms.get("compute_p_values")

    @compute_p_values.setter
    def compute_p_values(self, compute_p_values):
        assert_is_type(compute_p_values, None, bool)
        self._parms["compute_p_values"] = compute_p_values

    @property
    def remove_collinear_columns(self):
        """
        In case of linearly dependent columns, remove some of the dependent columns

        Type: ``bool``  (default: ``False``).
        """
        return self._parms.get("remove_collinear_columns")

    @remove_collinear_columns.setter
    def remove_collinear_columns(self, remove_collinear_columns):
        assert_is_type(remove_collinear_columns, None, bool)
        self._parms["remove_collinear_columns"] = remove_collinear_columns

    @property
    def intercept(self):
        """
        Include constant term in the model

        Type: ``bool``  (default: ``True``).
        """
        return self._parms.get("intercept")

    @intercept.setter
    def intercept(self, intercept):
        assert_is_type(intercept, None, bool)
        self._parms["intercept"] = intercept

    @property
    def non_negative(self):
        """
        Restrict coefficients (not intercept) to be non-negative

        Type: ``bool``  (default: ``False``).
        """
        return self._parms.get("non_negative")

    @non_negative.setter
    def non_negative(self, non_negative):
        assert_is_type(non_negative, None, bool)
        self._parms["non_negative"] = non_negative

    @property
    def max_iterations(self):
        """
        Maximum number of iterations

        Type: ``int``  (default: ``-1``).
        """
        return self._parms.get("max_iterations")

    @max_iterations.setter
    def max_iterations(self, max_iterations):
        assert_is_type(max_iterations, None, int)
        self._parms["max_iterations"] = max_iterations

    @property
    def objective_epsilon(self):
        """
        Converge if  objective value changes less than this. Default indicates: If lambda_search is set to True the
        value of objective_epsilon is set to .0001. If the lambda_search is set to False and lambda is equal to zero,
        the value of objective_epsilon is set to .000001, for any other value of lambda the default value of
        objective_epsilon is set to .0001.

        Type: ``float``  (default: ``-1``).
        """
        return self._parms.get("objective_epsilon")

    @objective_epsilon.setter
    def objective_epsilon(self, objective_epsilon):
        assert_is_type(objective_epsilon, None, numeric)
        self._parms["objective_epsilon"] = objective_epsilon

    @property
    def beta_epsilon(self):
        """
        Converge if  beta changes less (using L-infinity norm) than beta esilon, ONLY applies to IRLSM solver

        Type: ``float``  (default: ``0.0001``).
        """
        return self._parms.get("beta_epsilon")

    @beta_epsilon.setter
    def beta_epsilon(self, beta_epsilon):
        assert_is_type(beta_epsilon, None, numeric)
        self._parms["beta_epsilon"] = beta_epsilon

    @property
    def gradient_epsilon(self):
        """
        Converge if  objective changes less (using L-infinity norm) than this, ONLY applies to L-BFGS solver. Default
        indicates: If lambda_search is set to False and lambda is equal to zero, the default value of gradient_epsilon
        is equal to .000001, otherwise the default value is .0001. If lambda_search is set to True, the conditional
        values above are 1E-8 and 1E-6 respectively.

        Type: ``float``  (default: ``-1``).
        """
        return self._parms.get("gradient_epsilon")

    @gradient_epsilon.setter
    def gradient_epsilon(self, gradient_epsilon):
        assert_is_type(gradient_epsilon, None, numeric)
        self._parms["gradient_epsilon"] = gradient_epsilon

    @property
    def link(self):
        """

        One of: ``"family_default"``, ``"identity"``, ``"logit"``, ``"log"``, ``"inverse"``, ``"tweedie"``,
        ``"ologit"``, ``"oprobit"``, ``"ologlog"``  (default: ``"family_default"``).
        """
        return self._parms.get("link")

    @link.setter
    def link(self, link):
        assert_is_type(link, None, Enum("family_default", "identity", "logit", "log", "inverse", "tweedie", "ologit", "oprobit", "ologlog"))
        self._parms["link"] = link

    @property
    def prior(self):
        """
        Prior probability for y==1. To be used only for logistic regression iff the data has been sampled and the mean
        of response does not reflect reality.

        Type: ``float``  (default: ``-1``).
        """
        return self._parms.get("prior")

    @prior.setter
    def prior(self, prior):
        assert_is_type(prior, None, numeric)
        self._parms["prior"] = prior

    @property
    def lambda_min_ratio(self):
        """
        Minimum lambda used in lambda search, specified as a ratio of lambda_max (the smallest lambda that drives all
        coefficients to zero). Default indicates: if the number of observations is greater than the number of variables,
        then lambda_min_ratio is set to 0.0001; if the number of observations is less than the number of variables, then
        lambda_min_ratio is set to 0.01.

        Type: ``float``  (default: ``-1``).
        """
        return self._parms.get("lambda_min_ratio")

    @lambda_min_ratio.setter
    def lambda_min_ratio(self, lambda_min_ratio):
        assert_is_type(lambda_min_ratio, None, numeric)
        self._parms["lambda_min_ratio"] = lambda_min_ratio

    @property
    def beta_constraints(self):
        """
        Beta constraints

        Type: ``H2OFrame``.
        """
        return self._parms.get("beta_constraints")

    @beta_constraints.setter
    def beta_constraints(self, beta_constraints):
        assert_is_type(beta_constraints, None, H2OFrame)
        self._parms["beta_constraints"] = beta_constraints

    @property
    def max_active_predictors(self):
        """
        Maximum number of active predictors during computation. Use as a stopping criterion to prevent expensive model
        building with many predictors. Default indicates: If the IRLSM solver is used, the value of
        max_active_predictors is set to 5000 otherwise it is set to 100000000.

        Type: ``int``  (default: ``-1``).
        """
        return self._parms.get("max_active_predictors")

    @max_active_predictors.setter
    def max_active_predictors(self, max_active_predictors):
        assert_is_type(max_active_predictors, None, int)
        self._parms["max_active_predictors"] = max_active_predictors

    @property
    def interactions(self):
        """
        A list of predictor column indices to interact. All pairwise combinations will be computed for the list.

        Type: ``List[str]``.
        """
        return self._parms.get("interactions")

    @interactions.setter
    def interactions(self, interactions):
        assert_is_type(interactions, None, [str])
        self._parms["interactions"] = interactions

    @property
    def interaction_pairs(self):
        """
        A list of pairwise (first order) column interactions.

        Type: ``List[tuple]``.
        """
        return self._parms.get("interaction_pairs")

    @interaction_pairs.setter
    def interaction_pairs(self, interaction_pairs):
        assert_is_type(interaction_pairs, None, [tuple])
        self._parms["interaction_pairs"] = interaction_pairs

    @property
    def obj_reg(self):
        """
        Likelihood divider in objective value computation, default is 1/nobs

        Type: ``float``  (default: ``-1``).
        """
        return self._parms.get("obj_reg")

    @obj_reg.setter
    def obj_reg(self, obj_reg):
        assert_is_type(obj_reg, None, numeric)
        self._parms["obj_reg"] = obj_reg

    @property
    def export_checkpoints_dir(self):
        """
        Automatically export generated models to this directory.

        Type: ``str``.
        """
        return self._parms.get("export_checkpoints_dir")

    @export_checkpoints_dir.setter
    def export_checkpoints_dir(self, export_checkpoints_dir):
        assert_is_type(export_checkpoints_dir, None, str)
        self._parms["export_checkpoints_dir"] = export_checkpoints_dir

    @property
    def balance_classes(self):
        """
        Balance training data class counts via over/under-sampling (for imbalanced data).

        Type: ``bool``  (default: ``False``).
        """
        return self._parms.get("balance_classes")

    @balance_classes.setter
    def balance_classes(self, balance_classes):
        assert_is_type(balance_classes, None, bool)
        self._parms["balance_classes"] = balance_classes

    @property
    def class_sampling_factors(self):
        """
        Desired over/under-sampling ratios per class (in lexicographic order). If not specified, sampling factors will
        be automatically computed to obtain class balance during training. Requires balance_classes.

        Type: ``List[float]``.
        """
        return self._parms.get("class_sampling_factors")

    @class_sampling_factors.setter
    def class_sampling_factors(self, class_sampling_factors):
        assert_is_type(class_sampling_factors, None, [float])
        self._parms["class_sampling_factors"] = class_sampling_factors

    @property
    def max_after_balance_size(self):
        """
        Maximum relative size of the training data after balancing class counts (can be less than 1.0). Requires
        balance_classes.

        Type: ``float``  (default: ``5``).
        """
        return self._parms.get("max_after_balance_size")

    @max_after_balance_size.setter
    def max_after_balance_size(self, max_after_balance_size):
        assert_is_type(max_after_balance_size, None, float)
        self._parms["max_after_balance_size"] = max_after_balance_size

    @property
    def max_confusion_matrix_size(self):
        """
        [Deprecated] Maximum size (# classes) for confusion matrices to be printed in the Logs

        Type: ``int``  (default: ``20``).
        """
        return self._parms.get("max_confusion_matrix_size")

    @max_confusion_matrix_size.setter
    def max_confusion_matrix_size(self, max_confusion_matrix_size):
        assert_is_type(max_confusion_matrix_size, None, int)
        self._parms["max_confusion_matrix_size"] = max_confusion_matrix_size

    @property
    def max_hit_ratio_k(self):
        """
        Maximum number (top K) of predictions to use for hit ratio computation (for multi-class only, 0 to disable)

        Type: ``int``  (default: ``0``).
        """
        return self._parms.get("max_hit_ratio_k")

    @max_hit_ratio_k.setter
    def max_hit_ratio_k(self, max_hit_ratio_k):
        assert_is_type(max_hit_ratio_k, None, int)
        self._parms["max_hit_ratio_k"] = max_hit_ratio_k

    @property
    def max_runtime_secs(self):
        """
        Maximum allowed runtime in seconds for model training. Use 0 to disable.

        Type: ``float``  (default: ``0``).
        """
        return self._parms.get("max_runtime_secs")

    @max_runtime_secs.setter
    def max_runtime_secs(self, max_runtime_secs):
        assert_is_type(max_runtime_secs, None, numeric)
        self._parms["max_runtime_secs"] = max_runtime_secs

    @property
    def custom_metric_func(self):
        """
        Reference to custom evaluation function, format: `language:keyName=funcName`

        Type: ``str``.
        """
        return self._parms.get("custom_metric_func")

    @custom_metric_func.setter
    def custom_metric_func(self, custom_metric_func):
        assert_is_type(custom_metric_func, None, str)
        self._parms["custom_metric_func"] = custom_metric_func

    @property
    def Lambda(self):
        """DEPRECATED. Use ``self.lambda_`` instead"""
        return self._parms["lambda"] if "lambda" in self._parms else None

    @Lambda.setter
    def Lambda(self, value):
        self._parms["lambda"] = value

    @staticmethod
    def getGLMRegularizationPath(model):
        """
        Extract full regularization path explored during lambda search from glm model.

        :param model: source lambda search model
        """
        x = h2o.api("GET /3/GetGLMRegPath", data={"model": model._model_json["model_id"]["name"]})
        ns = x.pop("coefficient_names")
        res = {
            "lambdas": x["lambdas"],
            "explained_deviance_train": x["explained_deviance_train"],
            "explained_deviance_valid": x["explained_deviance_valid"],
            "coefficients": [dict(zip(ns, y)) for y in x["coefficients"]],
        }
        if "coefficients_std" in x:
            res["coefficients_std"] = [dict(zip(ns, y)) for y in x["coefficients_std"]]
        return res

    @staticmethod
    def makeGLMModel(model, coefs, threshold=.5):
        """
        Create a custom GLM model using the given coefficients.

        Needs to be passed source model trained on the dataset to extract the dataset information from.

        :param model: source model, used for extracting dataset information
        :param coefs: dictionary containing model coefficients
        :param threshold: (optional, only for binomial) decision threshold used for classification
        """
        model_json = h2o.api(
            "POST /3/MakeGLMModel",
            data={"model": model._model_json["model_id"]["name"],
                  "names": list(coefs.keys()),
                  "beta": list(coefs.values()),
                  "threshold": threshold}
        )
        m = H2OGeneralizedLinearEstimator()
        m._resolve_model(model_json["model_id"]["name"], model_json)
        return m


# -*- encoding: utf-8 -*-
"""Information about the backend H2O cluster."""
from __future__ import division, print_function, absolute_import, unicode_literals

import sys
import time

import h2o
from h2o.exceptions import H2OConnectionError, H2OServerError
from h2o.display import H2ODisplay
from h2o.utils.compatibility import *  # NOQA
from h2o.utils.typechecks import assert_is_type
from h2o.utils.shared_utils import get_human_readable_bytes, get_human_readable_time

class H2OCluster(object):
    """
    Information about the backend H2O cluster.

    This object is available from ``h2o.cluster()`` or ``h2o.connection().cluster``, and its purpose is to provide
    basic information / manipulation methods for the underlying cluster.
    """

    # If information is this many seconds old, it will be refreshed next time you call :meth:`status`.
    REFRESH_INTERVAL = 1.0

    def __init__(self):
        """Initialize new H2OCluster instance."""
        self._props = {}
        self._retrieved_at = None

    @staticmethod
    def from_kvs(keyvals):
        """
        Create H2OCluster object from a list of key-value pairs.

        TODO: This method should be moved into the base H2OResponse class.
        """
        obj = H2OCluster()
        obj._retrieved_at = time.time()
        for k, v in keyvals:
            if k in {"__meta", "_exclude_fields", "__schema"}: continue
            if k in _cloud_v3_valid_keys:
                obj._props[k] = v
            else:
                raise AttributeError("Attribute %s cannot be set on H2OCluster (= %r)" % (k, v))
        return obj

    @property
    def skip_ticks(self):
        return self._props.get("skip_ticks", None)

    @property
    def bad_nodes(self):
        return self._props["bad_nodes"]

    @property
    def branch_name(self):
        return self._props["branch_name"]

    @property
    def build_number(self):
        return self._props["build_number"]

    @property
    def build_age(self):
        # If the build age is unknown, then the cluster is so old it comes from "prehistoric" times when no
        # build number reporting was done...
        return self._props.get("build_age", "PREHISTORIC")

    @property
    def build_too_old(self):
        # If the prop "build_too_old" wasn't reported by the server, then it's definitely too old :)
        return self._props.get("build_too_old", True)

    @property
    def cloud_healthy(self):
        return self._props["cloud_healthy"]

    @property
    def cloud_name(self):
        return self._props["cloud_name"]

    @property
    def cloud_size(self):
        return self._props["cloud_size"]

    @property
    def cloud_uptime_millis(self):
        return self._props["cloud_uptime_millis"]

    @property
    def cloud_internal_timezone(self):
        return self._props["cloud_internal_timezone"]

    @property
    def datafile_parser_timezone(self):
        return self._props["datafile_parser_timezone"]

    @property
    def consensus(self):
        return self._props["consensus"]

    @property
    def is_client(self):
        return self._props["is_client"]

    @property
    def locked(self):
        return self._props["locked"]

    @property
    def node_idx(self):
        return self._props["node_idx"]

    @property
    def nodes(self):
        return self._props["nodes"]

    @property
    def version(self):
        return self._props["version"]

    @property
    def internal_security_enabled(self):
        return self._props["internal_security_enabled"]

    def node(self,node_idx):
        """
        Get information about a particular node in an H2O cluster (node index is 0 based)

        Information includes the following:

        nthreads: Number of threads
        pid: PID of current H2O process
        mem_value_size: Data on Node memory
        max_disk: Max disk
        free_disk: Free disk
        open_fds: Open File Descripters
        swap_mem: Size of data on node's disk
        tcps_active: Open TCP connections
        num_cpus: Number of cpus
        cpus_allowed: CPU's allowed
        gflops: Linpack GFlops
        fjthrds: F/J Thread count, by priority
        mem_bw: Memory bandwith
        fjqueue: F/J Task count, by priority
        my_cpu_pct: System CPU percentage used by this H2O process in last interval
        pojo_mem: Temp (non Data) memory
        num_keys: Number of local keys
        ip_port: IP address and port in the form a.b.c.d:e
        last_ping: Time (in msec) of last ping
        rpcs_active: Active Remote Procedure Calls
        max_mem: Maximum memory size for node
        healthy: (now-last_ping)<HeartbeatThread.TIMEOUT
        sys_load: System load; average #runnables/#cores
        sys_cpu_pct: System CPU percentage used by everything in last interval
        free_mem: Free heap
        h2o: IP

        :param node_idx: An int value indicating which node to extract information from
        :returns: Dictionary containing node info

        :examples:
          >>>import h2o
          >>>h2o.init()
          >>>node_one = h2o.cluster().node(0)
          >>>node_one["pid"] #Get PID for first node in H2O Cluster
        """
        return self.nodes[node_idx]

    def shutdown(self, prompt=False):
        """
        Shut down the server.

        This method checks if the H2O cluster is still running, and if it does shuts it down (via a REST API call).

        :param prompt: A logical value indicating whether to prompt the user before shutting down the H2O server.
        """
        if not self.is_running(): return
        assert_is_type(prompt, bool)
        if prompt:
            question = "Are you sure you want to shutdown the H2O instance running at %s (Y/N)? " \
                       % h2o.connection().base_url
            response = input(question)  # works in Py2 & Py3 because redefined in h2o.utils.compatibility module
        else:
            response = "Y"
        if response.lower() in {"y", "yes"}:
            h2o.api("POST /3/Shutdown")
            h2o.connection().close()

    def is_running(self):
        """
        Determine if the H2O cluster is running or not.

        :returns: True if the cluster is up; False otherwise
        """
        try:
            if h2o.connection().local_server and not h2o.connection().local_server.is_running(): return False
            h2o.api("GET /")
            return True
        except (H2OConnectionError, H2OServerError):
            return False

    def show_status(self, detailed=False):
        """
        Print current cluster status information.

        :param detailed: if True, then also print detailed information about each node.
        """
        if self._retrieved_at + self.REFRESH_INTERVAL < time.time():
            # Info is stale, need to refresh
            new_info = h2o.api("GET /3/Cloud")
            self._fill_from_h2ocluster(new_info)
        ncpus = sum(node["num_cpus"] for node in self.nodes)
        allowed_cpus = sum(node["cpus_allowed"] for node in self.nodes)
        free_mem = sum(node["free_mem"] for node in self.nodes)
        unhealthy_nodes = sum(not node["healthy"] for node in self.nodes)
        status = "locked" if self.locked else "accepting new members"
        if unhealthy_nodes == 0:
            status += ", healthy"
        else:
            status += ", %d nodes are not healthy" % unhealthy_nodes
        api_extensions = self.list_api_extensions()
        H2ODisplay([
            ["H2O cluster uptime:",        get_human_readable_time(self.cloud_uptime_millis)],
            ["H2O cluster timezone:",      self.cloud_internal_timezone],
            ["H2O data parsing timezone:", self.datafile_parser_timezone],
            ["H2O cluster version:",       self.version],
            ["H2O cluster version age:",   "{} {}".format(self.build_age, ("!!!" if self.build_too_old else ""))],
            ["H2O cluster name:",          self.cloud_name],
            ["H2O cluster total nodes:",   self.cloud_size],
            ["H2O cluster free memory:",   get_human_readable_bytes(free_mem)],
            ["H2O cluster total cores:",   str(ncpus)],
            ["H2O cluster allowed cores:", str(allowed_cpus)],
            ["H2O cluster status:",        status],
            ["H2O connection url:",        h2o.connection().base_url],
            ["H2O connection proxy:",      h2o.connection().proxy],
            ["H2O internal security:",     self.internal_security_enabled],
            ["H2O API Extensions:",        ', '.join(api_extensions)],
            ["Python version:",            "%d.%d.%d %s" % tuple(sys.version_info[:4])],
        ])

        if detailed:
            keys = ["h2o", "healthy", "last_ping", "num_cpus", "sys_load", "mem_value_size", "free_mem", "pojo_mem",
                    "swap_mem", "free_disk", "max_disk", "pid", "num_keys", "tcps_active", "open_fds", "rpcs_active"]
            header = ["Nodes info:"] + ["Node %d" % (i + 1) for i in range(len(self.nodes))]
            table = [[k] for k in keys]
            for node in self.nodes:
                for i, k in enumerate(keys):
                    table[i].append(node[k])
            H2ODisplay(table=table, header=header)

    def network_test(self):
        """Test network connectivity."""
        res = h2o.api("GET /3/NetworkTest")
        res["table"].show()

    def list_jobs(self):
        """List all jobs performed by the cluster."""
        res = h2o.api("GET /3/Jobs")
        table = [["type"], ["dest"], ["description"], ["status"]]
        for job in res["jobs"]:
            job_dest = job["dest"]
            table[0].append(self._translate_job_type(job_dest["type"]))
            table[1].append(job_dest["name"])
            table[2].append(job["description"])
            table[3].append(job["status"])
        return table

    def list_all_extensions(self):
        """List all available extensions on the h2o backend"""
        return self._list_extensions("Capabilities")

    def list_core_extensions(self):
        """List available core extensions on the h2o backend"""
        return self._list_extensions("Capabilities/Core")

    def list_api_extensions(self):
        """List available API extensions on the h2o backend"""
        return self._list_extensions("Capabilities/API")

    @property
    def timezone(self):
        """Current timezone of the H2O cluster."""
        return h2o.rapids("(getTimeZone)")["string"]

    @timezone.setter
    def timezone(self, tz):
        assert_is_type(tz, str)
        h2o.rapids('(setTimeZone "%s")' % tz)

    def list_timezones(self):
        """Return the list of all known timezones."""
        from h2o.expr import ExprNode
        return h2o.H2OFrame._expr(expr=ExprNode("listTimeZones"))._frame()

    #-------------------------------------------------------------------------------------------------------------------
    # Private
    #-------------------------------------------------------------------------------------------------------------------

    def _fill_from_h2ocluster(self, other):
        """
        Update information in this object from another H2OCluster instance.

        :param H2OCluster other: source of the new information for this object.
        """
        self._props = other._props
        self._retrieved_at = other._retrieved_at
        other._props = {}
        other._retrieved_at = None

    def _list_extensions(self, endpoint):
        res = h2o.api("GET /3/" + endpoint)["capabilities"]
        return [x["name"] for x in res]

    def _translate_job_type(self, type):
        if type is None:
            return ('Removed')
        type_mapping = dict([
            ('Key<Frame>', 'Frame'),
            ('Key<Model>', 'Model'),
            ('Key<Grid>', 'Grid'),
            ('Key<PartialDependence>', 'PartialDependence'),
            ('Key<AutoML>', 'Auto Model'),
            ('Key<ScalaCodeResult>', 'Scala Code Execution'),
            ('Key<KeyedVoid>', 'Void')
        ])
        if type in type_mapping:
            return type_mapping[type]
        else:
            return 'Unknown'

_cloud_v3_valid_keys = {"is_client", "build_number", "cloud_name", "locked", "node_idx", "consensus", "branch_name",
                        "version", "cloud_uptime_millis", "cloud_internal_timezone", "datafile_parser_timezone", "cloud_healthy", "bad_nodes", "cloud_size", "skip_ticks",
                        "nodes", "build_age", "build_too_old", "internal_security_enabled"}


#!/usr/bin/env python
# -*- encoding: utf-8 -*-
#
# This file is auto-generated by h2o-3/h2o-bindings/bin/gen_python.py
# Copyright 2016 H2O.ai;  Apache License Version 2.0 (see LICENSE for details)
#
from __future__ import absolute_import, division, print_function, unicode_literals

from h2o.estimators.estimator_base import H2OEstimator
from h2o.exceptions import H2OValueError
from h2o.frame import H2OFrame
from h2o.utils.typechecks import assert_is_type, Enum, numeric, is_type
import json
import ast

class H2OStackedEnsembleEstimator(H2OEstimator):
    """
    Stacked Ensemble

    Builds a stacked ensemble (aka "super learner") machine learning method that uses two
    or more H2O learning algorithms to improve predictive performance. It is a loss-based
    supervised learning method that finds the optimal combination of a collection of prediction
    algorithms.This method supports regression and binary classification.

    Examples
    --------
      >>> import h2o
      >>> h2o.init()
      >>> from h2o.estimators.random_forest import H2ORandomForestEstimator
      >>> from h2o.estimators.gbm import H2OGradientBoostingEstimator
      >>> from h2o.estimators.stackedensemble import H2OStackedEnsembleEstimator
      >>> col_types = ["numeric", "numeric", "numeric", "enum", "enum", "numeric", "numeric", "numeric", "numeric"]
      >>> data = h2o.import_file("http://h2o-public-test-data.s3.amazonaws.com/smalldata/prostate/prostate.csv", col_types=col_types)
      >>> train, test = data.split_frame(ratios=[.8], seed=1)
      >>> x = ["CAPSULE","GLEASON","RACE","DPROS","DCAPS","PSA","VOL"]
      >>> y = "AGE"
      >>> nfolds = 5
      >>> my_gbm = H2OGradientBoostingEstimator(nfolds=nfolds, fold_assignment="Modulo", keep_cross_validation_predictions=True)
      >>> my_gbm.train(x=x, y=y, training_frame=train)
      >>> my_rf = H2ORandomForestEstimator(nfolds=nfolds, fold_assignment="Modulo", keep_cross_validation_predictions=True)
      >>> my_rf.train(x=x, y=y, training_frame=train)
      >>> stack = H2OStackedEnsembleEstimator(model_id="my_ensemble", training_frame=train, validation_frame=test, base_models=[my_gbm.model_id, my_rf.model_id])
      >>> stack.train(x=x, y=y, training_frame=train, validation_frame=test)
      >>> stack.model_performance()
    """

    algo = "stackedensemble"

    def __init__(self, **kwargs):
        super(H2OStackedEnsembleEstimator, self).__init__()
        self._parms = {}
        names_list = {"model_id", "training_frame", "response_column", "validation_frame", "blending_frame",
                      "base_models", "metalearner_algorithm", "metalearner_nfolds", "metalearner_fold_assignment",
                      "metalearner_fold_column", "metalearner_params", "seed", "keep_levelone_frame",
                      "export_checkpoints_dir"}
        if "Lambda" in kwargs: kwargs["lambda_"] = kwargs.pop("Lambda")
        for pname, pvalue in kwargs.items():
            if pname == 'model_id':
                self._id = pvalue
                self._parms["model_id"] = pvalue
            elif pname in names_list:
                # Using setattr(...) will invoke type-checking of the arguments
                setattr(self, pname, pvalue)
            else:
                raise H2OValueError("Unknown parameter %s = %r" % (pname, pvalue))
        self._parms["_rest_version"] = 99

    @property
    def training_frame(self):
        """
        Id of the training data frame.

        Type: ``H2OFrame``.
        """
        return self._parms.get("training_frame")

    @training_frame.setter
    def training_frame(self, training_frame):
        assert_is_type(training_frame, None, H2OFrame)
        self._parms["training_frame"] = training_frame

    @property
    def response_column(self):
        """
        Response variable column.

        Type: ``str``.
        """
        return self._parms.get("response_column")

    @response_column.setter
    def response_column(self, response_column):
        assert_is_type(response_column, None, str)
        self._parms["response_column"] = response_column

    @property
    def validation_frame(self):
        """
        Id of the validation data frame.

        Type: ``H2OFrame``.
        """
        return self._parms.get("validation_frame")

    @validation_frame.setter
    def validation_frame(self, validation_frame):
        assert_is_type(validation_frame, None, H2OFrame)
        self._parms["validation_frame"] = validation_frame

    @property
    def blending_frame(self):
        """
        Frame used to compute the predictions that serve as the training frame for the metalearner (triggers blending
        mode if provided)

        Type: ``H2OFrame``.
        """
        return self._parms.get("blending_frame")

    @blending_frame.setter
    def blending_frame(self, blending_frame):
        assert_is_type(blending_frame, None, H2OFrame)
        self._parms["blending_frame"] = blending_frame

    @property
    def base_models(self):
        """
        List of models (or model ids) to ensemble/stack together. If not using blending frame, then models must have
        been cross-validated using nfolds > 1, and folds must be identical across models.

        Type: ``List[str]``  (default: ``[]``).
        """
        return self._parms.get("base_models")

    @base_models.setter
    def base_models(self, base_models):
         if is_type(base_models,[H2OEstimator]):
            base_models = [b.model_id for b in base_models]
            self._parms["base_models"] = base_models
         else:
            assert_is_type(base_models, None, [str])
            self._parms["base_models"] = base_models

    @property
    def metalearner_algorithm(self):
        """
        Type of algorithm to use as the metalearner. Options include 'AUTO' (GLM with non negative weights; if
        validation_frame is present, a lambda search is performed), 'glm' (GLM with default parameters), 'gbm' (GBM with
        default parameters), 'drf' (Random Forest with default parameters), or 'deeplearning' (Deep Learning with
        default parameters).

        One of: ``"auto"``, ``"glm"``, ``"gbm"``, ``"drf"``, ``"deeplearning"``  (default: ``"auto"``).
        """
        return self._parms.get("metalearner_algorithm")

    @metalearner_algorithm.setter
    def metalearner_algorithm(self, metalearner_algorithm):
        assert_is_type(metalearner_algorithm, None, Enum("auto", "glm", "gbm", "drf", "deeplearning"))
        self._parms["metalearner_algorithm"] = metalearner_algorithm

    @property
    def metalearner_nfolds(self):
        """
        Number of folds for K-fold cross-validation of the metalearner algorithm (0 to disable or >= 2).

        Type: ``int``  (default: ``0``).
        """
        return self._parms.get("metalearner_nfolds")

    @metalearner_nfolds.setter
    def metalearner_nfolds(self, metalearner_nfolds):
        assert_is_type(metalearner_nfolds, None, int)
        self._parms["metalearner_nfolds"] = metalearner_nfolds

    @property
    def metalearner_fold_assignment(self):
        """
        Cross-validation fold assignment scheme for metalearner cross-validation.  Defaults to AUTO (which is currently
        set to Random). The 'Stratified' option will stratify the folds based on the response variable, for
        classification problems.

        One of: ``"auto"``, ``"random"``, ``"modulo"``, ``"stratified"``.
        """
        return self._parms.get("metalearner_fold_assignment")

    @metalearner_fold_assignment.setter
    def metalearner_fold_assignment(self, metalearner_fold_assignment):
        assert_is_type(metalearner_fold_assignment, None, Enum("auto", "random", "modulo", "stratified"))
        self._parms["metalearner_fold_assignment"] = metalearner_fold_assignment

    @property
    def metalearner_fold_column(self):
        """
        Column with cross-validation fold index assignment per observation for cross-validation of the metalearner.

        Type: ``str``.
        """
        return self._parms.get("metalearner_fold_column")

    @metalearner_fold_column.setter
    def metalearner_fold_column(self, metalearner_fold_column):
        assert_is_type(metalearner_fold_column, None, str)
        self._parms["metalearner_fold_column"] = metalearner_fold_column

    @property
    def metalearner_params(self):
        """
        Parameters for metalearner algorithm

        Type: ``dict``  (default: ``None``).
        Example: metalearner_gbm_params = {'max_depth': 2, 'col_sample_rate': 0.3}
        """
        if self._parms.get("metalearner_params") != None:
            metalearner_params_dict =  ast.literal_eval(self._parms.get("metalearner_params"))
            for k in metalearner_params_dict:
                if len(metalearner_params_dict[k]) == 1: #single parameter
                    metalearner_params_dict[k] = metalearner_params_dict[k][0]
            return metalearner_params_dict
        else:
            return self._parms.get("metalearner_params")

    @metalearner_params.setter
    def metalearner_params(self, metalearner_params):
        assert_is_type(metalearner_params, None, dict)
        if metalearner_params is not None and metalearner_params != "":
            for k in metalearner_params:
                if ("[" and "]") not in str(metalearner_params[k]):
                    metalearner_params[k]=[metalearner_params[k]]
            self._parms["metalearner_params"] = str(json.dumps(metalearner_params))
        else:
            self._parms["metalearner_params"] = None

    @property
    def seed(self):
        """
        Seed for random numbers; passed through to the metalearner algorithm. Defaults to -1 (time-based random number)

        Type: ``int``  (default: ``-1``).
        """
        return self._parms.get("seed")

    @seed.setter
    def seed(self, seed):
        assert_is_type(seed, None, int)
        self._parms["seed"] = seed

    @property
    def keep_levelone_frame(self):
        """
        Keep level one frame used for metalearner training.

        Type: ``bool``  (default: ``False``).
        """
        return self._parms.get("keep_levelone_frame")

    @keep_levelone_frame.setter
    def keep_levelone_frame(self, keep_levelone_frame):
        assert_is_type(keep_levelone_frame, None, bool)
        self._parms["keep_levelone_frame"] = keep_levelone_frame

    @property
    def export_checkpoints_dir(self):
        """
        Automatically export generated models to this directory.

        Type: ``str``.
        """
        return self._parms.get("export_checkpoints_dir")

    @export_checkpoints_dir.setter
    def export_checkpoints_dir(self, export_checkpoints_dir):
        assert_is_type(export_checkpoints_dir, None, str)
        self._parms["export_checkpoints_dir"] = export_checkpoints_dir

    # Print the metalearner of an H2OStackedEnsembleEstimator.
    def metalearner(self):
        model = self._model_json["output"]
        if "metalearner" in model and model["metalearner"] is not None:
            return model["metalearner"]
        print("No metalearner for this model")

    #Fetch the levelone_frame_id for an H2OStackedEnsembleEstimator.
    def levelone_frame_id(self):
        model = self._model_json["output"]
        if "levelone_frame_id" in model and model["levelone_frame_id"] is not None:
            return model["levelone_frame_id"]
        print("No levelone_frame_id for this model")

    def stacking_strategy(self):
        model = self._model_json["output"]
        if "stacking_strategy" in model and model["stacking_strategy"] is not None:
            return model["stacking_strategy"]
        print("No stacking strategy for this model")

    # Override train method to support blending
    def train(self, x=None, y=None, training_frame=None, blending_frame=None, **kwargs):
        assert_is_type(blending_frame, None, H2OFrame)

        def extend_parms(parms):
            if blending_frame is not None:
                parms['blending_frame'] = blending_frame

        super(self.__class__, self)._train(x, y, training_frame, extend_parms_fn=extend_parms, **kwargs)