from .transform_base import H2OTransformer
import warnings

class H2OScaler(H2OTransformer):
    """
    Standardize an H2OFrame by demeaning and scaling each column.

    The default scaling will result in an H2OFrame with columns
    having zero mean and unit variance. Users may specify the
    centering and scaling values used in the standardization of
    the H2OFrame.
    """

    def __init__(self, center=True, scale=True):
        """
        :param center: A boolean or list of numbers. If True, then columns will be demeaned before scaling.
            If False, then columns will not be demeaned before scaling.
            If centers is an array of numbers, then len(centers) must match the number of
            columns in the dataset. Each value is removed from the respective column
            before scaling.
        :param scale: A boolean or list of numbers. If True, then columns will be scaled by the column's standard
            deviation. If False, then columns will not be scaled. If scales is an array, then len(scales) must match
            the number of columns in the dataset. Each column is scaled by the respective value in this array.
        :returns: An instance of H2OScaler.
        """
        self.parms = locals()
        self.parms = {k: v for k, v in self.parms.items() if k != "self"}
        if center is None or scale is None: raise ValueError("centers and scales must not be None.")
        self._means = None
        self._stds = None

    @property
    def means(self):
        return self._means

    @property
    def stds(self):
        return self._stds

    def fit(self, X, y=None, **params):
        """
        Fit this object by computing the means and standard deviations used by the transform method.

        :param X: An H2OFrame; may contain NAs and/or categoricals.
        :param y: None (Ignored)
        :param params: Ignored
        :returns: This H2OScaler instance
        """
        if isinstance(self.parms["center"], (tuple, list)): self._means = self.parms["center"]
        if isinstance(self.parms["scale"], (tuple, list)): self._stds = self.parms["scale"]
        if self.means is None and self.parms["center"]:
            self._means = X.mean(return_frame=True).getrow()
        else:
            self._means = False
        if self.stds is None and self.parms["scale"]:
            self._stds = X.sd()
        else:
            self._stds = False
        return self

    def transform(self, X, y=None, **params):
        """
        Scale an H2OFrame with the fitted means and standard deviations.

        :param X: An H2OFrame; may contain NAs and/or categoricals.
        :param y: None (Ignored)
        :param params: (Ignored)
        :returns: A scaled H2OFrame.
        """
        return X.scale(self.means, self.stds)

    def inverse_transform(self, X, y=None, **params):
        """
        Undo the scale transformation.

        :param X: An H2OFrame; may contain NAs and/or categoricals.
        :param y: None (Ignored)
        :param params: (Ignored)
        :returns: An H2OFrame
        """
        for i in range(X.ncol):
            X[i] = self.means[i] + self.stds[i] * X[i]
        return X

class H2OColSelect(H2OTransformer):

    def __init__(self, cols):
        self.cols = cols

    def fit(self, X, y=None, **params):
        return self

    def transform(self, X, y=None, **params):
        return X[self.cols]

    def to_rest(self, step_name):
        args = [step_name, "H2OColSelect", ("(cols_py dummy %r)" % self.cols), False, "|"]
        return super(H2OColSelect, self).to_rest(args)

class H2OColOp(H2OTransformer):
    """
    Perform a column operation. If inplace is True, then cbind the result onto original frame,
    otherwise, perform the operation in place.
    """

    def __init__(self, op, col=None, inplace=True, new_col_name=None, **params):
        self.fun = op
        self.col = col
        self.inplace = inplace
        self.params = params
        self.new_col_name = new_col_name
        if inplace and new_col_name is not None:
            warnings.warn("inplace was False, but new_col_name was not empty. Ignoring new_col_name.")
        if isinstance(col, (list, tuple)): raise ValueError("col must be None or a single column.")

    def fit(self, X, y=None, **params):
        return self

    def transform(self, X, y=None, **params):
        res = H2OColOp._transform_helper(X, params)
        if self.inplace:
            X[self.col] = res
        else:
            return X.cbind(res)
        return X

    def _transform_helper(self, X, **params):
        if not self.params:
            if self.col is not None:
                res = self.fun(X[self.col])
            else:
                res = self.fun(X)
        else:
            if self.col is not None:
                res = self.fun(X[self.col], **self.params)
            else:
                res = self.fun(X, **self.params)
        return res

    def to_rest(self, step_name):
        ast = self._transform_helper(self._dummy_frame())._ex._to_string()
        new_col_names = self.new_col_name
        if new_col_names is None:
            new_col_names = ["|"]
        elif not isinstance(new_col_names, (list, tuple)):
            new_col_names = [new_col_names]
        return super(H2OColOp, self).to_rest(
            [step_name, self.__class__.__name__, ast, self.inplace, "|".join(new_col_names)])

class H2OBinaryOp(H2OColOp):
    """Perform a binary operation on a column.

    If left is None, then the column will appear on the left in the operation; otherwise
    it will be appear on the right.

    A ValueError is raised if both left and right are None.
    """

    def __init__(self, op, col, inplace=True, new_col_name=None, left=None, right=None, **params):
        super(H2OBinaryOp, self).__init__(op, col, inplace, new_col_name, **params)
        self.left_is_col = isinstance(left, H2OCol)
        self.right_is_col = isinstance(right, H2OCol)
        self.left = left
        self.right = right
        if left is None and right is None:
            raise ValueError("left and right cannot both be None")

    def _transform_helper(self, X, **params):
        if self.left is None: return self.fun(X[self.col], X[self.right.col] if self.right_is_col else self.right)
        return self.fun(X[self.left.col] if self.left_is_col else self.left, X[self.col])

class H2OCol(object):
    """
    Wrapper class for H2OBinaryOp step's left/right args.

    Use if you want to signal that a column actually comes from the train to be fitted on.
    """

    def __init__(self, column):
        self.col = column

        # TODO: handle arbitrary (non H2OFrame) inputs -- sql, web, file, generated


#!/usr/bin/python

import sys
import os
import requests
import re
import dateutil.parser

g_user = None
g_pass = None
g_start_date = None

def get_issue_key(issue):
    return issue[u'key']

def get_issue_type_name(issue):
    name = issue[u'fields'][u'issuetype'][u'name']
    return name

def get_issue_summary(issue):
    summary = issue[u'fields'][u'summary']
    summary = summary.encode('ascii', 'ignore')
    return summary

def get_issue_component_name(issue):
    components = issue[u'fields'][u'components']
    if (components is None):
        return "No component"
    if (len(components) == 0):
        return "No component"
    name = components[0][u'name']
    return name

def usage():
    print("")
    print("usage:  " + g_script_name + " --user username --pass password --startdate startdate")
    print("")
    print("e.g.")
    print("")
    print("./" + g_script_name + " --startdate 2015-01-30")
    print("")
    sys.exit(1)

def unknown_arg(s):
    print("")
    print("ERROR: Unknown argument: " + s)
    print("")
    usage()

def parse_config_file():
    global g_user
    global g_pass
    global g_sprint
    global g_start_date
    home = os.path.expanduser("~")
    config_file = os.path.join(home, ".h2o_jira")
    if (os.path.exists(config_file)):
        with open(config_file) as f:
            for line in f:
                match_groups = re.search(r"\s*(\S+)\s*=\s*([\S\s]+)\s*", line.rstrip())
                if (match_groups is not None):
                    key = match_groups.group(1)
                    value = match_groups.group(2)
                    if (key == "user"):
                        g_user = value
                    elif (key == "pass"):
                        g_pass = value
                    elif (key == "sprint"):
                        g_sprint = value
                    elif (key == "startdate"):
                        g_start_date = dateutil.parser.parse(value)

def parse_args(argv):
    global g_user
    global g_pass
    global g_start_date

    i = 1
    while (i < len(argv)):
        s = argv[i]

        if (s == "--user"):
            i += 1
            if (i > len(argv)):
                usage()
            g_user = argv[i]
        elif (s == "--pass"):
            i += 1
            if (i > len(argv)):
                usage()
            g_pass = argv[i]
        elif (s == "--startdate"):
            i += 1
            if (i > len(argv)):
                usage()
            g_start_date = dateutil.parser.parse(argv[i])
        elif (s == "-h" or s == "--h" or s == "-help" or s == "--help"):
            usage()
        else:
            unknown_arg(s)

        i += 1

    if (g_user is None):
        print "ERROR: user is not specified"
        usage()

    if (g_pass is None):
        print "ERROR: pass is not specified"
        usage()

    if (g_start_date is None):
        print "ERROR: start_date is not specified"
        usage()

def main(argv):
    """
    Main program.

    @return: none
    """
    global g_script_name

    g_script_name = os.path.basename(argv[0])
    parse_config_file()
    parse_args(argv)

    url = 'https://0xdata.atlassian.net/rest/api/2/search?jql=' \
          + 'project+in+(PUBDEV,HEXDEV)' \
          + '+and+' \
          + 'resolutiondate+>=+' + g_start_date.strftime("%Y-%m-%d") \
          + '+and+' \
          + 'resolution+in+(Done,Fixed)' \
          + 'order+by+type,component,resolutiondate' \
          + '&maxResults=1000'

    r = requests.get(url, auth=(g_user, g_pass))
    if (r.status_code != 200):
        print("ERROR: status code is " + str(r.status_code))
        sys.exit(1)
    j = r.json()
    issues = j[u'issues']
    if len(issues) >= 1000:
        print("ERROR: len(issues) >= 1000.  Too many issues.")
        sys.exit(1)
    last_issue_type_name = ""
    last_component_name = ""
    for issue in issues:
        issue_type_name = get_issue_type_name(issue)
        component_name = get_issue_component_name(issue)
        if (issue_type_name != last_issue_type_name):
            print("")
            print "# " + issue_type_name
            last_issue_type_name = issue_type_name
        if (component_name != last_component_name):
            print("")
            print "## " + component_name
            last_component_name = component_name
        key = get_issue_key(issue)
        summary = get_issue_summary(issue)
        print "* " + key + ": " + summary

if __name__ == "__main__":
    main(sys.argv)


#!/usr/bin/python

import sys
import os
import json
import pickle

import copy
import subprocess

"""
In summary, this script given a jenkins job full console url and a summary log filename will
1. scrape the console output log, all unit tests outputs and all java_*_0.out.txt of the
   latest build.
2. From all the logs, it will generate potentially two log files: jenkins_job_name_build_number_failed_tests.log
   and jenkins_job_name_build_number_passed_tests.log.  Inside each log file, it contains the job name, build number,
   timestamp, git hash, git branch, node name, build failure and build timeout information.  In addition, it will list
   unit tests that failed/passed with the corresponding java WARN/ERRR/FATAL/STACKTRACE messages associated with the unit tests.
3. Users can choose to ignore certain java messages that are deemed okay.  These ignored java messages are stored in a pickle
   file with a default name and location.  However, if the user wants to use their own ignored java messages, they can do
   so by specifying a third optional argument to this script as the name to where their own personal pickle file name.
4. If there are okay ignored java messages stored in a pickle file, this script will not grab them and store them in
   any log files.
5. For details on how to generate ignore java messages and save them to a pickle file, please see addjavamessage2ignore.py.
"""

# --------------------------------------------------------------------
# Main program
# --------------------------------------------------------------------

g_test_root_dir = os.path.dirname(os.path.realpath(__file__)) # directory where we are running out code from
g_script_name = ''  # store script name.

g_node_name = "Building remotely on"   # the very next string is the name of the computer node that ran the test
g_git_hash_branch = "Checking out Revision"    # next string is git hash, and the next one is (origin/branch)
g_build_timeout = "Build timed out"             # phrase when tests run too long
g_build_success = ["Finished: SUCCESS",'BUILD SUCCESSFUL']   # sentence at the end that guarantee build success

g_build_success_tests = ['generate_rest_api_docs.py','generate_java_bindings.py'] # two functions that are usually performed after build success
g_build_id_text = 'Build id is'
g_view_name = ''

g_temp_filename = os.path.join(g_test_root_dir,'tempText')  # temp file to store data curled from Jenkins

# generate file names to store the final logs.
g_output_filename_failed_tests = os.path.join(g_test_root_dir,'failedMessage_failed_tests.log')
g_output_filename_passed_tests = os.path.join(g_test_root_dir,'failedMessage_passed_tests.log')
g_output_pickle_filename = os.path.join(g_test_root_dir,'failedMessage.pickle.log')

g_failed_test_info_dict = {}
g_failed_test_info_dict["7.build_failure"] = "No"   # initialize build_failure with no by default

# info used to generate timestamp
g_weekdays = 'Monday, Tuesday, Wednesday, Thursday, Friday, Saturday, Sunday'
g_months = 'January, Feburary, March, May, April, May, June, July, August, September, October, November, December'

g_failure_occurred = False  # denote when failure actually occurred

g_failed_jobs = []                      # record job names of failed jobs
g_failed_job_java_message_types = []    # java bad message types (can be WARN:, ERRR:, FATAL:, TRACE:)
g_failed_job_java_messages = []         # record failed job java message

g_success_jobs = []                     # record job names of passed jobs
g_success_job_java_message_types = []
g_success_job_java_messages = []        # record of successful jobs bad java messages

# text you will find before you can find your java_*_*.out.txt
g_before_java_file = ["H2O Cloud", "Node", "started with output file"]

g_java_filenames = []   # contains all java filenames for us to mine
g_java_message_type = ["WARN:", ":WARN:", "ERRR:", "FATAL:", "TRACE:"]    # bad java message types
g_all_java_message_type = ["WARN:", ":WARN:", "ERRR:", "FATAL:", "TRACE:", "DEBUG:","INFO:"]    # all java message types

g_java_general_bad_message_types = []
g_java_general_bad_messages = []        # store java messages that are not associated with any tests

g_jenkins_url = ''
g_toContinue = False

g_current_testname = ''                 # denote when we are in a test during java text scanning

g_java_start_text = 'STARTING TEST:'    # test being started in java

g_ok_java_messages = {} # dict that stores java bad messages that we can ignore
g_java_message_pickle_filename = "bad_java_messages_to_exclude.pickle"  # pickle file that store the dictionary structure that include Java error message to exclude
g_build_failed_message = ["Finished: FAILURE".lower(),'BUILD FAILED'.lower()]   # something has gone wrong.  No tests are performed.
g_summary_text_filename = ""     # filename to store the summary file (contains all logs) sent to user via email.

'''
The sole purpose of this function is to enable us to be able to call
any function that is specified as the first argument using the argument
list specified in second argument.
'''
def perform(function_name, *arguments):
    """

    Parameters
    ----------

    function_name :  python function handle
        name of functio we want to call and run
    *arguments :  Python list
        list of arguments to be passed to function_name

    :return: bool
    """
    return function_name(*arguments)

'''
This function is written to remove extra characters before the actual string we are
looking for.  The Jenkins console output is encoded using utf-8.  However, the stupid
redirect function can only encode using ASCII.  I have googled for half a day with no
results to how.  Hence, we are going to the heat and just manually get rid of the junk.
'''
def extract_true_string(string_content):
    """
    remove extra characters before the actual string we are
    looking for.  The Jenkins console output is encoded using utf-8.  However, the stupid
    redirect function can only encode using ASCII.  I have googled for half a day with no
    results to how to resolve the issue.  Hence, we are going to the heat and just manually
    get rid of the junk.

    Parameters
    ----------

    string_content :  str
        contains a line read in from jenkins console

    :return: str: contains the content of the line after the string '[0m'

    """

    startL,found,endL = string_content.partition('[0m')

    if found:
        return endL
    else:
        return string_content

"""
Function find_time is written to extract the timestamp when a job is built.
"""
def find_time(each_line,temp_func_list):
    """
    calculate the approximate date/time from the timestamp about when the job
    was built.  This information was then saved in dict g_failed_test_info_dict.
    In addition, it will delete this particular function handle off the temp_func_list
    as we do not need to perform this action again.

    Parameters
    ----------

    each_line :  str
        contains a line read in from jenkins console
    temp_func_list :  list of Python function handles
        contains a list of functions that we want to invoke to extract information from
        the Jenkins console text.

    :return: bool to determine if text mining should continue on the jenkins console text
    """
    global g_weekdays
    global g_months
    global g_failed_test_info_dict

    temp_strings = each_line.strip().split()

    if (len(temp_strings) > 2):
        if ((temp_strings[0] in g_weekdays) or (temp_strings[1] in g_weekdays)) and ((temp_strings[1] in g_months) or (temp_strings[2] in g_months)):
            g_failed_test_info_dict["3.timestamp"] = each_line.strip()
            temp_func_list.remove(find_time)    # found timestamp, don't need to look again for it

    return True

def find_node_name(each_line,temp_func_list):
    """
    Find the slave machine where a Jenkins job was executed on.  It will save this
    information in g_failed_test_info_dict.  In addition, it will
    delete this particular function handle off the temp_func_list as we do not need
    to perform this action again.

    Parameters
    ----------

    each_line :  str
        contains a line read in from jenkins console
    temp_func_list :  list of Python function handles
        contains a list of functions that we want to invoke to extract information from
        the Jenkins console text.

    :return: bool to determine if text mining should continue on the jenkins console text
    """
    global g_node_name
    global g_failed_test_info_dict

    if g_node_name in each_line:
        temp_strings = each_line.split()
        [start,found,endstr] = each_line.partition(g_node_name)

        if found:
            temp_strings = endstr.split()
            g_failed_test_info_dict["6.node_name"] = extract_true_string(temp_strings[1])
            temp_func_list.remove(find_node_name)

    return True

def find_git_hash_branch(each_line,temp_func_list):
    """
    Find the git hash and branch info that  a Jenkins job was taken from.  It will save this
    information in g_failed_test_info_dict.  In addition, it will delete this particular
    function handle off the temp_func_list as we do not need to perform this action again.

    Parameters
    ----------

    each_line :  str
        contains a line read in from jenkins console
    temp_func_list :  list of Python function handles
        contains a list of functions that we want to invoke to extract information from
        the Jenkins console text.

    :return: bool to determine if text mining should continue on the jenkins console text
    """
    global g_git_hash_branch
    global g_failed_test_info_dict

    if g_git_hash_branch in each_line:
        [start,found,endstr] = each_line.partition(g_git_hash_branch)
        temp_strings = endstr.strip().split()

        if len(temp_strings) > 1:
            g_failed_test_info_dict["4.git_hash"] = temp_strings[0]
            g_failed_test_info_dict["5.git_branch"] = temp_strings[1]

        temp_func_list.remove(find_git_hash_branch)

    return True

def find_build_timeout(each_line,temp_func_list):
    """
    Find if a Jenkins job has taken too long to finish and was killed.  It will save this
    information in g_failed_test_info_dict.

    Parameters
    ----------

    each_line :  str
        contains a line read in from jenkins console
    temp_func_list :  list of Python function handles
        contains a list of functions that we want to invoke to extract information from
        the Jenkins console text.

    :return: bool to determine if text mining should continue on the jenkins console text
"""
    global g_build_timeout
    global g_failed_test_info_dict
    global g_failure_occurred

    if g_build_timeout in each_line:
        g_failed_test_info_dict["8.build_timeout"] = 'Yes'
        g_failure_occurred = True
        return False    # build timeout was found, no need to continue mining the console text
    else:
        return True

def find_build_failure(each_line,temp_func_list):
    """
    Find if a Jenkins job has failed to build.  It will save this
    information in g_failed_test_info_dict.  In addition, it will delete this particular
    function handle off the temp_func_list as we do not need to perform this action again.

    Parameters
    ----------

    each_line :  str
        contains a line read in from jenkins console
    temp_func_list :  list of Python function handles
        contains a list of functions that we want to invoke to extract information from
        the Jenkins console text.

    :return: bool to determine if text mining should continue on the jenkins console text
    """
    global g_build_success
    global g_build_success_tests
    global g_failed_test_info_dict
    global g_failure_occurred
    global g_build_failed_message

    for ind in range(0,len(g_build_failed_message)):
        if g_build_failed_message[ind] in each_line.lower():
            if ((ind == 0) and (len(g_failed_jobs) > 0)):
                continue
            else:
                g_failure_occurred = True
                g_failed_test_info_dict["7.build_failure"] = 'Yes'
                temp_func_list.remove(find_build_failure)
                return False

    return True

def find_java_filename(each_line,temp_func_list):
    """
    Find if all the java_*_0.out.txt files that were mentioned in the console output.
    It will save this information in g_java_filenames as a list of strings.

    Parameters
    ----------

    each_line :  str
        contains a line read in from jenkins console
    temp_func_list :  list of Python function handles
        contains a list of functions that we want to invoke to extract information from
        the Jenkins console text.

    :return: bool to determine if text mining should continue on the jenkins console text
"""
    global g_before_java_file
    global g_java_filenames

    for each_word in g_before_java_file:
        if (each_word not in each_line):
            return True

    # line contains the name and location of java txt output filename
    temp_strings = each_line.split()
    g_java_filenames.append(temp_strings[-1])

    return True

def find_build_id(each_line,temp_func_list):
    """
    Find the build id of a jenkins job.  It will save this
    information in g_failed_test_info_dict.  In addition, it will delete this particular
    function handle off the temp_func_list as we do not need to perform this action again.

    Parameters
    ----------

    each_line :  str
        contains a line read in from jenkins console
    temp_func_list :  list of Python function handles
        contains a list of functions that we want to invoke to extract information from
        the Jenkins console text.

    :return: bool to determine if text mining should continue on the jenkins console text
    """
    global g_before_java_file
    global g_java_filenames
    global g_build_id_text
    global g_jenkins_url
    global g_output_filename
    global g_output_pickle_filename

    if g_build_id_text in each_line:
        [startStr,found,endStr] = each_line.partition(g_build_id_text)
        g_failed_test_info_dict["2.build_id"] = endStr.strip()

        temp_func_list.remove(find_build_id)
        g_jenkins_url = os.path.join('http://',g_jenkins_url,'view',g_view_name,'job',g_failed_test_info_dict["1.jobName"],g_failed_test_info_dict["2.build_id"],'artifact')

    return True

# global list of all functions that are performed extracting new build information.
g_build_func_list = [find_time,find_node_name,find_build_id,find_git_hash_branch,find_build_timeout,find_build_failure,find_java_filename]

def update_test_dict(each_line):
    """
    Extract unit tests information from the jenkins job console output.  It will save this
    information in g_failed_jobs list and setup a place holder for saving the bad java
    messages/message types in g_failed_job_java_messages, g_failed_job_java_message_types.

    Parameters
    ----------

    each_line :  str
        contains a line read in from jenkins console

    :return: bool to determine if text mining should continue on the jenkins console text
    """
    global g_ignore_test_names
    global g_failed_jobs
    global g_failed_job_java_messages
    global g_failure_occurred

    temp_strings = each_line.split()

    if (len(temp_strings) >= 5) and ("FAIL" in each_line) and ("FAILURE" not in each_line):   # found failed test

        test_name = temp_strings[-2]
        g_failed_jobs.append(test_name)
        g_failed_job_java_messages.append([]) # insert empty java messages for now
        g_failed_job_java_message_types.append([])
        g_failure_occurred = True

    return True

'''
This function is written to extract the error messages from console output and
possible from the java_*_*.out to warn users of potentially bad runs.

'''
def extract_test_results():
    """
    Extract error messages from jenkins console output and from java_*_0.out.txt if they exist to
    warn users of potentially bad tests.  In addition, it will grab the following info about the jenkins
    job from the console output and saved it into g_failed_test_info_dict:
    1.jobName
    2.build_id
    3.timestamp
    4.git_hash
    5.git_branch
    6.node_name
    7.build_failure
    8.build_timeout
    9.general_bad_java_messages
    failed_tests_info *********: list of failed tests and their associated bad java messages
    passed_tests_info *********: list of passed tests and their associated bad java messages

    This is achieved by calling various functions.

    :return: none
    """
    global g_test_root_dir
    global g_temp_filename
    global g_output_filename
    global g_build_func_list

    temp_func_list = copy.copy(g_build_func_list)

    if os.path.isfile(g_temp_filename):
        console_file = open(g_temp_filename,'r')  # open temp file that stored jenkins job console output

        for each_line in console_file:  # go through each line of console output to extract build ID, data/time ...
            each_line.strip()

            for each_function in temp_func_list:
                to_continue = perform(each_function,each_line,temp_func_list)

                if not(to_continue):
                    break

            if not(to_continue):    # something like build failure or built time out has occurred.  Stop
                break
            else:
                update_test_dict(each_line)  # update the test_dict with new tests if found

        console_file.close()
    else:
        print "Error: console output file "+g_temp_filename + " does not exist."
        sys.exit(1)

'''
This function is written to extract the console output that has already been stored
in a text file in a remote place and saved it in a local directory that we have accessed
to.  We want to be able to read in the local text file and proces it.
'''
def get_console_out(url_string):
    """
    Grab the console output from Jenkins and save the content into a temp file
     (g_temp_filename).

    Parameters
    ----------
    url_string :  str
        contains information on the jenkins job whose console output we are interested in.

    :return: none
    """
    global g_temp_filename

    full_command = 'curl ' + url_string + ' > ' + g_temp_filename
    subprocess.call(full_command,shell=True)

def extract_job_build_url(url_string):
    """
    From user input, grab the jenkins job name and saved it in g_failed_test_info_dict.
    In addition, it will grab the jenkins url and the view name into g_jenkins_url, and
    g_view_name.

    Parameters
    ----------
    url_string :  str
        contains information on the jenkins job whose console output we are interested in.

    :return: none
    """
    global g_failed_test_info_dict
    global g_jenkins_url
    global g_view_name

    tempString = url_string.strip('/').split('/')

    if len(tempString) < 6:
        print "Illegal URL resource address.\n"
        sys.exit(1)

    g_failed_test_info_dict["1.jobName"] = tempString[6]

    g_jenkins_url = tempString[2]
    g_view_name = tempString[4]

def grab_java_message():
    """scan through the java output text and extract the bad java messages that may or may not happened when
    unit tests are run.  It will not record any bad java messages that are stored in g_ok_java_messages.

    :return: none
    """

    global g_temp_filename
    global g_current_testname
    global g_java_start_text
    global g_ok_java_messages
    global g_java_general_bad_messages  # store bad java messages not associated with running a unit test
    global g_java_general_bad_message_types
    global g_failure_occurred
    global g_java_message_type
    global g_all_java_message_type
    global g_toContinue

    java_messages = []      # store all bad java messages associated with running a unit test
    java_message_types = [] # store all bad java message types associated with running a unit test

    if os.path.isfile(g_temp_filename): # open temp file containing content of some java_*_0.out.txt
        java_file = open(g_temp_filename,'r')

        g_toContinue = False    # denote if a multi-line message starts

        tempMessage = ""
        messageType = ""

        for each_line in java_file:

            if (g_java_start_text in each_line):
                startStr,found,endStr = each_line.partition(g_java_start_text)

                if len(found) > 0:
                    if len(g_current_testname) > 0: # a new unit test is being started.  Save old info and move on
                        associate_test_with_java(g_current_testname,java_messages,java_message_types)

                    g_current_testname = endStr.strip() # record the test name

                    java_messages = []
                    java_message_types = []

            temp_strings = each_line.strip().split()

            if (len(temp_strings) >= 6) and (temp_strings[5] in g_all_java_message_type):
                if g_toContinue == True:    # at the end of last message fragment
                    addJavaMessages(tempMessage,messageType,java_messages,java_message_types)
                    tempMessage = ""
                    messageType = ""

                # start of new message fragment
                g_toContinue = False
            else: # non standard output.  Continuation of last java message, add it to bad java message list
                if g_toContinue:

                    tempMessage += each_line    # add more java message here
                    # if len(g_current_testname) == 0:
                    #     addJavaMessages(each_line.strip(),"",java_messages,java_message_types)
                    # else:
                    #     addJavaMessages(each_line.strip(),"",java_messages,java_message_types)

            if ((len(temp_strings) > 5) and (temp_strings[5] in g_java_message_type)):  # find a bad java message
                startStr,found,endStr = each_line.partition(temp_strings[5])    # can be WARN,ERRR,FATAL,TRACE

                if found and (len(endStr.strip()) > 0):
                    tempMessage += endStr
                    messageType = temp_strings[5]
#                    if (tempMessage not in g_ok_java_messages["general"]):  # found new bad messages that cannot be ignored
                    g_toContinue = True

                        # add tempMessage to bad java message list
#                        addJavaMessages(tempMessage,temp_strings[5],java_messages,java_message_types)
        java_file.close()

def addJavaMessages(tempMessage,messageType,java_messages,java_message_types):
    """
    Insert Java messages into java_messages and java_message_types if they are associated
    with a unit test or into g_java_general_bad_messages/g_java_general_bad_message_types
    otherwise.

    Parameters
    ----------
    tempMessage :  str
        contains the bad java messages
    messageType :  str
        contains the bad java message type
    java_messages : list of str
        contains the bad java message list associated with a unit test
    java_message_tuypes :  list of str
        contains the bad java message type list associated with a unit test.

    :return: none
    """
    global g_current_testname
    global g_java_general_bad_messages
    global g_java_general_bad_message_types
    global g_failure_occurred

    tempMess = tempMessage.strip()

    if (tempMess not in g_ok_java_messages["general"]):
        if (len(g_current_testname) == 0):    # java message not associated with any test name
            g_java_general_bad_messages.append(tempMess)
            g_java_general_bad_message_types.append(messageType)
            g_failure_occurred = True
        else:   # java message found during a test
            write_test = False  # do not include java message for test if False
            if (g_current_testname in g_ok_java_messages.keys()) and (tempMess in g_ok_java_messages[g_current_testname]): # test name associated with ignored Java messages
                write_test = False
            else:   # not java ignored message for current unit test
                write_test = True

            if write_test:
                java_messages.append(tempMess)
                java_message_types.append(messageType)
                g_failure_occurred = True

def associate_test_with_java(testname,java_message,java_message_type):
    """
    When a new unit test is started as indicated in the java_*_0.out.txt file,
    update the data structures that are keeping track of unit tests being run and
    bad java messages/messages types associated with each unit test.  Since a new
    unit test is being started, save all the bad java messages associated with
    the previous unit test and start a new set for the new unit test.

    Parameters
    ----------
    testname :  str
        previous unit test testname
    java_message :  list of str
        bad java messages associated with testname
    java_message_type :  list of str
        bad java message types associated with testname

    :return :  none
    """
    global g_failed_jobs                # record job names of failed jobs
    global g_failed_job_java_messages   # record failed job java message
    global g_failed_job_java_message_types

    global g_success_jobs               # record job names of passed jobs
    global g_success_job_java_messages  # record of successful jobs bad java messages
    global g_success_job_java_message_types

    if len(java_message) > 0:
        if (testname in g_failed_jobs):
            message_index = g_failed_jobs.index(testname)
            g_failed_job_java_messages[message_index] = java_message
            g_failed_job_java_message_types[message_index] = java_message_type
        else:   # job has been sucessfully executed but something still has gone wrong
            g_success_jobs.append(testname)
            g_success_job_java_messages.append(java_message)
            g_success_job_java_message_types.append(java_message_type)

def extract_java_messages():
    """
    loop through java_*_0.out.txt and extract potentially dangerous WARN/ERRR/FATAL
    messages associated with a test.  The test may even pass but something terrible
    has actually happened.

    :return: none
    """
    global g_jenkins_url
    global g_failed_test_info_dict
    global g_java_filenames

    global g_failed_jobs  # record job names of failed jobs
    global g_failed_job_java_messages # record failed job java message
    global g_failed_job_java_message_types

    global g_success_jobs # record job names of passed jobs
    global g_success_job_java_messages # record of successful jobs bad java messages
    global g_success_job_java_message_types

    global g_java_general_bad_messages  # store java error messages when no job is running
    global g_java_general_bad_message_types # store java error message types when no job is running.

    if (len(g_failed_jobs) > 0):  # artifacts available only during failure of some sort
        for fname in g_java_filenames:  # grab java message from each java_*_*_.out file
            temp_strings = fname.split('/')

            start_url = g_jenkins_url

            for windex in range(6,len(temp_strings)):
                start_url = os.path.join(start_url,temp_strings[windex])
            try:    # first java file path is different.  Can ignore it.
                get_console_out(start_url)  # get java text and save it in local directory for processing
                grab_java_message()         # actually process the java text output and see if we found offensive stuff
            except:
                pass

    # build up the dict structure that we are storing our data in
    if len(g_failed_jobs) > 0:
        g_failed_test_info_dict["failed_tests_info *********"] = [g_failed_jobs,g_failed_job_java_messages,g_failed_job_java_message_types]
    if len(g_success_jobs) > 0:
        g_failed_test_info_dict["passed_tests_info *********"] = [g_success_jobs,g_success_job_java_messages,g_success_job_java_message_types]

    if len(g_java_general_bad_messages) > 0:
        g_failed_test_info_dict["9.general_bad_java_messages"] = [g_java_general_bad_messages,g_java_general_bad_message_types]

def save_dict():
    """
    Save the log scraping results into logs denoted by g_output_filename_failed_tests and
    g_output_filename_passed_tests.

    :return: none
    """

    global g_test_root_dir
    global g_output_filename_failed_tests
    global g_output_filename_passed_tests
    global g_output_pickle_filename
    global g_failed_test_info_dict

    # some build can fail really early that no buid id info is stored in the console text.
    if "2.build_id" not in g_failed_test_info_dict.keys():
        g_failed_test_info_dict["2.build_id"] = "unknown"

    build_id = g_failed_test_info_dict["2.build_id"]

    g_output_filename_failed_tests = g_output_filename_failed_tests+'_build_'+build_id+'_failed_tests.log'
    g_output_filename_passed_tests = g_output_filename_passed_tests+'_build_'+build_id+'_passed_tests.log'
    g_output_pickle_filename = g_output_pickle_filename+'_build_'+build_id+'.pickle'

    allKeys = sorted(g_failed_test_info_dict.keys())

    # write out the jenkins job info into log files.
    with open(g_output_pickle_filename,'wb') as test_file:
        pickle.dump(g_failed_test_info_dict,test_file)

    # write out the failure report as text into a text file
    text_file_failed_tests = open(g_output_filename_failed_tests,'w')
    text_file_passed_tests = None
    allKeys = sorted(g_failed_test_info_dict.keys())
    write_passed_tests = False

    if ("passed_tests_info *********" in allKeys):
        text_file_passed_tests = open(g_output_filename_passed_tests,'w')
        write_passed_tests = True

    for keyName in allKeys:
        val = g_failed_test_info_dict[keyName]
        if isinstance(val,list):    # writing one of the job lists
            if (len(val) == 3):     # it is a message for a test
                if keyName == "failed_tests_info *********":
                    write_test_java_message(keyName,val,text_file_failed_tests)

                if keyName == "passed_tests_info *********":
                    write_test_java_message(keyName,val,text_file_passed_tests)
            elif (len(val) == 2):                   # it is a general bad java message
                write_java_message(keyName,val,text_file_failed_tests)
                if write_passed_tests:
                    write_java_message(keyName,val,text_file_passed_tests)
        else:
            write_general_build_message(keyName,val,text_file_failed_tests)
            if write_passed_tests:
                write_general_build_message(keyName,val,text_file_passed_tests)

    text_file_failed_tests.close()
    if write_passed_tests:
        text_file_passed_tests.close()

def write_general_build_message(key,val,text_file):
    """
    Write key/value into log file when the value is a string and not a list.

    Parameters
    ----------
    key :  str
        key value in g_failed_test_info_dict
    value :  str
        corresponding value associated with the key in key
    text_file : file handle
        file handle of log file to write the info to.

    :return: none
    """
    text_file.write(key+": ")
    text_file.write(val)
    text_file.write('\n\n')

def write_test_java_message(key,val,text_file):
    """
   Write key/value into log file when the value is a list of strings
   or even a list of list of string.  These lists are associated with
   unit tests that are executed in the jenkins job.

    Parameters
    ----------
    key :  str
        key value in g_failed_test_info_dict
    value :  list of str or list of list of str
        corresponding value associated with the key in key
    text_file : file handle
        file handle of log file to write the info to.

   :return: none
   """
    global g_failed_jobs

    text_file.write(key)
    text_file.write('\n')

    # val is a tuple of 3 tuples
    for index in range(len(val[0])):

        if ((val[0][index] in g_failed_jobs) or ((val[0][index] not in g_failed_jobs) and (len(val[1][index]) > 0))):
            text_file.write("\nTest Name: ")
            text_file.write(val[0][index])
            text_file.write('\n')

        if (len(val[1][index]) > 0) and (len(val) >= 3):
            text_file.write("Java Message Type and Message: \n")
            for eleIndex in range(len(val[1][index])):
                text_file.write(val[2][index][eleIndex]+" ")
                text_file.write(val[1][index][eleIndex])
                text_file.write('\n\n')

    text_file.write('\n')
    text_file.write('\n')

def update_summary_file():
    """
    Concatecate all log file into a summary text file to be sent to users
    at the end of a daily log scraping.

    :return: none
    """
    global g_summary_text_filename
    global g_output_filename_failed_tests
    global g_output_filename_passed_tests

    with open(g_summary_text_filename,'a') as tempfile:
        write_file_content(tempfile,g_output_filename_failed_tests)
        write_file_content(tempfile,g_output_filename_passed_tests)

def write_file_content(fhandle,file2read):
    """
    Write one log file into the summary text file.

    Parameters
    ----------
    fhandle :  Python file handle
        file handle to the summary text file
    file2read : Python file handle
        file handle to log file where we want to add its content to the summary text file.

    :return: none
    """
    if os.path.isfile(file2read):

        # write summary of failed tests logs
        with open(file2read,'r') as tfile:
            fhandle.write('============ Content of '+ file2read)
            fhandle.write('\n')
            fhandle.write(tfile.read())
            fhandle.write('\n\n')

def write_java_message(key,val,text_file):
    """
    Loop through all java messages that are not associated with a unit test and
    write them into a log file.

    Parameters
    ----------
    key :  str
        9.general_bad_java_messages
    val : list of list of str
        contains the bad java messages and the message types.

    :return: none
    """

    text_file.write(key)
    text_file.write('\n')

    if (len(val[0]) > 0) and (len(val) >= 3):
        for index in range(len(val[0])):
            text_file.write("Java Message Type: ")
            text_file.write(val[1][index])
            text_file.write('\n')

            text_file.write("Java Message: ")

            for jmess in val[2][index]:
                text_file.write(jmess)
                text_file.write('\n')

        text_file.write('\n \n')

def load_java_messages_to_ignore():
    """
    Load in pickle file that contains dict structure with bad java messages to ignore per unit test
    or for all cases.  The ignored bad java info is stored in g_ok_java_messages dict.

    :return:
    """
    global g_ok_java_messages
    global g_java_message_pickle_filename

    if os.path.isfile(g_java_message_pickle_filename):
        with open(g_java_message_pickle_filename,'rb') as tfile:
            g_ok_java_messages = pickle.load(tfile)
    else:
        g_ok_java_messages["general"] = []

def main(argv):
    """
    Main program.

    @return: none
    """
    global g_script_name
    global g_test_root_dir
    global g_temp_filename
    global g_output_filename_failed_tests
    global g_output_filename_passed_tests
    global g_output_pickle_filename
    global g_failure_occurred
    global g_failed_test_info_dict
    global g_java_message_pickle_filename
    global g_summary_text_filename

    if len(argv) < 3:
        print "Must resource url like http://mr-0xa1:8080/view/wendy_jenkins/job/h2o_regression_pyunit_medium_large/lastBuild/consoleFull, filename of summary text, filename (optional ending in .pickle) to retrieve Java error messages to exclude.\n"
        sys.exit(1)
    else:   # we may be in business
        g_script_name = os.path.basename(argv[0])   # get name of script being run.
        resource_url = argv[1]

        g_temp_filename = os.path.join(g_test_root_dir,'tempText')
        g_summary_text_filename = os.path.join(g_test_root_dir,argv[2])

        if len(argv) == 4:
            g_java_message_pickle_filename  = argv[3]

        get_console_out(resource_url)       # save remote console output in local directory
        extract_job_build_url(resource_url) # extract the job name of build id for identification purposes

        log_filename = g_failed_test_info_dict["1.jobName"]
        log_pickle_filename = g_failed_test_info_dict["1.jobName"]

        # pickle file that store bad Java messages that we can ignore.
        g_java_message_pickle_filename = os.path.join(g_test_root_dir,g_java_message_pickle_filename)
        g_output_filename_failed_tests = os.path.join(g_test_root_dir,log_filename)
        g_output_filename_passed_tests = os.path.join(g_test_root_dir,log_filename)
        g_output_pickle_filename = os.path.join(g_test_root_dir,log_pickle_filename)

        load_java_messages_to_ignore()          # load in bad java messages to ignore and store in g_ok_java_messages
        extract_test_results()      # grab the console text and stored the failed tests.
        extract_java_messages()     # grab dangerous java messages that we found for the various unit tests
        if ((len(g_failed_jobs) > 0) or (g_failed_test_info_dict["7.build_failure"]=='Yes')):
            g_failure_occurred = True

        if g_failure_occurred:
            save_dict() # save the dict structure in a pickle file and a text file when failure is detected
            update_summary_file()   # join together all log files into one giant summary text.

            # output this info to console to form the list of failed jenkins jobs.
            print g_failed_test_info_dict["1.jobName"]+' build '+g_failed_test_info_dict["2.build_id"]+','
        else:
            print ""

if __name__ == "__main__":
    main(sys.argv)


#!/usr/bin/env python
# -*- encoding: utf-8 -*-
from __future__ import unicode_literals
import bindings as bi
import sys
PY3 = sys.version_info[0] == 3
str_type = str if PY3 else (str, unicode)

# We specify these not as real types, but as parameter annotations in the docstrings
class PythonTypeTranslatorForCheck(bi.TypeTranslator):
    def __init__(self):
        super(PythonTypeTranslatorForCheck, self).__init__()
        self.types["byte"] = "int"
        self.types["short"] = "int"
        self.types["long"] = "int"
        self.types["double"] = "numeric"
        self.types["string"] = "str"
        self.types["boolean"] = "bool"
        self.types["Polymorphic"] = "object"
        self.types["Object"] = "object"
        self.types["VecSpecifier"] = "str"
        self.types["StringPair"] = "tuple"
        self.types["KeyValue"] = "dict"
        self.make_array = lambda vtype: "dict" if vtype == "dict" else "[%s]" % vtype
        self.make_array2 = lambda vtype: "[[%s]]" % vtype
        self.make_map = lambda ktype, vtype: "{%s: %s}" % (ktype, vtype)
        self.make_key = lambda itype, schema: "H2OFrame" if schema == "Key<Frame>" else "str"
        self.make_enum = lambda schema, values: \
            "Enum(%s)" % ", ".join(stringify(v) for v in values) if values else schema

type_adapter1 = PythonTypeTranslatorForCheck()
def translate_type_for_check(h2o_type, values=None):
    schema = h2o_type.replace("[]", "")
    return type_adapter1.translate(h2o_type, schema, values)

# We specify these not as real types, but as parameter annotations in the docstrings
class PythonTypeTranslatorForDoc(bi.TypeTranslator):
    def __init__(self):
        super(PythonTypeTranslatorForDoc, self).__init__()
        self.types["byte"] = "int"
        self.types["short"] = "int"
        self.types["long"] = "int"
        self.types["double"] = "float"
        self.types["string"] = "str"
        self.types["boolean"] = "bool"
        self.types["Polymorphic"] = "object"
        self.types["Object"] = "object"
        self.types["VecSpecifier"] = "str"
        self.types["StringPair"] = "tuple"
        self.types["KeyValue"] = "dict"
        self.make_array = lambda vtype: "dict" if vtype == "dict" else "List[%s]" % vtype
        self.make_array2 = lambda vtype: "List[List[%s]]" % vtype
        self.make_map = lambda ktype, vtype: "Dict[%s, %s]" % (ktype, vtype)
        self.make_key = lambda itype, schema: "H2OFrame" if schema == "Key<Frame>" else "str"
        self.make_enum = lambda schema, values: \
            "Enum[%s]" % ", ".join(stringify(v) for v in values) if values else schema

type_adapter2 = PythonTypeTranslatorForDoc()
def translate_type_for_doc(h2o_type, values=None):
    schema = h2o_type.replace("[]", "")
    return type_adapter2.translate(h2o_type, schema, values)

def normalize_enum_constant(s):
    """Return enum constant `s` converted to a canonical snake-case."""
    if s.islower(): return s
    if s.isupper(): return s.lower()
    return "".join(ch if ch.islower() else "_" + ch.lower() for ch in s).strip("_")

def stringify(v):
    if v == "Infinity": return u'∞'
    if isinstance(v, str_type): return '"%s"' % v
    if isinstance(v, float): return '%.10g' % v
    return str(v)

def reindent_block(string, new_indent):
    if not string: return ""
    add_indent = " " * new_indent
    lines = string.split("\n")
    if len(lines) == 1:
        return lines[0].strip()
    line0_indent = len(lines[0]) - len(lines[0].lstrip())
    line1_indent = len(lines[1]) - len(lines[1].lstrip())
    remove_indent = max(line0_indent, line1_indent)
    out = ""
    for line in lines:
        dedented_line = line.lstrip()
        if dedented_line:
            extra_indent = " " * (len(line) - len(dedented_line) - remove_indent)
            out += add_indent + extra_indent + dedented_line + "\n"
        else:
            out += "\n"
    return out.strip()

# This is the list of all reserved keywords in Python. It is a syntax error to use any of them as an object's property.
# Currently we have only "lambda" in GLM as a violator, but keeping the whole list here just to be future-proof.
# For all such violating properties, we name the accessor with an underscore at the end (eg. lambda_), but at the same
# time keep the actual name in self.params (i.e. model.lambda_ ≡ model.params["lambda"]).
reserved_words = {
    "and", "del", "from", "not", "while", "as", "elif", "global", "or", "with", "assert", "else", "if", "pass",
    "yield", "break", "except", "import", "print", "class", "exec", "in", "raise", "continue", "finally", "is",
    "return", "def", "for", "lambda", "try"
}

# ----------------------------------------------------------------------------------------------------------------------
#   Generate per-model classes
# ----------------------------------------------------------------------------------------------------------------------
def gen_module(schema, algo):
    classname = algo_to_classname(algo)
    extra_imports = extra_imports_for(algo)
    help_preamble = help_preamble_for(algo)
    help_epilogue = help_epilogue_for(algo)
    init_extra = init_extra_for(algo)
    class_extra = class_extra_for(algo)
    module_extra = module_extra_for(algo)

    param_names = []
    for param in schema["parameters"]:
        assert (param["type"][:4] == "enum") == bool(param["values"]), "Values are expected for enum types only"
        if param["values"]:
            enum_values = [normalize_enum_constant(p) for p in param["values"]]
            if param["default_value"]:
                param["default_value"] = normalize_enum_constant(param["default_value"])
        else:
            enum_values = None
        pname = param["name"]
        if (pname==u'distribution') and (not(algo==u'glm') and not(algo==u'gbm')):    # quasibinomial only in glm, gbm
            enum_values.remove(u'quasibinomial')
        if (pname==u'distribution') and (not(algo==u'glm')):    # ordinal only in glm
            enum_values.remove(u'ordinal')
        if pname in reserved_words: pname += "_"
        param_names.append(pname)
        param["pname"] = pname
        param["ptype"] = translate_type_for_check(param["type"], enum_values)
        param["dtype"] = translate_type_for_doc(param["type"], enum_values)

    yield "#!/usr/bin/env python"
    yield "# -*- encoding: utf-8 -*-"
    yield "#"
    yield "# This file is auto-generated by h2o-3/h2o-bindings/bin/gen_python.py"
    yield "# Copyright 2016 H2O.ai;  Apache License Version 2.0 (see LICENSE for details)"
    yield "#"
    yield "from __future__ import absolute_import, division, print_function, unicode_literals"
    yield ""
    yield "from h2o.estimators.estimator_base import H2OEstimator"
    yield "from h2o.exceptions import H2OValueError"
    yield "from h2o.frame import H2OFrame"
    if classname == "H2OStackedEnsembleEstimator":
        yield "from h2o.utils.typechecks import assert_is_type, Enum, numeric, is_type"
        yield "import json"
        yield "import ast"
    else:
        yield "from h2o.utils.typechecks import assert_is_type, Enum, numeric"
    if extra_imports:
        yield reindent_block(extra_imports, 0) + ""
    yield ""
    yield ""
    yield "class %s(H2OEstimator):" % classname
    yield "    \"\"\""
    yield "    " + schema["algo_full_name"]
    yield ""
    if help_preamble:
        yield "    %s" % reindent_block(help_preamble, 4)
    if help_epilogue:
        yield ""
        yield "    %s" % reindent_block(help_epilogue, 4)
    yield "    \"\"\""
    yield ""
    yield '    algo = "%s"' % algo
    yield ""
    yield "    def __init__(self, **kwargs):"
    yield "        super(%s, self).__init__()" % classname
    yield "        self._parms = {}"
    yield "        names_list = {%s}" % bi.wrap(", ".join('"%s"' % p for p in param_names),
                                                indent=(" " * 22), indent_first=False)
    yield '        if "Lambda" in kwargs: kwargs["lambda_"] = kwargs.pop("Lambda")'
    yield "        for pname, pvalue in kwargs.items():"
    yield "            if pname == 'model_id':"
    yield "                self._id = pvalue"
    yield '                self._parms["model_id"] = pvalue'
    if algo == 'word2vec':
        yield '            elif pname == \'pre_trained\':'
        yield '                setattr(self, pname, pvalue)'
        yield '                self._determine_vec_size();'
        yield '                setattr(self, \'vec_size\', self.vec_size)'
    yield "            elif pname in names_list:"
    yield "                # Using setattr(...) will invoke type-checking of the arguments"
    yield "                setattr(self, pname, pvalue)"
    yield "            else:"
    yield '                raise H2OValueError("Unknown parameter %s = %r" % (pname, pvalue))'
    if init_extra:
        yield "        " + reindent_block(init_extra, 8)
    yield ""
    for param in schema["parameters"]:
        pname = param["pname"]
        ptype = param["ptype"]
        if pname == "model_id": continue  # The getter is already defined in ModelBase
        sname = pname[:-1] if pname[-1] == '_' else pname

        if param["dtype"].startswith("Enum"):
            vals = param["dtype"][5:-1].split(", ")
            extrahelp = "One of: " + ", ".join("``%s``" % v for v in vals)
        else:
            if pname == "metalearner_params":
                extrahelp = "Type: ``dict``"
            else:
                extrahelp = "Type: ``%s``" % param["dtype"]
        if param["default_value"] is None:
            extrahelp += "."
        else:
            if pname == "metalearner_params":
                extrahelp += "  (default: ``None``)."
            else:
                extrahelp += "  (default: ``%s``)." % stringify(param["default_value"])

        if (pname == "offset_column" or pname == "distribution") and algo == "drf":
            yield "    @property"
            yield "    def %s(self):" % pname
            yield '        """'
            yield "        [Deprecated] %s" % bi.wrap(param["help"], indent=(" " * 8), indent_first=False)
            yield ""
            yield "        %s" % bi.wrap(extrahelp, indent=(" " * 8), indent_first=False)
        else:
            yield "    @property"
            yield "    def %s(self):" % pname
            yield '        """'
            yield "        %s" % bi.wrap(param["help"], indent=(" " * 8), indent_first=False)
            yield ""
            yield "        %s" % bi.wrap(extrahelp, indent=(" " * 8), indent_first=False)
        if pname == "metalearner_params":
            yield "        Example: metalearner_gbm_params = {'max_depth': 2, 'col_sample_rate': 0.3}"
        yield '        """'
        if pname != "metalearner_params":
            yield "        return self._parms.get(\"%s\")" % sname
        else:
            yield "        if self._parms.get(\"%s\") != None:" % sname
            yield "            metalearner_params_dict =  ast.literal_eval(self._parms.get(\"%s\"))" % sname
            yield "            for k in metalearner_params_dict:"
            yield "                if len(metalearner_params_dict[k]) == 1: #single parameter"
            yield "                    metalearner_params_dict[k] = metalearner_params_dict[k][0]"
            yield "            return metalearner_params_dict"
            yield "        else:"
            yield "            return self._parms.get(\"%s\")" % sname
        yield ""
        yield "    @%s.setter" % pname
        yield "    def %s(self, %s):" % (pname, pname)
        if pname in {"initial_weights", "initial_biases"}:
            yield "        assert_is_type(%s, None, [H2OFrame, None])" % pname
        elif pname in {"alpha", "lambda_"} and ptype == "[numeric]":
            # For `alpha` and `lambda` the server reports type float[], while in practice simple floats are also ok
            yield "        assert_is_type(%s, None, numeric, [numeric])" % pname
        elif pname in {"checkpoint", "pretrained_autoencoder"}:
            yield "        assert_is_type(%s, None, str, H2OEstimator)" % pname
        elif pname in {"base_models"}:
            yield "         if is_type(base_models,[H2OEstimator]):"
            yield      "            %s = [b.model_id for b in %s]" % (pname,pname)
            yield      "            self._parms[\"%s\"] = %s" % (sname, pname)
            yield "         else:"
            yield "            assert_is_type(%s, None, %s)" % (pname, ptype)
            yield "            self._parms[\"%s\"] = %s" % (sname, pname)
        elif pname in {"metalearner_params"}:
            yield "        assert_is_type(%s, None, %s)" % (pname, "dict")
            yield '        if %s is not None and %s != "":' % (pname, pname)
            yield "            for k in %s:" % (pname)
            yield '                if ("[" and "]") not in str(metalearner_params[k]):'
            yield "                    metalearner_params[k]=[metalearner_params[k]]"
            yield "            self._parms[\"%s\"] = str(json.dumps(%s))" % (sname, pname)
            yield "        else:"
            yield "            self._parms[\"%s\"] = None" % (sname)
        else:
            yield "        assert_is_type(%s, None, %s)" % (pname, ptype)
        if pname not in {"base_models", "metalearner_params"}:
            yield "        self._parms[\"%s\"] = %s" % (sname, pname)
        yield ""
        yield ""
    if class_extra:
        yield ""
        yield "    " + reindent_block(class_extra, 4)
    if module_extra:
        yield ""
        yield reindent_block(module_extra, 0)

def algo_to_classname(algo):
    if algo == "deeplearning": return "H2ODeepLearningEstimator"
    if algo == "deepwater": return "H2ODeepWaterEstimator"
    if algo == "xgboost": return "H2OXGBoostEstimator"
    if algo == "gbm": return "H2OGradientBoostingEstimator"
    if algo == "glm": return "H2OGeneralizedLinearEstimator"
    if algo == "glrm": return "H2OGeneralizedLowRankEstimator"
    if algo == "kmeans": return "H2OKMeansEstimator"
    if algo == "naivebayes": return "H2ONaiveBayesEstimator"
    if algo == "drf": return "H2ORandomForestEstimator"
    if algo == "svd": return "H2OSingularValueDecompositionEstimator"
    if algo == "pca": return "H2OPrincipalComponentAnalysisEstimator"
    if algo == "stackedensemble": return "H2OStackedEnsembleEstimator"
    if algo == "isolationforest": return "H2OIsolationForestEstimator"
    return "H2O" + algo.capitalize() + "Estimator"

def extra_imports_for(algo):
    if algo == "glm" or algo == "deepwater" or algo == "xgboost":
        return "import h2o"

def help_preamble_for(algo):
    if algo == "deeplearning":
        return """
            Build a Deep Neural Network model using CPUs
            Builds a feed-forward multilayer artificial neural network on an H2OFrame"""
    if algo == "deepwater":
        return """
            Build a Deep Learning model using multiple native GPU backends
            Builds a deep neural network on an H2OFrame containing various data sources"""
    if algo == "kmeans":
        return """Performs k-means clustering on an H2O dataset."""
    if algo == "glrm":
        return """Builds a generalized low rank model of a H2O dataset."""
    if algo == "glm":
        return """
            Fits a generalized linear model, specified by a response variable, a set of predictors, and a
            description of the error distribution."""
    if algo == "gbm":
        return """
            Builds gradient boosted trees on a parsed data set, for regression or classification.
            The default distribution function will guess the model type based on the response column type.
            Otherwise, the response column must be an enum for "bernoulli" or "multinomial", and numeric
            for all other distributions."""
    if algo == "xgboost":
        return """Builds a eXtreme Gradient Boosting model using the native XGBoost backend."""
    if algo == "naivebayes":
        return """
            The naive Bayes classifier assumes independence between predictor variables
            conditional on the response, and a Gaussian distribution of numeric predictors with
            mean and standard deviation computed from the training dataset. When building a naive
            Bayes classifier, every row in the training dataset that contains at least one NA will
            be skipped completely. If the test dataset has missing values, then those predictors
            are omitted in the probability calculation during prediction."""
    if algo == "stackedensemble":
        return """
            Builds a stacked ensemble (aka "super learner") machine learning method that uses two
            or more H2O learning algorithms to improve predictive performance. It is a loss-based
            supervised learning method that finds the optimal combination of a collection of prediction
            algorithms.This method supports regression and binary classification. """
    if algo == "isolationforest":
        return """
            Builds an Isolation Forest model. Isolation Forest algorithm samples the training frame
            and in each iteration builds a tree that partitions the space of the sample observations until
            it isolates each observation. Length of the path from root to a leaf node of the resulting tree
            is used to calculate the anomaly score. Anomalies are easier to isolate and their average
            tree path is expected to be shorter than paths of regular observations.
        """

def help_epilogue_for(algo):
    if algo == "deeplearning":
        return """Examples
                       --------
                         >>> import h2o
                         >>> from h2o.estimators.deeplearning import H2ODeepLearningEstimator
                         >>> h2o.connect()
                         >>> rows = [[1,2,3,4,0], [2,1,2,4,1], [2,1,4,2,1], [0,1,2,34,1], [2,3,4,1,0]] * 50
                         >>> fr = h2o.H2OFrame(rows)
                         >>> fr[4] = fr[4].asfactor()
                         >>> model = H2ODeepLearningEstimator()
                         >>> model.train(x=range(4), y=4, training_frame=fr)"""
    if algo == "stackedensemble":
        return """Examples
                       --------
                         >>> import h2o
                         >>> h2o.init()
                         >>> from h2o.estimators.random_forest import H2ORandomForestEstimator
                         >>> from h2o.estimators.gbm import H2OGradientBoostingEstimator
                         >>> from h2o.estimators.stackedensemble import H2OStackedEnsembleEstimator
                         >>> col_types = ["numeric", "numeric", "numeric", "enum", "enum", "numeric", "numeric", "numeric", "numeric"]
                         >>> data = h2o.import_file("http://h2o-public-test-data.s3.amazonaws.com/smalldata/prostate/prostate.csv", col_types=col_types)
                         >>> train, test = data.split_frame(ratios=[.8], seed=1)
                         >>> x = ["CAPSULE","GLEASON","RACE","DPROS","DCAPS","PSA","VOL"]
                         >>> y = "AGE"
                         >>> nfolds = 5
                         >>> my_gbm = H2OGradientBoostingEstimator(nfolds=nfolds, fold_assignment="Modulo", keep_cross_validation_predictions=True)
                         >>> my_gbm.train(x=x, y=y, training_frame=train)
                         >>> my_rf = H2ORandomForestEstimator(nfolds=nfolds, fold_assignment="Modulo", keep_cross_validation_predictions=True)
                         >>> my_rf.train(x=x, y=y, training_frame=train)
                         >>> stack = H2OStackedEnsembleEstimator(model_id="my_ensemble", training_frame=train, validation_frame=test, base_models=[my_gbm.model_id, my_rf.model_id])
                         >>> stack.train(x=x, y=y, training_frame=train, validation_frame=test)
                         >>> stack.model_performance()"""
    if algo == "glm":
        return """
            A subclass of :class:`ModelBase` is returned. The specific subclass depends on the machine learning task
            at hand (if it's binomial classification, then an H2OBinomialModel is returned, if it's regression then a
            H2ORegressionModel is returned). The default print-out of the models is shown, but further GLM-specific
            information can be queried out of the object. Upon completion of the GLM, the resulting object has
            coefficients, normalized coefficients, residual/null deviance, aic, and a host of model metrics including
            MSE, AUC (for logistic regression), degrees of freedom, and confusion matrices."""

def init_extra_for(algo):
    if algo == "deeplearning":
        return "if isinstance(self, H2OAutoEncoderEstimator): self._parms['autoencoder'] = True"
    if algo == "glrm":
        return """self._parms["_rest_version"] = 3"""
    if algo == "stackedensemble":
        return """self._parms["_rest_version"] = 99"""
    if algo == "aggregator":
        return """self._parms["_rest_version"] = 99"""

def class_extra_for(algo):
    if algo == "aggregator":
        # Add aggregated_frame to model
        return """
            @property
            def aggregated_frame(self):
                if (self._model_json is not None and
                    self._model_json.get("output", {}).get("output_frame", {}).get("name") is not None):
                    out_frame_name = self._model_json["output"]["output_frame"]["name"]
                    return H2OFrame.get_frame(out_frame_name)"""
    if algo == "glm":
        # Before we were replacing .lambda property with .Lambda. However that violates Python naming conventions for
        # variables, so now we prefer to map that property to .lambda_. The old name remains, for compatibility reasons.
        return """
            @property
            def Lambda(self):
                \"""DEPRECATED. Use ``self.lambda_`` instead\"""
                return self._parms["lambda"] if "lambda" in self._parms else None

            @Lambda.setter
            def Lambda(self, value):
                self._parms["lambda"] = value

            @staticmethod
            def getGLMRegularizationPath(model):
                \"\"\"
                Extract full regularization path explored during lambda search from glm model.

                :param model: source lambda search model
                \"\"\"
                x = h2o.api("GET /3/GetGLMRegPath", data={"model": model._model_json["model_id"]["name"]})
                ns = x.pop("coefficient_names")
                res = {
                    "lambdas": x["lambdas"],
                    "explained_deviance_train": x["explained_deviance_train"],
                    "explained_deviance_valid": x["explained_deviance_valid"],
                    "coefficients": [dict(zip(ns, y)) for y in x["coefficients"]],
                }
                if "coefficients_std" in x:
                    res["coefficients_std"] = [dict(zip(ns, y)) for y in x["coefficients_std"]]
                return res

            @staticmethod
            def makeGLMModel(model, coefs, threshold=.5):
                \"\"\"
                Create a custom GLM model using the given coefficients.

                Needs to be passed source model trained on the dataset to extract the dataset information from.

                :param model: source model, used for extracting dataset information
                :param coefs: dictionary containing model coefficients
                :param threshold: (optional, only for binomial) decision threshold used for classification
                \"\"\"
                model_json = h2o.api(
                    "POST /3/MakeGLMModel",
                    data={"model": model._model_json["model_id"]["name"],
                          "names": list(coefs.keys()),
                          "beta": list(coefs.values()),
                          "threshold": threshold}
                )
                m = H2OGeneralizedLinearEstimator()
                m._resolve_model(model_json["model_id"]["name"], model_json)
                return m"""

    elif algo == "deepwater":
        return """
        # Ask the H2O server whether a Deep Water model can be built (depends on availability of native backends)
        @staticmethod
        def available():
            \"\"\"Returns True if a deep water model can be built, or False otherwise.\"\"\"
            builder_json = h2o.api("GET /3/ModelBuilders", data={"algo": "deepwater"})
            visibility = builder_json["model_builders"]["deepwater"]["visibility"]
            if visibility == "Experimental":
                print("Cannot build a Deep Water model - no backend found.")
                return False
            else:
                return True
        """

    elif algo == "xgboost":
        return """
        # Ask the H2O server whether a XGBoost model can be built (depends on availability of native backends)
        @staticmethod
        def available():
            \"\"\"
            Returns True if a XGBoost model can be built, or False otherwise.
            \"\"\"
            if "XGBoost" not in h2o.cluster().list_core_extensions():
                print("Cannot build an XGBoost model - no backend found.")
                return False
            else:
                return True
        """
    elif algo == "stackedensemble":
        return """
        # Print the metalearner of an H2OStackedEnsembleEstimator.
        def metalearner(self):
            model = self._model_json["output"]
            if "metalearner" in model and model["metalearner"] is not None:
                return model["metalearner"]
            print("No metalearner for this model")

        #Fetch the levelone_frame_id for an H2OStackedEnsembleEstimator.
        def levelone_frame_id(self):
            model = self._model_json["output"]
            if "levelone_frame_id" in model and model["levelone_frame_id"] is not None:
                return model["levelone_frame_id"]
            print("No levelone_frame_id for this model")

        def stacking_strategy(self):
            model = self._model_json["output"]
            if "stacking_strategy" in model and model["stacking_strategy"] is not None:
                return model["stacking_strategy"]
            print("No stacking strategy for this model")

        # Override train method to support blending
        def train(self, x=None, y=None, training_frame=None, blending_frame=None, **kwargs):
            assert_is_type(blending_frame, None, H2OFrame)

            def extend_parms(parms):
                if blending_frame is not None:
                    parms['blending_frame'] = blending_frame

            super(self.__class__, self)._train(x, y, training_frame, extend_parms_fn=extend_parms, **kwargs)
        """
    elif algo == "word2vec":
        return """
        def _requires_training_frame(self):
            \"\"\"
            Determines if Word2Vec algorithm requires a training frame.
            :return: False.
            \"\"\"
            return False

        @staticmethod
        def from_external(external=H2OFrame):
            \"\"\"
            Creates new H2OWord2vecEstimator based on an external model.
            :param external: H2OFrame with an external model
            :return: H2OWord2vecEstimator instance representing the external model
            \"\"\"
            w2v_model = H2OWord2vecEstimator(pre_trained=external)
            w2v_model.train()
            return w2v_model

        def _determine_vec_size(self):
            \"\"\"
            Determines vec_size for a pre-trained model after basic model verification.
            \"\"\"
            first_column = self.pre_trained.types[self.pre_trained.columns[0]]

            if first_column != 'string':
                raise H2OValueError("First column of given pre_trained model %s is required to be a String",
                                    self.pre_trained.frame_id)

            if list(self.pre_trained.types.values()).count('string') > 1:
                raise H2OValueError("There are multiple columns in given pre_trained model %s with a String type.",
                                    self.pre_trained.frame_id)

            self.vec_size = self.pre_trained.dim[1] - 1;
        """
    elif algo == "pca":
        return """
        def init_for_pipeline(self):
            \"\"\"
            Returns H2OPCA object which implements fit and transform method to be used in sklearn.Pipeline properly.
            All parameters defined in self.__params, should be input parameters in H2OPCA.__init__ method.

            :returns: H2OPCA object
            \"\"\"
            import inspect
            from h2o.transforms.decomposition import H2OPCA
            # check which parameters can be passed to H2OPCA init
            var_names = list(dict(inspect.getmembers(H2OPCA.__init__.__code__))['co_varnames'])
            parameters = {k: v for k, v in self._parms.items() if k in var_names}
            return H2OPCA(**parameters)
        """
    elif algo == "generic":
        return """
        def _requires_training_frame(self):
            \"\"\"
            Determines if Generic model requires a training frame.
            :return: False.
            \"\"\"
            return False

        @staticmethod
        def from_file(file=str):
            \"\"\"
            Creates new Generic model by loading existing embedded model into library, e.g. from H2O MOJO.
            The imported model must be supported by H2O.
            :param file: A string containing path to the file to create the model from
            :return: H2OGenericEstimator instance representing the generic model
            \"\"\"
            from h2o import lazy_import, get_frame
            model_key = lazy_import(file)
            model_bytes_frame = get_frame(model_key[0])
            model = H2OGenericEstimator(model_key = model_bytes_frame)
            model.train()

            return model
        """

def module_extra_for(algo):
    if algo == "deeplearning":
        return """
            class H2OAutoEncoderEstimator(H2ODeepLearningEstimator):
                \"\"\"
                Examples
                --------
                  >>> import h2o as ml
                  >>> from h2o.estimators.deeplearning import H2OAutoEncoderEstimator
                  >>> ml.init()
                  >>> rows = [[1,2,3,4,0]*50, [2,1,2,4,1]*50, [2,1,4,2,1]*50, [0,1,2,34,1]*50, [2,3,4,1,0]*50]
                  >>> fr = ml.H2OFrame(rows)
                  >>> fr[4] = fr[4].asfactor()
                  >>> model = H2OAutoEncoderEstimator()
                  >>> model.train(x=range(4), training_frame=fr)
                \"\"\"
                pass"""

def gen_init(modules):
    yield "#!/usr/bin/env python"
    yield "# -*- encoding: utf-8 -*-"
    yield "#"
    yield "# This file is auto-generated by h2o-3/h2o-bindings/bin/gen_python.py"
    yield "# Copyright 2016 H2O.ai;  Apache License Version 2.0 (see LICENSE for details)"
    yield "#"
    module_strs = []
    for module, clz, category in sorted(modules):
        if clz == "H2OGridSearch": continue
        module_strs.append('"%s"' % clz)
        if clz == "H2OAutoML": continue
        module_strs.append('"%s"' % clz)
        yield "from .%s import %s" % (module, clz)
    yield ""
    yield "__all__ = ("
    yield bi.wrap(", ".join(module_strs), indent="    ")
    yield ")"

def gen_models_docs(modules):
    yield ".. This file is autogenerated from gen_python.py, DO NOT MODIFY"
    yield ":tocdepth: 3"
    yield ""
    yield "Modeling In H2O"
    yield "==============="
    for cat in ["Supervised", "Unsupervised", "Miscellaneous"]:
        yield ""
        yield cat
        yield "+" * len(cat)
        yield ""
        for module, clz, category in sorted(modules):
            if category != cat: continue
            fullmodule = "h2o.estimators.%s.%s" % (module, clz)
            if clz == "H2OGridSearch":
                fullmodule = "h2o.grid.grid_search.H2OGridSearch"
            if clz == "H2OAutoML":
                fullmodule = "h2o.automl.autoh2o.H2OAutoML"
            yield ":mod:`%s`" % clz
            yield "-" * (7 + len(clz))
            yield ".. autoclass:: %s" % fullmodule
            yield "    :show-inheritance:"
            yield "    :members:"
            yield ""

# ----------------------------------------------------------------------------------------------------------------------
#   MAIN:
# ----------------------------------------------------------------------------------------------------------------------
def main():
    bi.init("Python", "../../../h2o-py/h2o/estimators", clear_dir=False)

    modules = [("deeplearning", "H2OAutoEncoderEstimator", "Unsupervised"),
               ("estimator_base", "H2OEstimator", "Miscellaneous"),
               ("grid_search", "H2OGridSearch", "Miscellaneous"),
               ("automl", "H2OAutoML", "Miscellaneous")]
    builders = filter(lambda b: b[0] != 'coxph', bi.model_builders().items()) # CoxPH is not supported in Python yet
    for name, mb in builders:
        module = name
        if name == "drf": module = "random_forest"
        if name == "naivebayes": module = "naive_bayes"
        if name == "isolationforest": module = "isolation_forest"
        bi.vprint("Generating model: " + name)
        bi.write_to_file("%s.py" % module, gen_module(mb, name))
        category = "Supervised" if mb["supervised"] else "Unsupervised"
        if name in {"svd", "word2vec"}:
            category = "Miscellaneous"
        modules.append((module, algo_to_classname(name), category))

    bi.write_to_file("__init__.py", gen_init(modules))
    bi.write_to_file("../../docs/modeling.rst", gen_models_docs(modules))

    type_adapter1.vprint_translation_map()

if __name__ == "__main__":
    main()


# -*- encoding: utf-8 -*-
from __future__ import absolute_import, division, print_function, unicode_literals
from collections import OrderedDict
from h2o.utils.compatibility import *  # NOQA

from .model_base import ModelBase
from .metrics_base import *  # NOQA
import h2o
from h2o.expr import ExprNode

class H2OWordEmbeddingModel(ModelBase):
    """
    Word embedding model.
    """

    def find_synonyms(self, word, count=20):
        """
        Find synonyms using a word2vec model.

        :param str word: A single word to find synonyms for.
        :param int count: The first "count" synonyms will be returned.

        :returns: the approximate reconstruction of the training data.
        """
        j = h2o.api("GET /3/Word2VecSynonyms", data={'model': self.model_id, 'word': word, 'count': count})
        return OrderedDict(sorted(zip(j['synonyms'], j['scores']), key=lambda t: t[1], reverse=True))

    def transform(self, words, aggregate_method):
        """
        Transform words (or sequences of words) to vectors using a word2vec model.

        :param str words: An H2OFrame made of a single column containing source words.
        :param str aggregate_method: Specifies how to aggregate sequences of words. If method is `NONE`
               then no aggregation is performed and each input word is mapped to a single word-vector.
               If method is 'AVERAGE' then input is treated as sequences of words delimited by NA.
               Each word of a sequences is internally mapped to a vector and vectors belonging to
               the same sentence are averaged and returned in the result.

        :returns: the approximate reconstruction of the training data.
        """
        j = h2o.api("GET /3/Word2VecTransform", data={'model': self.model_id, 'words_frame': words.frame_id, 'aggregate_method': aggregate_method})
        return h2o.get_frame(j["vectors_frame"]["name"])

    def to_frame(self):
        """
        Converts a given word2vec model into H2OFrame.

        :returns: a frame representing learned word embeddings.
        """
        return h2o.H2OFrame._expr(expr=ExprNode("word2vec.to.frame", self))


# -*- encoding: utf-8 -*-
"""
Handler to an asynchronous task executed on the remote server.

A job is an object with states: CREATED, RUNNING, DONE, FAILED, CANCELLED
A job can be polled for completion and reports the current progress as long as it is RUNNING.
"""
from __future__ import division, print_function, absolute_import, unicode_literals

import warnings

import h2o
from h2o.exceptions import H2OJobCancelled, H2OResponseError
from h2o.utils.progressbar import ProgressBar
from h2o.utils.shared_utils import clamp
from datetime import datetime
import time

class H2OJob(object):
    """A class representing an H2O Job."""

    __PROGRESS_BAR__ = True  # display & update progress bar while polling

    def __init__(self, jobs, job_type):
        """Initialize new H2OJob object."""
        if "jobs" in jobs:
            job = jobs["jobs"][0]
        elif "job" in jobs:
            job = jobs["job"]
        else:
            job = jobs

        self.job = job
        self.status = job["status"]
        self.job_key = job["key"]["name"]
        self.dest_key = job["dest"]["name"]
        self.warnings = None
        self.progress = 0
        self.exception = job["exception"] if "exception" in job else None
        self._job_type = job_type
        self._polling = False
        self._poll_count = 10**10

    def poll(self, verbose_model_scoring_history = False):
        """
        Wait until the job finishes.

        This method will continuously query the server about the status of the job, until the job reaches a
        completion. During this time we will display (in stdout) a progress bar with % completion status.
        """
        try:
            hidden = not H2OJob.__PROGRESS_BAR__
            pb = ProgressBar(title=self._job_type + " progress", hidden=hidden)
            if verbose_model_scoring_history:
                pb.execute(self._refresh_job_status, print_verbose_info=lambda x: self._print_verbose_info() if int(x * 10) % 5 == 0  else " ")
            else:
                pb.execute(self._refresh_job_status)
        except StopIteration as e:
            if str(e) == "cancelled":
                h2o.api("POST /3/Jobs/%s/cancel" % self.job_key)
                self.status = "CANCELLED"
            # Potentially we may want to re-raise the exception here

        assert self.status in {"DONE", "CANCELLED", "FAILED"} or self._poll_count <= 0, \
            "Polling finished while the job has status %s" % self.status
        if self.warnings:
            for w in self.warnings:
                warnings.warn(w)

        # check if failed... and politely print relevant message
        if self.status == "CANCELLED":
            raise H2OJobCancelled("Job<%s> was cancelled by the user." % self.job_key)
        if self.status == "FAILED":
            if (isinstance(self.job, dict)) and ("stacktrace" in list(self.job)):
                raise EnvironmentError("Job with key {} failed with an exception: {}\nstacktrace: "
                                       "\n{}".format(self.job_key, self.exception, self.job["stacktrace"]))
            else:
                raise EnvironmentError("Job with key %s failed with an exception: %s" % (self.job_key, self.exception))

        return self

    # TODO: this is not multi-client safe:
    def poll_once(self):
        """Query the job status and show the progress bar, but then cancel immediately."""
        self._poll_count = 1
        self.poll()
        self._poll_count = 10**10
        return self

    def _refresh_job_status(self):
        if self._poll_count <= 0: raise StopIteration("")
        jobs = h2o.api("GET /3/Jobs/%s" % self.job_key)
        self.job = jobs["jobs"][0] if "jobs" in jobs else jobs["job"][0]
        self.status = self.job["status"]
        self.progress = self.job["progress"]
        self.exception = self.job["exception"]
        self.warnings = self.job["warnings"] if "warnings" in self.job else None
        self._poll_count -= 1
        # Sometimes the server may report the job at 100% but still having status "RUNNING" -- we work around this
        # by showing progress at 99% instead. Sometimes the server may report the job at 0% but having status "DONE",
        # in this case we set the progress to 100% manually.
        if self.status == "CREATED": self.progress = 0
        if self.status == "RUNNING": self.progress = clamp(self.progress, 0, 0.99)
        if self.status == "DONE": self.progress = 1
        if self.status == "FAILED": raise StopIteration("failed")
        if self.status == "CANCELLED": raise StopIteration("cancelled by the server")
        return self.progress

    def _print_verbose_info(self):
        try:
            model = h2o.get_model(self.job['dest']['name'])
            print("\nScoring History for Model " + str(model.model_id) + " at " + str(datetime.now()))
            print("Model Build is {0:.0f}% done...".format(self.progress*100))
            print(model.scoring_history().tail())
            print("\n")
        except H2OResponseError: #To catch 400 error
            pass
            print("Model build is starting now...")
        except AttributeError: #To catch NoneType error if scoring history is not available
            pass
            print("Scoring History is not available yet...")

    def __repr__(self):
        if self.status in {"CREATED", "RUNNING"}:
            desc = "at %d%%" % int(self.progress * 100 + 0.5)
        else:
            desc = self.status.lower()
        return "<H2OJob id=%s %s>" % (self.job_key, desc)


# -*- encoding: utf-8 -*-
from __future__ import division, print_function, absolute_import, unicode_literals

import uuid

import h2o
from h2o.frame import H2OFrame
from h2o.transforms.transform_base import H2OTransformer
from h2o.utils.compatibility import *  # NOQA
from h2o.utils.shared_utils import urlopen, quoted
from h2o.utils.typechecks import assert_is_type
import os

class H2OAssembly(object):
    """
    H2OAssembly class can be used to specify multiple frame operations in one place.

    Sample usage:

    >>> iris = h2o.load_dataset("iris")
    >>> assembly = H2OAssembly(steps=[
    ... ("col_select",       H2OColSelect(["Sepal.Length", "Petal.Length", "Species"])),
    ... ("cos_Sepal.Length", H2OColOp(op=H2OFrame.cos, col="Sepal.Length", inplace=True)),
    ... ("str_cnt_Species",  H2OColOp(op=H2OFrame.countmatches, col="Species",
    ...                               inplace=False, pattern="s"))
    ... ])
    >>> result = assembly.fit(iris)  # fit the assembly and perform the munging operations

    In this example, we first load the iris frame.  Next, the following data munging operations are performed on the
    iris frame

        1. only select three columns out of the five columns;
        2. take the cosine of the column Sepal.Length and replace the original column with the cosine of the column;
        3. want to count the number of rows with the letter s in the class column.  Since inplace is set to
           False, a new column is generated to hold the result.

    Extension class of Pipeline implementing additional methods:

       - to_pojo: Exports the assembly to a self-contained Java POJO used in a per-row, high-throughput environment.

    In addition, H2OAssembly provides a few static methods that perform element to element operations between
    two frames. They all are called as

    >>> H2OAssembly.op(frame1, frame2)

    while frame1, frame2 are H2OFrame of the same size and same column types.  It will return a H2OFrame
    containing the element-wise result of operation op.  The following operations are currently supported

        - divide
        - plus
        - multiply
        - minus
        - less_than
        - less_than_equal
        - equal_equal
        - not_equal
        - greater_than
        - greater_than_equal
    """

    # static properties pointing to H2OFrame methods
    divide = H2OFrame.__truediv__
    plus = H2OFrame.__add__
    multiply = H2OFrame.__mul__
    minus = H2OFrame.__sub__
    less_than = H2OFrame.__lt__
    less_than_equal = H2OFrame.__le__
    equal_equal = H2OFrame.__eq__
    not_equal = H2OFrame.__ne__
    greater_than = H2OFrame.__gt__
    greater_than_equal = H2OFrame.__ge__

    def __init__(self, steps):
        """
        Build a new H2OAssembly.

        :param steps: A list of steps that sequentially transforms the input data. Each step is a
            tuple ``(name, operation)``, where each ``operation`` is an instance of an ``H2OTransformer`` class.
        """
        assert_is_type(steps, [(str, H2OTransformer)])
        self.id = None
        self.steps = steps
        self.fuzed = []
        self.in_colnames = None
        self.out_colnames = None

    @property
    def names(self):
        return list(zip(*self.steps))[0][:-1]

    def to_pojo(self, pojo_name="", path="", get_jar=True):
        """
        Convert the munging operations performed on H2OFrame into a POJO.

        :param pojo_name:  (str) Name of POJO
        :param path:  (str) path of POJO.
        :param get_jar: (bool) Whether to also download the h2o-genmodel.jar file needed to compile the POJO
        :return: None
        """
        assert_is_type(pojo_name, str)
        assert_is_type(path, str)
        assert_is_type(get_jar, bool)
        if pojo_name == "":
            pojo_name = "AssemblyPOJO_" + str(uuid.uuid4())
        java = h2o.api("GET /99/Assembly.java/%s/%s" % (self.id, pojo_name))
        file_path = path + "/" + pojo_name + ".java"
        if path == "":
            print(java)
        else:
            with open(file_path, 'w', encoding="utf-8") as f:
                f.write(java)  # this had better be utf-8 ?
        if get_jar and path != "":
            h2o.api("GET /3/h2o-genmodel.jar", save_to=os.path.join(path, "h2o-genmodel.jar"))

    # def union(self, assemblies):
    #   # fuse the assemblies onto this one, each is added to the end going left -> right
    #   # assemblies must be a list of namedtuples.
    #   #   [(H2OAssembly, X, y, {params}), ..., (H2OAssembly, X, y, {params})]
    #   for i in assemblies:
    #     if not isinstance(i, namedtuple):
    #       raise ValueError("Not a namedtuple. Assembly must be of type collections.namedtuple with fields [assembly, x, params].")
    #     if i._fields != ('assembly','x','params'):
    #       raise ValueError("Assembly must be a namedtuple with fields ('assembly', 'x', 'params').")
    #     self.fuzed.append(i)

    def fit(self, fr):
        """
        To perform the munging operations on a frame specified in steps on the frame fr.

        :param fr: H2OFrame where munging operations are to be performed on.
        :return: H2OFrame after munging operations are completed.
        """
        assert_is_type(fr, H2OFrame)
        steps = "[%s]" % ",".join(quoted(step[1].to_rest(step[0]).replace('"', "'")) for step in self.steps)
        j = h2o.api("POST /99/Assembly", data={"steps": steps, "frame": fr.frame_id})
        self.id = j["assembly"]["name"]
        return H2OFrame.get_frame(j["result"]["name"])