#!/usr/bin/python

import sys
import os
from os import listdir
from os.path import isfile, join
import time
import json
import datetime
from pytz import timezone
from dateutil import parser

"""
This script will summary failed tests results and determine if any of them may be intermittents.  For
tests that are determined to be intermittents, a dictionary structure will be generated to store information
about the intermittents.

Currently, a simple threshold test is used to determine if a test is intermittent.  If the failure count of
any test exceed the threshold, we will label it as intermittent.  In particular, the following information
will be stored for each intermittent failure:
        "JenkinsJobName"
        "BuildID"
        "Timestamp"
        "GitHash"
        "TestCategory"
        "NodeName"
        "FailureMessages"
        "FailureCount"

"""

# --------------------------------------------------------------------
# Main program
# --------------------------------------------------------------------

g_test_root_dir = os.path.dirname(os.path.realpath(__file__)) # directory where we are running out code from
g_threshold_failure = 0
g_summary_dict_name = ''
g_summary_csv_filename = ''
g_file_start = []

g_summary_dict_intermittents = dict()
g_summary_dict_all = dict()

def init_intermittents_dict(init_dict):
    """
    initialize the fields of dictionary storing failed tests.
    :return:
    """
    init_dict["TestName"] = []
    init_dict["TestInfo"] = []

def usage():
    """
    Print USAGE help.
    """
    print("")
    print("Usage:  ")
    print("python summarizeINtermittents threshold Filename_for_dict AWS_path Failed_PyUnits_summary_dict_from ....")
    print("- threshold is an integer for which a failed test is labeled intermittent if its number of "
          "failure exceeds it.")
    print("- Filename_for_dict is a string denoting the name of the dictionary that will store the final intermittents.")
    print("- Failed_PyUnits_summary_dict_from is a string denoting the beginning of pickle files that contains"
          "")
    print("- ... denotes extra strings that represent the beginning of pickle files that you want us to summarize"
          "for you.")

def summarizeFailedRuns():
    """
    This function will look at the local directory and pick out files that have the correct start name and
    summarize the results into one giant dict.

    :return: None
    """
    global g_summary_dict_all

    onlyFiles = [x for x in listdir(g_test_root_dir) if isfile(join(g_test_root_dir, x))]   # grab files

    for f in onlyFiles:
        for fileStart in g_file_start:
            if (fileStart in f) and (os.path.getsize(f) > 10):  # found the file containing failed tests
                fFullPath = os.path.join(g_test_root_dir, f)
                try:
                    temp_dict = json.load(open(fFullPath,'r'))

                    # scrape through temp_dict and see if we need to add the test to intermittents
                    for ind in range(len(temp_dict["TestName"])):
                        addFailedTests(g_summary_dict_all, temp_dict, ind)
                except:
                    continue
                break

def addFailedTests(summary_dict, temp_dict, index):
    testName = temp_dict["TestName"][index]
    testNameList = summary_dict["TestName"]
    # check if new intermittents or old ones
    if testName in testNameList:
        testIndex =testNameList.index(testName) # update the test
        updateFailedTestInfo(summary_dict, temp_dict["TestInfo"][index], testIndex, testName, False)
    else:    # new intermittent uncovered
        summary_dict["TestName"].append(testName)
        updateFailedTestInfo(summary_dict, temp_dict["TestInfo"][index], len(summary_dict["TestName"])-1, testName, True)

def updateFailedTestInfo(summary_dict, one_test_info, testIndex, testName, newTest):
    """
    For each test, a dictionary structure will be built to record the various info about that test's failure
    information.  In particular, for each failed tests, there will be a dictionary associated with that test
    stored in the field "TestInfo" of g_faiiled_tests_info_dict.  The following fields are included:
        "JenkinsJobName": job name
        "BuildID"
        "Timestamp": in seconds
        "GitHash"
        "TestCategory": JUnit, PyUnit, RUnit or HadoopPyUnit, HadoopRUnit
        "NodeName": name of machine that the job was run on
        "FailureCount": integer counting number of times this particular test has failed.  An intermittent can be
          determined as any test with FailureCount >= 2.
        "FailureMessages": contains failure messages for the test
    :return: a new dict for that test
    """
    if newTest: # setup the dict structure to store the new data
        summary_dict["TestInfo"].append(dict())
        summary_dict["TestInfo"][testIndex]["JenkinsJobName"]=[]
        summary_dict["TestInfo"][testIndex]["BuildID"]=[]
        summary_dict["TestInfo"][testIndex]["Timestamp"]=[]
        summary_dict["TestInfo"][testIndex]["GitHash"]=[]
        summary_dict["TestInfo"][testIndex]["TestCategory"]=[]
        summary_dict["TestInfo"][testIndex]["NodeName"]=[]
        summary_dict["TestInfo"][testIndex]["FailureCount"]=0
        summary_dict["TestInfo"][testIndex]["TestName"] = testName     # add test name
        summary_dict["TestInfo"][testIndex]["FailureMessages"] = [] # contains failure messages for the test

    summary_dict["TestInfo"][testIndex]["JenkinsJobName"].extend(one_test_info["JenkinsJobName"])
    summary_dict["TestInfo"][testIndex]["BuildID"].extend(one_test_info["BuildID"])
    summary_dict["TestInfo"][testIndex]["Timestamp"].extend(one_test_info["Timestamp"])
    summary_dict["TestInfo"][testIndex]["GitHash"].extend(one_test_info["GitHash"])
    summary_dict["TestInfo"][testIndex]["TestCategory"].extend(one_test_info["TestCategory"])
    summary_dict["TestInfo"][testIndex]["NodeName"].extend(one_test_info["NodeName"])
    summary_dict["TestInfo"][testIndex]["FailureMessages"].extend(one_test_info["FailureMessages"])
    summary_dict["TestInfo"][testIndex]["FailureCount"] += one_test_info["FailureCount"]

def extractPrintSaveIntermittens():
    """
    This function will print out the intermittents onto the screen for casual viewing.  It will also print out
    where the giant summary dictionary is going to be stored.

    :return: None
    """
    # extract intermittents from collected failed tests
    global g_summary_dict_intermittents

    localtz = time.tzname[0]

    for ind in range(len(g_summary_dict_all["TestName"])):
        if g_summary_dict_all["TestInfo"][ind]["FailureCount"] >= g_threshold_failure:
            addFailedTests(g_summary_dict_intermittents, g_summary_dict_all, ind)

    # save dict in file
    if len(g_summary_dict_intermittents["TestName"]) > 0:
        json.dump(g_summary_dict_intermittents, open(g_summary_dict_name, 'w'))

        with open(g_summary_csv_filename, 'w') as summaryFile:
            for ind in range(len(g_summary_dict_intermittents["TestName"])):
                testName = g_summary_dict_intermittents["TestName"][ind]
                numberFailure = g_summary_dict_intermittents["TestInfo"][ind]["FailureCount"]
                firstFailedTS  = parser.parse(time.ctime(min(g_summary_dict_intermittents["TestInfo"][ind]["Timestamp"]))+
                                              ' '+localtz)
                firstFailedStr = firstFailedTS.strftime("%a %b %d %H:%M:%S %Y %Z")
                recentFail = parser.parse(time.ctime(max(g_summary_dict_intermittents["TestInfo"][ind]["Timestamp"]))+
                                          ' '+localtz)
                recentFailStr = recentFail.strftime("%a %b %d %H:%M:%S %Y %Z")
                eachTest = "{0}, {1}, {2}, {3}\n".format(testName, recentFailStr, numberFailure,
                                                       g_summary_dict_intermittents["TestInfo"][ind]["TestCategory"][0])
                summaryFile.write(eachTest)
                print("Intermittent: {0}, Last failed: {1}, Failed {2} times since "
                      "{3}".format(testName, recentFailStr, numberFailure, firstFailedStr))

def main(argv):
    """
    Main program.  Expect script name plus  inputs in the following order:
    - This script name
    1. threshold: integer that will denote when a failed test will be declared an intermittent
    2. string denote filename of where our final dict structure will be stored.
    3. string that denote the beginning of a file containing failed tests info.
    4. Optional strings that denote the beginning of a file containing failed tests info.

    @return: none
    """
    global g_script_name
    global g_test_root_dir
    global g_threshold_failure
    global g_file_start
    global g_summary_dict_name
    global g_summary_dict_all
    global g_summary_dict_intermittents
    global g_summary_csv_filename

    if len(argv) < 5:
        print "Wrong call.  Not enough arguments.\n"
        usage()
        sys.exit(1)
    else:   # we may be in business
        g_threshold_failure = int(argv[1])
        g_summary_dict_name = os.path.join(g_test_root_dir, argv[2])
        g_summary_csv_filename = g_summary_dict_name+".csv"

        for ind in range(3, len(argv)):
            g_file_start.append(argv[ind])

        init_intermittents_dict(g_summary_dict_all)
        init_intermittents_dict(g_summary_dict_intermittents)
        summarizeFailedRuns()
        extractPrintSaveIntermittens()

if __name__ == "__main__":
    main(sys.argv)


# -*- encoding: utf-8 -*-
"""
Regression model.

:copyright: (c) 2016 H2O.ai
:license:   Apache License Version 2.0 (see LICENSE for details)
"""
from __future__ import absolute_import, division, print_function, unicode_literals

import imp

from h2o.model.confusion_matrix import ConfusionMatrix
from h2o.utils.backward_compatibility import backwards_compatible
from h2o.utils.compatibility import *  # NOQA
from h2o.utils.typechecks import assert_is_type, assert_satisfies, numeric

class MetricsBase(backwards_compatible()):
    """
    A parent class to house common metrics available for the various Metrics types.

    The methods here are available across different model categories.
    """

    def __init__(self, metric_json, on=None, algo=""):
        super(MetricsBase, self).__init__()
        # Yep, it's messed up...
        if isinstance(metric_json, MetricsBase): metric_json = metric_json._metric_json
        self._metric_json = metric_json
        # train and valid and xval are not mutually exclusive -- could have a test. train and
        # valid only make sense at model build time.
        self._on_train = False
        self._on_valid = False
        self._on_xval = False
        self._algo = algo
        if on == "training_metrics":
            self._on_train = True
        elif on == "validation_metrics":
            self._on_valid = True
        elif on == "cross_validation_metrics":
            self._on_xval = True
        elif on is None:
            pass
        else:
            raise ValueError("on expected to be train,valid,or xval. Got: " + str(on))

    @classmethod
    def make(cls, kvs):
        """Factory method to instantiate a MetricsBase object from the list of key-value pairs."""
        return cls(metric_json=dict(kvs))

    def __repr__(self):
        # FIXME !!!  __repr__ should never print anything, but return a string
        self.show()
        return ""

    # TODO: convert to actual fields list
    def __getitem__(self, key):
        return self._metric_json.get(key)

    @staticmethod
    def _has(dictionary, key):
        return key in dictionary and dictionary[key] is not None

    def show(self):
        """Display a short summary of the metrics."""
        if self._metric_json==None:
            print("WARNING: Model metrics cannot be calculated and metric_json is empty due to the absence of the response column in your dataset.")
            return
        metric_type = self._metric_json['__meta']['schema_type']
        types_w_glm = ['ModelMetricsRegressionGLM', 'ModelMetricsBinomialGLM']
        types_w_clustering = ['ModelMetricsClustering']
        types_w_mult = ['ModelMetricsMultinomial']
        types_w_ord = ['ModelMetricsOrdinal']
        types_w_bin = ['ModelMetricsBinomial', 'ModelMetricsBinomialGLM']
        types_w_r2 = ['ModelMetricsRegressionGLM']
        types_w_mean_residual_deviance = ['ModelMetricsRegressionGLM', 'ModelMetricsRegression']
        types_w_mean_absolute_error = ['ModelMetricsRegressionGLM', 'ModelMetricsRegression']
        types_w_logloss = types_w_bin + types_w_mult+types_w_ord
        types_w_dim = ["ModelMetricsGLRM"]
        types_w_anomaly = ['ModelMetricsAnomaly']

        print()
        print(metric_type + ": " + self._algo)
        reported_on = "** Reported on {} data. **"
        if self._on_train:
            print(reported_on.format("train"))
        elif self._on_valid:
            print(reported_on.format("validation"))
        elif self._on_xval:
            print(reported_on.format("cross-validation"))
        else:
            print(reported_on.format("test"))
        print()
        if metric_type not in types_w_anomaly:
            print("MSE: " + str(self.mse()))
            print("RMSE: " + str(self.rmse()))
        if metric_type in types_w_mean_absolute_error:
            print("MAE: " + str(self.mae()))
            print("RMSLE: " + str(self.rmsle()))
        if metric_type in types_w_r2:
            print("R^2: " + str(self.r2()))
        if metric_type in types_w_mean_residual_deviance:
            print("Mean Residual Deviance: " + str(self.mean_residual_deviance()))
        if metric_type in types_w_logloss:
            print("LogLoss: " + str(self.logloss()))
        if metric_type == 'ModelMetricsBinomial':
            # second element for first threshold is the actual mean per class error
            print("Mean Per-Class Error: %s" % self.mean_per_class_error()[0][1])
        if metric_type == 'ModelMetricsMultinomial' or metric_type == 'ModelMetricsOrdinal':
            print("Mean Per-Class Error: " + str(self.mean_per_class_error()))
        if metric_type in types_w_glm:
            print("Null degrees of freedom: " + str(self.null_degrees_of_freedom()))
            print("Residual degrees of freedom: " + str(self.residual_degrees_of_freedom()))
            print("Null deviance: " + str(self.null_deviance()))
            print("Residual deviance: " + str(self.residual_deviance()))
            print("AIC: " + str(self.aic()))
        if metric_type in types_w_bin:
            print("AUC: " + str(self.auc()))
            print("pr_auc: " + str(self.pr_auc()))
            print("Gini: " + str(self.gini()))
            self.confusion_matrix().show()
            self._metric_json["max_criteria_and_metric_scores"].show()
            if self.gains_lift():
                print(self.gains_lift())
        if metric_type in types_w_anomaly:
            print("Anomaly Score: " + str(self.mean_score()))
            print("Normalized Anomaly Score: " + str(self.mean_normalized_score()))
        if (metric_type in types_w_mult) or (metric_type in types_w_ord):
            self.confusion_matrix().show()
            self.hit_ratio_table().show()
        if metric_type in types_w_clustering:
            print("Total Within Cluster Sum of Square Error: " + str(self.tot_withinss()))
            print("Total Sum of Square Error to Grand Mean: " + str(self.totss()))
            print("Between Cluster Sum of Square Error: " + str(self.betweenss()))
            self._metric_json['centroid_stats'].show()

        if metric_type in types_w_dim:
            print("Sum of Squared Error (Numeric): " + str(self.num_err()))
            print("Misclassification Error (Categorical): " + str(self.cat_err()))
        if self.custom_metric_name():
            print("{}: {}".format(self.custom_metric_name(), self.custom_metric_value()))

    def r2(self):
        """The R squared coefficient."""
        return self._metric_json["r2"]

    def logloss(self):
        """Log loss."""
        return self._metric_json["logloss"]

    def nobs(self):
        """The number of observations."""
        return self._metric_json["nobs"]

    def mean_residual_deviance(self):
        """The mean residual deviance for this set of metrics."""
        return self._metric_json["mean_residual_deviance"]

    def auc(self):
        """The AUC for this set of metrics."""
        return self._metric_json['AUC']

    def pr_auc(self):
        return self._metric_json['pr_auc']

    def aic(self):
        """The AIC for this set of metrics."""
        return self._metric_json['AIC']

    def gini(self):
        """Gini coefficient."""
        return self._metric_json['Gini']

    def mse(self):
        """The MSE for this set of metrics."""
        return self._metric_json['MSE']

    def rmse(self):
        """The RMSE for this set of metrics."""
        return self._metric_json['RMSE']

    def mae(self):
        """The MAE for this set of metrics."""
        return self._metric_json['mae']

    def rmsle(self):
        """The RMSLE for this set of metrics."""
        return self._metric_json['rmsle']

    def residual_deviance(self):
        """The residual deviance if the model has it, otherwise None."""
        if MetricsBase._has(self._metric_json, "residual_deviance"):
            return self._metric_json["residual_deviance"]
        return None

    def residual_degrees_of_freedom(self):
        """The residual DoF if the model has residual deviance, otherwise None."""
        if MetricsBase._has(self._metric_json, "residual_degrees_of_freedom"):
            return self._metric_json["residual_degrees_of_freedom"]
        return None

    def null_deviance(self):
        """The null deviance if the model has residual deviance, otherwise None."""
        if MetricsBase._has(self._metric_json, "null_deviance"):
            return self._metric_json["null_deviance"]
        return None

    def null_degrees_of_freedom(self):
        """The null DoF if the model has residual deviance, otherwise None."""
        if MetricsBase._has(self._metric_json, "null_degrees_of_freedom"):
            return self._metric_json["null_degrees_of_freedom"]
        return None

    def mean_per_class_error(self):
        """The mean per class error."""
        return self._metric_json['mean_per_class_error']

    def custom_metric_name(self):
        """Name of custom metric or None."""
        if MetricsBase._has(self._metric_json, "custom_metric_name"):
            return self._metric_json['custom_metric_name']
        else:
            return None

    def custom_metric_value(self):
        """Value of custom metric or None."""
        if MetricsBase._has(self._metric_json, "custom_metric_value"):
            return self._metric_json['custom_metric_value']
        else:
            return None

    # Deprecated functions; left here for backward compatibility
    _bcim = {
        "giniCoef": lambda self, *args, **kwargs: self.gini(*args, **kwargs)
    }

class H2ORegressionModelMetrics(MetricsBase):
    """
    This class provides an API for inspecting the metrics returned by a regression model.

    It is possible to retrieve the R^2 (1 - MSE/variance) and MSE.
    """

    def __init__(self, metric_json, on=None, algo=""):
        super(H2ORegressionModelMetrics, self).__init__(metric_json, on, algo)

class H2OClusteringModelMetrics(MetricsBase):

    def __init__(self, metric_json, on=None, algo=""):
        super(H2OClusteringModelMetrics, self).__init__(metric_json, on, algo)

    def tot_withinss(self):
        """The Total Within Cluster Sum-of-Square Error, or None if not present."""
        if MetricsBase._has(self._metric_json, "tot_withinss"):
            return self._metric_json["tot_withinss"]
        return None

    def totss(self):
        """The Total Sum-of-Square Error to Grand Mean, or None if not present."""
        if MetricsBase._has(self._metric_json, "totss"):
            return self._metric_json["totss"]
        return None

    def betweenss(self):
        """The Between Cluster Sum-of-Square Error, or None if not present."""
        if MetricsBase._has(self._metric_json, "betweenss"):
            return self._metric_json["betweenss"]
        return None

class H2OMultinomialModelMetrics(MetricsBase):

    def __init__(self, metric_json, on=None, algo=""):
        super(H2OMultinomialModelMetrics, self).__init__(metric_json, on, algo)

    def confusion_matrix(self):
        """Returns a confusion matrix based of H2O's default prediction threshold for a dataset."""
        return self._metric_json['cm']['table']

    def hit_ratio_table(self):
        """Retrieve the Hit Ratios."""
        return self._metric_json['hit_ratio_table']

class H2OOrdinalModelMetrics(MetricsBase):

    def __init__(self, metric_json, on=None, algo=""):
        super(H2OOrdinalModelMetrics, self).__init__(metric_json, on, algo)

    def confusion_matrix(self):
        """Returns a confusion matrix based of H2O's default prediction threshold for a dataset."""
        return self._metric_json['cm']['table']

    def hit_ratio_table(self):
        """Retrieve the Hit Ratios."""
        return self._metric_json['hit_ratio_table']

class H2OBinomialModelMetrics(MetricsBase):
    """
    This class is essentially an API for the AUC object.
    This class contains methods for inspecting the AUC for different criteria.
    To input the different criteria, use the static variable `criteria`.
    """

    def __init__(self, metric_json, on=None, algo=""):
        """
          Create a new Binomial Metrics object (essentially a wrapper around some json)

          :param metric_json: A blob of json holding all of the needed information
          :param on_train: Metrics built on training data (default is False)
          :param on_valid: Metrics built on validation data (default is False)
          :param on_xval: Metrics built on cross validation data (default is False)
          :param algo: The algorithm the metrics are based off of (e.g. deeplearning, gbm, etc.)
          :returns: A new H2OBinomialModelMetrics object.
          """
        super(H2OBinomialModelMetrics, self).__init__(metric_json, on, algo)

    def F1(self, thresholds=None):
        """
        :param thresholds: thresholds parameter must be a list (i.e. [0.01, 0.5, 0.99]). If None, then the
            thresholds in this set of metrics will be used.
        :returns: The F1 for the given set of thresholds.
        """
        return self.metric("f1", thresholds=thresholds)

    def F2(self, thresholds=None):
        """
        :param thresholds: thresholds parameter must be a list (i.e. [0.01, 0.5, 0.99]). If None, then the
            thresholds in this set of metrics will be used.
        :returns: The F2 for this set of metrics and thresholds.
        """
        return self.metric("f2", thresholds=thresholds)

    def F0point5(self, thresholds=None):
        """
        :param thresholds: thresholds parameter must be a list (i.e. [0.01, 0.5, 0.99]). If None, then the
            thresholds in this set of metrics will be used.
        :returns: The F0.5 for this set of metrics and thresholds.
        """
        return self.metric("f0point5", thresholds=thresholds)

    def accuracy(self, thresholds=None):
        """
        :param thresholds: thresholds parameter must be a list (i.e. [0.01, 0.5, 0.99]). If None, then the
            thresholds in this set of metrics will be used.
        :returns: The accuracy for this set of metrics and thresholds.
        """
        return self.metric("accuracy", thresholds=thresholds)

    def error(self, thresholds=None):
        """
        :param thresholds: thresholds parameter must be a list (i.e. [0.01, 0.5, 0.99]). If None, then the
            thresholds in this set of metrics will be used.
        :returns: The error for this set of metrics and thresholds.
        """
        return 1 - self.metric("accuracy", thresholds=thresholds)

    def precision(self, thresholds=None):
        """
        :param thresholds: thresholds parameter must be a list (i.e. [0.01, 0.5, 0.99]). If None, then the
            thresholds in this set of metrics will be used.
        :returns: The precision for this set of metrics and thresholds.
        """
        return self.metric("precision", thresholds=thresholds)

    def tpr(self, thresholds=None):
        """
        :param thresholds: thresholds parameter must be a list (i.e. [0.01, 0.5, 0.99]). If None, then the
            thresholds in this set of metrics will be used.
        :returns: The True Postive Rate.
        """
        return self.metric("tpr", thresholds=thresholds)

    def tnr(self, thresholds=None):
        """
        :param thresholds: thresholds parameter must be a list (i.e. [0.01, 0.5, 0.99]). If None, then the
            thresholds in this set of metrics will be used.
        :returns: The True Negative Rate.
        """
        return self.metric("tnr", thresholds=thresholds)

    def fnr(self, thresholds=None):
        """
        :param thresholds: thresholds parameter must be a list (i.e. [0.01, 0.5, 0.99]). If None, then the
            thresholds in this set of metrics will be used.
        :returns: The False Negative Rate.
        """
        return self.metric("fnr", thresholds=thresholds)

    def fpr(self, thresholds=None):
        """
        :param thresholds: thresholds parameter must be a list (i.e. [0.01, 0.5, 0.99]). If None, then the
            thresholds in this set of metrics will be used.
        :returns: The False Positive Rate.
        """
        return self.metric("fpr", thresholds=thresholds)

    def recall(self, thresholds=None):
        """
        :param thresholds: thresholds parameter must be a list (i.e. [0.01, 0.5, 0.99]). If None, then the
            thresholds in this set of metrics will be used.
        :returns: Recall for this set of metrics and thresholds.
        """
        return self.metric("tpr", thresholds=thresholds)

    def sensitivity(self, thresholds=None):
        """
        :param thresholds: thresholds parameter must be a list (i.e. [0.01, 0.5, 0.99]). If None, then the
            thresholds in this set of metrics will be used.
        :returns: Sensitivity or True Positive Rate for this set of metrics and thresholds.
        """
        return self.metric("tpr", thresholds=thresholds)

    def fallout(self, thresholds=None):
        """
        :param thresholds: thresholds parameter must be a list (i.e. [0.01, 0.5, 0.99]). If None, then the
            thresholds in this set of metrics will be used.
        :returns: The fallout (same as False Positive Rate) for this set of metrics and thresholds.
        """
        return self.metric("fpr", thresholds=thresholds)

    def missrate(self, thresholds=None):
        """
        :param thresholds: thresholds parameter must be a list (i.e. [0.01, 0.5, 0.99]). If None, then the
            thresholds in this set of metrics will be used.
        :returns: THe miss rate (same as False Negative Rate).
        """
        return self.metric("fnr", thresholds=thresholds)

    def specificity(self, thresholds=None):
        """
        :param thresholds: thresholds parameter must be a list (i.e. [0.01, 0.5, 0.99]). If None, then the
            thresholds in this set of metrics will be used.
        :returns: The specificity (same as True Negative Rate).
        """
        return self.metric("tnr", thresholds=thresholds)

    def mcc(self, thresholds=None):
        """
        :param thresholds: thresholds parameter must be a list (i.e. [0.01, 0.5, 0.99]). If None, then the
            thresholds in this set of metrics will be used.
        :returns: The absolute MCC (a value between 0 and 1, 0 being totally dissimilar, 1 being identical).
        """
        return self.metric("absolute_mcc", thresholds=thresholds)

    def max_per_class_error(self, thresholds=None):
        """
        :param thresholds: thresholds parameter must be a list (i.e. [0.01, 0.5, 0.99]). If None, then the
            thresholds in this set of metrics will be used.
        :returns: Return 1 - min(per class accuracy).
        """
        return 1 - self.metric("min_per_class_accuracy", thresholds=thresholds)

    def mean_per_class_error(self, thresholds=None):
        """
        :param thresholds: thresholds parameter must be a list (i.e. [0.01, 0.5, 0.99]). If None, then the
            thresholds in this set of metrics will be used.
        :returns: mean per class error.
        """
        return [[x[0], 1 - x[1]] for x in self.metric("mean_per_class_accuracy", thresholds=thresholds)]

    def metric(self, metric, thresholds=None):
        """
        :param str metric: The desired metric.
        :param thresholds: thresholds parameter must be a list (i.e. [0.01, 0.5, 0.99]). If None, then
            the thresholds in this set of metrics will be used.
        :returns: The set of metrics for the list of thresholds.
        """
        assert_is_type(thresholds, None, [numeric])
        if not thresholds: thresholds = [self.find_threshold_by_max_metric(metric)]
        thresh2d = self._metric_json['thresholds_and_metric_scores']
        metrics = []
        for t in thresholds:
            idx = self.find_idx_by_threshold(t)
            metrics.append([t, thresh2d[metric][idx]])
        return metrics

    def plot(self, type="roc", server=False):
        """
        Produce the desired metric plot.

        :param type: the type of metric plot (currently, only ROC supported).
        :param server: if True, generate plot inline using matplotlib's "Agg" backend.
        :returns: None
        """
        # TODO: add more types (i.e. cutoffs)
        assert_is_type(type, "roc")
        # check for matplotlib. exit if absent.
        try:
            imp.find_module('matplotlib')
            import matplotlib
            if server: matplotlib.use('Agg', warn=False)
            import matplotlib.pyplot as plt
        except ImportError:
            print("matplotlib is required for this function!")
            return

        if type == "roc":
            plt.xlabel('False Positive Rate (FPR)')
            plt.ylabel('True Positive Rate (TPR)')
            plt.title('ROC Curve')
            plt.text(0.5, 0.5, r'AUC={0:.4f}'.format(self._metric_json["AUC"]))
            plt.plot(self.fprs, self.tprs, 'b--')
            plt.axis([0, 1, 0, 1])
            if not server: plt.show()

    @property
    def fprs(self):
        """
        Return all false positive rates for all threshold values.

        :returns: a list of false positive rates.
        """
        return self._metric_json["thresholds_and_metric_scores"]["fpr"]

    @property
    def tprs(self):
        """
        Return all true positive rates for all threshold values.

        :returns: a list of true positive rates.
        """
        return self._metric_json["thresholds_and_metric_scores"]["tpr"]

    #: metrics names allowed for confusion matrix
    max_metrics = ('absolute_mcc', 'accuracy',
                   'f0point5', 'f1', 'f2',
                   'mean_per_class_accuracy', 'min_per_class_accuracy',
                   'precision', 'recall', 'specificity',
                   # 'fpr', 'fallout',                    # could be enabled maybe once PUBDEV-6366 is fixed
                   # 'fnr', 'missrate', 'sensitivity',
                   # 'tpr', 'recall',
                   # 'tnr', 'specificity'
                   )

    def confusion_matrix(self, metrics=None, thresholds=None):
        """
        Get the confusion matrix for the specified metric

        :param metrics: A string (or list of strings) among metrics listed in :const:`max_metrics`. Defaults to 'f1'.
        :param thresholds: A value (or list of values) between 0 and 1.
        :returns: a list of ConfusionMatrix objects (if there are more than one to return), or a single ConfusionMatrix
            (if there is only one).
        """
        # make lists out of metrics and thresholds arguments
        if metrics is None and thresholds is None:
            metrics = ['f1']

        if isinstance(metrics, list):
            metrics_list = metrics
        elif metrics is None:
            metrics_list = []
        else:
            metrics_list = [metrics]

        if isinstance(thresholds, list):
            thresholds_list = thresholds
        elif thresholds is None:
            thresholds_list = []
        else:
            thresholds_list = [thresholds]

        # error check the metrics_list and thresholds_list
        assert_is_type(thresholds_list, [numeric])
        assert_satisfies(thresholds_list, all(0 <= t <= 1 for t in thresholds_list))

        if not all(m.lower() in H2OBinomialModelMetrics.max_metrics for m in metrics_list):
            raise ValueError("The only allowable metrics are {}", ', '.join(H2OBinomialModelMetrics.max_metrics))

        # make one big list that combines the thresholds and metric-thresholds
        metrics_thresholds = [self.find_threshold_by_max_metric(m) for m in metrics_list]
        for mt in metrics_thresholds:
            thresholds_list.append(mt)
        first_metrics_thresholds_offset = len(thresholds_list) - len(metrics_thresholds)

        thresh2d = self._metric_json['thresholds_and_metric_scores']
        actual_thresholds = [float(e[0]) for i, e in enumerate(thresh2d.cell_values)]
        cms = []
        for i, t in enumerate(thresholds_list):
            idx = self.find_idx_by_threshold(t)
            row = thresh2d.cell_values[idx]
            tns = row[11]
            fns = row[12]
            fps = row[13]
            tps = row[14]
            p = tps + fns
            n = tns + fps
            c0 = n - fps
            c1 = p - tps
            if t in metrics_thresholds:
                m = metrics_list[i - first_metrics_thresholds_offset]
                table_header = "Confusion Matrix (Act/Pred) for max {} @ threshold = {}".format(m, actual_thresholds[idx])
            else:
                table_header = "Confusion Matrix (Act/Pred) @ threshold = {}".format(actual_thresholds[idx])
            cms.append(ConfusionMatrix(cm=[[c0, fps], [c1, tps]], domains=self._metric_json['domain'],
                                       table_header=table_header))

        if len(cms) == 1:
            return cms[0]
        else:
            return cms

    def find_threshold_by_max_metric(self, metric):
        """
        :param metrics: A string among the metrics listed in :const:`max_metrics`.
        :returns: the threshold at which the given metric is maximal.
        """
        crit2d = self._metric_json['max_criteria_and_metric_scores']

        for e in crit2d.cell_values:
            if e[0] == "max " + metric.lower():
                return e[1]
        raise ValueError("No metric " + str(metric.lower()))

    def find_idx_by_threshold(self, threshold):
        """
        Retrieve the index in this metric's threshold list at which the given threshold is located.

        :param threshold: Find the index of this input threshold.
        :returns: the index
        :raises ValueError: if no such index can be found.
        """
        assert_is_type(threshold, numeric)
        thresh2d = self._metric_json['thresholds_and_metric_scores']
        for i, e in enumerate(thresh2d.cell_values):
            t = float(e[0])
            if abs(t - threshold) < 1e-8 * max(t, threshold):
                return i
        if 0 <= threshold <= 1:
            thresholds = [float(e[0]) for i, e in enumerate(thresh2d.cell_values)]
            threshold_diffs = [abs(t - threshold) for t in thresholds]
            closest_idx = threshold_diffs.index(min(threshold_diffs))
            closest_threshold = thresholds[closest_idx]
            print("Could not find exact threshold {0}; using closest threshold found {1}."
                  .format(threshold, closest_threshold))
            return closest_idx
        raise ValueError("Threshold must be between 0 and 1, but got {0} ".format(threshold))

    def gains_lift(self):
        """Retrieve the Gains/Lift table."""
        if 'gains_lift_table' in self._metric_json:
            return self._metric_json['gains_lift_table']
        return None

class H2OAutoEncoderModelMetrics(MetricsBase):

    def __init__(self, metric_json, on=None, algo=""):
        super(H2OAutoEncoderModelMetrics, self).__init__(metric_json, on, algo)

class H2ODimReductionModelMetrics(MetricsBase):

    def __init__(self, metric_json, on=None, algo=""):
        super(H2ODimReductionModelMetrics, self).__init__(metric_json, on, algo)

    def num_err(self):
        """Sum of Squared Error over non-missing numeric entries, or None if not present."""
        if MetricsBase._has(self._metric_json, "numerr"):
            return self._metric_json["numerr"]
        return None

    def cat_err(self):
        """The Number of Misclassified categories over non-missing categorical entries, or None if not present."""
        if MetricsBase._has(self._metric_json, "caterr"):
            return self._metric_json["caterr"]
        return None

class H2OWordEmbeddingModelMetrics(MetricsBase):

    def __init__(self, metric_json, on=None, algo=""):
        super(H2OWordEmbeddingModelMetrics, self).__init__(metric_json, on, algo)

class H2OAnomalyDetectionModelMetrics(MetricsBase):

    def __init__(self, metric_json, on=None, algo=""):
        super(H2OAnomalyDetectionModelMetrics, self).__init__(metric_json, on, algo)

    def mean_score(self):
        """Mean Anomaly Score. For Isolation Forest represents the average of all tree-path lengths."""
        if MetricsBase._has(self._metric_json, "mean_score"):
            return self._metric_json["mean_score"]
        return None

    def mean_normalized_score(self):
        """Mean Normalized Anomaly Score. For Isolation Forest - normalized average path length."""
        if MetricsBase._has(self._metric_json, "mean_normalized_score"):
            return self._metric_json["mean_normalized_score"]
        return None


#!/usr/bin/env python
# -*- encoding: utf-8 -*-
from __future__ import unicode_literals
import bindings as bi

class CSharpTypeTranslator(bi.TypeTranslator):
    def __init__(self):
        bi.TypeTranslator.__init__(self)
        self.types["boolean"] = "bool"
        self.types["Polymorphic"] = "object"
        self.types["Object"] = "object"
        self.make_map = lambda ktype, vtype: "Dictionary<%s,%s>" % (ktype, vtype)
        self.make_key = lambda itype, schema: "string"

type_adapter = CSharpTypeTranslator()
def translate_type(h2o_type, schema):
    return type_adapter.translate(h2o_type, schema)

# ----------------------------------------------------------------------------------------------------------------------
#   Generate Schema
# ----------------------------------------------------------------------------------------------------------------------
def generate_schema(class_name, schema):
    """
    Generate C# declaration file for a schema.
    """
    has_map = False
    for field in schema["fields"]:
        if field["type"].startswith("Map"): has_map = True

    superclass = schema["superclass"]
    if superclass == "Iced": superclass = "Object"

    yield "/**"
    yield " * This file is auto-generated by h2o-3/h2o-bindings/bin/gen_csharp.py"
    yield " * Copyright 2016 H2O.ai;  Apache License Version 2.0 (see LICENSE for details)"
    yield " */"
    yield "namespace ai.h2o"
    yield "{"
    yield "  using System;"
    yield "  using System.Collections.Generic;" if has_map else None
    yield ""
    yield "  public class {name}: {super} {{".format(name=class_name, super=superclass)

    for field in schema["fields"]:
        if field["name"] == "__meta": continue
        csharp_type = translate_type(field["type"], field["schema_name"])
        yield "    /// <summary>"
        yield bi.wrap(field["help"], "    ///   ")
        yield "    /// </summary>"
        yield "    public {type} {name} {{ get; set; }}".format(type=csharp_type, name=field["name"])
        yield ""

    yield "  }"
    yield "}"

# ----------------------------------------------------------------------------------------------------------------------
#   Generate Enum class
# ----------------------------------------------------------------------------------------------------------------------
def generate_enum(name, values):
    yield "/**"
    yield " * This file is auto-generated by h2o-3/h2o-bindings/bin/gen_csharp.py"
    yield " * Copyright 2016 H2O.ai;  Apache License Version 2.0 (see LICENSE for details)"
    yield " */"
    yield "namespace ai.h2o"
    yield "{"
    yield "  public enum " + name + " {"
    for value in values:
        yield "    %s," % value
    yield "  }"
    yield "}"

# ----------------------------------------------------------------------------------------------------------------------
#   MAIN:
# ----------------------------------------------------------------------------------------------------------------------
def main():
    bi.init("C#", "CSharp")

    for schema in bi.schemas():
        name = schema["name"]
        bi.vprint("Generating schema: " + name)
        bi.write_to_file("h2o/%s.cs" % name, generate_schema(name, schema))

    for name, values in bi.enums().items():
        bi.vprint("Generating enum: " + name)
        bi.write_to_file("h2o/%s.cs" % name, generate_enum(name, sorted(values)))

    type_adapter.vprint_translation_map()

if __name__ == "__main__":
    main()


#!/usr/bin/env python
# -*- encoding: utf-8 -*-
#
# This file is auto-generated by h2o-3/h2o-bindings/bin/gen_python.py
# Copyright 2016 H2O.ai;  Apache License Version 2.0 (see LICENSE for details)
#
from __future__ import absolute_import, division, print_function, unicode_literals

from h2o.estimators.estimator_base import H2OEstimator
from h2o.exceptions import H2OValueError
from h2o.frame import H2OFrame
from h2o.utils.typechecks import assert_is_type, Enum, numeric
import h2o

class H2ODeepWaterEstimator(H2OEstimator):
    """
    Deep Water

    Build a Deep Learning model using multiple native GPU backends
    Builds a deep neural network on an H2OFrame containing various data sources
    """

    algo = "deepwater"

    def __init__(self, **kwargs):
        super(H2ODeepWaterEstimator, self).__init__()
        self._parms = {}
        names_list = {"model_id", "checkpoint", "autoencoder", "training_frame", "validation_frame", "nfolds",
                      "balance_classes", "max_after_balance_size", "class_sampling_factors",
                      "keep_cross_validation_models", "keep_cross_validation_predictions",
                      "keep_cross_validation_fold_assignment", "fold_assignment", "fold_column", "response_column",
                      "offset_column", "weights_column", "ignored_columns", "score_each_iteration",
                      "categorical_encoding", "overwrite_with_best_model", "epochs", "train_samples_per_iteration",
                      "target_ratio_comm_to_comp", "seed", "standardize", "learning_rate", "learning_rate_annealing",
                      "momentum_start", "momentum_ramp", "momentum_stable", "distribution", "score_interval",
                      "score_training_samples", "score_validation_samples", "score_duty_cycle", "classification_stop",
                      "regression_stop", "stopping_rounds", "stopping_metric", "stopping_tolerance", "max_runtime_secs",
                      "ignore_const_cols", "shuffle_training_data", "mini_batch_size", "clip_gradient", "network",
                      "backend", "image_shape", "channels", "sparse", "gpu", "device_id", "cache_data",
                      "network_definition_file", "network_parameters_file", "mean_image_file",
                      "export_native_parameters_prefix", "activation", "hidden", "input_dropout_ratio",
                      "hidden_dropout_ratios", "problem_type", "export_checkpoints_dir"}
        if "Lambda" in kwargs: kwargs["lambda_"] = kwargs.pop("Lambda")
        for pname, pvalue in kwargs.items():
            if pname == 'model_id':
                self._id = pvalue
                self._parms["model_id"] = pvalue
            elif pname in names_list:
                # Using setattr(...) will invoke type-checking of the arguments
                setattr(self, pname, pvalue)
            else:
                raise H2OValueError("Unknown parameter %s = %r" % (pname, pvalue))

    @property
    def checkpoint(self):
        """
        Model checkpoint to resume training with.

        Type: ``str``.
        """
        return self._parms.get("checkpoint")

    @checkpoint.setter
    def checkpoint(self, checkpoint):
        assert_is_type(checkpoint, None, str, H2OEstimator)
        self._parms["checkpoint"] = checkpoint

    @property
    def autoencoder(self):
        """
        Auto-Encoder.

        Type: ``bool``  (default: ``False``).
        """
        return self._parms.get("autoencoder")

    @autoencoder.setter
    def autoencoder(self, autoencoder):
        assert_is_type(autoencoder, None, bool)
        self._parms["autoencoder"] = autoencoder

    @property
    def training_frame(self):
        """
        Id of the training data frame.

        Type: ``H2OFrame``.
        """
        return self._parms.get("training_frame")

    @training_frame.setter
    def training_frame(self, training_frame):
        assert_is_type(training_frame, None, H2OFrame)
        self._parms["training_frame"] = training_frame

    @property
    def validation_frame(self):
        """
        Id of the validation data frame.

        Type: ``H2OFrame``.
        """
        return self._parms.get("validation_frame")

    @validation_frame.setter
    def validation_frame(self, validation_frame):
        assert_is_type(validation_frame, None, H2OFrame)
        self._parms["validation_frame"] = validation_frame

    @property
    def nfolds(self):
        """
        Number of folds for K-fold cross-validation (0 to disable or >= 2).

        Type: ``int``  (default: ``0``).
        """
        return self._parms.get("nfolds")

    @nfolds.setter
    def nfolds(self, nfolds):
        assert_is_type(nfolds, None, int)
        self._parms["nfolds"] = nfolds

    @property
    def balance_classes(self):
        """
        Balance training data class counts via over/under-sampling (for imbalanced data).

        Type: ``bool``  (default: ``False``).
        """
        return self._parms.get("balance_classes")

    @balance_classes.setter
    def balance_classes(self, balance_classes):
        assert_is_type(balance_classes, None, bool)
        self._parms["balance_classes"] = balance_classes

    @property
    def max_after_balance_size(self):
        """
        Maximum relative size of the training data after balancing class counts (can be less than 1.0). Requires
        balance_classes.

        Type: ``float``  (default: ``5``).
        """
        return self._parms.get("max_after_balance_size")

    @max_after_balance_size.setter
    def max_after_balance_size(self, max_after_balance_size):
        assert_is_type(max_after_balance_size, None, float)
        self._parms["max_after_balance_size"] = max_after_balance_size

    @property
    def class_sampling_factors(self):
        """
        Desired over/under-sampling ratios per class (in lexicographic order). If not specified, sampling factors will
        be automatically computed to obtain class balance during training. Requires balance_classes.

        Type: ``List[float]``.
        """
        return self._parms.get("class_sampling_factors")

    @class_sampling_factors.setter
    def class_sampling_factors(self, class_sampling_factors):
        assert_is_type(class_sampling_factors, None, [float])
        self._parms["class_sampling_factors"] = class_sampling_factors

    @property
    def keep_cross_validation_models(self):
        """
        Whether to keep the cross-validation models.

        Type: ``bool``  (default: ``True``).
        """
        return self._parms.get("keep_cross_validation_models")

    @keep_cross_validation_models.setter
    def keep_cross_validation_models(self, keep_cross_validation_models):
        assert_is_type(keep_cross_validation_models, None, bool)
        self._parms["keep_cross_validation_models"] = keep_cross_validation_models

    @property
    def keep_cross_validation_predictions(self):
        """
        Whether to keep the predictions of the cross-validation models.

        Type: ``bool``  (default: ``False``).
        """
        return self._parms.get("keep_cross_validation_predictions")

    @keep_cross_validation_predictions.setter
    def keep_cross_validation_predictions(self, keep_cross_validation_predictions):
        assert_is_type(keep_cross_validation_predictions, None, bool)
        self._parms["keep_cross_validation_predictions"] = keep_cross_validation_predictions

    @property
    def keep_cross_validation_fold_assignment(self):
        """
        Whether to keep the cross-validation fold assignment.

        Type: ``bool``  (default: ``False``).
        """
        return self._parms.get("keep_cross_validation_fold_assignment")

    @keep_cross_validation_fold_assignment.setter
    def keep_cross_validation_fold_assignment(self, keep_cross_validation_fold_assignment):
        assert_is_type(keep_cross_validation_fold_assignment, None, bool)
        self._parms["keep_cross_validation_fold_assignment"] = keep_cross_validation_fold_assignment

    @property
    def fold_assignment(self):
        """
        Cross-validation fold assignment scheme, if fold_column is not specified. The 'Stratified' option will stratify
        the folds based on the response variable, for classification problems.

        One of: ``"auto"``, ``"random"``, ``"modulo"``, ``"stratified"``  (default: ``"auto"``).
        """
        return self._parms.get("fold_assignment")

    @fold_assignment.setter
    def fold_assignment(self, fold_assignment):
        assert_is_type(fold_assignment, None, Enum("auto", "random", "modulo", "stratified"))
        self._parms["fold_assignment"] = fold_assignment

    @property
    def fold_column(self):
        """
        Column with cross-validation fold index assignment per observation.

        Type: ``str``.
        """
        return self._parms.get("fold_column")

    @fold_column.setter
    def fold_column(self, fold_column):
        assert_is_type(fold_column, None, str)
        self._parms["fold_column"] = fold_column

    @property
    def response_column(self):
        """
        Response variable column.

        Type: ``str``.
        """
        return self._parms.get("response_column")

    @response_column.setter
    def response_column(self, response_column):
        assert_is_type(response_column, None, str)
        self._parms["response_column"] = response_column

    @property
    def offset_column(self):
        """
        Offset column. This will be added to the combination of columns before applying the link function.

        Type: ``str``.
        """
        return self._parms.get("offset_column")

    @offset_column.setter
    def offset_column(self, offset_column):
        assert_is_type(offset_column, None, str)
        self._parms["offset_column"] = offset_column

    @property
    def weights_column(self):
        """
        Column with observation weights. Giving some observation a weight of zero is equivalent to excluding it from the
        dataset; giving an observation a relative weight of 2 is equivalent to repeating that row twice. Negative
        weights are not allowed. Note: Weights are per-row observation weights and do not increase the size of the data
        frame. This is typically the number of times a row is repeated, but non-integer values are supported as well.
        During training, rows with higher weights matter more, due to the larger loss function pre-factor.

        Type: ``str``.
        """
        return self._parms.get("weights_column")

    @weights_column.setter
    def weights_column(self, weights_column):
        assert_is_type(weights_column, None, str)
        self._parms["weights_column"] = weights_column

    @property
    def ignored_columns(self):
        """
        Names of columns to ignore for training.

        Type: ``List[str]``.
        """
        return self._parms.get("ignored_columns")

    @ignored_columns.setter
    def ignored_columns(self, ignored_columns):
        assert_is_type(ignored_columns, None, [str])
        self._parms["ignored_columns"] = ignored_columns

    @property
    def score_each_iteration(self):
        """
        Whether to score during each iteration of model training.

        Type: ``bool``  (default: ``False``).
        """
        return self._parms.get("score_each_iteration")

    @score_each_iteration.setter
    def score_each_iteration(self, score_each_iteration):
        assert_is_type(score_each_iteration, None, bool)
        self._parms["score_each_iteration"] = score_each_iteration

    @property
    def categorical_encoding(self):
        """
        Encoding scheme for categorical features

        One of: ``"auto"``, ``"enum"``, ``"one_hot_internal"``, ``"one_hot_explicit"``, ``"binary"``, ``"eigen"``,
        ``"label_encoder"``, ``"sort_by_response"``, ``"enum_limited"``  (default: ``"auto"``).
        """
        return self._parms.get("categorical_encoding")

    @categorical_encoding.setter
    def categorical_encoding(self, categorical_encoding):
        assert_is_type(categorical_encoding, None, Enum("auto", "enum", "one_hot_internal", "one_hot_explicit", "binary", "eigen", "label_encoder", "sort_by_response", "enum_limited"))
        self._parms["categorical_encoding"] = categorical_encoding

    @property
    def overwrite_with_best_model(self):
        """
        If enabled, override the final model with the best model found during training.

        Type: ``bool``  (default: ``True``).
        """
        return self._parms.get("overwrite_with_best_model")

    @overwrite_with_best_model.setter
    def overwrite_with_best_model(self, overwrite_with_best_model):
        assert_is_type(overwrite_with_best_model, None, bool)
        self._parms["overwrite_with_best_model"] = overwrite_with_best_model

    @property
    def epochs(self):
        """
        How many times the dataset should be iterated (streamed), can be fractional.

        Type: ``float``  (default: ``10``).
        """
        return self._parms.get("epochs")

    @epochs.setter
    def epochs(self, epochs):
        assert_is_type(epochs, None, numeric)
        self._parms["epochs"] = epochs

    @property
    def train_samples_per_iteration(self):
        """
        Number of training samples (globally) per MapReduce iteration. Special values are 0: one epoch, -1: all
        available data (e.g., replicated training data), -2: automatic.

        Type: ``int``  (default: ``-2``).
        """
        return self._parms.get("train_samples_per_iteration")

    @train_samples_per_iteration.setter
    def train_samples_per_iteration(self, train_samples_per_iteration):
        assert_is_type(train_samples_per_iteration, None, int)
        self._parms["train_samples_per_iteration"] = train_samples_per_iteration

    @property
    def target_ratio_comm_to_comp(self):
        """
        Target ratio of communication overhead to computation. Only for multi-node operation and
        train_samples_per_iteration = -2 (auto-tuning).

        Type: ``float``  (default: ``0.05``).
        """
        return self._parms.get("target_ratio_comm_to_comp")

    @target_ratio_comm_to_comp.setter
    def target_ratio_comm_to_comp(self, target_ratio_comm_to_comp):
        assert_is_type(target_ratio_comm_to_comp, None, numeric)
        self._parms["target_ratio_comm_to_comp"] = target_ratio_comm_to_comp

    @property
    def seed(self):
        """
        Seed for random numbers (affects sampling) - Note: only reproducible when running single threaded.

        Type: ``int``  (default: ``-1``).
        """
        return self._parms.get("seed")

    @seed.setter
    def seed(self, seed):
        assert_is_type(seed, None, int)
        self._parms["seed"] = seed

    @property
    def standardize(self):
        """
        If enabled, automatically standardize the data. If disabled, the user must provide properly scaled input data.

        Type: ``bool``  (default: ``True``).
        """
        return self._parms.get("standardize")

    @standardize.setter
    def standardize(self, standardize):
        assert_is_type(standardize, None, bool)
        self._parms["standardize"] = standardize

    @property
    def learning_rate(self):
        """
        Learning rate (higher => less stable, lower => slower convergence).

        Type: ``float``  (default: ``0.001``).
        """
        return self._parms.get("learning_rate")

    @learning_rate.setter
    def learning_rate(self, learning_rate):
        assert_is_type(learning_rate, None, numeric)
        self._parms["learning_rate"] = learning_rate

    @property
    def learning_rate_annealing(self):
        """
        Learning rate annealing: rate / (1 + rate_annealing * samples).

        Type: ``float``  (default: ``1e-06``).
        """
        return self._parms.get("learning_rate_annealing")

    @learning_rate_annealing.setter
    def learning_rate_annealing(self, learning_rate_annealing):
        assert_is_type(learning_rate_annealing, None, numeric)
        self._parms["learning_rate_annealing"] = learning_rate_annealing

    @property
    def momentum_start(self):
        """
        Initial momentum at the beginning of training (try 0.5).

        Type: ``float``  (default: ``0.9``).
        """
        return self._parms.get("momentum_start")

    @momentum_start.setter
    def momentum_start(self, momentum_start):
        assert_is_type(momentum_start, None, numeric)
        self._parms["momentum_start"] = momentum_start

    @property
    def momentum_ramp(self):
        """
        Number of training samples for which momentum increases.

        Type: ``float``  (default: ``10000``).
        """
        return self._parms.get("momentum_ramp")

    @momentum_ramp.setter
    def momentum_ramp(self, momentum_ramp):
        assert_is_type(momentum_ramp, None, numeric)
        self._parms["momentum_ramp"] = momentum_ramp

    @property
    def momentum_stable(self):
        """
        Final momentum after the ramp is over (try 0.99).

        Type: ``float``  (default: ``0.9``).
        """
        return self._parms.get("momentum_stable")

    @momentum_stable.setter
    def momentum_stable(self, momentum_stable):
        assert_is_type(momentum_stable, None, numeric)
        self._parms["momentum_stable"] = momentum_stable

    @property
    def distribution(self):
        """
        Distribution function

        One of: ``"auto"``, ``"bernoulli"``, ``"multinomial"``, ``"gaussian"``, ``"poisson"``, ``"gamma"``,
        ``"tweedie"``, ``"laplace"``, ``"quantile"``, ``"huber"``  (default: ``"auto"``).
        """
        return self._parms.get("distribution")

    @distribution.setter
    def distribution(self, distribution):
        assert_is_type(distribution, None, Enum("auto", "bernoulli", "multinomial", "gaussian", "poisson", "gamma", "tweedie", "laplace", "quantile", "huber"))
        self._parms["distribution"] = distribution

    @property
    def score_interval(self):
        """
        Shortest time interval (in seconds) between model scoring.

        Type: ``float``  (default: ``5``).
        """
        return self._parms.get("score_interval")

    @score_interval.setter
    def score_interval(self, score_interval):
        assert_is_type(score_interval, None, numeric)
        self._parms["score_interval"] = score_interval

    @property
    def score_training_samples(self):
        """
        Number of training set samples for scoring (0 for all).

        Type: ``int``  (default: ``10000``).
        """
        return self._parms.get("score_training_samples")

    @score_training_samples.setter
    def score_training_samples(self, score_training_samples):
        assert_is_type(score_training_samples, None, int)
        self._parms["score_training_samples"] = score_training_samples

    @property
    def score_validation_samples(self):
        """
        Number of validation set samples for scoring (0 for all).

        Type: ``int``  (default: ``0``).
        """
        return self._parms.get("score_validation_samples")

    @score_validation_samples.setter
    def score_validation_samples(self, score_validation_samples):
        assert_is_type(score_validation_samples, None, int)
        self._parms["score_validation_samples"] = score_validation_samples

    @property
    def score_duty_cycle(self):
        """
        Maximum duty cycle fraction for scoring (lower: more training, higher: more scoring).

        Type: ``float``  (default: ``0.1``).
        """
        return self._parms.get("score_duty_cycle")

    @score_duty_cycle.setter
    def score_duty_cycle(self, score_duty_cycle):
        assert_is_type(score_duty_cycle, None, numeric)
        self._parms["score_duty_cycle"] = score_duty_cycle

    @property
    def classification_stop(self):
        """
        Stopping criterion for classification error fraction on training data (-1 to disable).

        Type: ``float``  (default: ``0``).
        """
        return self._parms.get("classification_stop")

    @classification_stop.setter
    def classification_stop(self, classification_stop):
        assert_is_type(classification_stop, None, numeric)
        self._parms["classification_stop"] = classification_stop

    @property
    def regression_stop(self):
        """
        Stopping criterion for regression error (MSE) on training data (-1 to disable).

        Type: ``float``  (default: ``0``).
        """
        return self._parms.get("regression_stop")

    @regression_stop.setter
    def regression_stop(self, regression_stop):
        assert_is_type(regression_stop, None, numeric)
        self._parms["regression_stop"] = regression_stop

    @property
    def stopping_rounds(self):
        """
        Early stopping based on convergence of stopping_metric. Stop if simple moving average of length k of the
        stopping_metric does not improve for k:=stopping_rounds scoring events (0 to disable)

        Type: ``int``  (default: ``5``).
        """
        return self._parms.get("stopping_rounds")

    @stopping_rounds.setter
    def stopping_rounds(self, stopping_rounds):
        assert_is_type(stopping_rounds, None, int)
        self._parms["stopping_rounds"] = stopping_rounds

    @property
    def stopping_metric(self):
        """
        Metric to use for early stopping (AUTO: logloss for classification, deviance for regression). Note that custom
        and custom_increasing can only be used in GBM and DRF with the Python client.

        One of: ``"auto"``, ``"deviance"``, ``"logloss"``, ``"mse"``, ``"rmse"``, ``"mae"``, ``"rmsle"``, ``"auc"``,
        ``"lift_top_group"``, ``"misclassification"``, ``"mean_per_class_error"``, ``"custom"``, ``"custom_increasing"``
        (default: ``"auto"``).
        """
        return self._parms.get("stopping_metric")

    @stopping_metric.setter
    def stopping_metric(self, stopping_metric):
        assert_is_type(stopping_metric, None, Enum("auto", "deviance", "logloss", "mse", "rmse", "mae", "rmsle", "auc", "lift_top_group", "misclassification", "mean_per_class_error", "custom", "custom_increasing"))
        self._parms["stopping_metric"] = stopping_metric

    @property
    def stopping_tolerance(self):
        """
        Relative tolerance for metric-based stopping criterion (stop if relative improvement is not at least this much)

        Type: ``float``  (default: ``0``).
        """
        return self._parms.get("stopping_tolerance")

    @stopping_tolerance.setter
    def stopping_tolerance(self, stopping_tolerance):
        assert_is_type(stopping_tolerance, None, numeric)
        self._parms["stopping_tolerance"] = stopping_tolerance

    @property
    def max_runtime_secs(self):
        """
        Maximum allowed runtime in seconds for model training. Use 0 to disable.

        Type: ``float``  (default: ``0``).
        """
        return self._parms.get("max_runtime_secs")

    @max_runtime_secs.setter
    def max_runtime_secs(self, max_runtime_secs):
        assert_is_type(max_runtime_secs, None, numeric)
        self._parms["max_runtime_secs"] = max_runtime_secs

    @property
    def ignore_const_cols(self):
        """
        Ignore constant columns.

        Type: ``bool``  (default: ``True``).
        """
        return self._parms.get("ignore_const_cols")

    @ignore_const_cols.setter
    def ignore_const_cols(self, ignore_const_cols):
        assert_is_type(ignore_const_cols, None, bool)
        self._parms["ignore_const_cols"] = ignore_const_cols

    @property
    def shuffle_training_data(self):
        """
        Enable global shuffling of training data.

        Type: ``bool``  (default: ``True``).
        """
        return self._parms.get("shuffle_training_data")

    @shuffle_training_data.setter
    def shuffle_training_data(self, shuffle_training_data):
        assert_is_type(shuffle_training_data, None, bool)
        self._parms["shuffle_training_data"] = shuffle_training_data

    @property
    def mini_batch_size(self):
        """
        Mini-batch size (smaller leads to better fit, larger can speed up and generalize better).

        Type: ``int``  (default: ``32``).
        """
        return self._parms.get("mini_batch_size")

    @mini_batch_size.setter
    def mini_batch_size(self, mini_batch_size):
        assert_is_type(mini_batch_size, None, int)
        self._parms["mini_batch_size"] = mini_batch_size

    @property
    def clip_gradient(self):
        """
        Clip gradients once their absolute value is larger than this value.

        Type: ``float``  (default: ``10``).
        """
        return self._parms.get("clip_gradient")

    @clip_gradient.setter
    def clip_gradient(self, clip_gradient):
        assert_is_type(clip_gradient, None, numeric)
        self._parms["clip_gradient"] = clip_gradient

    @property
    def network(self):
        """
        Network architecture.

        One of: ``"auto"``, ``"user"``, ``"lenet"``, ``"alexnet"``, ``"vgg"``, ``"googlenet"``, ``"inception_bn"``,
        ``"resnet"``  (default: ``"auto"``).
        """
        return self._parms.get("network")

    @network.setter
    def network(self, network):
        assert_is_type(network, None, Enum("auto", "user", "lenet", "alexnet", "vgg", "googlenet", "inception_bn", "resnet"))
        self._parms["network"] = network

    @property
    def backend(self):
        """
        Deep Learning Backend.

        One of: ``"mxnet"``, ``"caffe"``, ``"tensorflow"``  (default: ``"mxnet"``).
        """
        return self._parms.get("backend")

    @backend.setter
    def backend(self, backend):
        assert_is_type(backend, None, Enum("mxnet", "caffe", "tensorflow"))
        self._parms["backend"] = backend

    @property
    def image_shape(self):
        """
        Width and height of image.

        Type: ``List[int]``  (default: ``[0, 0]``).
        """
        return self._parms.get("image_shape")

    @image_shape.setter
    def image_shape(self, image_shape):
        assert_is_type(image_shape, None, [int])
        self._parms["image_shape"] = image_shape

    @property
    def channels(self):
        """
        Number of (color) channels.

        Type: ``int``  (default: ``3``).
        """
        return self._parms.get("channels")

    @channels.setter
    def channels(self, channels):
        assert_is_type(channels, None, int)
        self._parms["channels"] = channels

    @property
    def sparse(self):
        """
        Sparse data handling (more efficient for data with lots of 0 values).

        Type: ``bool``  (default: ``False``).
        """
        return self._parms.get("sparse")

    @sparse.setter
    def sparse(self, sparse):
        assert_is_type(sparse, None, bool)
        self._parms["sparse"] = sparse

    @property
    def gpu(self):
        """
        Whether to use a GPU (if available).

        Type: ``bool``  (default: ``True``).
        """
        return self._parms.get("gpu")

    @gpu.setter
    def gpu(self, gpu):
        assert_is_type(gpu, None, bool)
        self._parms["gpu"] = gpu

    @property
    def device_id(self):
        """
        Device IDs (which GPUs to use).

        Type: ``List[int]``  (default: ``[0]``).
        """
        return self._parms.get("device_id")

    @device_id.setter
    def device_id(self, device_id):
        assert_is_type(device_id, None, [int])
        self._parms["device_id"] = device_id

    @property
    def cache_data(self):
        """
        Whether to cache the data in memory (automatically disabled if data size is too large).

        Type: ``bool``  (default: ``True``).
        """
        return self._parms.get("cache_data")

    @cache_data.setter
    def cache_data(self, cache_data):
        assert_is_type(cache_data, None, bool)
        self._parms["cache_data"] = cache_data

    @property
    def network_definition_file(self):
        """
        Path of file containing network definition (graph, architecture).

        Type: ``str``.
        """
        return self._parms.get("network_definition_file")

    @network_definition_file.setter
    def network_definition_file(self, network_definition_file):
        assert_is_type(network_definition_file, None, str)
        self._parms["network_definition_file"] = network_definition_file

    @property
    def network_parameters_file(self):
        """
        Path of file containing network (initial) parameters (weights, biases).

        Type: ``str``.
        """
        return self._parms.get("network_parameters_file")

    @network_parameters_file.setter
    def network_parameters_file(self, network_parameters_file):
        assert_is_type(network_parameters_file, None, str)
        self._parms["network_parameters_file"] = network_parameters_file

    @property
    def mean_image_file(self):
        """
        Path of file containing the mean image data for data normalization.

        Type: ``str``.
        """
        return self._parms.get("mean_image_file")

    @mean_image_file.setter
    def mean_image_file(self, mean_image_file):
        assert_is_type(mean_image_file, None, str)
        self._parms["mean_image_file"] = mean_image_file

    @property
    def export_native_parameters_prefix(self):
        """
        Path (prefix) where to export the native model parameters after every iteration.

        Type: ``str``.
        """
        return self._parms.get("export_native_parameters_prefix")

    @export_native_parameters_prefix.setter
    def export_native_parameters_prefix(self, export_native_parameters_prefix):
        assert_is_type(export_native_parameters_prefix, None, str)
        self._parms["export_native_parameters_prefix"] = export_native_parameters_prefix

    @property
    def activation(self):
        """
        Activation function. Only used if no user-defined network architecture file is provided, and only for
        problem_type=dataset.

        One of: ``"rectifier"``, ``"tanh"``.
        """
        return self._parms.get("activation")

    @activation.setter
    def activation(self, activation):
        assert_is_type(activation, None, Enum("rectifier", "tanh"))
        self._parms["activation"] = activation

    @property
    def hidden(self):
        """
        Hidden layer sizes (e.g. [200, 200]). Only used if no user-defined network architecture file is provided, and
        only for problem_type=dataset.

        Type: ``List[int]``.
        """
        return self._parms.get("hidden")

    @hidden.setter
    def hidden(self, hidden):
        assert_is_type(hidden, None, [int])
        self._parms["hidden"] = hidden

    @property
    def input_dropout_ratio(self):
        """
        Input layer dropout ratio (can improve generalization, try 0.1 or 0.2).

        Type: ``float``  (default: ``0``).
        """
        return self._parms.get("input_dropout_ratio")

    @input_dropout_ratio.setter
    def input_dropout_ratio(self, input_dropout_ratio):
        assert_is_type(input_dropout_ratio, None, numeric)
        self._parms["input_dropout_ratio"] = input_dropout_ratio

    @property
    def hidden_dropout_ratios(self):
        """
        Hidden layer dropout ratios (can improve generalization), specify one value per hidden layer, defaults to 0.5.

        Type: ``List[float]``.
        """
        return self._parms.get("hidden_dropout_ratios")

    @hidden_dropout_ratios.setter
    def hidden_dropout_ratios(self, hidden_dropout_ratios):
        assert_is_type(hidden_dropout_ratios, None, [numeric])
        self._parms["hidden_dropout_ratios"] = hidden_dropout_ratios

    @property
    def problem_type(self):
        """
        Problem type, auto-detected by default. If set to image, the H2OFrame must contain a string column containing
        the path (URI or URL) to the images in the first column. If set to text, the H2OFrame must contain a string
        column containing the text in the first column. If set to dataset, Deep Water behaves just like any other H2O
        Model and builds a model on the provided H2OFrame (non-String columns).

        One of: ``"auto"``, ``"image"``, ``"dataset"``  (default: ``"auto"``).
        """
        return self._parms.get("problem_type")

    @problem_type.setter
    def problem_type(self, problem_type):
        assert_is_type(problem_type, None, Enum("auto", "image", "dataset"))
        self._parms["problem_type"] = problem_type

    @property
    def export_checkpoints_dir(self):
        """
        Automatically export generated models to this directory.

        Type: ``str``.
        """
        return self._parms.get("export_checkpoints_dir")

    @export_checkpoints_dir.setter
    def export_checkpoints_dir(self, export_checkpoints_dir):
        assert_is_type(export_checkpoints_dir, None, str)
        self._parms["export_checkpoints_dir"] = export_checkpoints_dir

    # Ask the H2O server whether a Deep Water model can be built (depends on availability of native backends)
    @staticmethod
    def available():
        """Returns True if a deep water model can be built, or False otherwise."""
        builder_json = h2o.api("GET /3/ModelBuilders", data={"algo": "deepwater"})
        visibility = builder_json["model_builders"]["deepwater"]["visibility"]
        if visibility == "Experimental":
            print("Cannot build a Deep Water model - no backend found.")
            return False
        else:
            return True


#!/usr/bin/python

import sys
import os
import json
import subprocess
import time
import datetime
from pytz import timezone
from dateutil import parser

"""
This script will be invoked if it is included in the post-build action of an jenkin job and the job has failed.

It will perform the following tasks:
1. attach the failure informaiton of all tests in the current build to a summary file including the following fields:
    - timestamp, jenkin_job_name, build_id, git_has, node_name, build_failure(test failed but to build failure),
    JUnit/PyUnit/RUnit/Hadoop, testName.
2. save the above summary file to s3 somewhere using the command: s3cmd put "$TEST_OUTPUT_FILE" s3://ai.h2o.tests/jenkins/
3. store the failed test info in a dictionary and save it to s3 as well;
4. for failed tests, save the txt failed test results to aid the debugging process.  Attach timestamp to file name
  in order to aid the process of cleaning out the file directory with obsolete files.
"""

# --------------------------------------------------------------------
# Main program
# --------------------------------------------------------------------

g_test_root_dir = os.path.dirname(os.path.realpath(__file__)) # directory where we are running out code from
g_script_name = ''  # store script name.
g_timestamp = ''
g_job_name = ''
g_build_id = ''
g_git_hash = ''
g_node_name = ''
g_unit_test_type = ''
g_jenkins_url = ''
g_temp_filename = os.path.join(g_test_root_dir,'tempText')  # temp file to store data curled from Jenkins
g_failed_testnames = []
g_failed_test_paths = []
g_failed_tests_dict = ''    # contains the filename that contains failed tests info in a dictionary
g_failed_tests_info_dict = dict()   # store failed tests info in a dictionary
g_resource_url = ''
g_timestring = ''
g_daily_failure_csv = ''

def init_failed_tests_dict():
    """
    initialize the fields of dictionary storing failed tests.
    :return:
    """
    global g_failed_test_info_dict
    g_failed_tests_info_dict["TestName"] = []
    g_failed_tests_info_dict["TestInfo"] = []

def init_update_each_failed_test_dict(one_test_info, failed_test_path, testName, newTest):
    """
    For each test, a dictionary structure will be built to record the various info about that test's failure
    information.  In particular, for each failed tests, there will be a dictionary associated with that test
    stored in the field "TestInfo" of g_faiiled_tests_info_dict.  The following fields are included:
        "JenkinsJobName": job name
        "BuildID"
        "Timestamp": in seconds
        "GitHash"
        "TestCategory": JUnit, PyUnit, RUnit or HadoopPyUnit, HadoopRUnit
        "NodeName": name of machine that the job was run on
        "FailureCount": integer counting number of times this particular test has failed.  An intermittent can be
          determined as any test with FailureCount >= 2.
        "FailureMessages": contains failure messages for the test
    :return: a new dict for that test
    """
    if newTest:
        one_test_info = dict()
        one_test_info["JenkinsJobName"] = []
        one_test_info["BuildID"] = []
        one_test_info["Timestamp"] = []
        one_test_info["GitHash"] = []
        one_test_info["TestCategory"] = []    # would be JUnit, PyUnit, RUnit or HadoopPyUnit, HadoopRUnit
        one_test_info["NodeName"] = []
        one_test_info["FailureMessages"] = [] # contains failure messages for the test
        one_test_info["FailureCount"] = 0
        one_test_info["TestName"] = testName

#    if g_timestamp not in one_test_info["Timestamp"]:
    one_test_info["JenkinsJobName"].append(g_job_name)
    one_test_info["BuildID"].append(g_build_id)
    one_test_info["Timestamp"].append(g_timestamp)
    one_test_info["GitHash"].append(g_git_hash)
    one_test_info["TestCategory"].append(g_unit_test_type) # would be JUnit, PyUnit, RUnit or HadoopPyUnit, HadoopRUnit
    one_test_info["NodeName"].append(g_node_name)
    one_test_info["FailureCount"] += 1

    error_url = '/'.join([g_resource_url, 'testReport', failed_test_path])
    get_console_out(error_url)      # store failure message in temp file

    if os.path.isfile(g_temp_filename):
        with open(g_temp_filename, 'r') as error_file:
            one_test_info["FailureMessages"].append(error_file.read())
    else:
        one_test_info["FailureMessages"].append("")   # append empty error message if file not found
    return one_test_info

def usage():
    """
    Print USAGE help.
    """
    print("")
    print("Usage:  ")
    print("python scrapeForIntermittents timestamp job_name build_id git_sha node_name unit_test_category jenkins_URL "
          "output_filename output_dict_name month_of_data_to_keep tests_info_per_build_failure")
    print(" The unit_test_category can be 'junit', 'pyunit' or 'runit'.")
    print(" The ouput_dict_name is the filename that we will save a dictionary structure of the failed unit tests.")
    print(" The month_of_data_to_keep is an integer indicating how many months that we want to kee the data "
          "starting from now.  Any data that is older than the value will be deleted.")
    print("tests_info_per_build_failure is the file name that will store failed test info per build failure.")

'''
This function is written to extract the console output that has already been stored
in a text file in a remote place and saved it in a local directory that we have accessed
to.  We want to be able to read in the local text file and proces it.
'''
def get_console_out(url_string):
    """
    Grab the console output from Jenkins and save the content into a temp file
     (g_temp_filename).  From the saved text file, we can grab the names of
     failed tests.

    Parameters
    ----------
    url_string :  str
        contains information on the jenkins job whose console output we are interested in.  It is in the context
        of resource_url/job/job_name/build_id/testReport/

    :return: none
    """
    full_command = 'curl ' + '"'+ url_string +'"'+ ' --user '+'"admin:admin"'+' > ' + g_temp_filename
    subprocess.call(full_command,shell=True)

def extract_failed_tests_info():
    """
    This method will scrape the console output for pyunit,runit and hadoop runs and grab the list of failed tests
    and their corresponding paths so that the test execution summary can be located later.

    :return: none
    """
    global g_failed_testnames
    global g_failed_test_paths

    if os.path.isfile(g_temp_filename):
        console_file = open(g_temp_filename,'r')  # open temp file that stored jenkins job console output
        try:
            for each_line in console_file:  # go through each line of console output to extract build ID, data/time ...
                each_line.strip()
                print(each_line)
                if ("Test Result" in each_line) and ("failure" in each_line): # the next few lines will contain failed tests
                    temp = each_line.split("testReport")
                    if ("Test Result" in temp[1]) and ("failure" in temp[1]):        # grab number of failed tests
                        try:
                            tempCount = int(temp[1].split("</a>")[1].split(" ")[0].split("(")[1])

                            if isinstance(tempCount, int) and tempCount > 0:  # temp[1], temp[2],... should contain failed tests
                                for findex in range(2,len(temp)):
                                    tempMess = temp[findex].split(">")
                                    g_failed_test_paths.append(tempMess[0].strip('"'))
                                    ftestname = tempMess[1].strip("</a")
                                    nameLen = len(ftestname)
                                    true_testname = ftestname[8:nameLen] if 'r_suite.' in ftestname else ftestname
                                    g_failed_testnames.append(true_testname)
                                break   # done.  Only one spot contains failed test info.
                        except:
                            break   # file probably does not have failures captured.
        finally:
            console_file.close()

def save_failed_tests_info():
    """
    Given the failed tests information in g_failed_testnames, add the new failed test to
    text file.  In addition, it will update the dictionary that stores all failed test info as well.

    :return: None
    """
    global g_failed_tests_info_dict
    if len(g_failed_testnames) > 0: # found failed test
        if os.path.isfile(g_failed_tests_dict) and os.path.getsize(g_failed_tests_dict) > 10:
            try:
                g_failed_tests_info_dict=json.load(open(g_failed_tests_dict, 'r'))
            except:
                init_failed_tests_dict()

            # with open(g_failed_tests_dict, 'rb') as dict_file:
            #     g_failed_tests_info_dict = pickle.load(dict_file)
        else:       # file not found, create new dict
            init_failed_tests_dict()

        with open(g_summary_text_filename, 'a') as failed_file:
            with open(g_daily_failure_csv, 'w') as daily_failure:
                for index in range(len(g_failed_testnames)):

                    testInfo = ','.join([g_timestring, g_job_name, str(g_build_id), g_git_hash, g_node_name,
                                         g_unit_test_type, g_failed_testnames[index]])
                    failed_file.write(testInfo+'\n')
                    daily_failure.write(testInfo+'\n')
                        # update failed tests dictionary
                    update_failed_test_info_dict(g_failed_testnames[index], g_failed_test_paths[index])
        json.dump(g_failed_tests_info_dict, open(g_failed_tests_dict, 'w'))

def update_failed_test_info_dict(failed_testname, failed_test_path):
    """
    Update the dictionary structure that stores failed unit test information.

    :param failed_testname: string containing name of failed test.
    :param failed_test_path: string containing the path to failed test url.
    :return: None
    """
    global g_failed_tests_info_dict

    if failed_testname in g_failed_tests_info_dict["TestName"]:     # existing test
        g_failed_tests_info_dict["TestInfo"][g_failed_tests_info_dict["TestName"].index(failed_testname)] = \
            init_update_each_failed_test_dict(
                g_failed_tests_info_dict["TestInfo"][g_failed_tests_info_dict["TestName"].index(failed_testname)],
                failed_test_path, failed_testname, False)
    else:   # next test
        g_failed_tests_info_dict["TestName"].append(failed_testname)
        g_failed_tests_info_dict["TestInfo"].append(init_update_each_failed_test_dict(dict(), failed_test_path,
                                                                                failed_testname, True))

def trim_data_back_to(monthToKeep):
    """
    This method will remove data from the summary text file and the dictionary file for tests that occurs before
    the number of months specified by monthToKeep.

    :param monthToKeep:
    :return:
    """
    global g_failed_tests_info_dict
    current_time = time.time()      # unit in seconds

    oldest_time_allowed = current_time - monthToKeep*30*24*3600 # in seconds

    clean_up_failed_test_dict(oldest_time_allowed)
    clean_up_summary_text(oldest_time_allowed)

def clean_up_failed_test_dict(oldest_time_allowed):
    # read in data from dictionary file
    global g_failed_tests_info_dict
    if os.path.isfile(g_failed_tests_dict) and os.path.getsize(g_failed_tests_dict)>10:
        try:
            g_failed_tests_info_dict=json.load(open(g_failed_tests_dict,'r'))
            test_index = 0
            while test_index < len(g_failed_tests_info_dict["TestName"]):
                test_dicts = g_failed_tests_info_dict["TestInfo"][test_index]   # a list of dictionary

                dict_index = 0
                while (len(test_dicts["Timestamp"]) > 0) and (dict_index < len(test_dicts["Timestamp"])):
                    if (test_dicts["Timestamp"][dict_index] < oldest_time_allowed):
                        del test_dicts["JenkinsJobName"][dict_index]
                        del test_dicts["BuildID"][dict_index]
                        del test_dicts["Timestamp"][dict_index]
                        del test_dicts["GitHash"][dict_index]
                        del test_dicts["TestCategory"][dict_index]
                        del test_dicts["NodeName"][dict_index]
                        test_dicts["FailureCount"] -= 1
                    else:
                        dict_index = dict_index+1

                if test_dicts["FailureCount"] <= 0: # remove test name with 0 counts of failure count
                    del g_failed_tests_info_dict["Testname"][test_index]
                    del g_failed_tests_info_dict["TestInfo"][test_index]
                else:
                    test_index = test_index+1

            json.dump(g_failed_tests_info_dict, open(g_failed_tests_dict, 'w'))
        except:
            pass

def clean_up_summary_text(oldest_time_allowed):
    # clean up the summary text data
    if os.path.isfile(g_summary_text_filename):
        with open(g_summary_text_filename, 'r') as text_file:
            with open(g_temp_filename, 'w') as temp_file:
                for each_line in text_file:
                    temp = each_line.split(',')
                    if len(temp) >= 7:
                        dateObj = parser.parse(temp[0]).timetuple()
                        timestamp = time.mktime(dateObj)

                        if (timestamp > oldest_time_allowed):
                            temp_file.write(each_line)

        with open(g_summary_text_filename, 'w') as text_file:   # write content back to original summary file
            with open(g_temp_filename, 'r') as temp_file:
                text_file.write(temp_file.read())

def main(argv):
    """
    Main program.  Expect script name plus 7 inputs in the following order:
    - This script name
    1. timestamp: time in s
    2. jenkins_job_name (JOB_NAME)
    3. build_id (BUILD_ID)
    4. git hash (GIT_COMMIT)
    5. node name (NODE_NAME)
    6. unit test category (JUnit, PyUnit, RUnit, Hadoop)
    7. Jenkins URL (JENKINS_URL)
    8. Text file name where failure summaries are stored
    9. Filename that stored all failed test info as a dictionary
    10. duration (month) to keep data: data older tghan this input will be removed

    @return: none
    """
    global g_script_name
    global g_test_root_dir
    global g_timestamp
    global g_job_name
    global g_build_id
    global g_git_hash
    global g_node_name
    global g_unit_test_type
    global g_jenkins_url
    global g_temp_filename
    global g_summary_text_filename  # store failed test info in csv format
    global g_failed_tests_dict      # store failed test info as a dictionary
    global g_resource_url
    global g_timestring
    global g_daily_failure_csv

    if len(argv) < 12:
        print "Wrong call.  Not enough arguments.\n"
        usage()
        sys.exit(1)
    else:   # we may be in business
        g_script_name = os.path.basename(argv[0])   # get name of script being run.
        g_timestamp = float(argv[1])
        g_job_name = argv[2]
        g_build_id = argv[3]
        g_git_hash = argv[4]
        g_node_name= argv[5]
        g_unit_test_type = argv[6]
        g_jenkins_url = argv[7]

        localtz = time.tzname[0]
        dt = parser.parse(time.ctime(g_timestamp)+ ' '+localtz)
        g_timestring = dt.strftime("%a %b %d %H:%M:%S %Y %Z")
        g_temp_filename = os.path.join(g_test_root_dir,'tempText')
        g_summary_text_filename = os.path.join(g_test_root_dir, argv[8])
        g_failed_tests_dict = os.path.join(g_test_root_dir, argv[9])
        monthToKeep = float(argv[10])
        g_daily_failure_csv = os.path.join(g_test_root_dir, argv[11])

        g_resource_url = '/'.join([g_jenkins_url, "job", g_job_name, g_build_id])
        get_console_out(g_resource_url+"/#showFailuresLink/")       # save remote console output in local directory
        extract_failed_tests_info()      # grab the console text and stored the failed tests/paths
        save_failed_tests_info()         # save new failed test info into a file
        if monthToKeep > 0:
            trim_data_back_to(monthToKeep)   # remove data that are too old to save space

if __name__ == "__main__":
    main(sys.argv)