def find_build_id(each_line,temp_func_list):
    """
    Find the build id of a jenkins job.  It will save this
    information in g_failed_test_info_dict.  In addition, it will delete this particular
    function handle off the temp_func_list as we do not need to perform this action again.

    Parameters
    ----------

    each_line :  str
        contains a line read in from jenkins console
    temp_func_list :  list of Python function handles
        contains a list of functions that we want to invoke to extract information from
        the Jenkins console text.

    :return: bool to determine if text mining should continue on the jenkins console text
    """
    global g_before_java_file
    global g_java_filenames
    global g_build_id_text
    global g_jenkins_url
    global g_output_filename
    global g_output_pickle_filename

    if g_build_id_text in each_line:
        [startStr,found,endStr] = each_line.partition(g_build_id_text)
        g_failed_test_info_dict["2.build_id"] = endStr.strip()

        temp_func_list.remove(find_build_id)
        g_jenkins_url = os.path.join('http://',g_jenkins_url,'view',g_view_name,'job',g_failed_test_info_dict["1.jobName"],g_failed_test_info_dict["2.build_id"],'artifact')

    return True

def find_build_id ( each_line , temp_func_list ) : global g_before_java_file global g_java_filenames global g_build_id_text global g_jenkins_url global g_output_filename global g_output_pickle_filename if g_build_id_text in each_line : [ startStr , found , endStr ] = each_line . partition ( g_build_id_text ) g_failed_test_info_dict [ "2.build_id" ] = endStr . strip ( ) temp_func_list . remove ( find_build_id ) g_jenkins_url = os . path . join ( 'http://' , g_jenkins_url , 'view' , g_view_name , 'job' , g_failed_test_info_dict [ "1.jobName" ] , g_failed_test_info_dict [ "2.build_id" ] , 'artifact' ) return True

Find the build id of a jenkins job . It will save this information in g_failed_test_info_dict . In addition it will delete this particular function handle off the temp_func_list as we do not need to perform this action again .


def extract_job_build_url(url_string):
    """
    From user input, grab the jenkins job name and saved it in g_failed_test_info_dict.
    In addition, it will grab the jenkins url and the view name into g_jenkins_url, and
    g_view_name.

    Parameters
    ----------
    url_string :  str
        contains information on the jenkins job whose console output we are interested in.

    :return: none
    """
    global g_failed_test_info_dict
    global g_jenkins_url
    global g_view_name

    tempString = url_string.strip('/').split('/')

    if len(tempString) < 6:
        print "Illegal URL resource address.\n"
        sys.exit(1)

    g_failed_test_info_dict["1.jobName"] = tempString[6]

    g_jenkins_url = tempString[2]
    g_view_name = tempString[4]

def
extract_job_build_url
(
url_string
)
:
global
g_failed_test_info_dict
global
g_jenkins_url
global
g_view_name
tempString
=
url_string
.
strip
(
'/'
)
.
split
(
'/'
)
if
len
(
tempString
)
<
6
:
print
"Illegal URL resource address.\n"
sys
.
exit
(
1
)
g_failed_test_info_dict
[
"1.jobName"
]
=
tempString
[
6
]
g_jenkins_url
=
tempString
[
2
]
g_view_name
=
tempString
[
4
]

From user input grab the jenkins job name and saved it in g_failed_test_info_dict . In addition it will grab the jenkins url and the view name into g_jenkins_url and g_view_name .


def grab_java_message():
    """scan through the java output text and extract the bad java messages that may or may not happened when
    unit tests are run.  It will not record any bad java messages that are stored in g_ok_java_messages.

    :return: none
    """

    global g_temp_filename
    global g_current_testname
    global g_java_start_text
    global g_ok_java_messages
    global g_java_general_bad_messages  # store bad java messages not associated with running a unit test
    global g_java_general_bad_message_types
    global g_failure_occurred
    global g_java_message_type
    global g_all_java_message_type
    global g_toContinue

    java_messages = []      # store all bad java messages associated with running a unit test
    java_message_types = [] # store all bad java message types associated with running a unit test

    if os.path.isfile(g_temp_filename): # open temp file containing content of some java_*_0.out.txt
        java_file = open(g_temp_filename,'r')

        g_toContinue = False    # denote if a multi-line message starts

        tempMessage = ""
        messageType = ""

        for each_line in java_file:

            if (g_java_start_text in each_line):
                startStr,found,endStr = each_line.partition(g_java_start_text)

                if len(found) > 0:
                    if len(g_current_testname) > 0: # a new unit test is being started.  Save old info and move on
                        associate_test_with_java(g_current_testname,java_messages,java_message_types)

                    g_current_testname = endStr.strip() # record the test name

                    java_messages = []
                    java_message_types = []

            temp_strings = each_line.strip().split()

            if (len(temp_strings) >= 6) and (temp_strings[5] in g_all_java_message_type):
                if g_toContinue == True:    # at the end of last message fragment
                    addJavaMessages(tempMessage,messageType,java_messages,java_message_types)
                    tempMessage = ""
                    messageType = ""

                # start of new message fragment
                g_toContinue = False
            else: # non standard output.  Continuation of last java message, add it to bad java message list
                if g_toContinue:

                    tempMessage += each_line    # add more java message here
                    # if len(g_current_testname) == 0:
                    #     addJavaMessages(each_line.strip(),"",java_messages,java_message_types)
                    # else:
                    #     addJavaMessages(each_line.strip(),"",java_messages,java_message_types)

            if ((len(temp_strings) > 5) and (temp_strings[5] in g_java_message_type)):  # find a bad java message
                startStr,found,endStr = each_line.partition(temp_strings[5])    # can be WARN,ERRR,FATAL,TRACE

                if found and (len(endStr.strip()) > 0):
                    tempMessage += endStr
                    messageType = temp_strings[5]
#                    if (tempMessage not in g_ok_java_messages["general"]):  # found new bad messages that cannot be ignored
                    g_toContinue = True

                        # add tempMessage to bad java message list
#                        addJavaMessages(tempMessage,temp_strings[5],java_messages,java_message_types)
        java_file.close()

scan through the java output text and extract the bad java messages that may or may not happened when unit tests are run . It will not record any bad java messages that are stored in g_ok_java_messages .


def save_dict():
    """
    Save the log scraping results into logs denoted by g_output_filename_failed_tests and
    g_output_filename_passed_tests.

    :return: none
    """

    global g_test_root_dir
    global g_output_filename_failed_tests
    global g_output_filename_passed_tests
    global g_output_pickle_filename
    global g_failed_test_info_dict

    # some build can fail really early that no buid id info is stored in the console text.
    if "2.build_id" not in g_failed_test_info_dict.keys():
        g_failed_test_info_dict["2.build_id"] = "unknown"

    build_id = g_failed_test_info_dict["2.build_id"]

    g_output_filename_failed_tests = g_output_filename_failed_tests+'_build_'+build_id+'_failed_tests.log'
    g_output_filename_passed_tests = g_output_filename_passed_tests+'_build_'+build_id+'_passed_tests.log'
    g_output_pickle_filename = g_output_pickle_filename+'_build_'+build_id+'.pickle'

    allKeys = sorted(g_failed_test_info_dict.keys())

    # write out the jenkins job info into log files.
    with open(g_output_pickle_filename,'wb') as test_file:
        pickle.dump(g_failed_test_info_dict,test_file)

    # write out the failure report as text into a text file
    text_file_failed_tests = open(g_output_filename_failed_tests,'w')
    text_file_passed_tests = None
    allKeys = sorted(g_failed_test_info_dict.keys())
    write_passed_tests = False

    if ("passed_tests_info *********" in allKeys):
        text_file_passed_tests = open(g_output_filename_passed_tests,'w')
        write_passed_tests = True

    for keyName in allKeys:
        val = g_failed_test_info_dict[keyName]
        if isinstance(val,list):    # writing one of the job lists
            if (len(val) == 3):     # it is a message for a test
                if keyName == "failed_tests_info *********":
                    write_test_java_message(keyName,val,text_file_failed_tests)

                if keyName == "passed_tests_info *********":
                    write_test_java_message(keyName,val,text_file_passed_tests)
            elif (len(val) == 2):                   # it is a general bad java message
                write_java_message(keyName,val,text_file_failed_tests)
                if write_passed_tests:
                    write_java_message(keyName,val,text_file_passed_tests)
        else:
            write_general_build_message(keyName,val,text_file_failed_tests)
            if write_passed_tests:
                write_general_build_message(keyName,val,text_file_passed_tests)

    text_file_failed_tests.close()
    if write_passed_tests:
        text_file_passed_tests.close()

Save the log scraping results into logs denoted by g_output_filename_failed_tests and g_output_filename_passed_tests .


def update_summary_file():
    """
    Concatecate all log file into a summary text file to be sent to users
    at the end of a daily log scraping.

    :return: none
    """
    global g_summary_text_filename
    global g_output_filename_failed_tests
    global g_output_filename_passed_tests

    with open(g_summary_text_filename,'a') as tempfile:
        write_file_content(tempfile,g_output_filename_failed_tests)
        write_file_content(tempfile,g_output_filename_passed_tests)

def update_summary_file ( ) : global g_summary_text_filename global g_output_filename_failed_tests global g_output_filename_passed_tests with open ( g_summary_text_filename , 'a' ) as tempfile : write_file_content ( tempfile , g_output_filename_failed_tests ) write_file_content ( tempfile , g_output_filename_passed_tests )

Concatecate all log file into a summary text file to be sent to users at the end of a daily log scraping .


def write_file_content(fhandle,file2read):
    """
    Write one log file into the summary text file.

    Parameters
    ----------
    fhandle :  Python file handle
        file handle to the summary text file
    file2read : Python file handle
        file handle to log file where we want to add its content to the summary text file.

    :return: none
    """
    if os.path.isfile(file2read):

        # write summary of failed tests logs
        with open(file2read,'r') as tfile:
            fhandle.write('============ Content of '+ file2read)
            fhandle.write('\n')
            fhandle.write(tfile.read())
            fhandle.write('\n\n')

def
write_file_content
(
fhandle
,
file2read
)
:
if
os
.
path
.
isfile
(
file2read
)
:
# write summary of failed tests logs
with
open
(
file2read
,
'r'
)
as
tfile
:
fhandle
.
write
(
'============ Content of '
+
file2read
)
fhandle
.
write
(
'\n'
)
fhandle
.
write
(
tfile
.
read
(
)
)
fhandle
.
write
(
'\n\n'
)

Write one log file into the summary text file .


def write_java_message(key,val,text_file):
    """
    Loop through all java messages that are not associated with a unit test and
    write them into a log file.

    Parameters
    ----------
    key :  str
        9.general_bad_java_messages
    val : list of list of str
        contains the bad java messages and the message types.

    :return: none
    """

    text_file.write(key)
    text_file.write('\n')

    if (len(val[0]) > 0) and (len(val) >= 3):
        for index in range(len(val[0])):
            text_file.write("Java Message Type: ")
            text_file.write(val[1][index])
            text_file.write('\n')

            text_file.write("Java Message: ")

            for jmess in val[2][index]:
                text_file.write(jmess)
                text_file.write('\n')

        text_file.write('\n \n')

def
write_java_message
(
key
,
val
,
text_file
)
:
text_file
.
write
(
key
)
text_file
.
write
(
'\n'
)
if
(
len
(
val
[
0
]
)
>
0
)
and
(
len
(
val
)
>=
3
)
:
for
index
in
range
(
len
(
val
[
0
]
)
)
:
text_file
.
write
(
"Java Message Type: "
)
text_file
.
write
(
val
[
1
]
[
index
]
)
text_file
.
write
(
'\n'
)
text_file
.
write
(
"Java Message: "
)
for
jmess
in
val
[
2
]
[
index
]
:
text_file
.
write
(
jmess
)
text_file
.
write
(
'\n'
)
text_file
.
write
(
'\n \n'
)

Loop through all java messages that are not associated with a unit test and write them into a log file .


def load_java_messages_to_ignore():
    """
    Load in pickle file that contains dict structure with bad java messages to ignore per unit test
    or for all cases.  The ignored bad java info is stored in g_ok_java_messages dict.

    :return:
    """
    global g_ok_java_messages
    global g_java_message_pickle_filename

    if os.path.isfile(g_java_message_pickle_filename):
        with open(g_java_message_pickle_filename,'rb') as tfile:
            g_ok_java_messages = pickle.load(tfile)
    else:
        g_ok_java_messages["general"] = []

def load_java_messages_to_ignore ( ) : global g_ok_java_messages global g_java_message_pickle_filename if os . path . isfile ( g_java_message_pickle_filename ) : with open ( g_java_message_pickle_filename , 'rb' ) as tfile : g_ok_java_messages = pickle . load ( tfile ) else : g_ok_java_messages [ "general" ] = [ ]

Load in pickle file that contains dict structure with bad java messages to ignore per unit test or for all cases . The ignored bad java info is stored in g_ok_java_messages dict .


def normalize_enum_constant(s):
    """Return enum constant `s` converted to a canonical snake-case."""
    if s.islower(): return s
    if s.isupper(): return s.lower()
    return "".join(ch if ch.islower() else "_" + ch.lower() for ch in s).strip("_")

def normalize_enum_constant ( s ) : if s . islower ( ) : return s if s . isupper ( ) : return s . lower ( ) return "" . join ( ch if ch . islower ( ) else "_" + ch . lower ( ) for ch in s ) . strip ( "_" )

Return enum constant s converted to a canonical snake - case .


def find_synonyms(self, word, count=20):
        """
        Find synonyms using a word2vec model.

        :param str word: A single word to find synonyms for.
        :param int count: The first "count" synonyms will be returned.

        :returns: the approximate reconstruction of the training data.
        """
        j = h2o.api("GET /3/Word2VecSynonyms", data={'model': self.model_id, 'word': word, 'count': count})
        return OrderedDict(sorted(zip(j['synonyms'], j['scores']), key=lambda t: t[1], reverse=True))

def
find_synonyms
(
self
,
word
,
count
=
20
)
:
j
=
h2o
.
api
(
"GET /3/Word2VecSynonyms"
,
data
=
{
'model'
:
self
.
model_id
,
'word'
:
word
,
'count'
:
count
}
)
return
OrderedDict
(
sorted
(
zip
(
j
[
'synonyms'
]
,
j
[
'scores'
]
)
,
key
=
lambda
t
:
t
[
1
]
,
reverse
=
True
)
)


def poll(self, verbose_model_scoring_history = False):
        """
        Wait until the job finishes.

        This method will continuously query the server about the status of the job, until the job reaches a
        completion. During this time we will display (in stdout) a progress bar with % completion status.
        """
        try:
            hidden = not H2OJob.__PROGRESS_BAR__
            pb = ProgressBar(title=self._job_type + " progress", hidden=hidden)
            if verbose_model_scoring_history:
                pb.execute(self._refresh_job_status, print_verbose_info=lambda x: self._print_verbose_info() if int(x * 10) % 5 == 0  else " ")
            else:
                pb.execute(self._refresh_job_status)
        except StopIteration as e:
            if str(e) == "cancelled":
                h2o.api("POST /3/Jobs/%s/cancel" % self.job_key)
                self.status = "CANCELLED"
            # Potentially we may want to re-raise the exception here

        assert self.status in {"DONE", "CANCELLED", "FAILED"} or self._poll_count <= 0, \
            "Polling finished while the job has status %s" % self.status
        if self.warnings:
            for w in self.warnings:
                warnings.warn(w)

        # check if failed... and politely print relevant message
        if self.status == "CANCELLED":
            raise H2OJobCancelled("Job<%s> was cancelled by the user." % self.job_key)
        if self.status == "FAILED":
            if (isinstance(self.job, dict)) and ("stacktrace" in list(self.job)):
                raise EnvironmentError("Job with key {} failed with an exception: {}\nstacktrace: "
                                       "\n{}".format(self.job_key, self.exception, self.job["stacktrace"]))
            else:
                raise EnvironmentError("Job with key %s failed with an exception: %s" % (self.job_key, self.exception))

        return self


def to_pojo(self, pojo_name="", path="", get_jar=True):
        """
        Convert the munging operations performed on H2OFrame into a POJO.

        :param pojo_name:  (str) Name of POJO
        :param path:  (str) path of POJO.
        :param get_jar: (bool) Whether to also download the h2o-genmodel.jar file needed to compile the POJO
        :return: None
        """
        assert_is_type(pojo_name, str)
        assert_is_type(path, str)
        assert_is_type(get_jar, bool)
        if pojo_name == "":
            pojo_name = "AssemblyPOJO_" + str(uuid.uuid4())
        java = h2o.api("GET /99/Assembly.java/%s/%s" % (self.id, pojo_name))
        file_path = path + "/" + pojo_name + ".java"
        if path == "":
            print(java)
        else:
            with open(file_path, 'w', encoding="utf-8") as f:
                f.write(java)  # this had better be utf-8 ?
        if get_jar and path != "":
            h2o.api("GET /3/h2o-genmodel.jar", save_to=os.path.join(path, "h2o-genmodel.jar"))

def
to_pojo
(
self
,
pojo_name
=
""
,
path
=
""
,
get_jar
=
True
)
:
assert_is_type
(
pojo_name
,
str
)
assert_is_type
(
path
,
str
)
assert_is_type
(
get_jar
,
bool
)
if
pojo_name
==
""
:
pojo_name
=
"AssemblyPOJO_"
+
str
(
uuid
.
uuid4
(
)
)
java
=
h2o
.
api
(
"GET /99/Assembly.java/%s/%s"
%
(
self
.
id
,
pojo_name
)
)
file_path
=
path
+
"/"
+
pojo_name
+
".java"
if
path
==
""
:
print
(
java
)
else
:
with
open
(
file_path
,
'w'
,
encoding
=
"utf-8"
)
as
f
:
f
.
write
(
java
)
# this had better be utf-8 ?
if
get_jar
and
path
!=
""
:
h2o
.
api
(
"GET /3/h2o-genmodel.jar"
,
save_to
=
os
.
path
.
join
(
path
,
"h2o-genmodel.jar"
)
)

Convert the munging operations performed on H2OFrame into a POJO .


def fit(self, fr):
        """
        To perform the munging operations on a frame specified in steps on the frame fr.

        :param fr: H2OFrame where munging operations are to be performed on.
        :return: H2OFrame after munging operations are completed.
        """
        assert_is_type(fr, H2OFrame)
        steps = "[%s]" % ",".join(quoted(step[1].to_rest(step[0]).replace('"', "'")) for step in self.steps)
        j = h2o.api("POST /99/Assembly", data={"steps": steps, "frame": fr.frame_id})
        self.id = j["assembly"]["name"]
        return H2OFrame.get_frame(j["result"]["name"])

def
fit
(
self
,
fr
)
:
assert_is_type
(
fr
,
H2OFrame
)
steps
=
"[%s]"
%
","
.
join
(
quoted
(
step
[
1
]
.
to_rest
(
step
[
0
]
)
.
replace
(
'"'
,
"'"
)
)
for
step
in
self
.
steps
)
j
=
h2o
.
api
(
"POST /99/Assembly"
,
data
=
{
"steps"
:
steps
,
"frame"
:
fr
.
frame_id
}
)
self
.
id
=
j
[
"assembly"
]
[
"name"
]
return
H2OFrame
.
get_frame
(
j
[
"result"
]
[
"name"
]
)

To perform the munging operations on a frame specified in steps on the frame fr .


def percentileOnSortedList(N, percent, key=lambda x:x, interpolate='mean'):
    # 5 ways of resolving fractional
    # floor, ceil, funky, linear, mean
    interpolateChoices = ['floor', 'ceil', 'funky', 'linear', 'mean']
    if interpolate not in interpolateChoices:
        print "Bad choice for interpolate:", interpolate
        print "Supported choices:", interpolateChoices
    """
    Find the percentile of a list of values.

    @parameter N - is a list of values. Note N MUST BE already sorted.
    @parameter percent - a float value from 0.0 to 1.0.
    @parameter key - optional key function to compute value from each element of N.

    @return - the percentile of the values
    """
    if N is None:
        return None
    k = (len(N)-1) * percent
    f = int(math.floor(k))
    c = int(math.ceil(k))

    if f == c:
        d = key(N[f])
        msg = "aligned:"

    elif interpolate=='floor':
        d = key(N[f])
        msg = "fractional with floor:"

    elif interpolate=='ceil':
        d = key(N[c])
        msg = "fractional with ceil:"

    elif interpolate=='funky':
        d0 = key(N[f]) * (c-k)
        d1 = key(N[c]) * (k-f)
        d = d0+d1
        msg = "fractional with Tung(floor and ceil) :"

    elif interpolate=='linear':
        assert (c-f)==1
        assert (k>=f) and (k<=c)
        pctDiff = k-f
        dDiff = pctDiff * (key(N[c]) - key(N[f]))
        d = key(N[f] + dDiff)
        msg = "fractional %s with linear(floor and ceil):" % pctDiff

    elif interpolate=='mean':
        d = (key(N[c]) + key(N[f])) / 2.0
        msg = "fractional with mean(floor and ceil):"

    # print 3 around the floored k, for eyeballing when we're close
    flooredK = int(f)
    # print the 3 around the median
    if flooredK > 0:
        print "prior->", key(N[flooredK-1]), " "
    else:
        print "prior->", "<bof>"
    print "floor->", key(N[flooredK]), " ", msg, 'result:', d, "f:", f, "len(N):", len(N)
    if flooredK+1 < len(N):
        print " ceil->", key(N[flooredK+1]), "c:", c
    else:
        print " ceil-> <eof>", "c:", c

    return d

Find the percentile of a list of values .


def default_params(self):
        """Dictionary of the default parameters of the model."""
        params = {}
        for p in self.parms:
            params[p] = self.parms[p]["default_value"]
        return params

def default_params ( self ) : params = { } for p in self . parms : params [ p ] = self . parms [ p ] [ "default_value" ] return params

Dictionary of the default parameters of the model .


def actual_params(self):
        """Dictionary of actual parameters of the model."""
        params_to_select = {"model_id": "name",
                            "response_column": "column_name",
                            "training_frame": "name",
                            "validation_frame": "name"}
        params = {}
        for p in self.parms:
            if p in params_to_select.keys():
                params[p] = self.parms[p]["actual_value"].get(params_to_select[p], None)
            else:
                params[p] = self.parms[p]["actual_value"]
        return params

def actual_params ( self ) : params_to_select = { "model_id" : "name" , "response_column" : "column_name" , "training_frame" : "name" , "validation_frame" : "name" } params = { } for p in self . parms : if p in params_to_select . keys ( ) : params [ p ] = self . parms [ p ] [ "actual_value" ] . get ( params_to_select [ p ] , None ) else : params [ p ] = self . parms [ p ] [ "actual_value" ] return params

Dictionary of actual parameters of the model .


def deepfeatures(self, test_data, layer):
        """
        Return hidden layer details.

        :param test_data: Data to create a feature space on
        :param layer: 0 index hidden layer
        """
        if test_data is None: raise ValueError("Must specify test data")
        if str(layer).isdigit():
            j = H2OJob(h2o.api("POST /4/Predictions/models/%s/frames/%s" % (self._id, test_data.frame_id),
                               data={"deep_features_hidden_layer": layer}), "deepfeatures")
        else:
            j = H2OJob(h2o.api("POST /4/Predictions/models/%s/frames/%s" % (self._id, test_data.frame_id),
                               data={"deep_features_hidden_layer_name": layer}), "deepfeatures")
        j.poll()
        return h2o.get_frame(j.dest_key)


def scoring_history(self):
        """
        Retrieve Model Score History.

        :returns: The score history as an H2OTwoDimTable or a Pandas DataFrame.
        """
        model = self._model_json["output"]
        if "scoring_history" in model and model["scoring_history"] is not None:
            return model["scoring_history"].as_data_frame()
        print("No score history for this model")

def
scoring_history
(
self
)
:
model
=
self
.
_model_json
[
"output"
]
if
"scoring_history"
in
model
and
model
[
"scoring_history"
]
is
not
:
return
model
[
"scoring_history"
]
.
as_data_frame
(
)
print
(
"No score history for this model"
)


def show(self):
        """Print innards of model, without regards to type."""
        if self._future:
            self._job.poll_once()
            return
        if self._model_json is None:
            print("No model trained yet")
            return
        if self.model_id is None:
            print("This H2OEstimator has been removed.")
            return
        model = self._model_json["output"]
        print("Model Details")
        print("=============")

        print(self.__class__.__name__, ": ", self._model_json["algo_full_name"])
        print("Model Key: ", self._id)

        self.summary()

        print()
        # training metrics
        tm = model["training_metrics"]
        if tm: tm.show()
        vm = model["validation_metrics"]
        if vm: vm.show()
        xm = model["cross_validation_metrics"]
        if xm: xm.show()
        xms = model["cross_validation_metrics_summary"]
        if xms: xms.show()

        if "scoring_history" in model and model["scoring_history"]:
            model["scoring_history"].show()
        if "variable_importances" in model and model["variable_importances"]:
            model["variable_importances"].show()

def
show
(
self
)
:
if
self
.
_future
:
self
.
_job
.
poll_once
(
)
return
if
self
.
_model_json
is
:
print
(
"No model trained yet"
)
return
if
self
.
model_id
is
:
print
(
"This H2OEstimator has been removed."
)
return
model
=
self
.
_model_json
[
"output"
]
print
(
"Model Details"
)
print
(
"============="
)
print
(
self
.
__class__
.
__name__
,
": "
,
self
.
_model_json
[
"algo_full_name"
]
)
print
(
"Model Key: "
,
self
.
_id
)
self
.
summary
(
)
print
(
)
# training metrics
tm
=
model
[
"training_metrics"
]
if
tm
:
tm
.
show
(
)
vm
=
model
[
"validation_metrics"
]
if
vm
:
vm
.
show
(
)
xm
=
model
[
"cross_validation_metrics"
]
if
xm
:
xm
.
show
(
)
xms
=
model
[
"cross_validation_metrics_summary"
]
if
xms
:
xms
.
show
(
)
if
"scoring_history"
in
model
and
model
[
"scoring_history"
]
:
model
[
"scoring_history"
]
.
show
(
)
if
"variable_importances"
in
model
and
model
[
"variable_importances"
]
:
model
[
"variable_importances"
]
.
show
(
)

Print innards of model without regards to type .


def varimp(self, use_pandas=False):
        """
        Pretty print the variable importances, or return them in a list.

        :param use_pandas: If True, then the variable importances will be returned as a pandas data frame.

        :returns: A list or Pandas DataFrame.
        """
        model = self._model_json["output"]
        if self.algo=='glm' or "variable_importances" in list(model.keys()) and model["variable_importances"]:
            if self.algo=='glm':
                tempvals = model["standardized_coefficient_magnitudes"].cell_values
                maxVal = 0
                sum=0
                for item in tempvals:
                    sum=sum+item[1]
                    if item[1]>maxVal:
                        maxVal = item[1]
                vals = []
                for item in tempvals:
                    tempT = (item[0], item[1], item[1]/maxVal, item[1]/sum)
                    vals.append(tempT)
                header = ["variable", "relative_importance", "scaled_importance", "percentage"]
            else:
                vals = model["variable_importances"].cell_values
                header = model["variable_importances"].col_header

            if use_pandas and can_use_pandas():
                import pandas
                return pandas.DataFrame(vals, columns=header)
            else:
                return vals
        else:
            print("Warning: This model doesn't have variable importances")

Pretty print the variable importances or return them in a list .


def residual_degrees_of_freedom(self, train=False, valid=False, xval=False):
        """
        Retreive the residual degress of freedom if this model has the attribute, or None otherwise.

        :param bool train: Get the residual dof for the training set. If both train and valid are False, then train
            is selected by default.
        :param bool valid: Get the residual dof for the validation set. If both train and valid are True, then train
            is selected by default.

        :returns: Return the residual dof, or None if it is not present.
        """
        if xval: raise H2OValueError("Cross-validation metrics are not available.")
        if not train and not valid: train = True
        if train and valid:         train = True
        if train:
            return self._model_json["output"]["training_metrics"].residual_degrees_of_freedom()
        else:
            return self._model_json["output"]["validation_metrics"].residual_degrees_of_freedom()

Retreive the residual degress of freedom if this model has the attribute or None otherwise .


def coef(self):
        """
        Return the coefficients which can be applied to the non-standardized data.

        Note: standardize = True by default, if set to False then coef() return the coefficients which are fit directly.
        """
        tbl = self._model_json["output"]["coefficients_table"]
        if tbl is None:
            return None
        return {name: coef for name, coef in zip(tbl["names"], tbl["coefficients"])}

def coef ( self ) : tbl = self . _model_json [ "output" ] [ "coefficients_table" ] if tbl is None : return None return { name : coef for name , coef in zip ( tbl [ "names" ] , tbl [ "coefficients" ] ) }

Return the coefficients which can be applied to the non - standardized data .


def download_pojo(self, path="", get_genmodel_jar=False, genmodel_name=""):
        """
        Download the POJO for this model to the directory specified by path.

        If path is an empty string, then dump the output to screen.

        :param path:  An absolute path to the directory where POJO should be saved.
        :param get_genmodel_jar: if True, then also download h2o-genmodel.jar and store it in folder ``path``.
        :param genmodel_name Custom name of genmodel jar
        :returns: name of the POJO file written.
        """
        assert_is_type(path, str)
        assert_is_type(get_genmodel_jar, bool)
        path = path.rstrip("/")
        return h2o.download_pojo(self, path, get_jar=get_genmodel_jar, jar_name=genmodel_name)

def download_pojo ( self , path = "" , get_genmodel_jar = False , genmodel_name = "" ) : assert_is_type ( path , str ) assert_is_type ( get_genmodel_jar , bool ) path = path . rstrip ( "/" ) return h2o . download_pojo ( self , path , get_jar = get_genmodel_jar , jar_name = genmodel_name )

Download the POJO for this model to the directory specified by path .


def download_mojo(self, path=".", get_genmodel_jar=False, genmodel_name=""):
        """
        Download the model in MOJO format.

        :param path: the path where MOJO file should be saved.
        :param get_genmodel_jar: if True, then also download h2o-genmodel.jar and store it in folder ``path``.
        :param genmodel_name Custom name of genmodel jar
        :returns: name of the MOJO file written.
        """
        assert_is_type(path, str)
        assert_is_type(get_genmodel_jar, bool)

        if not self.have_mojo:
            raise H2OValueError("Export to MOJO not supported")

        if get_genmodel_jar:
            if genmodel_name == "":
                h2o.api("GET /3/h2o-genmodel.jar", save_to=os.path.join(path, "h2o-genmodel.jar"))
            else:
                h2o.api("GET /3/h2o-genmodel.jar", save_to=os.path.join(path, genmodel_name))
        return h2o.api("GET /3/Models/%s/mojo" % self.model_id, save_to=path)

def
download_mojo
(
self
,
path
=
"."
,
get_genmodel_jar
=
False
,
genmodel_name
=
""
)
:
assert_is_type
(
path
,
str
)
assert_is_type
(
get_genmodel_jar
,
bool
)
if
not
self
.
have_mojo
:
raise
H2OValueError
(
"Export to MOJO not supported"
)
if
get_genmodel_jar
:
if
genmodel_name
==
""
:
h2o
.
api
(
"GET /3/h2o-genmodel.jar"
,
save_to
=
os
.
path
.
join
(
path
,
"h2o-genmodel.jar"
)
)
else
:
h2o
.
api
(
"GET /3/h2o-genmodel.jar"
,
save_to
=
os
.
path
.
join
(
path
,
genmodel_name
)
)
return
h2o
.
api
(
"GET /3/Models/%s/mojo"
%
self
.
model_id
,
save_to
=
path
)


def save_model_details(self, path="", force=False):
        """
        Save Model Details of an H2O Model in JSON Format to disk.

        :param model: The model object to save.
        :param path: a path to save the model details at (hdfs, s3, local)
        :param force: if True overwrite destination directory in case it exists, or throw exception if set to False.

        :returns str: the path of the saved model details
        """
        assert_is_type(path, str)
        assert_is_type(force, bool)
        path = os.path.join(os.getcwd() if path == "" else path, self.model_id + ".json")
        return h2o.api("GET /99/Models/%s/json" % self.model_id, data={"dir": path, "force": force})["dir"]

def
save_model_details
(
self
,
path
=
""
,
force
=
False
)
:
assert_is_type
(
path
,
str
)
assert_is_type
(
force
,
bool
)
path
=
os
.
path
.
join
(
os
.
getcwd
(
)
if
path
==
""
else
path
,
self
.
model_id
+
".json"
)
return
h2o
.
api
(
"GET /99/Models/%s/json"
%
self
.
model_id
,
data
=
{
"dir"
:
path
,
"force"
:
force
}
)
[
"dir"
]

Save Model Details of an H2O Model in JSON Format to disk .


def _check_targets(y_actual, y_predicted):
        """Check that y_actual and y_predicted have the same length.

        :param H2OFrame y_actual:
        :param H2OFrame y_predicted:

        :returns: None
        """
        if len(y_actual) != len(y_predicted):
            raise ValueError("Row mismatch: [{},{}]".format(len(y_actual), len(y_predicted)))

def
_check_targets
(
y_actual
,
y_predicted
)
:
if
len
(
y_actual
)
!=
len
(
y_predicted
)
:
raise
ValueError
(
"Row mismatch: [{},{}]"
.
format
(
len
(
y_actual
)
,
len
(
y_predicted
)
)
)

Check that y_actual and y_predicted have the same length .


def cross_validation_models(self):
        """
        Obtain a list of cross-validation models.

        :returns: list of H2OModel objects.
        """
        cvmodels = self._model_json["output"]["cross_validation_models"]
        if cvmodels is None: return None
        m = []
        for p in cvmodels: m.append(h2o.get_model(p["name"]))
        return m

def cross_validation_models ( self ) : cvmodels = self . _model_json [ "output" ] [ "cross_validation_models" ] if cvmodels is None : return None m = [ ] for p in cvmodels : m . append ( h2o . get_model ( p [ "name" ] ) ) return m

Obtain a list of cross - validation models .


def gbm(interactive=True, echo=True, testing=False):
    """GBM model demo."""

    def demo_body(go):
        """
        Demo of H2O's Gradient Boosting estimator.

        This demo uploads a dataset to h2o, parses it, and shows a description.
        Then it divides the dataset into training and test sets, builds a GLM
        from the training set, and makes predictions for the test set.
        Finally, default performance metrics are displayed.
        """
        go()
        # Connect to H2O
        h2o.init()

        go()
        # Upload the prostate dataset that comes included in the h2o python package
        prostate = h2o.load_dataset("prostate")

        go()
        # Print a description of the prostate data
        prostate.describe()

        go()
        # Randomly split the dataset into ~70/30, training/test sets
        train, test = prostate.split_frame(ratios=[0.70])

        go()
        # Convert the response columns to factors (for binary classification problems)
        train["CAPSULE"] = train["CAPSULE"].asfactor()
        test["CAPSULE"] = test["CAPSULE"].asfactor()

        go()
        # Build a (classification) GLM
        from h2o.estimators import H2OGradientBoostingEstimator
        prostate_gbm = H2OGradientBoostingEstimator(distribution="bernoulli", ntrees=10, max_depth=8,
                                                    min_rows=10, learn_rate=0.2)
        prostate_gbm.train(x=["AGE", "RACE", "PSA", "VOL", "GLEASON"],
                           y="CAPSULE", training_frame=train)

        go()
        # Show the model
        prostate_gbm.show()

        go()
        # Predict on the test set and show the first ten predictions
        predictions = prostate_gbm.predict(test)
        predictions.show()

        go()
        # Fetch a tree, print number of tree nodes, show root node description
        from h2o.tree import H2OTree, H2ONode
        tree = H2OTree(prostate_gbm, 0, "0")
        len(tree)
        tree.left_children
        tree.right_children
        tree.root_node.show()

        go()
        # Show default performance metrics
        performance = prostate_gbm.model_performance(test)
        performance.show()

    # Execute:
    _run_demo(demo_body, interactive, echo, testing)


def deeplearning(interactive=True, echo=True, testing=False):
    """Deep Learning model demo."""

    def demo_body(go):
        """
        Demo of H2O's Deep Learning model.

        This demo uploads a dataset to h2o, parses it, and shows a description.
        Then it divides the dataset into training and test sets, builds a GLM
        from the training set, and makes predictions for the test set.
        Finally, default performance metrics are displayed.
        """
        go()
        # Connect to H2O
        h2o.init()

        go()
        # Upload the prostate dataset that comes included in the h2o python package
        prostate = h2o.load_dataset("prostate")

        go()
        # Print a description of the prostate data
        prostate.describe()

        go()
        # Randomly split the dataset into ~70/30, training/test sets
        train, test = prostate.split_frame(ratios=[0.70])

        go()
        # Convert the response columns to factors (for binary classification problems)
        train["CAPSULE"] = train["CAPSULE"].asfactor()
        test["CAPSULE"] = test["CAPSULE"].asfactor()

        go()
        # Build a (classification) GLM
        from h2o.estimators import H2ODeepLearningEstimator
        prostate_dl = H2ODeepLearningEstimator(activation="Tanh", hidden=[10, 10, 10], epochs=10000)
        prostate_dl.train(x=list(set(prostate.col_names) - {"ID", "CAPSULE"}),
                          y="CAPSULE", training_frame=train)

        go()
        # Show the model
        prostate_dl.show()

        go()
        # Predict on the test set and show the first ten predictions
        predictions = prostate_dl.predict(test)
        predictions.show()

        go()
        # Show default performance metrics
        performance = prostate_dl.model_performance(test)
        performance.show()

    # Execute:
    _run_demo(demo_body, interactive, echo, testing)


def glm(interactive=True, echo=True, testing=False):
    """GLM model demo."""

    def demo_body(go):
        """
        Demo of H2O's Generalized Linear Estimator.

        This demo uploads a dataset to h2o, parses it, and shows a description.
        Then it divides the dataset into training and test sets, builds a GLM
        from the training set, and makes predictions for the test set.
        Finally, default performance metrics are displayed.
        """
        go()
        # Connect to H2O
        h2o.init()

        go()
        # Upload the prostate dataset that comes included in the h2o python package
        prostate = h2o.load_dataset("prostate")

        go()
        # Print a description of the prostate data
        prostate.describe()

        go()
        # Randomly split the dataset into ~70/30, training/test sets
        train, test = prostate.split_frame(ratios=[0.70])

        go()
        # Convert the response columns to factors (for binary classification problems)
        train["CAPSULE"] = train["CAPSULE"].asfactor()
        test["CAPSULE"] = test["CAPSULE"].asfactor()

        go()
        # Build a (classification) GLM
        from h2o.estimators import H2OGeneralizedLinearEstimator
        prostate_glm = H2OGeneralizedLinearEstimator(family="binomial", alpha=[0.5])
        prostate_glm.train(x=["AGE", "RACE", "PSA", "VOL", "GLEASON"],
                           y="CAPSULE", training_frame=train)

        go()
        # Show the model
        prostate_glm.show()

        go()
        # Predict on the test set and show the first ten predictions
        predictions = prostate_glm.predict(test)
        predictions.show()

        go()
        # Show default performance metrics
        performance = prostate_glm.model_performance(test)
        performance.show()

    # Execute:
    _run_demo(demo_body, interactive, echo, testing)


def _wait_for_keypress():
    """
    Wait for a key press on the console and return it.

    Borrowed from http://stackoverflow.com/questions/983354/how-do-i-make-python-to-wait-for-a-pressed-key
    """
    result = None
    if os.name == "nt":
        # noinspection PyUnresolvedReferences
        import msvcrt
        result = msvcrt.getch()
    else:
        import termios
        fd = sys.stdin.fileno()

        oldterm = termios.tcgetattr(fd)
        newattr = termios.tcgetattr(fd)
        newattr[3] = newattr[3] & ~termios.ICANON & ~termios.ECHO
        termios.tcsetattr(fd, termios.TCSANOW, newattr)

        try:
            result = sys.stdin.read(1)
        except IOError:
            pass
        finally:
            termios.tcsetattr(fd, termios.TCSAFLUSH, oldterm)

    return result

def
_wait_for_keypress
(
)
:
result
=
if
os
.
name
==
"nt"
:
# noinspection PyUnresolvedReferences
import
msvcrt
result
=
msvcrt
.
getch
(
)
else
:
import
termios
fd
=
sys
.
stdin
.
fileno
(
)
oldterm
=
termios
.
tcgetattr
(
fd
)
newattr
=
termios
.
tcgetattr
(
fd
)
newattr
[
3
]
=
newattr
[
3
]
&
~
termios
.
ICANON
&
~
termios
.
ECHO
termios
.
tcsetattr
(
fd
,
termios
.
TCSANOW
,
newattr
)
try
:
result
=
sys
.
stdin
.
read
(
1
)
except
IOError
:
pass
finally
:
termios
.
tcsetattr
(
fd
,
termios
.
TCSAFLUSH
,
oldterm
)
return
result

Wait for a key press on the console and return it .


def as_data_frame(self):
        """Convert to a python 'data frame'."""
        if can_use_pandas():
            import pandas
            pandas.options.display.max_colwidth = 70
            return pandas.DataFrame(self._cell_values, columns=self._col_header)
        return self

def as_data_frame ( self ) : if can_use_pandas ( ) : import pandas pandas . options . display . max_colwidth = 70 return pandas . DataFrame ( self . _cell_values , columns = self . _col_header ) return self


def show(self, header=True):
        """Print the contents of this table."""
        # if h2o.can_use_pandas():
        #  import pandas
        #  pandas.options.display.max_rows = 20
        #  print pandas.DataFrame(self._cell_values,columns=self._col_header)
        #  return
        if header and self._table_header:
            print(self._table_header + ":", end=' ')
            if self._table_description: print(self._table_description)
        print()
        table = copy.deepcopy(self._cell_values)
        nr = 0
        if _is_list_of_lists(table): nr = len(
            table)  # only set if we truly have multiple rows... not just one long row :)
        if nr > 20:  # create a truncated view of the table, first/last 5 rows
            trunc_table = []
            trunc_table += [v for v in table[:5]]
            trunc_table.append(["---"] * len(table[0]))
            trunc_table += [v for v in table[(nr - 5):]]
            table = trunc_table

        H2ODisplay(table, self._col_header, numalign="left", stralign="left")
        if nr > 20 and can_use_pandas(): print('\nSee the whole table with table.as_data_frame()')


def start(jar_path=None, nthreads=-1, enable_assertions=True, max_mem_size=None, min_mem_size=None,
              ice_root=None, log_dir=None, log_level=None, port="54321+", name=None, extra_classpath=None,
              verbose=True, jvm_custom_args=None, bind_to_localhost=True):
        """
        Start new H2O server on the local machine.

        :param jar_path: Path to the h2o.jar executable. If not given, then we will search for h2o.jar in the
            locations returned by `._jar_paths()`.
        :param nthreads: Number of threads in the thread pool. This should be related to the number of CPUs used.
            -1 means use all CPUs on the host. A positive integer specifies the number of CPUs directly.
        :param enable_assertions: If True, pass `-ea` option to the JVM.
        :param max_mem_size: Maximum heap size (jvm option Xmx), in bytes.
        :param min_mem_size: Minimum heap size (jvm option Xms), in bytes.
        :param log_dir: Directory for H2O logs to be stored if a new instance is started. Default directory is determined
        by H2O internally.
        :param log_level: The logger level for H2O if a new instance is started.
        :param ice_root: A directory where H2O stores its temporary files. Default location is determined by
            tempfile.mkdtemp().
        :param port: Port where to start the new server. This could be either an integer, or a string of the form
            "DDDDD+", indicating that the server should start looking for an open port starting from DDDDD and up.
        :param name: name of the h2o cluster to be started
        :param extra_classpath List of paths to libraries that should be included on the Java classpath.
        :param verbose: If True, then connection info will be printed to the stdout.
        :param jvm_custom_args Custom, user-defined arguments for the JVM H2O is instantiated in
        :param bind_to_localhost A flag indicating whether access to the H2O instance should be restricted to the local
            machine (default) or if it can be reached from other computers on the network.
            Only applicable when H2O is started from the Python client.

        :returns: a new H2OLocalServer instance
        """
        assert_is_type(jar_path, None, str)
        assert_is_type(port, None, int, str)
        assert_is_type(name, None, str)
        assert_is_type(nthreads, -1, BoundInt(1, 4096))
        assert_is_type(enable_assertions, bool)
        assert_is_type(min_mem_size, None, int)
        assert_is_type(max_mem_size, None, BoundInt(1 << 25))
        assert_is_type(log_dir, str, None)
        assert_is_type(log_level, str, None)
        assert_satisfies(log_level, log_level in [None, "TRACE", "DEBUG", "INFO", "WARN", "ERRR", "FATA"])
        assert_is_type(ice_root, None, I(str, os.path.isdir))
        assert_is_type(extra_classpath, None, [str])
        assert_is_type(jvm_custom_args, list, None)
        assert_is_type(bind_to_localhost, bool)
        if jar_path:
            assert_satisfies(jar_path, jar_path.endswith("h2o.jar"))

        if min_mem_size is not None and max_mem_size is not None and min_mem_size > max_mem_size:
            raise H2OValueError("`min_mem_size`=%d is larger than the `max_mem_size`=%d" % (min_mem_size, max_mem_size))
        if port is None: port = "54321+"
        baseport = None
        # TODO: get rid of this port gimmick and have 2 separate parameters.
        if is_type(port, str):
            if port.isdigit():
                port = int(port)
            else:
                if not(port[-1] == "+" and port[:-1].isdigit()):
                    raise H2OValueError("`port` should be of the form 'DDDD+', where D is a digit. Got: %s" % port)
                baseport = int(port[:-1])
                port = 0

        hs = H2OLocalServer()
        hs._verbose = bool(verbose)
        hs._jar_path = hs._find_jar(jar_path)
        hs._extra_classpath = extra_classpath
        hs._ice_root = ice_root
        hs._name = name
        if not ice_root:
            hs._ice_root = tempfile.mkdtemp()
            hs._tempdir = hs._ice_root

        if verbose: print("Attempting to start a local H2O server...")
        hs._launch_server(port=port, baseport=baseport, nthreads=int(nthreads), ea=enable_assertions,
                          mmax=max_mem_size, mmin=min_mem_size, jvm_custom_args=jvm_custom_args,
                          bind_to_localhost=bind_to_localhost, log_dir=log_dir, log_level=log_level)
        if verbose: print("  Server is running at %s://%s:%d" % (hs.scheme, hs.ip, hs.port))
        atexit.register(lambda: hs.shutdown())
        return hs

Start new H2O server on the local machine .


def _find_jar(self, path0=None):
        """
        Return the location of an h2o.jar executable.

        :param path0: Explicitly given h2o.jar path. If provided, then we will simply check whether the file is there,
            otherwise we will search for an executable in locations returned by ._jar_paths().

        :raises H2OStartupError: if no h2o.jar executable can be found.
        """
        jar_paths = [path0] if path0 else self._jar_paths()
        searched_paths = []
        for jp in jar_paths:
            searched_paths.append(jp)
            if os.path.exists(jp):
                return jp
        raise H2OStartupError("Cannot start local server: h2o.jar not found. Paths searched:\n" +
                              "".join("    %s\n" % s for s in searched_paths))

Return the location of an h2o . jar executable .


def _jar_paths():
        """Produce potential paths for an h2o.jar executable."""

        # PUBDEV-3534 hook to use arbitrary h2o.jar
        own_jar = os.getenv("H2O_JAR_PATH", "")
        if own_jar != "":
            if not os.path.isfile(own_jar):
                raise H2OStartupError("Environment variable H2O_JAR_PATH is set to '%d' but file does not exists, unset environment variable or provide valid path to h2o.jar file." % own_jar)
            yield own_jar

        # Check if running from an h2o-3 src folder (or any subfolder), in which case use the freshly-built h2o.jar
        cwd_chunks = os.path.abspath(".").split(os.path.sep)
        for i in range(len(cwd_chunks), 0, -1):
            if cwd_chunks[i - 1] == "h2o-3":
                yield os.path.sep.join(cwd_chunks[:i] + ["build", "h2o.jar"])
        # Then check the backend/bin folder:
        # (the following works assuming this code is located in h2o/backend/server.py file)
        backend_dir = os.path.split(os.path.realpath(__file__))[0]
        yield os.path.join(backend_dir, "bin", "h2o.jar")

        # Then try several old locations where h2o.jar might have been installed
        prefix1 = prefix2 = sys.prefix
        # On Unix-like systems Python typically gets installed into /Library/... or /System/Library/... If one of
        # those paths is sys.prefix, then we also build its counterpart.
        if prefix1.startswith(os.path.sep + "Library"):
            prefix2 = os.path.join("", "System", prefix1)
        elif prefix1.startswith(os.path.sep + "System"):
            prefix2 = prefix1[len(os.path.join("", "System")):]
        yield os.path.join(prefix1, "h2o_jar", "h2o.jar")
        yield os.path.join(os.path.abspath(os.sep), "usr", "local", "h2o_jar", "h2o.jar")
        yield os.path.join(prefix1, "local", "h2o_jar", "h2o.jar")
        yield os.path.join(get_config_var("userbase"), "h2o_jar", "h2o.jar")
        yield os.path.join(prefix2, "h2o_jar", "h2o.jar")

Produce potential paths for an h2o . jar executable .


def hit_ratio_table(self, train=False, valid=False, xval=False):
        """
        Retrieve the Hit Ratios.

        If all are False (default), then return the training metric value.
        If more than one options is set to True, then return a dictionary of metrics where the keys are "train",
        "valid", and "xval".

        :param train: If train is True, then return the hit ratio value for the training data.
        :param valid: If valid is True, then return the hit ratio value for the validation data.
        :param xval:  If xval is True, then return the hit ratio value for the cross validation data.
        :return: The hit ratio for this regression model.
        """
        tm = ModelBase._get_metrics(self, train, valid, xval)
        m = {}
        for k, v in zip(list(tm.keys()), list(tm.values())): m[k] = None if v is None else v.hit_ratio_table()
        return list(m.values())[0] if len(m) == 1 else m

def hit_ratio_table ( self , train = False , valid = False , xval = False ) : tm = ModelBase . _get_metrics ( self , train , valid , xval ) m = { } for k , v in zip ( list ( tm . keys ( ) ) , list ( tm . values ( ) ) ) : m [ k ] = None if v is None else v . hit_ratio_table ( ) return list ( m . values ( ) ) [ 0 ] if len ( m ) == 1 else m


def csv_dict_writer(f, fieldnames, **kwargs):
    """Equivalent of csv.DictWriter, but allows `delimiter` to be a unicode string on Py2."""
    import csv
    if "delimiter" in kwargs:
        kwargs["delimiter"] = str(kwargs["delimiter"])
    return csv.DictWriter(f, fieldnames, **kwargs)

def csv_dict_writer ( f , fieldnames , * * kwargs ) : import csv if "delimiter" in kwargs : kwargs [ "delimiter" ] = str ( kwargs [ "delimiter" ] ) return csv . DictWriter ( f , fieldnames , * * kwargs )

Equivalent of csv . DictWriter but allows delimiter to be a unicode string on Py2 .


def _uri2path(self, uri):
        ''' Convert uri to absolute filepath

        Parameters
        ----------
        uri : string
            URI of python module to return path for

        Returns
        -------
        path : None or string
            Returns None if there is no valid path for this URI
            Otherwise returns absolute file system path for URI

        Examples
        --------
        >>> docwriter = ApiDocWriter('sphinx')
        >>> import sphinx
        >>> modpath = sphinx.__path__[0]
        >>> res = docwriter._uri2path('sphinx.builder')
        >>> res == os.path.join(modpath, 'builder.py')
        True
        >>> res = docwriter._uri2path('sphinx')
        >>> res == os.path.join(modpath, '__init__.py')
        True
        >>> docwriter._uri2path('sphinx.does_not_exist')

        '''
        if uri == self.package_name:
            return os.path.join(self.root_path, '__init__.py')
        path = uri.replace('.', os.path.sep)
        path = path.replace(self.package_name + os.path.sep, '')
        path = os.path.join(self.root_path, path)
        # XXX maybe check for extensions as well?
        if os.path.exists(path + '.py'): # file
            path += '.py'
        elif os.path.exists(os.path.join(path, '__init__.py')):
            path = os.path.join(path, '__init__.py')
        else:
            return None
        return path


def _path2uri(self, dirpath):
        ''' Convert directory path to uri '''
        relpath = dirpath.replace(self.root_path, self.package_name)
        if relpath.startswith(os.path.sep):
            relpath = relpath[1:]
        return relpath.replace(os.path.sep, '.')

def _path2uri ( self , dirpath ) : relpath = dirpath . replace ( self . root_path , self . package_name ) if relpath . startswith ( os . path . sep ) : relpath = relpath [ 1 : ] return relpath . replace ( os . path . sep , '.' )


def _parse_lines(self, linesource):
        ''' Parse lines of text for functions and classes '''
        functions = []
        classes = []
        for line in linesource:
            if line.startswith('def ') and line.count('('):
                # exclude private stuff
                name = self._get_object_name(line)
                if not name.startswith('_'):
                    functions.append(name)
            elif line.startswith('class '):
                # exclude private stuff
                name = self._get_object_name(line)
                if not name.startswith('_'):
                    classes.append(name)
            else:
                pass
        functions.sort()
        classes.sort()
        return functions, classes

def
_parse_lines
(
self
,
linesource
)
:
functions
=
[
]
classes
=
[
]
for
line
in
linesource
:
if
line
.
startswith
(
'def '
)
and
line
.
count
(
'('
)
:
# exclude private stuff
name
=
self
.
_get_object_name
(
line
)
if
not
name
.
startswith
(
'_'
)
:
functions
.
append
(
name
)
elif
line
.
startswith
(
'class '
)
:
# exclude private stuff
name
=
self
.
_get_object_name
(
line
)
if
not
name
.
startswith
(
'_'
)
:
classes
.
append
(
name
)
else
:
pass
functions
.
sort
(
)
classes
.
sort
(
)
return
functions
,
classes


def generate_api_doc(self, uri):
        '''Make autodoc documentation template string for a module

        Parameters
        ----------
        uri : string
            python location of module - e.g 'sphinx.builder'

        Returns
        -------
        S : string
            Contents of API doc
        '''
        # get the names of all classes and functions
        functions, classes = self._parse_module(uri)
        if not len(functions) and not len(classes):
            print 'WARNING: Empty -',uri  # dbg
            return ''

        # Make a shorter version of the uri that omits the package name for
        # titles
        uri_short = re.sub(r'^%s\.' % self.package_name,'',uri)

        ad = '.. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n'

        chap_title = uri_short
        ad += (chap_title+'\n'+ self.rst_section_levels[1] * len(chap_title)
               + '\n\n')

        # Set the chapter title to read 'module' for all modules except for the
        # main packages
        if '.' in uri:
            title = 'Module: :mod:`' + uri_short + '`'
        else:
            title = ':mod:`' + uri_short + '`'
        ad += title + '\n' + self.rst_section_levels[2] * len(title)

        if len(classes):
            ad += '\nInheritance diagram for ``%s``:\n\n' % uri
            ad += '.. inheritance-diagram:: %s \n' % uri
            ad += '   :parts: 3\n'

        ad += '\n.. automodule:: ' + uri + '\n'
        ad += '\n.. currentmodule:: ' + uri + '\n'
        multi_class = len(classes) > 1
        multi_fx = len(functions) > 1
        if multi_class:
            ad += '\n' + 'Classes' + '\n' + \
                  self.rst_section_levels[2] * 7 + '\n'
        elif len(classes) and multi_fx:
            ad += '\n' + 'Class' + '\n' + \
                  self.rst_section_levels[2] * 5 + '\n'
        for c in classes:
            ad += '\n:class:`' + c + '`\n' \
                  + self.rst_section_levels[multi_class + 2 ] * \
                  (len(c)+9) + '\n\n'
            ad += '\n.. autoclass:: ' + c + '\n'
            # must NOT exclude from index to keep cross-refs working
            ad += '  :members:\n' \
                  '  :undoc-members:\n' \
                  '  :show-inheritance:\n' \
                  '  :inherited-members:\n' \
                  '\n' \
                  '  .. automethod:: __init__\n'
        if multi_fx:
            ad += '\n' + 'Functions' + '\n' + \
                  self.rst_section_levels[2] * 9 + '\n\n'
        elif len(functions) and multi_class:
            ad += '\n' + 'Function' + '\n' + \
                  self.rst_section_levels[2] * 8 + '\n\n'
        for f in functions:
            # must NOT exclude from index to keep cross-refs working
            ad += '\n.. autofunction:: ' + uri + '.' + f + '\n\n'
        return ad


def discover_modules(self):
        ''' Return module sequence discovered from ``self.package_name``

        Parameters
        ----------
        None

        Returns
        -------
        mods : sequence
            Sequence of module names within ``self.package_name``

        Examples
        --------
        >>> dw = ApiDocWriter('sphinx')
        >>> mods = dw.discover_modules()
        >>> 'sphinx.util' in mods
        True
        >>> dw.package_skip_patterns.append('\.util$')
        >>> 'sphinx.util' in dw.discover_modules()
        False
        >>>
        '''
        modules = [self.package_name]
        # raw directory parsing
        for dirpath, dirnames, filenames in os.walk(self.root_path):
            # Check directory names for packages
            root_uri = self._path2uri(os.path.join(self.root_path,
                                                   dirpath))
            for dirname in dirnames[:]: # copy list - we modify inplace
                package_uri = '.'.join((root_uri, dirname))
                if (self._uri2path(package_uri) and
                    self._survives_exclude(package_uri, 'package')):
                    modules.append(package_uri)
                else:
                    dirnames.remove(dirname)
            # Check filenames for modules
            for filename in filenames:
                module_name = filename[:-3]
                module_uri = '.'.join((root_uri, module_name))
                if (self._uri2path(module_uri) and
                    self._survives_exclude(module_uri, 'module')):
                    modules.append(module_uri)
        return sorted(modules)

def
discover_modules
(
self
)
:
modules
=
[
self
.
package_name
]
# raw directory parsing
for
dirpath
,
dirnames
,
filenames
in
os
.
walk
(
self
.
root_path
)
:
# Check directory names for packages
root_uri
=
self
.
_path2uri
(
os
.
path
.
join
(
self
.
root_path
,
dirpath
)
)
for
dirname
in
dirnames
[
:
]
:
# copy list - we modify inplace
package_uri
=
'.'
.
join
(
(
root_uri
,
dirname
)
)
if
(
self
.
_uri2path
(
package_uri
)
and
self
.
_survives_exclude
(
package_uri
,
'package'
)
)
:
modules
.
append
(
package_uri
)
else
:
dirnames
.
remove
(
dirname
)
# Check filenames for modules
for
filename
in
filenames
:
module_name
=
filename
[
:
3
]
module_uri
=
'.'
.
join
(
(
root_uri
,
module_name
)
)
if
(
self
.
_uri2path
(
module_uri
)
and
self
.
_survives_exclude
(
module_uri
,
'module'
)
)
:
modules
.
append
(
module_uri
)
return
sorted
(
modules
)

Return module sequence discovered from self . package_name


def write_api_docs(self, outdir):
        """Generate API reST files.

        Parameters
        ----------
        outdir : string
            Directory name in which to store files
            We create automatic filenames for each module

        Returns
        -------
        None

        Notes
        -----
        Sets self.written_modules to list of written modules
        """
        if not os.path.exists(outdir):
            os.mkdir(outdir)
        # compose list of modules
        modules = self.discover_modules()
        self.write_modules_api(modules,outdir)

def
write_api_docs
(
self
,
outdir
)
:
if
not
os
.
path
.
exists
(
outdir
)
:
os
.
mkdir
(
outdir
)
# compose list of modules
modules
=
self
.
discover_modules
(
)
self
.
write_modules_api
(
modules
,
outdir
)


def write_index(self, outdir, froot='gen', relative_to=None):
        """Make a reST API index file from written files

        Parameters
        ----------
        path : string
            Filename to write index to
        outdir : string
            Directory to which to write generated index file
        froot : string, optional
            root (filename without extension) of filename to write to
            Defaults to 'gen'.  We add ``self.rst_extension``.
        relative_to : string
            path to which written filenames are relative.  This
            component of the written file path will be removed from
            outdir, in the generated index.  Default is None, meaning,
            leave path as it is.
        """
        if self.written_modules is None:
            raise ValueError('No modules written')
        # Get full filename path
        path = os.path.join(outdir, froot+self.rst_extension)
        # Path written into index is relative to rootpath
        if relative_to is not None:
            relpath = outdir.replace(relative_to + os.path.sep, '')
        else:
            relpath = outdir
        idx = open(path,'wt')
        w = idx.write
        w('.. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n')
        w('.. toctree::\n\n')
        for f in self.written_modules:
            w('   %s\n' % os.path.join(relpath,f))
        idx.close()


def to_list(self):
        """Convert this confusion matrix into a 2x2 plain list of values."""
        return [[int(self.table.cell_values[0][1]), int(self.table.cell_values[0][2])],
                [int(self.table.cell_values[1][1]), int(self.table.cell_values[1][2])]]

def to_list ( self ) : return [ [ int ( self . table . cell_values [ 0 ] [ 1 ] ) , int ( self . table . cell_values [ 0 ] [ 2 ] ) ] , [ int ( self . table . cell_values [ 1 ] [ 1 ] ) , int ( self . table . cell_values [ 1 ] [ 2 ] ) ] ]

Convert this confusion matrix into a 2x2 plain list of values .


def load_dict():
    """
    Load java messages that can be ignored pickle file into a dict structure g_ok_java_messages.

    :return: none
    """
    global g_load_java_message_filename
    global g_ok_java_messages

    if os.path.isfile(g_load_java_message_filename):
            # only load dict from file if it exists.
        with open(g_load_java_message_filename,'rb') as ofile:
            g_ok_java_messages = pickle.load(ofile)
    else:   # no previous java messages to be excluded are found
        g_ok_java_messages["general"] = []

Load java messages that can be ignored pickle file into a dict structure g_ok_java_messages .


def add_new_message():
    """
    Add new java messages to ignore from user text file.  It first reads in the new java ignored messages
    from the user text file and generate a dict structure to out of the new java ignored messages.  This
    is achieved by function extract_message_to_dict.  Next, new java messages will be added to the original
    ignored java messages dict g_ok_java_messages.  Again, this is achieved by function update_message_dict.

    :return: none
    """
    global g_new_messages_to_exclude    # filename containing text file from user containing new java ignored messages
    global g_dict_changed               # True if new ignored java messages are added.

    new_message_dict = extract_message_to_dict(g_new_messages_to_exclude)

    if new_message_dict:
        g_dict_changed = True
        update_message_dict(new_message_dict,1)

Add new java messages to ignore from user text file . It first reads in the new java ignored messages from the user text file and generate a dict structure to out of the new java ignored messages . This is achieved by function extract_message_to_dict . Next new java messages will be added to the original ignored java messages dict g_ok_java_messages . Again this is achieved by function update_message_dict .


def update_message_dict(message_dict,action):
    """
    Update the g_ok_java_messages dict structure by
    1. add the new java ignored messages stored in message_dict if action == 1
    2. remove the java ignored messages stired in message_dict if action == 2.

    Parameters
    ----------

    message_dict :  Python dict
      key: unit test name or "general"
      value: list of java messages that are to be ignored if they are found when running the test stored as the key.  If
        the key is "general", the list of java messages are to be ignored when running all tests.
    action : int
      if 1: add java ignored messages stored in message_dict to g_ok_java_messages dict;
      if 2: remove java ignored messages stored in message_dict from g_ok_java_messages dict.

    :return: none
    """
    global g_ok_java_messages

    allKeys = g_ok_java_messages.keys()

    for key in message_dict.keys():
        if key in allKeys:  # key already exists, just add to it
            for message in message_dict[key]:

                if action == 1:
                    if message not in g_ok_java_messages[key]:
                        g_ok_java_messages[key].append(message)

                if action == 2:
                    if message in g_ok_java_messages[key]:
                        g_ok_java_messages[key].remove(message)
        else:   # new key here.  Can only add and cannot remove
            if action == 1:
                g_ok_java_messages[key] = message_dict[key]

Update the g_ok_java_messages dict structure by 1 . add the new java ignored messages stored in message_dict if action == 1 2 . remove the java ignored messages stired in message_dict if action == 2 .


def extract_message_to_dict(filename):
    """
    Read in a text file that java messages to be ignored and generate a dictionary structure out of
    it with key and value pairs.  The keys are test names and the values are lists of java message
    strings associated with that test name where we are either going to add to the existing java messages
    to ignore or remove them from g_ok_java_messages.

    Parameters
    ----------

    filename :  Str
       filename that contains ignored java messages.  The text file shall contain something like this:
        keyName = general
        Message = nfolds: nfolds cannot be larger than the number of rows (406).
        KeyName = pyunit_cv_cars_gbm.py
        Message = Caught exception: Illegal argument(s) for GBM model: GBM_model_python_1452503348770_2586.  \
            Details: ERRR on field: _nfolds: nfolds must be either 0 or >1.
        ...

    :return:
    message_dict : dict
        contains java message to be ignored with key as unit test name or "general" and values as list of ignored java
        messages.
    """
    message_dict = {}

    if os.path.isfile(filename):
        # open file to read in new exclude messages if it exists
        with open(filename,'r') as wfile:

            key = ""
            val = ""
            startMess = False

            while 1:
                each_line = wfile.readline()

                if not each_line:   # reached EOF
                    if startMess:
                        add_to_dict(val.strip(),key,message_dict)
                    break

                # found a test name or general with values to follow
                if "keyname" in each_line.lower():  # name of test file or the word "general"
                    temp_strings = each_line.strip().split('=')

                    if (len(temp_strings) > 1): # make sure the line is formatted sort of correctly
                        if startMess:   # this is the start of a new key/value pair
                            add_to_dict(val.strip(),key,message_dict)
                            val = ""

                        key = temp_strings[1].strip()
                        startMess = False

                if (len(each_line) > 1) and startMess:
                    val += each_line

                if "ignoredmessage" in each_line.lower():
                    startMess = True    # start of a Java message.
                    temp_mess = each_line.split('=')

                    if (len(temp_mess) > 1):
                        val = temp_mess[1]

    return message_dict

Read in a text file that java messages to be ignored and generate a dictionary structure out of it with key and value pairs . The keys are test names and the values are lists of java message strings associated with that test name where we are either going to add to the existing java messages to ignore or remove them from g_ok_java_messages .


def save_dict():
    """
    Save the ignored java message dict stored in g_ok_java_messages into a pickle file for future use.

    :return: none
    """
    global g_ok_java_messages
    global g_save_java_message_filename
    global g_dict_changed

    if g_dict_changed:
        with open(g_save_java_message_filename,'wb') as ofile:
            pickle.dump(g_ok_java_messages,ofile)

def save_dict ( ) : global g_ok_java_messages global g_save_java_message_filename global g_dict_changed if g_dict_changed : with open ( g_save_java_message_filename , 'wb' ) as ofile : pickle . dump ( g_ok_java_messages , ofile )

Save the ignored java message dict stored in g_ok_java_messages into a pickle file for future use .


def print_dict():
    """
    Write the java ignored messages in g_ok_java_messages into a text file for humans to read.

    :return: none
    """
    global g_ok_java_messages
    global g_java_messages_to_ignore_text_filename

    allKeys = sorted(g_ok_java_messages.keys())

    with open(g_java_messages_to_ignore_text_filename,'w') as ofile:
        for key in allKeys:

            for mess in g_ok_java_messages[key]:
                ofile.write('KeyName: '+key+'\n')
                ofile.write('IgnoredMessage: '+mess+'\n')

            print('KeyName: ',key)
            print('IgnoredMessage: ',g_ok_java_messages[key])
            print('\n')

def
print_dict
(
)
:
global
g_ok_java_messages
global
g_java_messages_to_ignore_text_filename
allKeys
=
sorted
(
g_ok_java_messages
.
keys
(
)
)
with
open
(
g_java_messages_to_ignore_text_filename
,
'w'
)
as
ofile
:
for
key
in
allKeys
:
for
mess
in
g_ok_java_messages
[
key
]
:
ofile
.
write
(
'KeyName: '
+
key
+
'\n'
)
ofile
.
write
(
'IgnoredMessage: '
+
mess
+
'\n'
)
print
(
'KeyName: '
,
key
)
print
(
'IgnoredMessage: '
,
g_ok_java_messages
[
key
]
)
print
(
'\n'
)

Write the java ignored messages in g_ok_java_messages into a text file for humans to read .


def parse_args(argv):
    """
    Parse user inputs and set the corresponing global variables to perform the
    necessary tasks.

    Parameters
    ----------

    argv : string array
        contains flags and input options from users

    :return:
    """
    global g_new_messages_to_exclude
    global g_old_messages_to_remove
    global g_load_java_message_filename
    global g_save_java_message_filename
    global g_print_java_messages

    if len(argv) < 2:   # print out help menu if user did not enter any arguments.
        usage()

    i = 1
    while (i < len(argv)):
        s = argv[i]

        if (s == "--inputfileadd"):         # input text file where new java messages are stored
            i += 1
            if (i > len(argv)):
                usage()
            g_new_messages_to_exclude = argv[i]
        elif (s == "--inputfilerm"):        # input text file containing java messages to be removed from the ignored list
            i += 1
            if (i > len(argv)):
                usage()
            g_old_messages_to_remove = argv[i]
        elif (s == "--loadjavamessage"):    # load previously saved java message pickle file from file other than
            i += 1                          # the default one before performing update
            if i > len(argv):
                usage()
            g_load_java_message_filename = argv[i]
        elif (s == "--savejavamessage"):    # save updated java message in this file instead of default file
            i += 1
            if (i > len(argv)):
                usage()
            g_save_java_message_filename = argv[i]
        elif (s == '--printjavamessage'):   # will print java message out to console and save in a text file
            i += 1
            g_print_java_messages = True
            g_load_java_message_filename = argv[i]
        elif (s == '--help'):               # print help menu and exit
            usage()
        else:
            unknown_arg(s)

        i += 1

Parse user inputs and set the corresponing global variables to perform the necessary tasks .


def usage():
    """
    Illustrate what the various input flags are and the options should be.

    :return: none
    """
    global g_script_name    # name of the script being run.

    print("")
    print("Usage:  " + g_script_name + " [...options...]")
    print("")
    print("     --help print out this help menu and show all the valid flags and inputs.")
    print("")
    print("    --inputfileadd filename where the new java messages to ignore are stored in.")
    print("")
    print("    --inputfilerm filename where the java messages are removed from the ignored list.")
    print("")
    print("    --loadjavamessage filename pickle file that stores the dict structure containing java messages to include.")
    print("")
    print("    --savejavamessage filename pickle file that saves the final dict structure after update.")
    print("")
    print("    --printjavamessage filename print java ignored java messages stored in pickle file filenam onto console and save into a text file.")
    print("")
    sys.exit(1)

Illustrate what the various input flags are and the options should be .


def locate_files(root_dir):
    """Find all python files in the given directory and all subfolders."""
    all_files = []
    root_dir = os.path.abspath(root_dir)
    for dir_name, subdirs, files in os.walk(root_dir):
        for f in files:
            if f.endswith(".py"):
                all_files.append(os.path.join(dir_name, f))
    return all_files

def locate_files ( root_dir ) : all_files = [ ] root_dir = os . path . abspath ( root_dir ) for dir_name , subdirs , files in os . walk ( root_dir ) : for f in files : if f . endswith ( ".py" ) : all_files . append ( os . path . join ( dir_name , f ) ) return all_files

Find all python files in the given directory and all subfolders .


def find_magic_in_file(filename):
    """
    Search the file for any magic incantations.

    :param filename: file to search
    :returns: a tuple containing the spell and then maybe some extra words (or None if no magic present)
    """
    with open(filename, "rt", encoding="utf-8") as f:
        for line in f:
            if line.startswith("#"):
                comment = line[1:].strip()
                if comment.startswith("~~~~* ") or comment.startswith("----* ") or comment.startswith("====* "):
                    spell = comment[5:].strip()
                    return tuple(spell.split())
            else:
                break
    return None

def
find_magic_in_file
(
filename
)
:
with
open
(
filename
,
"rt"
,
encoding
=
"utf-8"
)
as
f
:
for
line
in
f
:
if
line
.
startswith
(
"#"
)
:
comment
=
line
[
1
:
]
.
strip
(
)
if
comment
.
startswith
(
"~~~~* "
)
or
comment
.
startswith
(
"----* "
)
or
comment
.
startswith
(
"====* "
)
:
spell
=
comment
[
5
:
]
.
strip
(
)
return
tuple
(
spell
.
split
(
)
)
else
:
break
return

Search the file for any magic incantations .


def main():
    """Executed when script is run as-is."""
    # magic_files = {}
    for filename in locate_files(ROOT_DIR):
        print("Processing %s" % filename)
        with open(filename, "rt") as f:
            tokens = list(tokenize.generate_tokens(f.readline))
            text1 = tokenize.untokenize(tokens)
            ntokens = normalize_tokens(tokens)
            text2 = tokenize.untokenize(ntokens)
            assert text1 == text2

def
main
(
)
:
# magic_files = {}
for
filename
in
locate_files
(
ROOT_DIR
)
:
print
(
"Processing %s"
%
filename
)
with
open
(
filename
,
"rt"
)
as
f
:
tokens
=
list
(
tokenize
.
generate_tokens
(
f
.
readline
)
)
text1
=
tokenize
.
untokenize
(
tokens
)
ntokens
=
normalize_tokens
(
tokens
)
text2
=
tokenize
.
untokenize
(
ntokens
)
assert
text1
==
text2


def transform(self, data, allow_timestamps=False):
        """
        Transform H2OFrame using a MOJO Pipeline.

        :param data: Frame to be transformed.
        :param allow_timestamps: Allows datetime columns to be used directly with MOJO pipelines. It is recommended
        to parse your datetime columns as Strings when using pipelines because pipelines can interpret certain datetime
        formats in a different way. If your H2OFrame is parsed from a binary file format (eg. Parquet) instead of CSV
        it is safe to turn this option on and use datetime columns directly.

        :returns: A new H2OFrame.
        """
        assert_is_type(data, H2OFrame)
        assert_is_type(allow_timestamps, bool)
        return H2OFrame._expr(ExprNode("mojo.pipeline.transform", self.pipeline_id[0], data, allow_timestamps))

def transform ( self , data , allow_timestamps = False ) : assert_is_type ( data , H2OFrame ) assert_is_type ( allow_timestamps , bool ) return H2OFrame . _expr ( ExprNode ( "mojo.pipeline.transform" , self . pipeline_id [ 0 ] , data , allow_timestamps ) )

Transform H2OFrame using a MOJO Pipeline .


def summarizeFailedRuns():
    """
    This function will look at the local directory and pick out files that have the correct start name and
    summarize the results into one giant dict.

    :return: None
    """
    global g_summary_dict_all

    onlyFiles = [x for x in listdir(g_test_root_dir) if isfile(join(g_test_root_dir, x))]   # grab files

    for f in onlyFiles:
        for fileStart in g_file_start:
            if (fileStart in f) and (os.path.getsize(f) > 10):  # found the file containing failed tests
                fFullPath = os.path.join(g_test_root_dir, f)
                try:
                    temp_dict = json.load(open(fFullPath,'r'))

                    # scrape through temp_dict and see if we need to add the test to intermittents
                    for ind in range(len(temp_dict["TestName"])):
                        addFailedTests(g_summary_dict_all, temp_dict, ind)
                except:
                    continue
                break

This function will look at the local directory and pick out files that have the correct start name and summarize the results into one giant dict .


def extractPrintSaveIntermittens():
    """
    This function will print out the intermittents onto the screen for casual viewing.  It will also print out
    where the giant summary dictionary is going to be stored.

    :return: None
    """
    # extract intermittents from collected failed tests
    global g_summary_dict_intermittents

    localtz = time.tzname[0]

    for ind in range(len(g_summary_dict_all["TestName"])):
        if g_summary_dict_all["TestInfo"][ind]["FailureCount"] >= g_threshold_failure:
            addFailedTests(g_summary_dict_intermittents, g_summary_dict_all, ind)

    # save dict in file
    if len(g_summary_dict_intermittents["TestName"]) > 0:
        json.dump(g_summary_dict_intermittents, open(g_summary_dict_name, 'w'))

        with open(g_summary_csv_filename, 'w') as summaryFile:
            for ind in range(len(g_summary_dict_intermittents["TestName"])):
                testName = g_summary_dict_intermittents["TestName"][ind]
                numberFailure = g_summary_dict_intermittents["TestInfo"][ind]["FailureCount"]
                firstFailedTS  = parser.parse(time.ctime(min(g_summary_dict_intermittents["TestInfo"][ind]["Timestamp"]))+
                                              ' '+localtz)
                firstFailedStr = firstFailedTS.strftime("%a %b %d %H:%M:%S %Y %Z")
                recentFail = parser.parse(time.ctime(max(g_summary_dict_intermittents["TestInfo"][ind]["Timestamp"]))+
                                          ' '+localtz)
                recentFailStr = recentFail.strftime("%a %b %d %H:%M:%S %Y %Z")
                eachTest = "{0}, {1}, {2}, {3}\n".format(testName, recentFailStr, numberFailure,
                                                       g_summary_dict_intermittents["TestInfo"][ind]["TestCategory"][0])
                summaryFile.write(eachTest)
                print("Intermittent: {0}, Last failed: {1}, Failed {2} times since "
                      "{3}".format(testName, recentFailStr, numberFailure, firstFailedStr))

This function will print out the intermittents onto the screen for casual viewing . It will also print out where the giant summary dictionary is going to be stored .


def plot(self, type="roc", server=False):
        """
        Produce the desired metric plot.

        :param type: the type of metric plot (currently, only ROC supported).
        :param server: if True, generate plot inline using matplotlib's "Agg" backend.
        :returns: None
        """
        # TODO: add more types (i.e. cutoffs)
        assert_is_type(type, "roc")
        # check for matplotlib. exit if absent.
        try:
            imp.find_module('matplotlib')
            import matplotlib
            if server: matplotlib.use('Agg', warn=False)
            import matplotlib.pyplot as plt
        except ImportError:
            print("matplotlib is required for this function!")
            return

        if type == "roc":
            plt.xlabel('False Positive Rate (FPR)')
            plt.ylabel('True Positive Rate (TPR)')
            plt.title('ROC Curve')
            plt.text(0.5, 0.5, r'AUC={0:.4f}'.format(self._metric_json["AUC"]))
            plt.plot(self.fprs, self.tprs, 'b--')
            plt.axis([0, 1, 0, 1])
            if not server: plt.show()


def confusion_matrix(self, metrics=None, thresholds=None):
        """
        Get the confusion matrix for the specified metric

        :param metrics: A string (or list of strings) among metrics listed in :const:`max_metrics`. Defaults to 'f1'.
        :param thresholds: A value (or list of values) between 0 and 1.
        :returns: a list of ConfusionMatrix objects (if there are more than one to return), or a single ConfusionMatrix
            (if there is only one).
        """
        # make lists out of metrics and thresholds arguments
        if metrics is None and thresholds is None:
            metrics = ['f1']

        if isinstance(metrics, list):
            metrics_list = metrics
        elif metrics is None:
            metrics_list = []
        else:
            metrics_list = [metrics]

        if isinstance(thresholds, list):
            thresholds_list = thresholds
        elif thresholds is None:
            thresholds_list = []
        else:
            thresholds_list = [thresholds]

        # error check the metrics_list and thresholds_list
        assert_is_type(thresholds_list, [numeric])
        assert_satisfies(thresholds_list, all(0 <= t <= 1 for t in thresholds_list))

        if not all(m.lower() in H2OBinomialModelMetrics.max_metrics for m in metrics_list):
            raise ValueError("The only allowable metrics are {}", ', '.join(H2OBinomialModelMetrics.max_metrics))

        # make one big list that combines the thresholds and metric-thresholds
        metrics_thresholds = [self.find_threshold_by_max_metric(m) for m in metrics_list]
        for mt in metrics_thresholds:
            thresholds_list.append(mt)
        first_metrics_thresholds_offset = len(thresholds_list) - len(metrics_thresholds)

        thresh2d = self._metric_json['thresholds_and_metric_scores']
        actual_thresholds = [float(e[0]) for i, e in enumerate(thresh2d.cell_values)]
        cms = []
        for i, t in enumerate(thresholds_list):
            idx = self.find_idx_by_threshold(t)
            row = thresh2d.cell_values[idx]
            tns = row[11]
            fns = row[12]
            fps = row[13]
            tps = row[14]
            p = tps + fns
            n = tns + fps
            c0 = n - fps
            c1 = p - tps
            if t in metrics_thresholds:
                m = metrics_list[i - first_metrics_thresholds_offset]
                table_header = "Confusion Matrix (Act/Pred) for max {} @ threshold = {}".format(m, actual_thresholds[idx])
            else:
                table_header = "Confusion Matrix (Act/Pred) @ threshold = {}".format(actual_thresholds[idx])
            cms.append(ConfusionMatrix(cm=[[c0, fps], [c1, tps]], domains=self._metric_json['domain'],
                                       table_header=table_header))

        if len(cms) == 1:
            return cms[0]
        else:
            return cms


def available():
        """Returns True if a deep water model can be built, or False otherwise."""
        builder_json = h2o.api("GET /3/ModelBuilders", data={"algo": "deepwater"})
        visibility = builder_json["model_builders"]["deepwater"]["visibility"]
        if visibility == "Experimental":
            print("Cannot build a Deep Water model - no backend found.")
            return False
        else:
            return True

Returns True if a deep water model can be built or False otherwise .


def trim_data_back_to(monthToKeep):
    """
    This method will remove data from the summary text file and the dictionary file for tests that occurs before
    the number of months specified by monthToKeep.

    :param monthToKeep:
    :return:
    """
    global g_failed_tests_info_dict
    current_time = time.time()      # unit in seconds

    oldest_time_allowed = current_time - monthToKeep*30*24*3600 # in seconds

    clean_up_failed_test_dict(oldest_time_allowed)
    clean_up_summary_text(oldest_time_allowed)

This method will remove data from the summary text file and the dictionary file for tests that occurs before the number of months specified by monthToKeep .


def endpoint_groups():
    """Return endpoints, grouped by the class which handles them."""
    groups = defaultdict(list)
    for e in endpoints():
        groups[e["class_name"]].append(e)
    return groups

def endpoint_groups ( ) : groups = defaultdict ( list ) for e in endpoints ( ) : groups [ e [ "class_name" ] ] . append ( e ) return groups

Return endpoints grouped by the class which handles them .


def update_site_forward(apps, schema_editor):
    """Set site domain and name."""
    Site = apps.get_model("sites", "Site")
    Site.objects.update_or_create(
        id=settings.SITE_ID,
        defaults={
            "domain": "{{cookiecutter.domain_name}}",
            "name": "{{cookiecutter.project_name}}",
        },
    )

def update_site_forward ( apps , schema_editor ) : Site = apps . get_model ( "sites" , "Site" ) Site . objects . update_or_create ( id = settings . SITE_ID , defaults = { "domain" : "{{cookiecutter.domain_name}}" , "name" : "{{cookiecutter.project_name}}" , } , )


def json_data(self, data=None):
        """Adds the default_data to data and dumps it to a json."""
        if data is None:
            data = {}
        data.update(self.default_data)
        return json.dumps(data)

def json_data ( self , data = None ) : if data is None : data = { } data . update ( self . default_data ) return json . dumps ( data )

Adds the default_data to data and dumps it to a json .


def comment_user(self, user_id, amount=None):
    """ Comments last user_id's medias """
    if not self.check_user(user_id, filter_closed_acc=True):
        return False
    self.logger.info("Going to comment user_%s's feed:" % user_id)
    user_id = self.convert_to_user_id(user_id)
    medias = self.get_user_medias(user_id, is_comment=True)
    if not medias:
        self.logger.info(
            "None medias received: account is closed or medias have been filtered.")
        return False
    return self.comment_medias(medias[:amount])


def get_credentials(username=None):
    """Returns login and password stored in `secret.txt`."""
    while not check_secret():
        pass
    while True:
        try:
            with open(SECRET_FILE, "r") as f:
                lines = [line.strip().split(":", 2) for line in f.readlines()]
        except ValueError:
            msg = 'Problem with opening `{}`, will remove the file.'
            raise Exception(msg.format(SECRET_FILE))
        if username is not None:
            for login, password in lines:
                if login == username.strip():
                    return login, password
        print("Which account do you want to use? (Type number)")
        for ind, (login, password) in enumerate(lines):
            print("%d: %s" % (ind + 1, login))
        print("%d: %s" % (0, "add another account."))
        print("%d: %s" % (-1, "delete all accounts."))
        try:
            ind = int(sys.stdin.readline())
            if ind == 0:
                add_credentials()
                continue
            elif ind == -1:
                delete_credentials()
                check_secret()
                continue
            elif 0 <= ind - 1 < len(lines):
                return lines[ind - 1]
        except Exception:
            print("Wrong input, enter the number of the account to use.")

Returns login and password stored in secret . txt .


def like_user(self, user_id, amount=None, filtration=True):
    """ Likes last user_id's medias """
    if filtration:
        if not self.check_user(user_id):
            return False
    self.logger.info("Liking user_%s's feed:" % user_id)
    user_id = self.convert_to_user_id(user_id)
    medias = self.get_user_medias(user_id, filtration=filtration)
    if not medias:
        self.logger.info(
            "None medias received: account is closed or medias have been filtered.")
        return False
    return self.like_medias(medias[:amount])


def like_hashtag(self, hashtag, amount=None):
    """ Likes last medias from hashtag """
    self.logger.info("Going to like media with hashtag #%s." % hashtag)
    medias = self.get_total_hashtag_medias(hashtag, amount)
    return self.like_medias(medias)

def
like_hashtag
(
self
,
hashtag
,
amount
=
)
:
self
.
logger
.
info
(
"Going to like media with hashtag #%s."
%
hashtag
)
medias
=
self
.
get_total_hashtag_medias
(
hashtag
,
amount
)
return
self
.
like_medias
(
medias
)


def check_not_bot(self, user_id):
    """ Filter bot from real users. """
    self.small_delay()
    user_id = self.convert_to_user_id(user_id)
    if not user_id:
        return False
    if user_id in self.whitelist:
        return True
    if user_id in self.blacklist:
        return False

    user_info = self.get_user_info(user_id)
    if not user_info:
        return True  # closed acc

    skipped = self.skipped_file
    if "following_count" in user_info and user_info["following_count"] > self.max_following_to_block:
        msg = 'following_count > bot.max_following_to_block, skipping!'
        self.console_print(msg, 'red')
        skipped.append(user_id)
        return False  # massfollower

    if search_stop_words_in_user(self, user_info):
        msg = '`bot.search_stop_words_in_user` found in user, skipping!'
        skipped.append(user_id)
        return False

    return True


def read_list_from_file(file_path, quiet=False):
    """
        Reads list from file. One line - one item.
        Returns the list if file items.
    """
    try:
        if not check_if_file_exists(file_path, quiet=quiet):
            return []
        with codecs.open(file_path, "r", encoding="utf-8") as f:
            content = f.readlines()
            if sys.version_info[0] < 3:
                content = [str(item.encode('utf8')) for item in content]
            content = [item.strip() for item in content]
            return [i for i in content if i]
    except Exception as exception:
        print(str(exception))
        return []

def read_list_from_file ( file_path , quiet = False ) : try : if not check_if_file_exists ( file_path , quiet = quiet ) : return [ ] with codecs . open ( file_path , "r" , encoding = "utf-8" ) as f : content = f . readlines ( ) if sys . version_info [ 0 ] < 3 : content = [ str ( item . encode ( 'utf8' ) ) for item in content ] content = [ item . strip ( ) for item in content ] return [ i for i in content if i ] except Exception as exception : print ( str ( exception ) ) return [ ]

Reads list from file . One line - one item . Returns the list if file items .


def schedule(self, schedule_time):
        """Add a specific enqueue time to the message.

        :param schedule_time: The scheduled time to enqueue the message.
        :type schedule_time: ~datetime.datetime
        """
        if not self.properties.message_id:
            self.properties.message_id = str(uuid.uuid4())
        if not self.message.annotations:
            self.message.annotations = {}
        self.message.annotations[types.AMQPSymbol(self._x_OPT_SCHEDULED_ENQUEUE_TIME)] = schedule_time

def schedule ( self , schedule_time ) : if not self . properties . message_id : self . properties . message_id = str ( uuid . uuid4 ( ) ) if not self . message . annotations : self . message . annotations = { } self . message . annotations [ types . AMQPSymbol ( self . _x_OPT_SCHEDULED_ENQUEUE_TIME ) ] = schedule_time

Add a specific enqueue time to the message .


def defer(self):
        """Defer the message.

        This message will remain in the queue but must be received
        specifically by its sequence number in order to be processed.

        :raises: ~azure.servicebus.common.errors.MessageAlreadySettled if the message has been settled.
        :raises: ~azure.servicebus.common.errors.MessageLockExpired if message lock has already expired.
        :raises: ~azure.servicebus.common.errors.SessionLockExpired if session lock has already expired.
        :raises: ~azure.servicebus.common.errors.MessageSettleFailed if message settle operation fails.
        """
        self._is_live('defer')
        try:
            self.message.modify(True, True)
        except Exception as e:
            raise MessageSettleFailed("defer", e)

def defer ( self ) : self . _is_live ( 'defer' ) try : self . message . modify ( True , True ) except Exception as e : raise MessageSettleFailed ( "defer" , e )