# -*- encoding: utf-8 -*-
"""
Local server.

`H2OLocalServer` allows to start H2O servers on your local machine:
    hs = H2OLocalServer.start() : start a new local server
    hs.is_running() : check if the server is running
    hs.shutdown() : shut down the server

:copyright: (c) 2016 H2O.ai
:license:   Apache License Version 2.0 (see LICENSE for details)
"""
from __future__ import absolute_import, division, print_function, unicode_literals

import atexit
import os
import subprocess
import sys
import tempfile
import time
from random import choice
from sysconfig import get_config_var
from warnings import warn

from h2o.exceptions import H2OServerError, H2OStartupError, H2OValueError
from h2o.utils.compatibility import *  # NOQA
from h2o.utils.typechecks import assert_is_type, assert_satisfies, BoundInt, I, is_type

__all__ = ("H2OLocalServer", )

class H2OLocalServer(object):
    """
    Handle to an H2O server launched locally.

    Public interface::

        hs = H2OLocalServer.start(...)  # launch a new local H2O server
        hs.is_running()                 # check if the server is running
        hs.shutdown()                   # shut down the server
        hs.scheme                       # either "http" or "https"
        hs.ip                           # ip address of the server, typically "127.0.0.1"
        hs.port                         # port on which the server is listening

    Once started, the server will run until the script terminates, or until you call `.shutdown()` on it. Moreover,
    if the server terminates with an exception, then the server will not stop and will continue to run even after
    Python process exits. This runaway process may end up being in a bad shape (e.g. frozen), then the only way to
    terminate it is to kill the java process from the terminal.

    Alternatively, it is possible to start the server as a context manager, in which case it will be automatically
    shut down even if an exception occurs in Python (but not if the Python process is killed)::

        with H2OLocalServer.start() as hs:
            # do something with the server -- probably connect to it
    """

    ## (Avkash) Changing the maximum time to wait as 60 seconds to match with the same time in to R API. ##
    _TIME_TO_START = 60  # Maximum time we wait for the server to start up (in seconds)
    _TIME_TO_KILL = 3    # Maximum time we wait for the server to shut down until we kill it (in seconds)

    @staticmethod
    def start(jar_path=None, nthreads=-1, enable_assertions=True, max_mem_size=None, min_mem_size=None,
              ice_root=None, log_dir=None, log_level=None, port="54321+", name=None, extra_classpath=None,
              verbose=True, jvm_custom_args=None, bind_to_localhost=True):
        """
        Start new H2O server on the local machine.

        :param jar_path: Path to the h2o.jar executable. If not given, then we will search for h2o.jar in the
            locations returned by `._jar_paths()`.
        :param nthreads: Number of threads in the thread pool. This should be related to the number of CPUs used.
            -1 means use all CPUs on the host. A positive integer specifies the number of CPUs directly.
        :param enable_assertions: If True, pass `-ea` option to the JVM.
        :param max_mem_size: Maximum heap size (jvm option Xmx), in bytes.
        :param min_mem_size: Minimum heap size (jvm option Xms), in bytes.
        :param log_dir: Directory for H2O logs to be stored if a new instance is started. Default directory is determined
        by H2O internally.
        :param log_level: The logger level for H2O if a new instance is started.
        :param ice_root: A directory where H2O stores its temporary files. Default location is determined by
            tempfile.mkdtemp().
        :param port: Port where to start the new server. This could be either an integer, or a string of the form
            "DDDDD+", indicating that the server should start looking for an open port starting from DDDDD and up.
        :param name: name of the h2o cluster to be started
        :param extra_classpath List of paths to libraries that should be included on the Java classpath.
        :param verbose: If True, then connection info will be printed to the stdout.
        :param jvm_custom_args Custom, user-defined arguments for the JVM H2O is instantiated in
        :param bind_to_localhost A flag indicating whether access to the H2O instance should be restricted to the local
            machine (default) or if it can be reached from other computers on the network.
            Only applicable when H2O is started from the Python client.

        :returns: a new H2OLocalServer instance
        """
        assert_is_type(jar_path, None, str)
        assert_is_type(port, None, int, str)
        assert_is_type(name, None, str)
        assert_is_type(nthreads, -1, BoundInt(1, 4096))
        assert_is_type(enable_assertions, bool)
        assert_is_type(min_mem_size, None, int)
        assert_is_type(max_mem_size, None, BoundInt(1 << 25))
        assert_is_type(log_dir, str, None)
        assert_is_type(log_level, str, None)
        assert_satisfies(log_level, log_level in [None, "TRACE", "DEBUG", "INFO", "WARN", "ERRR", "FATA"])
        assert_is_type(ice_root, None, I(str, os.path.isdir))
        assert_is_type(extra_classpath, None, [str])
        assert_is_type(jvm_custom_args, list, None)
        assert_is_type(bind_to_localhost, bool)
        if jar_path:
            assert_satisfies(jar_path, jar_path.endswith("h2o.jar"))

        if min_mem_size is not None and max_mem_size is not None and min_mem_size > max_mem_size:
            raise H2OValueError("`min_mem_size`=%d is larger than the `max_mem_size`=%d" % (min_mem_size, max_mem_size))
        if port is None: port = "54321+"
        baseport = None
        # TODO: get rid of this port gimmick and have 2 separate parameters.
        if is_type(port, str):
            if port.isdigit():
                port = int(port)
            else:
                if not(port[-1] == "+" and port[:-1].isdigit()):
                    raise H2OValueError("`port` should be of the form 'DDDD+', where D is a digit. Got: %s" % port)
                baseport = int(port[:-1])
                port = 0

        hs = H2OLocalServer()
        hs._verbose = bool(verbose)
        hs._jar_path = hs._find_jar(jar_path)
        hs._extra_classpath = extra_classpath
        hs._ice_root = ice_root
        hs._name = name
        if not ice_root:
            hs._ice_root = tempfile.mkdtemp()
            hs._tempdir = hs._ice_root

        if verbose: print("Attempting to start a local H2O server...")
        hs._launch_server(port=port, baseport=baseport, nthreads=int(nthreads), ea=enable_assertions,
                          mmax=max_mem_size, mmin=min_mem_size, jvm_custom_args=jvm_custom_args,
                          bind_to_localhost=bind_to_localhost, log_dir=log_dir, log_level=log_level)
        if verbose: print("  Server is running at %s://%s:%d" % (hs.scheme, hs.ip, hs.port))
        atexit.register(lambda: hs.shutdown())
        return hs

    def is_running(self):
        """Return True if the server process is still running, False otherwise."""
        return self._process is not None and self._process.poll() is None

    def shutdown(self):
        """
        Shut down the server by trying to terminate/kill its process.

        First we attempt to terminate the server process gracefully (sending SIGTERM signal). However after
        _TIME_TO_KILL seconds if the process didn't shutdown, we forcefully kill it with a SIGKILL signal.
        """
        if not self._process: return
        try:
            kill_time = time.time() + self._TIME_TO_KILL
            while self._process.poll() is None and time.time() < kill_time:
                self._process.terminate()
                time.sleep(0.2)
            if self._process().poll() is None:
                self._process.kill()
                time.sleep(0.2)
            if self._verbose:
                print("Local H2O server %s:%s stopped." % (self.ip, self.port))
        except:
            pass
        self._process = None

    @property
    def scheme(self):
        """Connection scheme, 'http' or 'https'."""
        return self._scheme

    @property
    def ip(self):
        """IP address of the server."""
        return self._ip

    @property
    def port(self):
        """Port that the server is listening to."""
        return self._port

    @property
    def name(self):
        """H2O cluster name."""
        return self._name

    #-------------------------------------------------------------------------------------------------------------------
    # Private
    #-------------------------------------------------------------------------------------------------------------------

    def __init__(self):
        """[Internal] please use H2OLocalServer.start() to launch a new server."""
        self._scheme = None   # "http" or "https"
        self._ip = None
        self._port = None
        self._name = None
        self._process = None
        self._verbose = None
        self._jar_path = None
        self._extra_classpath = None
        self._ice_root = None
        self._stdout = None
        self._stderr = None
        self._tempdir = None

    def _find_jar(self, path0=None):
        """
        Return the location of an h2o.jar executable.

        :param path0: Explicitly given h2o.jar path. If provided, then we will simply check whether the file is there,
            otherwise we will search for an executable in locations returned by ._jar_paths().

        :raises H2OStartupError: if no h2o.jar executable can be found.
        """
        jar_paths = [path0] if path0 else self._jar_paths()
        searched_paths = []
        for jp in jar_paths:
            searched_paths.append(jp)
            if os.path.exists(jp):
                return jp
        raise H2OStartupError("Cannot start local server: h2o.jar not found. Paths searched:\n" +
                              "".join("    %s\n" % s for s in searched_paths))

    @staticmethod
    def _jar_paths():
        """Produce potential paths for an h2o.jar executable."""

        # PUBDEV-3534 hook to use arbitrary h2o.jar
        own_jar = os.getenv("H2O_JAR_PATH", "")
        if own_jar != "":
            if not os.path.isfile(own_jar):
                raise H2OStartupError("Environment variable H2O_JAR_PATH is set to '%d' but file does not exists, unset environment variable or provide valid path to h2o.jar file." % own_jar)
            yield own_jar

        # Check if running from an h2o-3 src folder (or any subfolder), in which case use the freshly-built h2o.jar
        cwd_chunks = os.path.abspath(".").split(os.path.sep)
        for i in range(len(cwd_chunks), 0, -1):
            if cwd_chunks[i - 1] == "h2o-3":
                yield os.path.sep.join(cwd_chunks[:i] + ["build", "h2o.jar"])
        # Then check the backend/bin folder:
        # (the following works assuming this code is located in h2o/backend/server.py file)
        backend_dir = os.path.split(os.path.realpath(__file__))[0]
        yield os.path.join(backend_dir, "bin", "h2o.jar")

        # Then try several old locations where h2o.jar might have been installed
        prefix1 = prefix2 = sys.prefix
        # On Unix-like systems Python typically gets installed into /Library/... or /System/Library/... If one of
        # those paths is sys.prefix, then we also build its counterpart.
        if prefix1.startswith(os.path.sep + "Library"):
            prefix2 = os.path.join("", "System", prefix1)
        elif prefix1.startswith(os.path.sep + "System"):
            prefix2 = prefix1[len(os.path.join("", "System")):]
        yield os.path.join(prefix1, "h2o_jar", "h2o.jar")
        yield os.path.join(os.path.abspath(os.sep), "usr", "local", "h2o_jar", "h2o.jar")
        yield os.path.join(prefix1, "local", "h2o_jar", "h2o.jar")
        yield os.path.join(get_config_var("userbase"), "h2o_jar", "h2o.jar")
        yield os.path.join(prefix2, "h2o_jar", "h2o.jar")

    def _launch_server(self, port, baseport, mmax, mmin, ea, nthreads, jvm_custom_args, bind_to_localhost, log_dir=None, log_level=None):
        """Actually start the h2o.jar executable (helper method for `.start()`)."""
        self._ip = "127.0.0.1"

        # Find Java and check version. (Note that subprocess.check_output returns the output as a bytes object)
        java = self._find_java()
        self._check_java(java, self._verbose)

        if self._verbose:
            print("  Starting server from " + self._jar_path)
            print("  Ice root: " + self._ice_root)

        # Combine jar path with the optional extra classpath
        classpath = [self._jar_path] if self._extra_classpath is None else [self._jar_path] + self._extra_classpath

        # Construct java command to launch the process
        cmd = [java]

        # ...add JVM options
        cmd += ["-ea"] if ea else []
        for (mq, num) in [("-Xms", mmin), ("-Xmx", mmax)]:
            if num is None: continue
            numstr = "%dG" % (num >> 30) if num == (num >> 30) << 30 else \
                     "%dM" % (num >> 20) if num == (num >> 20) << 20 else \
                     str(num)
            cmd += [mq + numstr]
        if jvm_custom_args is not None:
            for arg in jvm_custom_args:
                assert type(arg) is str
                cmd += [arg]

        cmd += ["-cp", os.pathsep.join(classpath), "water.H2OApp"]  # This should be the last JVM option

        # ...add H2O options
        cmd += ["-ip", self._ip]
        if bind_to_localhost:
            cmd += ["-web_ip", self._ip]
        cmd += ["-port", str(port)] if port else []
        cmd += ["-baseport", str(baseport)] if baseport else []
        cmd += ["-ice_root", self._ice_root]
        cmd += ["-nthreads", str(nthreads)] if nthreads > 0 else []

        if log_dir:
            cmd += ["-log_dir", log_dir]

        if log_level:
            cmd += ["-log_level", log_level]

        if not self._name:
            self._name = "H2O_from_python_%s" % self._tmp_file("salt")
        cmd += ["-name", self._name]

        # Warning: do not change to any higher log-level, otherwise we won't be able to know which port the
        # server is listening to.
        cmd += ["-log_level", "INFO"]

        # Create stdout and stderr files
        self._stdout = self._tmp_file("stdout")
        self._stderr = self._tmp_file("stderr")
        cwd = os.path.abspath(os.getcwd())
        out = open(self._stdout, "w")
        err = open(self._stderr, "w")
        if self._verbose:
            print("  JVM stdout: " + out.name)
            print("  JVM stderr: " + err.name)

        # Launch the process
        win32 = sys.platform == "win32"
        flags = getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0) if win32 else 0
        prex = os.setsid if not win32 else None
        try:
            proc = subprocess.Popen(args=cmd, stdout=out, stderr=err, cwd=cwd, creationflags=flags, preexec_fn=prex)
        except OSError as e:
            traceback = getattr(e, "child_traceback", None)
            raise H2OServerError("Unable to start server: %s" % e, traceback)

        # Wait until the server is up-and-running
        giveup_time = time.time() + self._TIME_TO_START
        while True:
            if proc.poll() is not None:
                raise H2OServerError("Server process terminated with error code %d" % proc.returncode)
            ret = self._get_server_info_from_logs()
            if ret:
                self._scheme = ret[0]
                self._ip = ret[1]
                self._port = ret[2]
                self._process = proc
                break
            if time.time() > giveup_time:
                elapsed_time = time.time() - (giveup_time - self._TIME_TO_START)
                raise H2OServerError("Server wasn't able to start in %f seconds." % elapsed_time)
            time.sleep(0.2)

    @staticmethod
    def _check_java(java, verbose):
        jver_bytes = subprocess.check_output([java, "-version"], stderr=subprocess.STDOUT)
        jver = jver_bytes.decode(encoding="utf-8", errors="ignore")
        if verbose:
            print("  Java Version: " + jver.strip().replace("\n", "; "))
        if "GNU libgcj" in jver:
            raise H2OStartupError("Sorry, GNU Java is not supported for H2O.\n"
                                  "Please download the latest 64-bit Java SE JDK from Oracle.")
        if "Client VM" in jver:
            warn("  You have a 32-bit version of Java. H2O works best with 64-bit Java.\n"
                 "  Please download the latest 64-bit Java SE JDK from Oracle.\n")

    @staticmethod
    def _find_java():
        """
        Find location of the java executable (helper for `._launch_server()`).

        This method is not particularly robust, and may require additional tweaking for different platforms...
        :return: Path to the java executable.
        :raises H2OStartupError: if java cannot be found.
        """
        # is java callable directly (doesn't work on windows it seems)?
        java = "java.exe" if sys.platform == "win32" else "java"
        if os.access(java, os.X_OK):
            return java

        # Can Java be found on the PATH?
        for path in os.getenv("PATH").split(os.pathsep):  # not same as os.path.sep!
            full_path = os.path.join(path, java)
            if os.access(full_path, os.X_OK):
                return full_path

        # check if JAVA_HOME is set (for Windows)
        if os.getenv("JAVA_HOME"):
            full_path = os.path.join(os.getenv("JAVA_HOME"), "bin", java)
            if os.path.exists(full_path):
                return full_path

        # check "/Program Files" and "/Program Files (x86)" on Windows
        if sys.platform == "win32":
            # On Windows, backslash on the drive letter is necessary, otherwise os.path.join produces an invalid path
            program_folders = [os.path.join("C:\\", "Program Files", "Java"),
                               os.path.join("C:\\", "Program Files (x86)", "Java"),
                               os.path.join("C:\\", "ProgramData", "Oracle", "Java")]
            for folder in program_folders:
                for dirpath, dirnames, filenames in os.walk(folder):
                    if java in filenames:
                        return os.path.join(dirpath, java)

        # not found...
        raise H2OStartupError("Cannot find Java. Please install the latest JRE from\n"
                              "http://www.oracle.com/technetwork/java/javase/downloads/index.html")

    def _tmp_file(self, kind):
        """
        Generate names for temporary files (helper method for `._launch_server()`).

        :param kind: one of "stdout", "stderr" or "salt". The "salt" kind is used for process name, not for a
            file, so it doesn't contain a path. All generated names are based on the user name of the currently
            logged-in user.
        """
        if sys.platform == "win32":
            username = os.getenv("USERNAME")
        else:
            username = os.getenv("USER")
        if not username:
            username = "unknownUser"
        usr = "".join(ch if ch.isalnum() else "_" for ch in username)

        if kind == "salt":
            return usr + "_" + "".join(choice("0123456789abcdefghijklmnopqrstuvwxyz") for _ in range(6))
        else:
            if not self._tempdir:
                self._tempdir = tempfile.mkdtemp()
            return os.path.join(self._tempdir, "h2o_%s_started_from_python.%s" % (usr, kind[3:]))

    def _get_server_info_from_logs(self):
        """
        Check server's output log, and determine its scheme / IP / port (helper method for `._launch_server()`).

        This method is polled during process startup. It looks at the server output log and checks for a presence of
        a particular string ("INFO: Open H2O Flow in your web browser:") which indicates that the server is
        up-and-running. If the method detects this string, it extracts the server's scheme, ip and port and returns
        them; otherwise it returns None.

        :returns: (scheme, ip, port) tuple if the server has already started, None otherwise.
        """
        searchstr = "INFO: Open H2O Flow in your web browser:"
        with open(self._stdout, "rt") as f:
            for line in f:
                if searchstr in line:
                    url = line[line.index(searchstr) + len(searchstr):].strip().rstrip("/")
                    parts = url.split(":")
                    assert len(parts) == 3 and (parts[0] == "http" or parts[1] == "https") and parts[2].isdigit(), \
                        "Unexpected URL: %s" % url
                    return parts[0], parts[1][2:], int(parts[2])
        return None

    def __enter__(self):
        return self

    def __exit__(self, *args):
        self.shutdown()
        assert len(args) == 3  # Avoid warning about unused args...
        return False  # ensure that any exception will be re-raised

    # Do not stop child process when the object is garbage collected!
    # This ensures that simple code such as
    #     for _ in range(5):
    #         h2o.H2OConnection.start()
    # will launch 5 servers, and they will not be closed down immediately (only when the program exits).


# -*- encoding: utf-8 -*-
from __future__ import absolute_import, division, print_function, unicode_literals
from h2o.utils.compatibility import *  # NOQA

from ..frame import H2OFrame
import h2o
from .model_base import ModelBase
from h2o.utils.typechecks import assert_is_type

class H2OMultinomialModel(ModelBase):

    def _make_model(self):
        return H2OMultinomialModel()

    def confusion_matrix(self, data):
        """
        Returns a confusion matrix based of H2O's default prediction threshold for a dataset.

        :param H2OFrame data: the frame with the prediction results for which the confusion matrix should be extracted.
        """
        assert_is_type(data, H2OFrame)
        j = h2o.api("POST /3/Predictions/models/%s/frames/%s" % (self._id, data.frame_id))
        return j["model_metrics"][0]["cm"]["table"]

    def hit_ratio_table(self, train=False, valid=False, xval=False):
        """
        Retrieve the Hit Ratios.

        If all are False (default), then return the training metric value.
        If more than one options is set to True, then return a dictionary of metrics where the keys are "train",
        "valid", and "xval".

        :param train: If train is True, then return the hit ratio value for the training data.
        :param valid: If valid is True, then return the hit ratio value for the validation data.
        :param xval:  If xval is True, then return the hit ratio value for the cross validation data.
        :return: The hit ratio for this regression model.
        """
        tm = ModelBase._get_metrics(self, train, valid, xval)
        m = {}
        for k, v in zip(list(tm.keys()), list(tm.values())): m[k] = None if v is None else v.hit_ratio_table()
        return list(m.values())[0] if len(m) == 1 else m

    def mean_per_class_error(self, train=False, valid=False, xval=False):
        """
        Retrieve the mean per class error across all classes

        If all are False (default), then return the training metric value.
        If more than one options is set to True, then return a dictionary of metrics where the keys are "train",
        "valid", and "xval".

        :param bool train: If True, return the mean_per_class_error value for the training data.
        :param bool valid: If True, return the mean_per_class_error value for the validation data.
        :param bool xval:  If True, return the mean_per_class_error value for each of the cross-validated splits.

        :returns: The mean_per_class_error values for the specified key(s).
        """
        tm = ModelBase._get_metrics(self, train, valid, xval)
        m = {}
        for k, v in zip(list(tm.keys()), list(tm.values())): m[k] = None if v is None else v.mean_per_class_error()
        return list(m.values())[0] if len(m) == 1 else m

    def plot(self, timestep="AUTO", metric="AUTO", **kwargs):
        """
        Plots training set (and validation set if available) scoring history for an H2OMultinomialModel. The timestep
        and metric arguments are restricted to what is available in its scoring history.

        :param timestep: A unit of measurement for the x-axis.
        :param metric: A unit of measurement for the y-axis.

        :returns: A scoring history plot.
        """

        if self._model_json["algo"] in ("deeplearning", "deepwater", "xgboost", "drf", "gbm"):
            if metric == "AUTO":
                metric = "classification_error"
            elif metric not in ("logloss", "classification_error", "rmse"):
                raise ValueError(
                    "metric for H2OMultinomialModel must be one of: AUTO, logloss, classification_error, rmse")

        self._plot(timestep=timestep, metric=metric, **kwargs)


# -*- encoding: utf-8 -*-
"""
Python 2 / 3 compatibility module.

This module gathers common declarations needed to ensure Python 2 / Python 3 compatibility.
It has to be imported from all other files, so that the common header looks like this:

from __future__ import absolute_import, division, print_function, unicode_literals
from h2o.utils.compatibility import *  # NOQA

------------------------------------------------------------------------------------------------------------------------
1. Strings
    In Py2 `str` is a byte string (`bytes is str == True`), and `unicode` is a unicode string.
    In Py3 `str` is a unicode string, `bytes` is a byte string, and symbol `unicode` is not defined.
    Iterating over a `bytes` string in Py2 produces characters, in Py3 character codes.

    For consistent results, use
        test_type(s, str)  to test whether an argument is a string
        bytes_iterator(s)  to iterate over byte-codes of strins s (which could be bytes or unicode)

2. Integers
    In Py2 there are two integer types: `int` and `long`. The latter has suffix "L" when stringified with repr().
    In Py3 `int` is a single integer type, `long` doesn't exist.

    For consistent results, use
        test_type(x, int)  to test whether an argument is an integer
        str(x)  to convert x to string (don't use repr()!)

3. Iterators
    In Py2 `range`, `filter`, `map` and `zip` produced lists.
    In Py3 same functions return iterators.

    This module backports these functions into Py2, so that they produce iterators both in Py2 and Py3.

4. Dictionaries
    There are multiple inconsistencies between Py2 and Py3 in the behavior of dict.keys(), dict.values(),
    dict.iteritems() / dict.items(), etc.

    This module installs methods `viewkeys`, `viewvalues`, `viewitems` that are preferrable for dict iteration /
    manipulation. These have same semantic as in Py3.

5. Obsolete functions
    Several functions that existed in Py2 were removed in Py3. These are:
        apply, cmp, coerce, execfile, file, long, raw_input, reduce, reload, unicode, xrange, StandardError

    This module replaces all these functions with stubs that produce runtime errors when called. Do not use any of
    these functions if you want your code to be compatible across Py2 / Py3!

6. Miscellaneous
    chr(): In Py2 returns a byte, in Py3 a unicode character.
    input():
    open(): Py3's `open` is equivalent to Py2's `io.open`.
    next(): In Py3 invokes __next__ method of an object, in Py2 doesn't.
    round(): In Py3 returns an integer, in Py2 a float.
    super(): In Py3 a no-argument form is supported, not in Py2.

    All these functions are redefined here to have Py3's behavior on Py2.

------------------------------------------------------------------------------------------------------------------------
:copyright: (c) 2016 H2O.ai
:license:   Apache License Version 2.0 (see LICENSE for details)
"""
from __future__ import absolute_import, division, print_function, unicode_literals
from future.utils import PY2, PY3, with_metaclass

__all__ = ("PY2", "PY3", "with_metaclass",  "bytes_iterator",
           "range", "filter", "map", "zip", "viewitems", "viewkeys", "viewvalues",
           "apply", "cmp", "coerce", "execfile", "file", "long", "raw_input", "reduce", "reload", "unicode", "xrange",
           "StandardError", "chr", "input", "open", "next", "round", "super", "csv_dict_writer", "repr2")

#-----------------------------------------------------------------------------------------------------------------------
# Iterators
#-----------------------------------------------------------------------------------------------------------------------
if True:
    from future.builtins.iterators import (range, filter, map, zip)
    from future.utils import (viewitems, viewkeys, viewvalues)

    def next_method(gen):
        """Return the 'next' method of the given generator."""
        if PY2:
            return gen.next
        else:
            return gen.__next__

#-----------------------------------------------------------------------------------------------------------------------
# Disabled functions
#   -- attempt to use any of these functions will raise an AssertionError now!
#-----------------------------------------------------------------------------------------------------------------------
def _disabled_function(name):
    """Make a function that cannot be called."""
    # noinspection PyUnusedLocal
    def disabled(*args, **kwargs):
        """Disabled function, DO NOT USE."""
        raise NameError("Function %s is not available in Python 3, and was disabled in Python 2 as well." % name)
    return disabled

if PY2:
    _native_unicode = unicode
    _native_long = long

# noinspection PyShadowingBuiltins
apply = _disabled_function("apply")
# noinspection PyShadowingBuiltins
cmp = _disabled_function("cmp")
# noinspection PyShadowingBuiltins
coerce = _disabled_function("coerce")
# noinspection PyShadowingBuiltins
execfile = _disabled_function("execfile")
# noinspection PyShadowingBuiltins
file = _disabled_function("file")
# noinspection PyShadowingBuiltins
long = _disabled_function("long")
# noinspection PyShadowingBuiltins
raw_input = _disabled_function("raw_input")
# noinspection PyShadowingBuiltins
reduce = _disabled_function("reduce")
# noinspection PyShadowingBuiltins
reload = _disabled_function("reload")
# noinspection PyShadowingBuiltins
unicode = _disabled_function("unicode")
# noinspection PyShadowingBuiltins
xrange = _disabled_function("xrange")
# noinspection PyShadowingBuiltins
StandardError = _disabled_function("StandardError")

#-----------------------------------------------------------------------------------------------------------------------
# Miscellaneous
#-----------------------------------------------------------------------------------------------------------------------
if True:
    from future.builtins.misc import (chr, input, open, next, round, super)

def csv_dict_writer(f, fieldnames, **kwargs):
    """Equivalent of csv.DictWriter, but allows `delimiter` to be a unicode string on Py2."""
    import csv
    if "delimiter" in kwargs:
        kwargs["delimiter"] = str(kwargs["delimiter"])
    return csv.DictWriter(f, fieldnames, **kwargs)

def bytes_iterator(s):
    """Given a string, return an iterator over this string's bytes (as ints)."""
    if s is None: return
    if PY2 or PY3 and isinstance(s, str):
        for ch in s:
            yield ord(ch)
    elif PY3 and isinstance(s, bytes):
        for ch in s:
            yield ch
    else:
        raise TypeError("String argument expected, got %s" % type(s))

def repr2(x):
    """Analogous to repr(), but will suppress 'u' prefix when repr-ing a unicode string."""
    s = repr(x)
    if len(s) >= 2 and s[0] == "u" and (s[1] == "'" or s[1] == '"'):
        s = s[1:]
    return s


"""Attempt to generate templates for module reference with Sphinx

XXX - we exclude extension modules

To include extension modules, first identify them as valid in the
``_uri2path`` method, then handle them in the ``_parse_module`` script.

We get functions and classes by parsing the text of .py files.
Alternatively we could import the modules for discovery, and we'd have
to do that for extension modules.  This would involve changing the
``_parse_module`` method to work via import and introspection, and
might involve changing ``discover_modules`` (which determines which
files are modules, and therefore which module URIs will be passed to
``_parse_module``).

NOTE: this is a modified version of a script originally shipped with the
PyMVPA project, which we've adapted for NIPY use.  PyMVPA is an MIT-licensed
project."""

# Stdlib imports
import os
import re

# Functions and classes
class ApiDocWriter(object):
    ''' Class for automatic detection and parsing of API docs
    to Sphinx-parsable reST format'''

    # only separating first two levels
    rst_section_levels = ['*', '=', '-', '~', '^']

    def __init__(self,
                 package_name,
                 rst_extension='.rst',
                 package_skip_patterns=None,
                 module_skip_patterns=None,
                 ):
        ''' Initialize package for parsing

        Parameters
        ----------
        package_name : string
            Name of the top-level package.  *package_name* must be the
            name of an importable package
        rst_extension : string, optional
            Extension for reST files, default '.rst'
        package_skip_patterns : None or sequence of {strings, regexps}
            Sequence of strings giving URIs of packages to be excluded
            Operates on the package path, starting at (including) the
            first dot in the package path, after *package_name* - so,
            if *package_name* is ``sphinx``, then ``sphinx.util`` will
            result in ``.util`` being passed for earching by these
            regexps.  If is None, gives default. Default is:
            ['\.tests$']
        module_skip_patterns : None or sequence
            Sequence of strings giving URIs of modules to be excluded
            Operates on the module name including preceding URI path,
            back to the first dot after *package_name*.  For example
            ``sphinx.util.console`` results in the string to search of
            ``.util.console``
            If is None, gives default. Default is:
            ['\.setup$', '\._']
        '''
        if package_skip_patterns is None:
            package_skip_patterns = ['\\.tests$']
        if module_skip_patterns is None:
            module_skip_patterns = ['\\.setup$', '\\._']
        self.package_name = package_name
        self.rst_extension = rst_extension
        self.package_skip_patterns = package_skip_patterns
        self.module_skip_patterns = module_skip_patterns

    def get_package_name(self):
        return self._package_name

    def set_package_name(self, package_name):
        ''' Set package_name

        >>> docwriter = ApiDocWriter('sphinx')
        >>> import sphinx
        >>> docwriter.root_path == sphinx.__path__[0]
        True
        >>> docwriter.package_name = 'docutils'
        >>> import docutils
        >>> docwriter.root_path == docutils.__path__[0]
        True
        '''
        # It's also possible to imagine caching the module parsing here
        self._package_name = package_name
        self.root_module = __import__(package_name)
        self.root_path = self.root_module.__path__[0]
        self.written_modules = None

    package_name = property(get_package_name, set_package_name, None,
                            'get/set package_name')

    def _get_object_name(self, line):
        ''' Get second token in line
        >>> docwriter = ApiDocWriter('sphinx')
        >>> docwriter._get_object_name("  def func():  ")
        'func'
        >>> docwriter._get_object_name("  class Klass(object):  ")
        'Klass'
        >>> docwriter._get_object_name("  class Klass:  ")
        'Klass'
        '''
        name = line.split()[1].split('(')[0].strip()
        # in case we have classes which are not derived from object
        # ie. old style classes
        return name.rstrip(':')

    def _uri2path(self, uri):
        ''' Convert uri to absolute filepath

        Parameters
        ----------
        uri : string
            URI of python module to return path for

        Returns
        -------
        path : None or string
            Returns None if there is no valid path for this URI
            Otherwise returns absolute file system path for URI

        Examples
        --------
        >>> docwriter = ApiDocWriter('sphinx')
        >>> import sphinx
        >>> modpath = sphinx.__path__[0]
        >>> res = docwriter._uri2path('sphinx.builder')
        >>> res == os.path.join(modpath, 'builder.py')
        True
        >>> res = docwriter._uri2path('sphinx')
        >>> res == os.path.join(modpath, '__init__.py')
        True
        >>> docwriter._uri2path('sphinx.does_not_exist')

        '''
        if uri == self.package_name:
            return os.path.join(self.root_path, '__init__.py')
        path = uri.replace('.', os.path.sep)
        path = path.replace(self.package_name + os.path.sep, '')
        path = os.path.join(self.root_path, path)
        # XXX maybe check for extensions as well?
        if os.path.exists(path + '.py'): # file
            path += '.py'
        elif os.path.exists(os.path.join(path, '__init__.py')):
            path = os.path.join(path, '__init__.py')
        else:
            return None
        return path

    def _path2uri(self, dirpath):
        ''' Convert directory path to uri '''
        relpath = dirpath.replace(self.root_path, self.package_name)
        if relpath.startswith(os.path.sep):
            relpath = relpath[1:]
        return relpath.replace(os.path.sep, '.')

    def _parse_module(self, uri):
        ''' Parse module defined in *uri* '''
        filename = self._uri2path(uri)
        if filename is None:
            # nothing that we could handle here.
            return ([],[])
        f = open(filename, 'rt')
        functions, classes = self._parse_lines(f)
        f.close()
        return functions, classes

    def _parse_lines(self, linesource):
        ''' Parse lines of text for functions and classes '''
        functions = []
        classes = []
        for line in linesource:
            if line.startswith('def ') and line.count('('):
                # exclude private stuff
                name = self._get_object_name(line)
                if not name.startswith('_'):
                    functions.append(name)
            elif line.startswith('class '):
                # exclude private stuff
                name = self._get_object_name(line)
                if not name.startswith('_'):
                    classes.append(name)
            else:
                pass
        functions.sort()
        classes.sort()
        return functions, classes

    def generate_api_doc(self, uri):
        '''Make autodoc documentation template string for a module

        Parameters
        ----------
        uri : string
            python location of module - e.g 'sphinx.builder'

        Returns
        -------
        S : string
            Contents of API doc
        '''
        # get the names of all classes and functions
        functions, classes = self._parse_module(uri)
        if not len(functions) and not len(classes):
            print 'WARNING: Empty -',uri  # dbg
            return ''

        # Make a shorter version of the uri that omits the package name for
        # titles
        uri_short = re.sub(r'^%s\.' % self.package_name,'',uri)

        ad = '.. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n'

        chap_title = uri_short
        ad += (chap_title+'\n'+ self.rst_section_levels[1] * len(chap_title)
               + '\n\n')

        # Set the chapter title to read 'module' for all modules except for the
        # main packages
        if '.' in uri:
            title = 'Module: :mod:`' + uri_short + '`'
        else:
            title = ':mod:`' + uri_short + '`'
        ad += title + '\n' + self.rst_section_levels[2] * len(title)

        if len(classes):
            ad += '\nInheritance diagram for ``%s``:\n\n' % uri
            ad += '.. inheritance-diagram:: %s \n' % uri
            ad += '   :parts: 3\n'

        ad += '\n.. automodule:: ' + uri + '\n'
        ad += '\n.. currentmodule:: ' + uri + '\n'
        multi_class = len(classes) > 1
        multi_fx = len(functions) > 1
        if multi_class:
            ad += '\n' + 'Classes' + '\n' + \
                  self.rst_section_levels[2] * 7 + '\n'
        elif len(classes) and multi_fx:
            ad += '\n' + 'Class' + '\n' + \
                  self.rst_section_levels[2] * 5 + '\n'
        for c in classes:
            ad += '\n:class:`' + c + '`\n' \
                  + self.rst_section_levels[multi_class + 2 ] * \
                  (len(c)+9) + '\n\n'
            ad += '\n.. autoclass:: ' + c + '\n'
            # must NOT exclude from index to keep cross-refs working
            ad += '  :members:\n' \
                  '  :undoc-members:\n' \
                  '  :show-inheritance:\n' \
                  '  :inherited-members:\n' \
                  '\n' \
                  '  .. automethod:: __init__\n'
        if multi_fx:
            ad += '\n' + 'Functions' + '\n' + \
                  self.rst_section_levels[2] * 9 + '\n\n'
        elif len(functions) and multi_class:
            ad += '\n' + 'Function' + '\n' + \
                  self.rst_section_levels[2] * 8 + '\n\n'
        for f in functions:
            # must NOT exclude from index to keep cross-refs working
            ad += '\n.. autofunction:: ' + uri + '.' + f + '\n\n'
        return ad

    def _survives_exclude(self, matchstr, match_type):
        ''' Returns True if *matchstr* does not match patterns

        ``self.package_name`` removed from front of string if present

        Examples
        --------
        >>> dw = ApiDocWriter('sphinx')
        >>> dw._survives_exclude('sphinx.okpkg', 'package')
        True
        >>> dw.package_skip_patterns.append('^\\.badpkg$')
        >>> dw._survives_exclude('sphinx.badpkg', 'package')
        False
        >>> dw._survives_exclude('sphinx.badpkg', 'module')
        True
        >>> dw._survives_exclude('sphinx.badmod', 'module')
        True
        >>> dw.module_skip_patterns.append('^\\.badmod$')
        >>> dw._survives_exclude('sphinx.badmod', 'module')
        False
        '''
        if match_type == 'module':
            patterns = self.module_skip_patterns
        elif match_type == 'package':
            patterns = self.package_skip_patterns
        else:
            raise ValueError('Cannot interpret match type "%s"'
                             % match_type)
        # Match to URI without package name
        L = len(self.package_name)
        if matchstr[:L] == self.package_name:
            matchstr = matchstr[L:]
        for pat in patterns:
            try:
                pat.search
            except AttributeError:
                pat = re.compile(pat)
            if pat.search(matchstr):
                return False
        return True

    def discover_modules(self):
        ''' Return module sequence discovered from ``self.package_name``

        Parameters
        ----------
        None

        Returns
        -------
        mods : sequence
            Sequence of module names within ``self.package_name``

        Examples
        --------
        >>> dw = ApiDocWriter('sphinx')
        >>> mods = dw.discover_modules()
        >>> 'sphinx.util' in mods
        True
        >>> dw.package_skip_patterns.append('\.util$')
        >>> 'sphinx.util' in dw.discover_modules()
        False
        >>>
        '''
        modules = [self.package_name]
        # raw directory parsing
        for dirpath, dirnames, filenames in os.walk(self.root_path):
            # Check directory names for packages
            root_uri = self._path2uri(os.path.join(self.root_path,
                                                   dirpath))
            for dirname in dirnames[:]: # copy list - we modify inplace
                package_uri = '.'.join((root_uri, dirname))
                if (self._uri2path(package_uri) and
                    self._survives_exclude(package_uri, 'package')):
                    modules.append(package_uri)
                else:
                    dirnames.remove(dirname)
            # Check filenames for modules
            for filename in filenames:
                module_name = filename[:-3]
                module_uri = '.'.join((root_uri, module_name))
                if (self._uri2path(module_uri) and
                    self._survives_exclude(module_uri, 'module')):
                    modules.append(module_uri)
        return sorted(modules)

    def write_modules_api(self, modules,outdir):
        # write the list
        written_modules = []
        for m in modules:
            api_str = self.generate_api_doc(m)
            if not api_str:
                continue
            # write out to file
            outfile = os.path.join(outdir,
                                   m + self.rst_extension)
            fileobj = open(outfile, 'wt')
            fileobj.write(api_str)
            fileobj.close()
            written_modules.append(m)
        self.written_modules = written_modules

    def write_api_docs(self, outdir):
        """Generate API reST files.

        Parameters
        ----------
        outdir : string
            Directory name in which to store files
            We create automatic filenames for each module

        Returns
        -------
        None

        Notes
        -----
        Sets self.written_modules to list of written modules
        """
        if not os.path.exists(outdir):
            os.mkdir(outdir)
        # compose list of modules
        modules = self.discover_modules()
        self.write_modules_api(modules,outdir)

    def write_index(self, outdir, froot='gen', relative_to=None):
        """Make a reST API index file from written files

        Parameters
        ----------
        path : string
            Filename to write index to
        outdir : string
            Directory to which to write generated index file
        froot : string, optional
            root (filename without extension) of filename to write to
            Defaults to 'gen'.  We add ``self.rst_extension``.
        relative_to : string
            path to which written filenames are relative.  This
            component of the written file path will be removed from
            outdir, in the generated index.  Default is None, meaning,
            leave path as it is.
        """
        if self.written_modules is None:
            raise ValueError('No modules written')
        # Get full filename path
        path = os.path.join(outdir, froot+self.rst_extension)
        # Path written into index is relative to rootpath
        if relative_to is not None:
            relpath = outdir.replace(relative_to + os.path.sep, '')
        else:
            relpath = outdir
        idx = open(path,'wt')
        w = idx.write
        w('.. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n')
        w('.. toctree::\n\n')
        for f in self.written_modules:
            w('   %s\n' % os.path.join(relpath,f))
        idx.close()


#!/usr/bin/python

import sys
import os
import requests
import urllib
import re
import dateutil.parser

g_user = None
g_pass = None
g_sprint = None
g_csv = False
g_verbose = False
g_burndown = False
g_start_date = None

def get_issue_key(issue):
    return issue[u'key']

def get_issue_assignee(issue):
    return issue[u'fields'][u'assignee']

def get_issue_assignee_name(issue):
    return issue[u'fields'][u'assignee'][u'name']

def get_issue_story_points(issue):
    story_points = issue[u'fields'][u'customfield_10004']
    if story_points is None:
        story_points = 0
    else:
        story_points = float(story_points)
    return story_points

def get_issue_resolution_date(issue):
    d = dateutil.parser.parse(issue[u'fields'][u'resolutiondate'], ignoretz=True)
    return d

class Person:
    def __init__(self, name):
        self.name = name
        self.resolved_list = []
        self.resolved_story_points = 0
        self.unresolved_list = []
        self.unresolved_story_points = 0

    def add(self, issue):
        story_points = get_issue_story_points(issue)
        resolution = issue[u'fields'][u'resolution']
        if resolution is None:
            self.unresolved_story_points += story_points
            self.unresolved_list.append(issue)
        else:
            self.resolved_story_points += story_points
            self.resolved_list.append(issue)

    def get_story_points(self):
        return (self.resolved_story_points + self.unresolved_story_points)

    def get_total_issues_count(self):
        return (len(self.resolved_list) + len(self.unresolved_list))

    def emit_issues_csv(self):
        print (self.name + "," + str(len(self.resolved_list)) + "," + str(len(self.unresolved_list)))

    def emit_story_points_csv(self):
        print (self.name + "," + str(self.resolved_story_points) + "," + str(self.unresolved_story_points))

    def emit_barchart(self):
        print("")
        print("-----" + self.name + "-----")
        Person._print_bar("  resolved", self.resolved_story_points, "R")
        if (g_verbose):
            print("")
            for issue in self.resolved_list:
                story_points = get_issue_story_points(issue)
                summary = issue[u'fields'][u'summary']
                summary = summary.encode('ascii', 'ignore')
                print("{0:14s}{1:11s} ({2:.1f}): {3}".format("", issue[u'key'], story_points, summary))
        if (g_verbose):
            print("")
        Person._print_bar("unresolved", self.unresolved_story_points, "U")
        if (g_verbose):
            print("")
            for issue in self.unresolved_list:
                story_points = get_issue_story_points(issue)
                summary = issue[u'fields'][u'summary']
                summary = summary.encode('ascii', 'ignore')
                print("{0:14s}{1:11s} ({2:.1f}): {3}".format("", issue[u'key'], story_points, summary))

    @staticmethod
    def _print_bar(label, value, char):
        bars_per_day = 4
        sys.stdout.write(label + ":  ")
        num_bars = int(value * bars_per_day)
        i = 0
        while (num_bars > 0):
            if (i % bars_per_day == 0):
                sys.stdout.write(" ")
            sys.stdout.write(char)
            num_bars -= 1
            i += 1
        sys.stdout.write("\n")

class PeopleManager:
    def __init__(self):
        self.people_map = {}

    def add(self, issue):
        assignee = get_issue_assignee(issue)
        if (assignee is None):
            print("ERROR: assignee is none for issue: " + str(issue))
            sys.exit(1)
        assignee_name = issue[u'fields'][u'assignee'][u'name']
        person = self.find(assignee_name)
        person.add(issue)

    def find(self, name):
        if name not in self.people_map:
            person = Person(name)
            self.people_map[name] = person
        person = self.people_map[name]
        return person

    def emit(self):
        if g_burndown:
            total_story_points = 0
            total_issues = 0
            for key in sorted(self.people_map.keys()):
                person = self.people_map[key]
                total_story_points += person.get_story_points()
                total_issues += person.get_total_issues_count()
            issues = self.get_resolved_issues_sorted_by_date()
            if (len(issues) == 0):
                print("No resolved issues")
                return
            after_total_story_points = total_story_points
            after_total_issues = total_issues
            earliest_resolution_date = get_issue_resolution_date(issues[0])
            if (g_start_date is not None):
                start_date = g_start_date
            else:
                start_date = earliest_resolution_date
            if g_csv:
                print("key,assignee,story_points,before_total_story_points,resolution_date,resolution_date_days,"
                      + "after_total_story_points,after_total_issues")
                if (start_date < earliest_resolution_date):
                    tmp = start_date
                else:
                    tmp = earliest_resolution_date
                date_str = tmp.strftime("%Y-%m-%d %H:%M")
                date_days = (tmp - start_date).total_seconds() / 3600.0 / 24
                print(",,,,{},{},{},{}".format(date_str, date_days, total_story_points, total_issues))
            for issue in issues:
                before_total_story_points = after_total_story_points
                before_total_issues = after_total_issues
                key = get_issue_key(issue)
                assignee = get_issue_assignee_name(issue)
                story_points = get_issue_story_points(issue)
                resolution_date = get_issue_resolution_date(issue)
                date_days = (resolution_date - start_date).total_seconds() / 3600.0 / 24
                after_total_story_points = before_total_story_points - story_points
                after_total_issues = before_total_issues - 1
                if g_csv:
                    date_str = resolution_date.strftime("%Y-%m-%d %H:%M")
                    print("{},{},{},{},{},{},{},{}"
                          .format(key, assignee,
                                  story_points, before_total_story_points,
                                  date_str, date_days,
                                  after_total_story_points,
                                  after_total_issues))
                else:
                    date_str = resolution_date.strftime("%Y-%m-%d %H:%M")
                    print("{:11s}  {:10s}  {:.1f}  {:.1f}  {:s}  {:.2f}  {:.1f}  {:.1f}"
                          .format(key, assignee,
                                  story_points, before_total_story_points,
                                  date_str, date_days,
                                  after_total_story_points,
                                  after_total_issues))
            return

        if g_csv:
            print("STORY_POINTS")
            print("name,resolved,unresolved")
            for key in sorted(self.people_map.keys()):
                person = self.people_map[key]
                person.emit_story_points_csv()
            print("")
            print("NUMBER_OF_ISSUES")
            print("name,resolved,unresolved")
            for key in sorted(self.people_map.keys()):
                person = self.people_map[key]
                person.emit_issues_csv()
        else:
            for key in sorted(self.people_map.keys()):
                person = self.people_map[key]
                person.emit_barchart()

    def get_resolved_issues_sorted_by_date(self):
        issues = []
        for person in self.people_map.values():
            for i in person.resolved_list:
                issues.append(i)
        sorted_issues = sorted(issues, key=lambda x: get_issue_resolution_date(x))
        return sorted_issues

def usage():
    print("")
    print("usage:  " + g_script_name + " --user username --pass password --sprint sprintname")
    print("")
    sys.exit(1)

def unknown_arg(s):
    print("")
    print("ERROR: Unknown argument: " + s)
    print("")
    usage()

def parse_config_file():
    global g_user
    global g_pass
    global g_sprint
    global g_start_date
    home = os.path.expanduser("~")
    config_file = os.path.join(home, ".h2o_jira")
    if (os.path.exists(config_file)):
        with open(config_file) as f:
            for line in f:
                match_groups = re.search(r"\s*(\S+)\s*=\s*([\S\s]+)\s*", line.rstrip())
                if (match_groups is not None):
                    key = match_groups.group(1)
                    value = match_groups.group(2)
                    if (key == "user"):
                        g_user = value
                    elif (key == "pass"):
                        g_pass = value
                    elif (key == "sprint"):
                        g_sprint = value
                    elif (key == "startdate"):
                        g_start_date = dateutil.parser.parse(value)

def parse_args(argv):
    global g_user
    global g_pass
    global g_sprint
    global g_csv
    global g_verbose
    global g_burndown
    global g_start_date

    i = 1
    while (i < len(argv)):
        s = argv[i]

        if (s == "--user"):
            i += 1
            if (i > len(argv)):
                usage()
            g_user = argv[i]
        elif (s == "--pass"):
            i += 1
            if (i > len(argv)):
                usage()
            g_pass = argv[i]
        elif (s == "--sprint"):
            i += 1
            if (i > len(argv)):
                usage()
            g_sprint = argv[i]
        elif (s == "--csv"):
            g_csv = True
        elif (s == "-v" or s == "--verbose"):
            g_verbose = True
        elif (s == "--burndown"):
            g_burndown = True
        elif (s == "--startdate"):
            i += 1
            if (i > len(argv)):
                usage()
            g_start_date = dateutil.parser.parse(argv[i])
        elif (s == "-h" or s == "--h" or s == "-help" or s == "--help"):
            usage()
        else:
            unknown_arg(s)

        i += 1

    if (g_user is None):
        print "ERROR: user is not specified"
        usage()

    if (g_pass is None):
        print "ERROR: pass is not specified"
        usage()

    if (g_sprint is None):
        print "ERROR: sprint is not specified"
        usage()

def main(argv):
    """
    Main program.

    @return: none
    """
    global g_script_name

    g_script_name = os.path.basename(argv[0])
    parse_config_file()
    parse_args(argv)

    url = 'https://0xdata.atlassian.net/rest/api/2/search?jql=sprint="' + urllib.quote(g_sprint) + '"&maxResults=1000'
    r = requests.get(url, auth=(g_user, g_pass))
    if (r.status_code != 200):
        print("ERROR: status code is " + str(r.status_code))
        sys.exit(1)
    j = r.json()
    issues = j[u'issues']
    pm = PeopleManager()
    for issue in issues:
        pm.add(issue)

    pm.emit()
    print("")

if __name__ == "__main__":
    main(sys.argv)


# -*- encoding: utf-8 -*-
from __future__ import absolute_import, division, print_function, unicode_literals

from h2o.two_dim_table import H2OTwoDimTable
from h2o.utils.compatibility import *  # NOQA
from h2o.utils.typechecks import assert_is_type

class ConfusionMatrix(object):
    ROUND = 4  # round count_errs / sum

    def __init__(self, cm, domains=None, table_header=None):
        assert_is_type(cm, list)

        if len(cm) == 2: cm = list(zip(*cm))  # transpose if 2x2
        nclass = len(cm)
        class_errs = [0] * nclass
        class_sums = [0] * nclass
        class_err_strings = [0] * nclass
        cell_values = [[0] * (1 + nclass)] * (1 + nclass)
        totals = [sum(c) for c in cm]
        total_errs = 0
        for i in range(nclass):
            class_errs[i] = sum([v[i] for v in cm[:i] + cm[(i + 1):]])
            total_errs += class_errs[i]
            class_sums[i] = sum([v[i] for v in cm])  # row sums
            class_err_strings[i] = \
                " (" + str(class_errs[i]) + "/" + str(class_sums[i]) + ")"
            class_errs[i] = float("nan") if class_sums[i] == 0 else round(class_errs[i] / class_sums[i], self.ROUND)
            # and the cell_values are
            cell_values[i] = [v[i] for v in cm] + [str(class_errs[i])] + [class_err_strings[i]]

        # tally up the totals
        class_errs += [sum(class_errs)]
        totals += [sum(class_sums)]
        class_err_strings += [" (" + str(total_errs) + "/" + str(totals[-1]) + ")"]

        class_errs[-1] = float("nan") if totals[-1] == 0 else round(total_errs / totals[-1], self.ROUND)

        # do the last row of cell_values ... the "totals" row
        cell_values[-1] = totals[0:-1] + [str(class_errs[-1])] + [class_err_strings[-1]]

        if table_header is None: table_header = "Confusion Matrix (Act/Pred)"
        col_header = [""]  # no column label for the "rows" column
        if domains is not None:
            import copy
            row_header = copy.deepcopy(domains)
            col_header += copy.deepcopy(domains)
        else:
            row_header = [str(i) for i in range(nclass)]
            col_header += [str(i) for i in range(nclass)]

        row_header += ["Total"]
        col_header += ["Error", "Rate"]

        for i in range(len(row_header)):
            cell_values[i].insert(0, row_header[i])

        self.table = H2OTwoDimTable(row_header=row_header, col_header=col_header,
                                    table_header=table_header, cell_values=cell_values)

    def show(self):
        """Print the confusion matrix into the console."""
        self.table.show()

    def __repr__(self):
        self.show()
        return ""

    def to_list(self):
        """Convert this confusion matrix into a 2x2 plain list of values."""
        return [[int(self.table.cell_values[0][1]), int(self.table.cell_values[0][2])],
                [int(self.table.cell_values[1][1]), int(self.table.cell_values[1][2])]]

    @staticmethod
    def read_cms(cms=None, domains=None):
        """Read confusion matrices from the list of sources (?)."""
        assert_is_type(cms, [list])
        return [ConfusionMatrix(cm, domains) for cm in cms]


#!/usr/bin/python

import sys
import os
import pickle

import copy
import subprocess

"""
This script is written for a user
1. to add new java messages that we can ignore during a log scraping session;
2. to remove old java messages that are okay to ignore in the past but cannot be ignored anymore.

To see how to call this script correctly, see usage().

To exclude java messages, the user can edit a text file that contains the following:
keyName = general
IgnoredMessage = nfolds: nfolds cannot be larger than the number of rows (406).
KeyName = pyunit_cv_cars_gbm.py
IgnoredMessage = Caught exception: Illegal argument(s) for GBM model: GBM_model_python_1452503348770_2586.  Details: ERRR on field: _nfolds: nfolds must be either 0 or >1.
...
KeyName = pyunit_cv_cars_gbm.py
IgnoredMessage = Stacktrace: [water.exceptions.H2OModelBuilderIllegalArgumentException.makeFromBuilder(H2OModelBuilderIllegalArgumentException.java:19), \
water.api.ModelBuilderHandler.handle(ModelBuilderHandler.java:45), water.api.RequestServer.handle(RequestServer.java:617), \
water.api.RequestServer.serve(RequestServer.java:558), water.JettyHTTPD$H2oDefaultServlet.doGeneric(JettyHTTPD.java:616), \
water.JettyHTTPD$H2oDefaultServlet.doPost(JettyHTTPD.java:564), javax.servlet.http.HttpServlet.service(HttpServlet.java:755), \
javax.servlet.http.HttpServlet.service(HttpServlet.java:848), org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:684)]; \
Values: {"messages":[{"_log_level":1,"_field_name":"_nfolds","_message":"nfolds must be either 0 or >1."},\
{"_log_level":5,"_field_name":"_tweedie_power","_message":"Only for Tweedie Distribution."},{"_log_level":5,"_field_name":"_max_after_balance_size",\
"_message":"Balance classes is false, hide max_after_balance_size"},{"_log_level":5,"_field_name":"_max_after_balance_size","_message":"Only used with balanced classes"},\
{"_log_level":5,"_field_name":"_class_sampling_factors","_message":"Class sampling factors is only applicable if balancing classes."}], "algo":"GBM", \
"parameters":{"_train":{"name":"py_3","type":"Key"},"_valid":null,"_nfolds":-1,"_keep_cross_validation_predictions":false,"_fold_assignment":"AUTO",\
"_distribution":"multinomial","_tweedie_power":1.5,"_ignored_columns":["economy_20mpg","fold_assignments","name","economy"],"_ignore_const_cols":true,\
"_weights_column":null,"_offset_column":null,"_fold_column":null,"_score_each_iteration":false,"_stopping_rounds":0,"_stopping_metric":"AUTO",\
"_stopping_tolerance":0.001,"_response_column":"cylinders","_balance_classes":false,"_max_after_balance_size":5.0,"_class_sampling_factors":null,\
"_max_confusion_matrix_size":20,"_checkpoint":null,"_ntrees":5,"_max_depth":5,"_min_rows":10.0,"_nbins":20,"_nbins_cats":1024,"_r2_stopping":0.999999,\
"_seed":-1,"_nbins_top_level":1024,"_build_tree_one_node":false,"_initial_score_interval":4000,"_score_interval":4000,"_sample_rate":1.0,\
"_col_sample_rate_per_tree":1.0,"_learn_rate":0.1,"_col_sample_rate":1.0}, "error_count":1}

Given the above text file, this script will build a dict structure (g_ok_java_message_dict) that contains the
following key/value pairs:
g_ok_java_message_dict["general"] = ["nfolds: nfolds cannot be larger than the number of rows (406)."]
g_ok_java_message_dict["pyunit_cv_cars_gbm.py"] = ["Caught exception: Illegal argument(s) for GBM model: GBM_model_python_1452503348770_2586.  \
    Details: ERRR on field: _nfolds: nfolds must be either 0 or >1.","Stacktrace: [water.exceptions.H2OModelBuilderIllegalArgumentException.makeFromBuilder(H2OModelBuilderIllegalArgumentException.java:19), \
water.api.ModelBuilderHandler.handle(ModelBuilderHandler.java:45), water.api.RequestServer.handle(RequestServer.java:617), \
water.api.RequestServer.serve(RequestServer.java:558), water.JettyHTTPD$H2oDefaultServlet.doGeneric(JettyHTTPD.java:616), \
water.JettyHTTPD$H2oDefaultServlet.doPost(JettyHTTPD.java:564), javax.servlet.http.HttpServlet.service(HttpServlet.java:755), \
javax.servlet.http.HttpServlet.service(HttpServlet.java:848), org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:684)]; \
Values: {"messages":[{"_log_level":1,"_field_name":"_nfolds","_message":"nfolds must be either 0 or >1."},\
{"_log_level":5,"_field_name":"_tweedie_power","_message":"Only for Tweedie Distribution."},{"_log_level":5,"_field_name":"_max_after_balance_size",\
"_message":"Balance classes is false, hide max_after_balance_size"},{"_log_level":5,"_field_name":"_max_after_balance_size","_message":"Only used with balanced classes"},\
{"_log_level":5,"_field_name":"_class_sampling_factors","_message":"Class sampling factors is only applicable if balancing classes."}], "algo":"GBM", \
"parameters":{"_train":{"name":"py_3","type":"Key"},"_valid":null,"_nfolds":-1,"_keep_cross_validation_predictions":false,"_fold_assignment":"AUTO",\
"_distribution":"multinomial","_tweedie_power":1.5,"_ignored_columns":["economy_20mpg","fold_assignments","name","economy"],"_ignore_const_cols":true,\
"_weights_column":null,"_offset_column":null,"_fold_column":null,"_score_each_iteration":false,"_stopping_rounds":0,"_stopping_metric":"AUTO",\
"_stopping_tolerance":0.001,"_response_column":"cylinders","_balance_classes":false,"_max_after_balance_size":5.0,"_class_sampling_factors":null,\
"_max_confusion_matrix_size":20,"_checkpoint":null,"_ntrees":5,"_max_depth":5,"_min_rows":10.0,"_nbins":20,"_nbins_cats":1024,"_r2_stopping":0.999999,\
"_seed":-1,"_nbins_top_level":1024,"_build_tree_one_node":false,"_initial_score_interval":4000,"_score_interval":4000,"_sample_rate":1.0,\
"_col_sample_rate_per_tree":1.0,"_learn_rate":0.1,"_col_sample_rate":1.0}, "error_count":1"]

The key value "general" implies that the java message stored in g_ok_java_message_dict["general"] will be ignored
for all unit tests.  The java messages stored by the specific unit test name is only ignored for that particular tests.

For each key value in the g_ok_java_message_dict, the values are stored as a list.

"""

# --------------------------------------------------------------------
# Main program
# --------------------------------------------------------------------

g_test_root_dir = os.path.dirname(os.path.realpath(__file__)) # directory where we are running out code from
g_load_java_message_filename = "bad_java_messages_to_exclude.pickle" # default pickle filename that store previous java messages that we wanted to exclude
g_save_java_message_filename = "bad_java_messages_to_exclude.pickle" # pickle filename that we are going to store our added java messages to
g_new_messages_to_exclude = ""  # user file that stores the new java messages to ignore
g_old_messages_to_remove = ""   # user file that stores java messages that are to be removed from the ignore list.
g_dict_changed = False          # True if dictionary has changed and False otherwise
g_java_messages_to_ignore_text_filename = "java_messages_to_ignore.txt"     # store all rules for humans to read
g_print_java_messages = False

# store java bad messages that we can ignore.  The keys are "general",testnames that we
# want to add exclude messages for.  The values will all be a list of java messages that we want to ignore.
g_ok_java_messages = {}

def load_dict():
    """
    Load java messages that can be ignored pickle file into a dict structure g_ok_java_messages.

    :return: none
    """
    global g_load_java_message_filename
    global g_ok_java_messages

    if os.path.isfile(g_load_java_message_filename):
            # only load dict from file if it exists.
        with open(g_load_java_message_filename,'rb') as ofile:
            g_ok_java_messages = pickle.load(ofile)
    else:   # no previous java messages to be excluded are found
        g_ok_java_messages["general"] = []

def add_new_message():
    """
    Add new java messages to ignore from user text file.  It first reads in the new java ignored messages
    from the user text file and generate a dict structure to out of the new java ignored messages.  This
    is achieved by function extract_message_to_dict.  Next, new java messages will be added to the original
    ignored java messages dict g_ok_java_messages.  Again, this is achieved by function update_message_dict.

    :return: none
    """
    global g_new_messages_to_exclude    # filename containing text file from user containing new java ignored messages
    global g_dict_changed               # True if new ignored java messages are added.

    new_message_dict = extract_message_to_dict(g_new_messages_to_exclude)

    if new_message_dict:
        g_dict_changed = True
        update_message_dict(new_message_dict,1) # update g_ok_java_messages with new message_dict, 1 to add, 2 to remove

def remove_old_message():
    """
    Remove java messages from ignored list if users desired it.  It first reads in the java ignored messages
    from user stored in g_old_messages_to_remove and build a dict structure (old_message_dict) out of it.  Next, it removes the
    java messages contained in old_message_dict from g_ok_java_messages.
    :return: none
    """
    global g_old_messages_to_remove
    global g_dict_changed

    # extract old java ignored messages to be removed in old_message_dict
    old_message_dict = extract_message_to_dict(g_old_messages_to_remove)

    if old_message_dict:
        g_dict_changed = True
        update_message_dict(old_message_dict,2) # remove the java messages stored in old_message_dict from g_ok_java_messages

def update_message_dict(message_dict,action):
    """
    Update the g_ok_java_messages dict structure by
    1. add the new java ignored messages stored in message_dict if action == 1
    2. remove the java ignored messages stired in message_dict if action == 2.

    Parameters
    ----------

    message_dict :  Python dict
      key: unit test name or "general"
      value: list of java messages that are to be ignored if they are found when running the test stored as the key.  If
        the key is "general", the list of java messages are to be ignored when running all tests.
    action : int
      if 1: add java ignored messages stored in message_dict to g_ok_java_messages dict;
      if 2: remove java ignored messages stored in message_dict from g_ok_java_messages dict.

    :return: none
    """
    global g_ok_java_messages

    allKeys = g_ok_java_messages.keys()

    for key in message_dict.keys():
        if key in allKeys:  # key already exists, just add to it
            for message in message_dict[key]:

                if action == 1:
                    if message not in g_ok_java_messages[key]:
                        g_ok_java_messages[key].append(message)

                if action == 2:
                    if message in g_ok_java_messages[key]:
                        g_ok_java_messages[key].remove(message)
        else:   # new key here.  Can only add and cannot remove
            if action == 1:
                g_ok_java_messages[key] = message_dict[key]

def extract_message_to_dict(filename):
    """
    Read in a text file that java messages to be ignored and generate a dictionary structure out of
    it with key and value pairs.  The keys are test names and the values are lists of java message
    strings associated with that test name where we are either going to add to the existing java messages
    to ignore or remove them from g_ok_java_messages.

    Parameters
    ----------

    filename :  Str
       filename that contains ignored java messages.  The text file shall contain something like this:
        keyName = general
        Message = nfolds: nfolds cannot be larger than the number of rows (406).
        KeyName = pyunit_cv_cars_gbm.py
        Message = Caught exception: Illegal argument(s) for GBM model: GBM_model_python_1452503348770_2586.  \
            Details: ERRR on field: _nfolds: nfolds must be either 0 or >1.
        ...

    :return:
    message_dict : dict
        contains java message to be ignored with key as unit test name or "general" and values as list of ignored java
        messages.
    """
    message_dict = {}

    if os.path.isfile(filename):
        # open file to read in new exclude messages if it exists
        with open(filename,'r') as wfile:

            key = ""
            val = ""
            startMess = False

            while 1:
                each_line = wfile.readline()

                if not each_line:   # reached EOF
                    if startMess:
                        add_to_dict(val.strip(),key,message_dict)
                    break

                # found a test name or general with values to follow
                if "keyname" in each_line.lower():  # name of test file or the word "general"
                    temp_strings = each_line.strip().split('=')

                    if (len(temp_strings) > 1): # make sure the line is formatted sort of correctly
                        if startMess:   # this is the start of a new key/value pair
                            add_to_dict(val.strip(),key,message_dict)
                            val = ""

                        key = temp_strings[1].strip()
                        startMess = False

                if (len(each_line) > 1) and startMess:
                    val += each_line

                if "ignoredmessage" in each_line.lower():
                    startMess = True    # start of a Java message.
                    temp_mess = each_line.split('=')

                    if (len(temp_mess) > 1):
                        val = temp_mess[1]

    return message_dict

def add_to_dict(val,key,message_dict):
    """
    Add new key, val (ignored java message) to dict message_dict.

    Parameters
    ----------

    val :  Str
       contains ignored java messages.
    key :  Str
        key for the ignored java messages.  It can be "general" or any R or Python unit
        test names
    message_dict :  dict
        stored ignored java message for key ("general" or any R or Python unit test names)

    :return: none
    """
    allKeys = message_dict.keys()
    if (len(val) > 0):    # got a valid message here
        if (key in allKeys) and (val not in message_dict[key]):
            message_dict[key].append(val)   # only include this message if it has not been added before
        else:
            message_dict[key] = [val]

def save_dict():
    """
    Save the ignored java message dict stored in g_ok_java_messages into a pickle file for future use.

    :return: none
    """
    global g_ok_java_messages
    global g_save_java_message_filename
    global g_dict_changed

    if g_dict_changed:
        with open(g_save_java_message_filename,'wb') as ofile:
            pickle.dump(g_ok_java_messages,ofile)

def print_dict():
    """
    Write the java ignored messages in g_ok_java_messages into a text file for humans to read.

    :return: none
    """
    global g_ok_java_messages
    global g_java_messages_to_ignore_text_filename

    allKeys = sorted(g_ok_java_messages.keys())

    with open(g_java_messages_to_ignore_text_filename,'w') as ofile:
        for key in allKeys:

            for mess in g_ok_java_messages[key]:
                ofile.write('KeyName: '+key+'\n')
                ofile.write('IgnoredMessage: '+mess+'\n')

            print('KeyName: ',key)
            print('IgnoredMessage: ',g_ok_java_messages[key])
            print('\n')

def parse_args(argv):
    """
    Parse user inputs and set the corresponing global variables to perform the
    necessary tasks.

    Parameters
    ----------

    argv : string array
        contains flags and input options from users

    :return:
    """
    global g_new_messages_to_exclude
    global g_old_messages_to_remove
    global g_load_java_message_filename
    global g_save_java_message_filename
    global g_print_java_messages

    if len(argv) < 2:   # print out help menu if user did not enter any arguments.
        usage()

    i = 1
    while (i < len(argv)):
        s = argv[i]

        if (s == "--inputfileadd"):         # input text file where new java messages are stored
            i += 1
            if (i > len(argv)):
                usage()
            g_new_messages_to_exclude = argv[i]
        elif (s == "--inputfilerm"):        # input text file containing java messages to be removed from the ignored list
            i += 1
            if (i > len(argv)):
                usage()
            g_old_messages_to_remove = argv[i]
        elif (s == "--loadjavamessage"):    # load previously saved java message pickle file from file other than
            i += 1                          # the default one before performing update
            if i > len(argv):
                usage()
            g_load_java_message_filename = argv[i]
        elif (s == "--savejavamessage"):    # save updated java message in this file instead of default file
            i += 1
            if (i > len(argv)):
                usage()
            g_save_java_message_filename = argv[i]
        elif (s == '--printjavamessage'):   # will print java message out to console and save in a text file
            i += 1
            g_print_java_messages = True
            g_load_java_message_filename = argv[i]
        elif (s == '--help'):               # print help menu and exit
            usage()
        else:
            unknown_arg(s)

        i += 1

def usage():
    """
    Illustrate what the various input flags are and the options should be.

    :return: none
    """
    global g_script_name    # name of the script being run.

    print("")
    print("Usage:  " + g_script_name + " [...options...]")
    print("")
    print("     --help print out this help menu and show all the valid flags and inputs.")
    print("")
    print("    --inputfileadd filename where the new java messages to ignore are stored in.")
    print("")
    print("    --inputfilerm filename where the java messages are removed from the ignored list.")
    print("")
    print("    --loadjavamessage filename pickle file that stores the dict structure containing java messages to include.")
    print("")
    print("    --savejavamessage filename pickle file that saves the final dict structure after update.")
    print("")
    print("    --printjavamessage filename print java ignored java messages stored in pickle file filenam onto console and save into a text file.")
    print("")
    sys.exit(1)

def unknown_arg(s):
    print("")
    print("ERROR: Unknown argument: " + s)
    print("")
    usage()

def main(argv):
    """
    Main program.

    @return: none
    """
    global g_script_name
    global g_test_root_dir
    global g_new_messages_to_exclude
    global g_old_messages_to_remove
    global g_load_java_message_filename
    global g_save_java_message_filename
    global g_print_java_messages
    global g_java_messages_to_ignore_text_filename

    g_script_name = os.path.basename(argv[0])   # get name of script being run.

    # Override any defaults with the user's choices.
    parse_args(argv)

    g_load_java_message_filename = os.path.join(g_test_root_dir,g_load_java_message_filename)
    load_dict() # load previously stored java messages to g_ok_java_messages

    if len(g_new_messages_to_exclude) > 0:
        g_new_messages_to_exclude = os.path.join(g_test_root_dir,g_new_messages_to_exclude)
        add_new_message()   # add new java messages to exclude to dictionary

    if len(g_old_messages_to_remove) > 0:
        g_old_messages_to_remove = os.path.join(g_test_root_dir,g_old_messages_to_remove)
        remove_old_message()    # remove java messages from ignored list if users desired it

    g_save_java_message_filename = os.path.join(g_test_root_dir,g_save_java_message_filename)
    save_dict()                 # save the updated dict g_ok_java_messages to pickle file

    if g_print_java_messages:   # print java ignored messages to console and text file
        g_java_messages_to_ignore_text_filename = os.path.join(g_test_root_dir,g_java_messages_to_ignore_text_filename)
        print_dict()

if __name__ == "__main__":
    main(sys.argv)


#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
This module enables magical transformations in h2o-py.

(more info to follow)
"""
from __future__ import absolute_import, division, print_function
#, unicode_literals

import os
import sys
import tokenize

from future.builtins.misc import open

ROOT_DIR = "../../h2o-py/h2o"

def test(x):
    """identity."""
    return x

def locate_files(root_dir):
    """Find all python files in the given directory and all subfolders."""
    all_files = []
    root_dir = os.path.abspath(root_dir)
    for dir_name, subdirs, files in os.walk(root_dir):
        for f in files:
            if f.endswith(".py"):
                all_files.append(os.path.join(dir_name, f))
    return all_files

@test
def find_magic_in_file(filename):
    """
    Search the file for any magic incantations.

    :param filename: file to search
    :returns: a tuple containing the spell and then maybe some extra words (or None if no magic present)
    """
    with open(filename, "rt", encoding="utf-8") as f:
        for line in f:
            if line.startswith("#"):
                comment = line[1:].strip()
                if comment.startswith("~~~~* ") or comment.startswith("----* ") or comment.startswith("====* "):
                    spell = comment[5:].strip()
                    return tuple(spell.split())
            else:
                break
    return None  # No magic found in file

# Parser
def parse_python_file(filename):
    """Parse file into chunks / objects."""

    with open(filename, "rt", encoding="utf-8") as f:
        tokens = list(tokenize.generate_tokens(f.readline))
        tokens = normalize_tokens(tokens)
        module = ChunkCode(tokens, 0, len(tokens))
        module.parse()
        print(module)

def main():
    """Executed when script is run as-is."""
    # magic_files = {}
    for filename in locate_files(ROOT_DIR):
        print("Processing %s" % filename)
        with open(filename, "rt") as f:
            tokens = list(tokenize.generate_tokens(f.readline))
            text1 = tokenize.untokenize(tokens)
            ntokens = normalize_tokens(tokens)
            text2 = tokenize.untokenize(ntokens)
            assert text1 == text2

        # magic = find_magic_in_file(f)
        # if magic:
        #     magic_files[f] = magic

    # print(magic_files)
    # parse_python_file("pymagic.py")

if __name__ == "__main__":
    main()


#!/usr/bin/env python
# -*- encoding: utf-8 -*-
#
# This file is auto-generated by h2o-3/h2o-bindings/bin/gen_python.py
# Copyright 2016 H2O.ai;  Apache License Version 2.0 (see LICENSE for details)
#
from __future__ import absolute_import, division, print_function, unicode_literals

from h2o.estimators.estimator_base import H2OEstimator
from h2o.exceptions import H2OValueError
from h2o.frame import H2OFrame
from h2o.utils.typechecks import assert_is_type, Enum, numeric

class H2OPrincipalComponentAnalysisEstimator(H2OEstimator):
    """
    Principal Components Analysis

    """

    algo = "pca"

    def __init__(self, **kwargs):
        super(H2OPrincipalComponentAnalysisEstimator, self).__init__()
        self._parms = {}
        names_list = {"model_id", "training_frame", "validation_frame", "ignored_columns", "ignore_const_cols",
                      "score_each_iteration", "transform", "pca_method", "pca_impl", "k", "max_iterations",
                      "use_all_factor_levels", "compute_metrics", "impute_missing", "seed", "max_runtime_secs",
                      "export_checkpoints_dir"}
        if "Lambda" in kwargs: kwargs["lambda_"] = kwargs.pop("Lambda")
        for pname, pvalue in kwargs.items():
            if pname == 'model_id':
                self._id = pvalue
                self._parms["model_id"] = pvalue
            elif pname in names_list:
                # Using setattr(...) will invoke type-checking of the arguments
                setattr(self, pname, pvalue)
            else:
                raise H2OValueError("Unknown parameter %s = %r" % (pname, pvalue))

    @property
    def training_frame(self):
        """
        Id of the training data frame.

        Type: ``H2OFrame``.
        """
        return self._parms.get("training_frame")

    @training_frame.setter
    def training_frame(self, training_frame):
        assert_is_type(training_frame, None, H2OFrame)
        self._parms["training_frame"] = training_frame

    @property
    def validation_frame(self):
        """
        Id of the validation data frame.

        Type: ``H2OFrame``.
        """
        return self._parms.get("validation_frame")

    @validation_frame.setter
    def validation_frame(self, validation_frame):
        assert_is_type(validation_frame, None, H2OFrame)
        self._parms["validation_frame"] = validation_frame

    @property
    def ignored_columns(self):
        """
        Names of columns to ignore for training.

        Type: ``List[str]``.
        """
        return self._parms.get("ignored_columns")

    @ignored_columns.setter
    def ignored_columns(self, ignored_columns):
        assert_is_type(ignored_columns, None, [str])
        self._parms["ignored_columns"] = ignored_columns

    @property
    def ignore_const_cols(self):
        """
        Ignore constant columns.

        Type: ``bool``  (default: ``True``).
        """
        return self._parms.get("ignore_const_cols")

    @ignore_const_cols.setter
    def ignore_const_cols(self, ignore_const_cols):
        assert_is_type(ignore_const_cols, None, bool)
        self._parms["ignore_const_cols"] = ignore_const_cols

    @property
    def score_each_iteration(self):
        """
        Whether to score during each iteration of model training.

        Type: ``bool``  (default: ``False``).
        """
        return self._parms.get("score_each_iteration")

    @score_each_iteration.setter
    def score_each_iteration(self, score_each_iteration):
        assert_is_type(score_each_iteration, None, bool)
        self._parms["score_each_iteration"] = score_each_iteration

    @property
    def transform(self):
        """
        Transformation of training data

        One of: ``"none"``, ``"standardize"``, ``"normalize"``, ``"demean"``, ``"descale"``  (default: ``"none"``).
        """
        return self._parms.get("transform")

    @transform.setter
    def transform(self, transform):
        assert_is_type(transform, None, Enum("none", "standardize", "normalize", "demean", "descale"))
        self._parms["transform"] = transform

    @property
    def pca_method(self):
        """
        Specify the algorithm to use for computing the principal components: GramSVD - uses a distributed computation of
        the Gram matrix, followed by a local SVD; Power - computes the SVD using the power iteration method
        (experimental); Randomized - uses randomized subspace iteration method; GLRM - fits a generalized low-rank model
        with L2 loss function and no regularization and solves for the SVD using local matrix algebra (experimental)

        One of: ``"gram_s_v_d"``, ``"power"``, ``"randomized"``, ``"glrm"``  (default: ``"gram_s_v_d"``).
        """
        return self._parms.get("pca_method")

    @pca_method.setter
    def pca_method(self, pca_method):
        assert_is_type(pca_method, None, Enum("gram_s_v_d", "power", "randomized", "glrm"))
        self._parms["pca_method"] = pca_method

    @property
    def pca_impl(self):
        """
        Specify the implementation to use for computing PCA (via SVD or EVD): MTJ_EVD_DENSEMATRIX - eigenvalue
        decompositions for dense matrix using MTJ; MTJ_EVD_SYMMMATRIX - eigenvalue decompositions for symmetric matrix
        using MTJ; MTJ_SVD_DENSEMATRIX - singular-value decompositions for dense matrix using MTJ; JAMA - eigenvalue
        decompositions for dense matrix using JAMA. References: JAMA - http://math.nist.gov/javanumerics/jama/; MTJ -
        https://github.com/fommil/matrix-toolkits-java/

        One of: ``"mtj_evd_densematrix"``, ``"mtj_evd_symmmatrix"``, ``"mtj_svd_densematrix"``, ``"jama"``.
        """
        return self._parms.get("pca_impl")

    @pca_impl.setter
    def pca_impl(self, pca_impl):
        assert_is_type(pca_impl, None, Enum("mtj_evd_densematrix", "mtj_evd_symmmatrix", "mtj_svd_densematrix", "jama"))
        self._parms["pca_impl"] = pca_impl

    @property
    def k(self):
        """
        Rank of matrix approximation

        Type: ``int``  (default: ``1``).
        """
        return self._parms.get("k")

    @k.setter
    def k(self, k):
        assert_is_type(k, None, int)
        self._parms["k"] = k

    @property
    def max_iterations(self):
        """
        Maximum training iterations

        Type: ``int``  (default: ``1000``).
        """
        return self._parms.get("max_iterations")

    @max_iterations.setter
    def max_iterations(self, max_iterations):
        assert_is_type(max_iterations, None, int)
        self._parms["max_iterations"] = max_iterations

    @property
    def use_all_factor_levels(self):
        """
        Whether first factor level is included in each categorical expansion

        Type: ``bool``  (default: ``False``).
        """
        return self._parms.get("use_all_factor_levels")

    @use_all_factor_levels.setter
    def use_all_factor_levels(self, use_all_factor_levels):
        assert_is_type(use_all_factor_levels, None, bool)
        self._parms["use_all_factor_levels"] = use_all_factor_levels

    @property
    def compute_metrics(self):
        """
        Whether to compute metrics on the training data

        Type: ``bool``  (default: ``True``).
        """
        return self._parms.get("compute_metrics")

    @compute_metrics.setter
    def compute_metrics(self, compute_metrics):
        assert_is_type(compute_metrics, None, bool)
        self._parms["compute_metrics"] = compute_metrics

    @property
    def impute_missing(self):
        """
        Whether to impute missing entries with the column mean

        Type: ``bool``  (default: ``False``).
        """
        return self._parms.get("impute_missing")

    @impute_missing.setter
    def impute_missing(self, impute_missing):
        assert_is_type(impute_missing, None, bool)
        self._parms["impute_missing"] = impute_missing

    @property
    def seed(self):
        """
        RNG seed for initialization

        Type: ``int``  (default: ``-1``).
        """
        return self._parms.get("seed")

    @seed.setter
    def seed(self, seed):
        assert_is_type(seed, None, int)
        self._parms["seed"] = seed

    @property
    def max_runtime_secs(self):
        """
        Maximum allowed runtime in seconds for model training. Use 0 to disable.

        Type: ``float``  (default: ``0``).
        """
        return self._parms.get("max_runtime_secs")

    @max_runtime_secs.setter
    def max_runtime_secs(self, max_runtime_secs):
        assert_is_type(max_runtime_secs, None, numeric)
        self._parms["max_runtime_secs"] = max_runtime_secs

    @property
    def export_checkpoints_dir(self):
        """
        Automatically export generated models to this directory.

        Type: ``str``.
        """
        return self._parms.get("export_checkpoints_dir")

    @export_checkpoints_dir.setter
    def export_checkpoints_dir(self, export_checkpoints_dir):
        assert_is_type(export_checkpoints_dir, None, str)
        self._parms["export_checkpoints_dir"] = export_checkpoints_dir

    def init_for_pipeline(self):
        """
        Returns H2OPCA object which implements fit and transform method to be used in sklearn.Pipeline properly.
        All parameters defined in self.__params, should be input parameters in H2OPCA.__init__ method.

        :returns: H2OPCA object
        """
        import inspect
        from h2o.transforms.decomposition import H2OPCA
        # check which parameters can be passed to H2OPCA init
        var_names = list(dict(inspect.getmembers(H2OPCA.__init__.__code__))['co_varnames'])
        parameters = {k: v for k, v in self._parms.items() if k in var_names}
        return H2OPCA(**parameters)


# -*- encoding: utf-8 -*-
"""
H2O MOJO Pipeline.

:copyright: (c) 2018 H2O.ai
:license:   Apache License Version 2.0 (see LICENSE for details)
"""
from __future__ import absolute_import, division, print_function, unicode_literals

import h2o
from h2o.expr import ExprNode
from h2o.frame import H2OFrame
from h2o.utils.typechecks import assert_is_type

__all__ = ("H2OMojoPipeline", )

class H2OMojoPipeline(object):
    """
    Representation of a MOJO Pipeline. This is currently an experimental feature.
    """

    #-------------------------------------------------------------------------------------------------------------------
    # Construction
    #-------------------------------------------------------------------------------------------------------------------

    def __init__(self, mojo_path=None):
        """
        Create a new H2OMojoPipeline object.

        :param mojo_path path to a MOJO file.
        """
        assert_is_type(mojo_path, str)

        self.pipeline_id = h2o.lazy_import(mojo_path)

    def transform(self, data, allow_timestamps=False):
        """
        Transform H2OFrame using a MOJO Pipeline.

        :param data: Frame to be transformed.
        :param allow_timestamps: Allows datetime columns to be used directly with MOJO pipelines. It is recommended
        to parse your datetime columns as Strings when using pipelines because pipelines can interpret certain datetime
        formats in a different way. If your H2OFrame is parsed from a binary file format (eg. Parquet) instead of CSV
        it is safe to turn this option on and use datetime columns directly.

        :returns: A new H2OFrame.
        """
        assert_is_type(data, H2OFrame)
        assert_is_type(allow_timestamps, bool)
        return H2OFrame._expr(ExprNode("mojo.pipeline.transform", self.pipeline_id[0], data, allow_timestamps))

    # Ask the H2O server whether MOJO Pipelines are enabled (depends on availability of MOJO Runtime)
    @staticmethod
    def available():
        """
        Returns True if a MOJO Pipelines can be used, or False otherwise.
        """
        if "MojoPipeline" not in h2o.cluster().list_core_extensions():
            print("Cannot use MOJO Pipelines - runtime was not found.")
            return False
        else:
            return True