#!/usr/bin/env python
# -*- encoding: utf-8 -*-
from __future__ import unicode_literals
import re
import textwrap
import bindings as bi

class JavaTypeTranslator(bi.TypeTranslator):
    def __init__(self):
        bi.TypeTranslator.__init__(self)
        self.types["string"] = "String"

type_adapter = JavaTypeTranslator()
def translate_type(h2o_type, schema):
    return type_adapter.translate(h2o_type, schema)

def get_java_value(field):
    value = field["value"]
    h2o_type = field["type"]
    java_type = translate_type(h2o_type, field["schema_name"])

    if java_type == "float" and value == "Infinity": return "Float.POSITIVE_INFINITY"
    if java_type == "double" and value == "Infinity": return "Double.POSITIVE_INFINITY"
    if java_type == "long": return str(value) + "L"
    if java_type == "float": return str(value) + "f"
    if java_type == "boolean": return str(value).lower()
    if java_type == "String" and (value == "" or value is None): return '""'
    if java_type == "String": return '"%s"' % value
    if value is None: return "null"
    if h2o_type.startswith("enum"): return field["schema_name"] + "." + value
    if h2o_type.endswith("[][]"): return "null"  # TODO
    if h2o_type.endswith("[]"):
        basetype = field["schema_name"] if field["is_schema"] else h2o_type.partition("[")[0]
        if basetype == "Iced": basetype = "Object"
        return "new %s[]{%s}" % (basetype, str(value)[1:-1])
    if h2o_type.startswith("Map"): return "null"  # TODO: handle Map
    if h2o_type.startswith("Key"): return "null"  # TODO: handle Key
    return value

def translate_name(name):
    """
    Convert names with underscores into camelcase.

    For example:
        "num_rows" => "numRows"
        "very_long_json_name" => "veryLongJsonName"
        "build_GBM_model" => "buildGbmModel"
        "KEY" => "key"
        "middle___underscores" => "middleUnderscores"
        "_exclude_fields" => "_excludeFields" (retain initial/trailing underscores)
        "__http_status__" => "__httpStatus__"

    :param name: name to be converted
    """
    parts = name.split("_")
    i = 0
    while parts[i] == "":
        parts[i] = "_"
        i += 1
    parts[i] = parts[i].lower()
    for j in range(i + 1, len(parts)):
        parts[j] = parts[j].capitalize()
    i = len(parts) - 1
    while parts[i] == "":
        parts[i] = "_"
        i -= 1
    return "".join(parts)

def dedent(ind, text):
    """
    Dedent text to the specific indentation level.

    :param ind: common indentation level for the resulting text (number of spaces to append to every line)
    :param text: text that should be transformed.
    :return: ``text`` with all common indentation removed, and then the specified amount of indentation added.
    """
    text2 = textwrap.dedent(text)
    if ind == 0:
        return text2
    indent_str = " " * ind
    return "\n".join(indent_str + line for line in text2.split("\n"))

#-----------------------------------------------------------------------------------------------------------------------
# Generate Schemas
#-----------------------------------------------------------------------------------------------------------------------

def generate_schema(class_name, schema):
    """
    Generate schema Java class.

    :param class_name: name of the class
    :param schema: information about the class
    """
    superclass = schema["superclass"]
    if superclass == "Schema": superclass = "Object"

    has_map = False
    is_model_builder = False
    has_inherited = False
    for field in schema["fields"]:
        if field["name"] == "__meta": continue
        if field["is_inherited"]:
            has_inherited = True
            continue
        if field["type"].startswith("Map"): has_map = True
        if field["name"] == "can_build": is_model_builder = True

    fields = []
    for field in schema["fields"]:
        if field["name"] == "__meta": continue
        java_type = translate_type(field["type"], field["schema_name"])
        java_value = get_java_value(field)

        # hackery: we flatten the parameters up into the ModelBuilder schema, rather than nesting them in the
        # parameters schema class...
        if False and is_model_builder and field["name"] == "parameters":
            fields.append(("parameters", "null", "ModelParameterSchemaV3[]", field["help"], field["is_inherited"]))
        else:
            fields.append((field["name"], java_value, java_type, field["help"], field["is_inherited"]))

    class_decl = class_name
    if "generics" in schema:
        class_decl += "<" + ", ".join("%s extends %s" % (t, long_type) for t, long_type in schema["generics"]) + ">"
    super_decl = superclass
    if "super_generics" in schema:
        super_decl += "<" + ", ".join(schema["super_generics"]) + ">"

    yield "/*"
    yield " * This file is auto-generated by h2o-3/h2o-bindings/bin/gen_java.py"
    yield " * Copyright 2016 H2O.ai;  Apache License Version 2.0 (see LICENSE for details)"
    yield " */"
    yield "package water.bindings.pojos;"
    yield ""
    yield "import com.google.gson.Gson;"
    yield "import com.google.gson.annotations.*;"
    yield "import java.util.Map;" if has_map else None
    yield ""
    yield ""
    yield "public class %s extends %s {" % (class_decl, super_decl) if super_decl != "Object" else None
    yield "public class %s {" % (class_decl) if super_decl == "Object" else None
    yield ""
    for name, value, ftype, fhelp, inherited in fields:
        if inherited: continue
        ccname = translate_name(name)
        yield "    /**"
        yield bi.wrap(fhelp, indent="     * ")
        yield "     */"
        yield "    @SerializedName(\"%s\")" % name if name != ccname else None
        yield "    public %s %s;" % (ftype, ccname)
        yield ""
    if has_inherited:
        yield ""
        yield "    /*" + ("-" * 114)
        yield "    //" + (" " * 50) + "INHERITED"
        yield "    //" + ("-" * 114)
        yield ""
        for name, value, ftype, fhelp, inherited in fields:
            if not inherited: continue
            yield bi.wrap(fhelp, "    // ")
            yield "    public %s %s;" % (ftype, translate_name(name))
            yield ""
        yield "    */"
        yield ""
    yield "    /**"
    yield "     * Public constructor"
    yield "     */"
    yield "    public %s() {" % class_name
    for name, value, _, _, _ in fields:
        if name == "parameters": continue
        if value == "null": continue
        yield "        %s = %s;" % (translate_name(name), value)
    yield "    }"
    yield ""
    yield "    /**"
    yield "     * Return the contents of this object as a JSON String."
    yield "     */"
    yield "    @Override"
    yield "    public String toString() {"
    yield "        return new Gson().toJson(this);"
    yield "    }"
    yield ""
    yield "}"

#-----------------------------------------------------------------------------------------------------------------------
# Generate Enum classes
#-----------------------------------------------------------------------------------------------------------------------

def generate_enum(name, values):
    # If values contain some reserved Java keywords, switch to all-uppercase naming scheme
    if {"enum", "int", "double", "boolean", "long", "byte", "class", "lambda", "null"} & set(values):
        values = [v.upper() for v in values]
    yield "/*"
    yield " * This file is auto-generated by h2o-3/h2o-bindings/bin/gen_java.py"
    yield " * Copyright 2016 H2O.ai;  Apache License Version 2.0 (see LICENSE for details)"
    yield " */"
    yield "package water.bindings.pojos;"
    yield ""
    yield "public enum " + name + " {"
    for value in values:
        yield "    %s," % value
    yield "}"

#-----------------------------------------------------------------------------------------------------------------------
#  Generate Retrofit proxy classes
#-----------------------------------------------------------------------------------------------------------------------

def generate_proxy(classname, endpoints):
    """
    Generate a Retrofit Proxy class.

    Retrofit interfaces look like this:
        public interface GitHubService {
            @GET("/users/{user}/repos")
            Call<List<Repo>> listRepos(@Path("user") String user);
        }
      :param classname: name of the class
      :param endpoints: list of endpoints served by this class
    """
    # Replace path vars like (?<schemaname>.*) with {schemaname} for Retrofit's annotation
    var_pattern = re.compile(r"\{(\w+)\}")

    helper_class = []
    found_key_array_parameter = False

    yield "/*"
    yield " * This file is auto-generated by h2o-3/h2o-bindings/bin/gen_java.py"
    yield " * Copyright 2016 H2O.ai;  Apache License Version 2.0 (see LICENSE for details)"
    yield " */"
    yield "package water.bindings.proxies.retrofit;"
    yield ""
    yield "import water.bindings.pojos.*;"
    yield "import retrofit2.*;"
    yield "import retrofit2.http.*;"
    yield "import java.util.Map;" if classname == "Grid" or classname == "ModelBuilders" else None
    yield ""
    yield "public interface " + classname + " {"
    yield ""

    for e in endpoints:
        method = e["handler_method"]
        # should we always use e.api_name ?
        if method == "exec":
            method = e["api_name"]

        param_strs = []
        required_param_strs = []
        for field in e["input_params"]:
            fname = field["name"]
            ftype = "Path" if field["is_path_param"] else "Field"
            ptype = translate_type(field["type"], field["schema_name"])
            if ptype.endswith("KeyV3") or ptype == "ColSpecifierV3": ptype = "String"
            if ptype.endswith("KeyV3[]"): ptype = "String[]"
            param_str = "@{ftype}(\"{fname}\") {ptype} {fname}".format(**locals())
            param_strs.append(param_str)
            if field["required"]:
                required_param_strs.append(param_str)
        if len(param_strs) == len(required_param_strs): required_param_strs = None

        yield u"  /** "
        yield bi.wrap(e["summary"], indent="   * ")
        for field in e["input_params"]:
            s = "   *   @param %s " % field["name"]
            yield s + bi.wrap(field["help"], indent="   *" + " " * (len(s) - 4), indent_first=False)
        yield u"   */"
        # Create 2 versions of each call: first with all input parameters present, and then only with required params
        for params in [param_strs, required_param_strs]:
            if params is None: continue
            yield u"  @FormUrlEncoded" if e["http_method"] == "POST" else None
            yield u"  @{method}(\"{path}\")".format(method=e["http_method"], path=e["url_pattern"])
            if len(params) <= 1:
                args = params[0] if params else ""
                yield "  Call<{schema}> {method}({args});".format(schema=e["output_schema"], method=method, args=args)
            else:
                yield "  Call<{schema}> {method}(".format(schema=e["output_schema"], method=method)
                for arg in params:
                    yield "    " + arg + ("" if arg == params[-1] else ",")
                yield "  );"
            yield ""

        # Make special static Helper class for Grid and ModelBuilders.
        if "algo" in e:
            # We make two train_ and validate_ methods.  One (built here) takes the parameters schema, the other
            # (built above) takes each parameter.
            helper_class.append("    /**")
            helper_class.append(bi.wrap(e["summary"], indent="     * "))
            helper_class.append("     */")
            helper_class.append("    public static Call<{oschema}> {method}({outer_class} z, {ischema} p) {{"
                                .format(ischema=e["input_schema"], oschema=e["output_schema"], method=method,
                                        outer_class=classname))
            helper_class.append("      return z.{method}(".format(method=method))
            for field in e["input_params"]:
                ptype = translate_type(field["type"], field["schema_name"])
                pname = translate_name(field["name"])
                if ptype.endswith("KeyV3"):
                    s = "(p.{parm} == null? null : p.{parm}.name)".format(parm=pname)
                elif ptype.endswith("KeyV3[]"):
                    found_key_array_parameter = True
                    s = "(p.{parm} == null? null : keyArrayToStringArray(p.{parm}))".format(parm=pname)
                elif ptype.startswith("ColSpecifier"):
                    s = "(p.{parm} == null? null : p.{parm}.columnName)".format(parm=pname)
                else:
                    s = "p." + pname
                if field != e["input_params"][-1]:
                    s += ","
                helper_class.append("        " + s)
            helper_class.append("      );")
            helper_class.append("    }")
            helper_class.append("")

    if helper_class:
        yield ""
        yield "  @SuppressWarnings(\"unused\")"
        yield "  class Helper {"
        for line in helper_class:
            yield line
        if found_key_array_parameter:
            yield "    /**"
            yield "     * Return an array of Strings for an array of keys."
            yield "     */"
            yield "    public static String[] keyArrayToStringArray(KeyV3[] keys) {"
            yield "      if (keys == null) return null;"
            yield "      String[] ids = new String[keys.length];"
            yield "      int i = 0;"
            yield "      for (KeyV3 key : keys) ids[i++] = key.name;"
            yield "      return ids;"
            yield "    }"
        yield "  }"
        yield ""

    yield "}"

# -----------------------------------------------------------------------------------------------------------------------
#  Generate main Retrofit interface class
# -----------------------------------------------------------------------------------------------------------------------
def generate_main_class(endpoints):
    yield dedent(0, """
        /*
         * This file is auto-generated by h2o-3/h2o-bindings/bin/gen_java.py
         * Copyright 2016 H2O.ai;  Apache License Version 2.0 (see LICENSE for details)
         */
        package water.bindings;

        import water.bindings.pojos.*;
        import water.bindings.proxies.retrofit.*;
        import retrofit2.*;
        import retrofit2.converter.gson.GsonConverterFactory;
        import com.google.gson.*;
        import com.google.gson.reflect.TypeToken;
        import com.google.gson.stream.JsonReader;
        import com.google.gson.stream.JsonWriter;
        import okhttp3.OkHttpClient;
        import java.io.IOException;
        import java.util.concurrent.TimeUnit;
        import java.lang.reflect.Array;
        import java.lang.reflect.Field;
        import java.lang.reflect.InvocationTargetException;
        import java.lang.reflect.Type;

        @SuppressWarnings("unused")
        public class H2oApi {

          public static String DEFAULT_URL = \"http://localhost:54321/\";

          public H2oApi() {
            this(DEFAULT_URL);
          }
          public H2oApi(String url) {
            _url = url;
          }

          public H2oApi setUrl(String url) {
            _url = url;
            retrofit = null;
            return this;
          }

          public H2oApi setTimeout(int t) {
            timeout_s = t;
            retrofit = null;
            return this;
          }

          /**
           * Set time interval for job polling in {@link #waitForJobCompletion(JobKeyV3)}.
           *   @param millis time interval, in milliseconds
           */
          public H2oApi setJobPollInterval(int millis) {
            pollInterval_ms = millis;
            return this;
          }

          /**
           * Continuously poll server for the status of the given job, until it completes.
           *   @param jobKey job to query
           *   @return the finished job
           */
          public JobV3 waitForJobCompletion(JobKeyV3 jobKey) {
            return waitForJobCompletion(keyToString(jobKey));
          }
          public JobV3 waitForJobCompletion(String jobId) {
            Jobs jobService = getService(Jobs.class);
            Response<JobsV3> jobsResponse = null;
            int retries = 3;
            JobsV3 jobs = null;
            do {
              try {
                Thread.sleep(pollInterval_ms);
                jobsResponse = jobService.fetch(jobId).execute();
              } catch (IOException e) {
                System.err.println(\"Caught exception: \" + e);
              } catch (InterruptedException e) { /* pass */ }
              if (jobsResponse == null || !jobsResponse.isSuccessful())
                if (retries-- > 0)
                  continue;
                else
                  throw new RuntimeException(\"/3/Jobs/\" + jobId + \" failed 3 times.\");
              jobs = jobsResponse.body();
              if (jobs.jobs == null || jobs.jobs.length != 1)
                throw new RuntimeException(\"Failed to find Job: \" + jobId);
            } while (jobs != null && jobs.jobs[0].status.equals(\"RUNNING\"));
            return jobs == null? null : jobs.jobs[0];
          }

        """)

    for route in endpoints:
        apiname = route["api_name"]
        class_name = route["class_name"]
        outtype = route["output_schema"]
        input_fields = route["input_params"]
        required_fields = [field for field in input_fields if field["required"]]
        input_fields_wo_excluded = [field for field in input_fields if field["name"] != "_exclude_fields"]

        yield "  /**"
        yield bi.wrap(route["summary"], indent="   * ")
        yield "   */"
        # Make several versions of each API call:
        #  (1) Only the required parameters
        #  (2) All parameters except the _excluded_fields
        #  (3) All parameters
        li = len(input_fields)
        le = len(input_fields_wo_excluded)
        lr = len(required_fields)
        assert lr <= 3, "Too many required fields in method " + apiname
        if lr == li:
            # The set of required fields is the same as the set of input fields. No need for (2) and (3).
            input_fields = None
            input_fields_wo_excluded = None
        elif le == li or le == lr or li >= 4:
            # If set (2) coincides with either (1) or (3), then we will not generate code for it.
            # Additionally, if there are too many input params so that we will put them into a container class,
            # then there will be no need for separate case (2) either.
            input_fields_wo_excluded = None

        # Temporary hack, since Parse requires all the fields from ParseSetup: don't generate the required-fields-only
        # method
        if 'parse' == apiname:
            required_fields = None

        for fields in [required_fields, input_fields_wo_excluded, input_fields]:
            if fields is None: continue
            use_schema_param = (len(fields) >= 4)
            value_field_strs = []
            typed_field_strs = []
            for field in fields:
                ftype = translate_type(field["type"], field["schema_name"])
                fname = translate_name(field["name"])
                typed_field_strs.append("%s %s" % (ftype, fname))
                if use_schema_param: fname = "params." + fname
                if ftype.endswith("KeyV3"):
                    s = "keyToString(%s)" % fname
                elif ftype.endswith("KeyV3[]"):
                    s = "keyArrayToStringArray(%s)" % fname
                elif ftype.startswith("ColSpecifier"):
                    s = "colToString(%s)" % fname
                else:
                    s = fname
                value_field_strs.append(s)

            if use_schema_param:
                args = route["input_schema"] + " params"
                values = "\n      " + ",\n      ".join(value_field_strs) + "\n    "
            else:
                args = ", ".join(typed_field_strs)
                values = ", ".join(value_field_strs)
                if fields == input_fields_wo_excluded:
                    values += ", \"\""

            method = apiname if route["handler_method"] == "exec" else route["handler_method"]
            yield "  public {type} {method}({args}) throws IOException {{".\
                  format(type=outtype, method=apiname, args=args)
            yield "    {clazz} s = getService({clazz}.class);".format(clazz=class_name)
            yield "    return s.{method}({values}).execute().body();".format(method=method, values=values)
            yield "  }"
        yield ""

    yield dedent(2, """

          //--------- PRIVATE --------------------------------------------------------------------------------------------------

          private Retrofit retrofit;
          private String _url = DEFAULT_URL;
          private int timeout_s = 60;
          private int pollInterval_ms = 1000;

          private void initializeRetrofit() {
            Gson gson = new GsonBuilder()
              .registerTypeAdapterFactory(new ModelV3TypeAdapter())
              .registerTypeAdapter(KeyV3.class, new KeySerializer())
              .registerTypeAdapter(ColSpecifierV3.class, new ColSerializer())
              .registerTypeAdapter(ModelBuilderSchema.class, new ModelDeserializer())
              .registerTypeAdapter(ModelSchemaBaseV3.class, new ModelSchemaDeserializer())
              .registerTypeAdapter(ModelOutputSchemaV3.class, new ModelOutputDeserializer())
              .registerTypeAdapter(ModelParametersSchemaV3.class, new ModelParametersDeserializer())
              .create();

            OkHttpClient client = new OkHttpClient.Builder()
              .connectTimeout(timeout_s, TimeUnit.SECONDS)
              .writeTimeout(timeout_s, TimeUnit.SECONDS)
              .readTimeout(timeout_s, TimeUnit.SECONDS)
              .build();

            this.retrofit = new Retrofit.Builder()
              .client(client)
              .baseUrl(_url)
              .addConverterFactory(GsonConverterFactory.create(gson))
              .build();
          }

          private Retrofit getRetrofit() {
            if (retrofit == null) initializeRetrofit();
            return retrofit;
          }

          private <T> T getService(Class<T> clazz) {
            return getRetrofit().create(clazz);
          }

          /**
           * Keys get sent as Strings and returned as objects also containing the type and URL,
           * so they need a custom GSON serializer.
           */
          private static class KeySerializer implements JsonSerializer<KeyV3>, JsonDeserializer<KeyV3> {
            @Override
            public JsonElement serialize(KeyV3 key, Type typeOfKey, JsonSerializationContext context) {
              return new JsonPrimitive(key.name);
            }
            @Override
            public KeyV3 deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) {
              if (json.isJsonNull()) return null;
              JsonObject jobj = json.getAsJsonObject();
              String type = jobj.get(\"type\").getAsString();
              switch (type) {
                // TODO: dynamically generate all possible cases
                case \"Key<Model>\": return context.deserialize(jobj, ModelKeyV3.class);
                case \"Key<Job>\":   return context.deserialize(jobj, JobKeyV3.class);
                case \"Key<Grid>\":  return context.deserialize(jobj, GridKeyV3.class);
                case \"Key<Frame>\": return context.deserialize(jobj, FrameKeyV3.class);
                default: throw new JsonParseException(\"Unable to deserialize key of type \" + type);
              }
            }
          }

          private static class ColSerializer implements JsonSerializer<ColSpecifierV3> {
            @Override
            public JsonElement serialize(ColSpecifierV3 col, Type typeOfCol, JsonSerializationContext context) {
              return new JsonPrimitive(col.columnName);
            }
          }

        """)

    def inj(middle):
        return lambda schema, algo: schema[:len(algo)] + middle + schema[len(algo):]

    for clz, base, target_clz_factory in [
            ("ModelDeserializer", "ModelBuilderSchema", lambda schema, algo: schema),
            ("ModelSchemaDeserializer", "ModelSchemaBaseV3", inj("Model")),
            ("ModelOutputDeserializer", "ModelOutputSchemaV3", inj("ModelOutput")),
            ("ModelParametersDeserializer", "ModelParametersSchemaV3", inj("Parameters"))]:
        yield "  /**"
        yield "   * Factory method for parsing a %s json object into an instance of the model-specific subclass." % base
        yield "   */"
        yield "  private static class %s implements JsonDeserializer<%s> {" % (clz, base)
        yield "    @Override"
        yield "    public %s deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context)" % base
        yield "      throws JsonParseException {"
        yield "      if (json.isJsonNull()) return null;"
        yield "      if (json.isJsonObject()) {"
        yield "        JsonObject jobj = json.getAsJsonObject();"
        yield "        if (jobj.has(\"algo\")) {"
        yield "          String algo = jobj.get(\"algo\").getAsJsonPrimitive().getAsString().toLowerCase();"
        yield "          switch (algo) {"
        for route in endpoints:
            if route["class_name"] == "ModelBuilders" and route["api_name"].startswith("train"):
                algo = route["algo"]
                oschema = route["output_schema"]
                assert oschema.lower()[:len(algo)] == algo, "Wrong output schema for algo %s: %s" % (algo, oschema)
                model = target_clz_factory(oschema, algo)
                yield "            case \"{algo}\": return context.deserialize(json, {model}.class);".format(**locals())
        yield "            default:"
        yield "              throw new JsonParseException(\"Unable to deserialize model of type \" + algo);"
        yield "          }"
        yield "        }"
        yield "      }"
        yield "      throw new JsonParseException(\"Invalid %s element \" + json.toString());" % base
        yield "    }"
        yield "  }"
        yield ""

    yield dedent(2, """
          private static class ModelV3TypeAdapter implements TypeAdapterFactory {
            @Override
            public <T> TypeAdapter<T> create(Gson gson, TypeToken<T> type) {
              final Class<? super T> rawType = type.getRawType();
              if (!ModelBuilderSchema.class.isAssignableFrom(rawType) &&
                  !ModelSchemaBaseV3.class.isAssignableFrom(rawType)) return null;
              final TypeAdapter<T> delegate = gson.getDelegateAdapter(this, type);
              return new TypeAdapter<T>() {
                @Override
                public void write(JsonWriter out, T value) throws IOException {
                  delegate.write(out, value);
                }
                @Override
                public T read(JsonReader in) throws IOException {
                  JsonObject jobj = new JsonParser().parse(in).getAsJsonObject();
                  if (jobj.has(\"parameters\") && jobj.get(\"parameters\").isJsonArray()) {
                    JsonArray jarr = jobj.get(\"parameters\").getAsJsonArray();
                    JsonObject paramsNew = new JsonObject();
                    for (JsonElement item : jarr) {
                      JsonObject itemObj = item.getAsJsonObject();
                      paramsNew.add(itemObj.get(\"name\").getAsString(), itemObj.get(\"actual_value\"));
                    }
                    jobj.add(\"parameters\", paramsNew);
                  }
                  // noinspection unchecked
                  return (T) new Gson().fromJson(jobj, rawType);
                }
              };
            }
          }

          /**
           * Return an array of Strings for an array of keys.
           */
          public static String[] keyArrayToStringArray(KeyV3[] keys) {
            if (keys == null) return null;
            String[] ids = new String[keys.length];
            int i = 0;
            for (KeyV3 key : keys) ids[i++] = key.name;
            return ids;
          }

          /**
           * Return an array of keys from an array of Strings.
           * @param ids array of string ids to convert to KeyV3's
           * @param clz class of key objects to create. Since we have JobKeyV3, FrameKeyV3, ModelKeyV3, etc -- this
           *            method needs to know which of these keys you want to create
           */
          public static <T extends KeyV3> T[] stringArrayToKeyArray(String[] ids, Class<T> clz) {
            if (ids == null) return null;
            // noinspection unchecked
            T[] keys = (T[]) Array.newInstance(clz, ids.length);
            String keyType = clz.getSimpleName();
            if (keyType.endsWith(\"KeyV3\")) keyType = keyType.substring(0, keyType.length()-5);
            try {
              int i = 0;
              for (String id: ids) {
                keys[i] = clz.getConstructor().newInstance();
                keys[i].name = id;
                keys[i].type = keyType;
                i++;
              }
            }
            catch (Exception e) {
              e.printStackTrace();
            }
            return keys;
          }

          /**
           *
           */
          public static String keyToString(KeyV3 key) {
            return key == null? null : key.name;
          }

          /**
           *
           */
          public static FrameKeyV3 stringToFrameKey(String key) {
            if (key == null) return null;
            FrameKeyV3 k = new FrameKeyV3();
            k.name = key;
            return k;
          }

          /**
           *
           */
          private static String colToString(ColSpecifierV3 col) {
            return col == null? null : col.columnName;
          }

          /**
           *
           */
          public static String stringToCol(String col) {
            if (col == null) return null;
            ColSpecifierV3 c = new ColSpecifierV3();
            c.columnName = col;
            return col;
          }

          public static void copyFields(Object to, Object from) {
            Field[] fromFields = from.getClass().getDeclaredFields();
            Field[] toFields   = to.getClass().getDeclaredFields();

            for (Field fromField : fromFields){
              Field toField;
              try {
                toField = to.getClass().getDeclaredField(fromField.getName());
                fromField.setAccessible(true);
                toField.setAccessible(true);
                toField.set(to, fromField.get(from));
              }
              catch (Exception ignored) {
                // NoSuchField is the normal case
              }
            }
          }
        }
        """)

# -----------------------------------------------------------------------------------------------------------------------
# MAIN:
# -----------------------------------------------------------------------------------------------------------------------
def main():
    bi.init("Java", "java")
    schemas_map = bi.schemas_map(add_generics=True)

    for name, schema in schemas_map.items():
        bi.vprint("Generating schema: " + name)
        bi.write_to_file("water/bindings/pojos/%s.java" % name, generate_schema(name, schema))

    for name, values in bi.enums().items():
        bi.vprint("Generating enum: " + name)
        bi.write_to_file("water/bindings/pojos/%s.java" % name, generate_enum(name, sorted(values)))

    for name, endpoints in bi.endpoint_groups().items():
        bi.vprint("Generating proxy: " + name)
        bi.write_to_file("water/bindings/proxies/retrofit/%s.java" % name, generate_proxy(name, endpoints))

    bi.vprint("Generating H2oApi.java")
    bi.write_to_file("water/bindings/H2oApi.java", generate_main_class(bi.endpoints()))

    type_adapter.vprint_translation_map()

if __name__ == "__main__":
    main()


#!/usr/bin/python

import sys
import os
import re
from random import random
from random import randrange
from random import uniform

class RealColumn:
    """
    One real number column in a dataset.  May or may not be integer.
    """

    def __init__(self, col_num, col_name, min_val, max_val, na_fraction, is_integer):
        self.col_num = col_num
        self.col_name = col_name
        self.min_val = min_val
        self.max_val = max_val
        self.na_fraction = na_fraction
        self.is_integer = is_integer

    def emit_header(self):
        sys.stdout.write(self.col_name)

    def emit(self):
        if (self.na_fraction == 0):
            is_na = False
        else:
            na_prob = random()
            is_na = na_prob < self.na_fraction

        if (is_na):
            sys.stdout.write("NA")
            return

        if (self.min_val == self.max_val):
            val = self.min_val
            if (self.is_integer):
                val = int(val)
        else:
            if (self.is_integer):
                val = randrange(self.min_val, self.max_val + 1)
            else:
                val = uniform(self.min_val, self.max_val)

        sys.stdout.write(str(val))

class CategoricalColumn:
    """
    One categorical column in a dataset.
    """

    def __init__(self, col_num, col_name, num_levels, na_fraction):
        self.col_num = col_num
        self.col_name = col_name
        self.num_levels = num_levels
        self.na_fraction = na_fraction

    def emit_header(self):
        sys.stdout.write(self.col_name)

    def emit(self):
        if (self.na_fraction == 0):
            is_na = False
        else:
            na_prob = random()
            is_na = na_prob < self.na_fraction

        if (is_na):
            sys.stdout.write("NA")
            return

        if (self.num_levels == 1):
            sys.stdout.write("level0")
            return

        val = randrange(self.num_levels)
        sys.stdout.write("level" + str(val))

class Dataset:
    """
    Object representing the anonymized parsed dataset.
    """

    def __init__(self, parse_log_path):
        """
        Constructor.

        @param parse_log_path: Filesystem path to the parse log output we will use to generate the dataset.
        """
        self.parse_log_path = parse_log_path
        self.dataset_name = None
        self.num_rows = None
        self.columns = []

    def parse(self):
        """
        Parse file specified by constructor.
        """
        f = open(self.parse_log_path, "r")
        self.parse2(f)
        f.close()

    def parse2(self, f):
        """
        Parse file specified by constructor.
        """
        line_num = 0
        s = f.readline()
        while (len(s) > 0):
            line_num += 1

            # Check for beginning of parsed data set.
            match_groups = re.search(r"Parse result for (.*) .(\d+) rows.:", s)
            if (match_groups is not None):
                dataset_name = match_groups.group(1)
                if (self.dataset_name is not None):
                    print("ERROR: Too many datasets found on file {} line {}".format(self.parse_log_path, line_num))
                    sys.exit(1)
                self.dataset_name = dataset_name
                num_rows = int(match_groups.group(2))
                self.num_rows = num_rows
                s = f.readline()
                continue

            # Check for numeric column.
            match_groups = re.search(r"INFO WATER:" +
                                     r"\s*C(\d+):" +
                                     r"\s*numeric" +
                                     r"\s*min\((\S*)\)" +
                                     r"\s*max\((\S*).\)" +
                                     r"\s*(na\((\S+)\))?" +
                                     r"\s*(constant)?",
                                     s)
            if (match_groups is not None):
                col_num = int(match_groups.group(1))
                min_val = float(match_groups.group(2))
                max_val = float(match_groups.group(3))
                # four = match_groups.group(4)
                na_count = match_groups.group(5)
                if (na_count is None):
                    na_count = 0
                else:
                    na_count = int(na_count)
                constant_str = match_groups.group(6)
                is_constant = constant_str is not None
                if (is_constant):
                    if (min_val != max_val):
                        print("ERROR: is_constant mismatch on file {} line {}".format(self.parse_log_path, line_num))
                        sys.exit(1)

                na_fraction = float(na_count) / float(self.num_rows)

                is_min_integer = float(int(min_val)) == float(min_val)
                is_max_integer = float(int(min_val)) == float(min_val)
                is_integer = is_min_integer and is_max_integer

                c = RealColumn(col_num, "C" + str(col_num), min_val, max_val, na_fraction, is_integer)
                self.add_col(c)

                s = f.readline()
                continue

            # Check for categorical column.
            match_groups = re.search(r"INFO WATER:" +
                                     r"\s*C(\d+):" +
                                     r"\s*categorical" +
                                     r"\s*min\((\S*)\)" +
                                     r"\s*max\((\S*).\)" +
                                     r"\s*(na\((\S+)\))?" +
                                     r"\s*(constant)?" +
                                     r"\s*cardinality\((\d+)\)",
                                     s)
            if (match_groups is not None):
                col_num = int(match_groups.group(1))
                min_val = float(match_groups.group(2))
                max_val = float(match_groups.group(3))
                # four = match_groups.group(4)
                na_count = match_groups.group(5)
                if (na_count is None):
                    na_count = 0
                else:
                    na_count = int(na_count)
                constant_str = match_groups.group(6)
                is_constant = constant_str is not None
                if (is_constant):
                    if (min_val != max_val):
                        print("ERROR: is_constant mismatch on file {} line {}".format(self.parse_log_path, line_num))
                        sys.exit(1)
                num_levels = int(match_groups.group(7))
                if (is_constant):
                    if (num_levels != 1):
                        print("ERROR: num_levels mismatch on file {} line {}".format(self.parse_log_path, line_num))
                        sys.exit(1)
                na_fraction = float(na_count) / float(self.num_rows)

                c = CategoricalColumn(col_num, "C" + str(col_num), num_levels, na_fraction)
                self.add_col(c)

                s = f.readline()
                continue

            print("ERROR: Unrecognized regexp pattern on file {} line {}".format(self.parse_log_path, line_num))
            sys.exit(1)

    def add_col(self, c):
        self.columns.append(c)

    def emit_header(self):
        columns = self.columns
        first = True
        for c in columns:
            if (not first):
                sys.stdout.write(",")
            c.emit_header()
            first = False
        sys.stdout.write("\n")
        sys.stdout.flush()

    def emit_one_row(self):
        columns = self.columns
        first = True
        for c in columns:
            if (not first):
                sys.stdout.write(",")
            c.emit()
            first = False
        sys.stdout.write("\n")
        sys.stdout.flush()

# --------------------------------------------------------------------
# Main program
#--------------------------------------------------------------------

# Global variables that can be set by the user.
g_script_name = None
g_parse_log_path = None
g_num_rows = 1

def usage():
    print("")
    print("This program takes the log output from H2O's parse and generates")
    print("a random dataset with similar characteristics.")
    print("")
    print("Usage:  " + g_script_name +
          " [--rows num_rows_to_generate]" +
          " -f parse_log_output_file")
    print("")
    sys.exit(1)

def unknown_arg(s):
    print("")
    print("ERROR: Unknown argument: " + s)
    print("")
    usage()

def bad_arg(s):
    print("")
    print("ERROR: Illegal use of (otherwise valid) argument: " + s)
    print("")
    usage()

def parse_args(argv):
    global g_parse_log_path
    global g_num_rows

    i = 1
    while (i < len(argv)):
        s = argv[i]

        if (s == "-f"):
            i += 1
            if (i > len(argv)):
                usage()
            g_parse_log_path = argv[i]
        elif (s == "--rows"):
            i += 1
            if (i > len(argv)):
                usage()
            g_num_rows = int(argv[i])
        elif (s == "-h" or s == "--h" or s == "-help" or s == "--help"):
            usage()
        else:
            unknown_arg(s)

        i += 1

def main(argv):
    """
    Main program.

    @return: none
    """
    global g_script_name
    global g_parse_log_path

    g_script_name = os.path.basename(argv[0])

    parse_args(argv)

    if (g_parse_log_path is None):
        print("")
        print("ERROR: -f not specified")
        usage()

    d = Dataset(g_parse_log_path)
    d.parse()

    d.emit_header()
    for i in range(0, g_num_rows):
        d.emit_one_row()

if __name__ == "__main__":
    main(sys.argv)


# -*- encoding: utf-8 -*-
from __future__ import absolute_import, division, print_function, unicode_literals
from h2o.utils.compatibility import *  # NOQA

import h2o
from .model_base import ModelBase

class H2OAutoEncoderModel(ModelBase):

    def anomaly(self, test_data, per_feature=False):
        """
        Obtain the reconstruction error for the input test_data.

        :param H2OFrame test_data: The dataset upon which the reconstruction error is computed.
        :param bool per_feature: Whether to return the square reconstruction error per feature.
            Otherwise, return the mean square error.

        :returns: the reconstruction error.
        """
        if test_data is None or test_data.nrow == 0: raise ValueError("Must specify test data")
        j = h2o.api("POST /3/Predictions/models/%s/frames/%s" % (self.model_id, test_data.frame_id),
                    data={"reconstruction_error": True, "reconstruction_error_per_feature": per_feature})
        return h2o.get_frame(j["model_metrics"][0]["predictions"]["frame_id"]["name"])


#!/usr/bin/python
import sys
import os
import json
import subprocess
import time
import numpy as np

"""
This script is written to extract the the time (ms) taken to perform the various operations in GLRM
model iteration.  You should not use this for anything else.  Provide the absolute path to the data
file if it is not in the same directory as this script.
"""

# --------------------------------------------------------------------
# Main program
# --------------------------------------------------------------------

g_test_root_dir = os.path.dirname(os.path.realpath(__file__))   # directory where we are running out code from

g_initialXY = "Time taken (ms) to initializeXY with"   # text of interest
g_reguarlize_Y = "Time taken (ms) to calculate regularize_y"
g_regularize_X_objective = "Time taken (ms) to calculate regularize_x and calculate"
g_updateX = "Time taken (ms) to updateX"
g_updateY = "Time taken (ms) to updateY"
g_objective = "Time taken (ms) to calculate new objective function value"
g_stepsize = "Time taken (ms) to set the step size"
g_history = "Time taken (ms) to history of run"

def extractRunInto(javaLogText):
    """
    This function will extract the various operation time for GLRM model building iterations.

    :param javaLogText:
    :return:
    """
    global g_initialXY
    global g_reguarlize_Y
    global g_regularize_X_objective
    global g_updateX
    global g_updateY
    global g_objective
    global g_stepsize
    global g_history

    if os.path.isfile(javaLogText):

        run_result = dict()
        run_result["total time (ms)"] = []
        run_result["initialXY (ms)"] = []
        run_result["regularize Y (ms)"] = []
        run_result["regularize X and objective (ms)"] = []
        run_result["update X (ms)"] = []
        run_result["update Y (ms)"] = []
        run_result["objective (ms)"] = []
        run_result["step size (ms)"] = []
        run_result["update history (ms)"] = []

        total_run_time = -1
        val = 0.0
        with open(javaLogText, 'r') as thefile:   # go into tempfile and grab test run info
            for each_line in thefile:
                temp_string = each_line.split()

                if len(temp_string) > 0:
                    val = temp_string[-1].replace('\\','')

                if g_initialXY in each_line:    # start of a new file
                    if total_run_time > 0:  # update total run time
                        run_result["total time (ms)"].append(total_run_time)
                        total_run_time = 0.0
                    else:
                        total_run_time = 0.0

                    run_result["initialXY (ms)"].append(float(val))
                    total_run_time = total_run_time+float(val)

                if g_reguarlize_Y in each_line:
                    run_result["regularize Y (ms)"].append(float(val))
                    total_run_time = total_run_time+float(val)

                if g_regularize_X_objective in each_line:
                    run_result["regularize X and objective (ms)"].append(float(val))
                    total_run_time = total_run_time+float(val)

                if g_updateX in each_line:
                    run_result["update X (ms)"].append(float(val))
                    total_run_time = total_run_time+float(val)

                if g_updateY in each_line:
                    run_result["update Y (ms)"].append(float(val))
                    total_run_time = total_run_time+float(val)

                if g_objective in each_line:
                    run_result["objective (ms)"].append(float(val))
                    total_run_time = total_run_time+float(val)

                if g_stepsize in each_line:
                    run_result["step size (ms)"].append(float(val))
                    total_run_time = total_run_time+float(val)

                if g_history in each_line:
                    run_result["update history (ms)"].append(float(val))
                    total_run_time = total_run_time+float(val)

        run_result["total time (ms)"].append(total_run_time)    # save the last one
        print("Run result summary: \n {0}".format(run_result))

    else:
        print("Cannot find your java log file.  Nothing is done.\n")

def main(argv):
    """
    Main program.  Take user input, parse it and call other functions to execute the commands
    and extract run summary and store run result in json file

    @return: none
    """
    global g_test_root_dir
    global g_temp_filename

    if len(argv) < 2:
        print("invoke this script as python extractGLRMRuntimeJavaLog.py javatextlog.\n")
        sys.exit(1)
    else:   # we may be in business
        javaLogText = argv[1]         # filename while java log is stored

        print("your java text is {0}".format(javaLogText))
        extractRunInto(javaLogText)

if __name__ == "__main__":
    main(sys.argv)


"""Extract reference documentation from the NumPy source tree.

"""

import inspect
import textwrap
import re
import pydoc
from StringIO import StringIO
from warnings import warn
4
class Reader(object):
    """A line-based string reader.

    """
    def __init__(self, data):
        """
        Parameters
        ----------
        data : str
           String with lines separated by '\n'.

        """
        if isinstance(data,list):
            self._str = data
        else:
            self._str = data.split('\n') # store string as list of lines

        self.reset()

    def __getitem__(self, n):
        return self._str[n]

    def reset(self):
        self._l = 0 # current line nr

    def read(self):
        if not self.eof():
            out = self[self._l]
            self._l += 1
            return out
        else:
            return ''

    def seek_next_non_empty_line(self):
        for l in self[self._l:]:
            if l.strip():
                break
            else:
                self._l += 1

    def eof(self):
        return self._l >= len(self._str)

    def read_to_condition(self, condition_func):
        start = self._l
        for line in self[start:]:
            if condition_func(line):
                return self[start:self._l]
            self._l += 1
            if self.eof():
                return self[start:self._l+1]
        return []

    def read_to_next_empty_line(self):
        self.seek_next_non_empty_line()
        def is_empty(line):
            return not line.strip()
        return self.read_to_condition(is_empty)

    def read_to_next_unindented_line(self):
        def is_unindented(line):
            return (line.strip() and (len(line.lstrip()) == len(line)))
        return self.read_to_condition(is_unindented)

    def peek(self,n=0):
        if self._l + n < len(self._str):
            return self[self._l + n]
        else:
            return ''

    def is_empty(self):
        return not ''.join(self._str).strip()

class NumpyDocString(object):
    def __init__(self,docstring):
        docstring = textwrap.dedent(docstring).split('\n')

        self._doc = Reader(docstring)
        self._parsed_data = {
            'Signature': '',
            'Summary': [''],
            'Extended Summary': [],
            'Parameters': [],
            'Returns': [],
            'Raises': [],
            'Warns': [],
            'Other Parameters': [],
            'Attributes': [],
            'Methods': [],
            'See Also': [],
            'Notes': [],
            'Warnings': [],
            'References': '',
            'Examples': '',
            'index': {}
            }

        self._parse()

    def __getitem__(self,key):
        return self._parsed_data[key]

    def __setitem__(self,key,val):
        if not self._parsed_data.has_key(key):
            warn("Unknown section %s" % key)
        else:
            self._parsed_data[key] = val

    def _is_at_section(self):
        self._doc.seek_next_non_empty_line()

        if self._doc.eof():
            return False

        l1 = self._doc.peek().strip()  # e.g. Parameters

        if l1.startswith('.. index::'):
            return True

        l2 = self._doc.peek(1).strip() #    ---------- or ==========
        return l2.startswith('-'*len(l1)) or l2.startswith('='*len(l1))

    def _strip(self,doc):
        i = 0
        j = 0
        for i,line in enumerate(doc):
            if line.strip(): break

        for j,line in enumerate(doc[::-1]):
            if line.strip(): break

        return doc[i:len(doc)-j]

    def _read_to_next_section(self):
        section = self._doc.read_to_next_empty_line()

        while not self._is_at_section() and not self._doc.eof():
            if not self._doc.peek(-1).strip(): # previous line was empty
                section += ['']

            section += self._doc.read_to_next_empty_line()

        return section

    def _read_sections(self):
        while not self._doc.eof():
            data = self._read_to_next_section()
            name = data[0].strip()

            if name.startswith('..'): # index section
                yield name, data[1:]
            elif len(data) < 2:
                yield StopIteration
            else:
                yield name, self._strip(data[2:])

    def _parse_param_list(self,content):
        r = Reader(content)
        params = []
        while not r.eof():
            header = r.read().strip()
            if ' : ' in header:
                arg_name, arg_type = header.split(' : ')[:2]
            else:
                arg_name, arg_type = header, ''

            desc = r.read_to_next_unindented_line()
            desc = dedent_lines(desc)

            params.append((arg_name,arg_type,desc))

        return params

    _name_rgx = re.compile(r"^\s*(:(?P<role>\w+):`(?P<name>[a-zA-Z0-9_.-]+)`|"
                           r" (?P<name2>[a-zA-Z0-9_.-]+))\s*", re.X)
    def _parse_see_also(self, content):
        """
        func_name : Descriptive text
            continued text
        another_func_name : Descriptive text
        func_name1, func_name2, :meth:`func_name`, func_name3

        """
        items = []

        def parse_item_name(text):
            """Match ':role:`name`' or 'name'"""
            m = self._name_rgx.match(text)
            if m:
                g = m.groups()
                if g[1] is None:
                    return g[3], None
                else:
                    return g[2], g[1]
            raise ValueError("%s is not a item name" % text)

        def push_item(name, rest):
            if not name:
                return
            name, role = parse_item_name(name)
            items.append((name, list(rest), role))
            del rest[:]

        current_func = None
        rest = []

        for line in content:
            if not line.strip(): continue

            m = self._name_rgx.match(line)
            if m and line[m.end():].strip().startswith(':'):
                push_item(current_func, rest)
                current_func, line = line[:m.end()], line[m.end():]
                rest = [line.split(':', 1)[1].strip()]
                if not rest[0]:
                    rest = []
            elif not line.startswith(' '):
                push_item(current_func, rest)
                current_func = None
                if ',' in line:
                    for func in line.split(','):
                        push_item(func, [])
                elif line.strip():
                    current_func = line
            elif current_func is not None:
                rest.append(line.strip())
        push_item(current_func, rest)
        return items

    def _parse_index(self, section, content):
        """
        .. index: default
           :refguide: something, else, and more

        """
        def strip_each_in(lst):
            return [s.strip() for s in lst]

        out = {}
        section = section.split('::')
        if len(section) > 1:
            out['default'] = strip_each_in(section[1].split(','))[0]
        for line in content:
            line = line.split(':')
            if len(line) > 2:
                out[line[1]] = strip_each_in(line[2].split(','))
        return out

    def _parse_summary(self):
        """Grab signature (if given) and summary"""
        if self._is_at_section():
            return

        summary = self._doc.read_to_next_empty_line()
        summary_str = " ".join([s.strip() for s in summary]).strip()
        if re.compile('^([\w., ]+=)?\s*[\w\.]+\(.*\)$').match(summary_str):
            self['Signature'] = summary_str
            if not self._is_at_section():
                self['Summary'] = self._doc.read_to_next_empty_line()
        else:
            self['Summary'] = summary

        if not self._is_at_section():
            self['Extended Summary'] = self._read_to_next_section()

    def _parse(self):
        self._doc.reset()
        self._parse_summary()

        for (section,content) in self._read_sections():
            if not section.startswith('..'):
                section = ' '.join([s.capitalize() for s in section.split(' ')])
            if section in ('Parameters', 'Attributes', 'Methods',
                           'Returns', 'Raises', 'Warns'):
                self[section] = self._parse_param_list(content)
            elif section.startswith('.. index::'):
                self['index'] = self._parse_index(section, content)
            elif section == 'See Also':
                self['See Also'] = self._parse_see_also(content)
            else:
                self[section] = content

    # string conversion routines

    def _str_header(self, name, symbol='-'):
        return [name, len(name)*symbol]

    def _str_indent(self, doc, indent=4):
        out = []
        for line in doc:
            out += [' '*indent + line]
        return out

    def _str_signature(self):
        if self['Signature']:
            return [self['Signature'].replace('*','\*')] + ['']
        else:
            return ['']

    def _str_summary(self):
        if self['Summary']:
            return self['Summary'] + ['']
        else:
            return []

    def _str_extended_summary(self):
        if self['Extended Summary']:
            return self['Extended Summary'] + ['']
        else:
            return []

    def _str_param_list(self, name):
        out = []
        if self[name]:
            out += self._str_header(name)
            for param,param_type,desc in self[name]:
                out += ['%s : %s' % (param, param_type)]
                out += self._str_indent(desc)
            out += ['']
        return out

    def _str_section(self, name):
        out = []
        if self[name]:
            out += self._str_header(name)
            out += self[name]
            out += ['']
        return out

    def _str_see_also(self, func_role):
        if not self['See Also']: return []
        out = []
        out += self._str_header("See Also")
        last_had_desc = True
        for func, desc, role in self['See Also']:
            if role:
                link = ':%s:`%s`' % (role, func)
            elif func_role:
                link = ':%s:`%s`' % (func_role, func)
            else:
                link = "`%s`_" % func
            if desc or last_had_desc:
                out += ['']
                out += [link]
            else:
                out[-1] += ", %s" % link
            if desc:
                out += self._str_indent([' '.join(desc)])
                last_had_desc = True
            else:
                last_had_desc = False
        out += ['']
        return out

    def _str_index(self):
        idx = self['index']
        out = []
        out += ['.. index:: %s' % idx.get('default','')]
        for section, references in idx.iteritems():
            if section == 'default':
                continue
            out += ['   :%s: %s' % (section, ', '.join(references))]
        return out

    def __str__(self, func_role=''):
        out = []
        out += self._str_signature()
        out += self._str_summary()
        out += self._str_extended_summary()
        for param_list in ('Parameters','Returns','Raises'):
            out += self._str_param_list(param_list)
        out += self._str_section('Warnings')
        out += self._str_see_also(func_role)
        for s in ('Notes','References','Examples'):
            out += self._str_section(s)
        out += self._str_index()
        return '\n'.join(out)

def indent(str,indent=4):
    indent_str = ' '*indent
    if str is None:
        return indent_str
    lines = str.split('\n')
    return '\n'.join(indent_str + l for l in lines)

def dedent_lines(lines):
    """Deindent a list of lines maximally"""
    return textwrap.dedent("\n".join(lines)).split("\n")

def header(text, style='-'):
    return text + '\n' + style*len(text) + '\n'

class FunctionDoc(NumpyDocString):
    def __init__(self, func, role='func', doc=None):
        self._f = func
        self._role = role # e.g. "func" or "meth"
        if doc is None:
            doc = inspect.getdoc(func) or ''
        try:
            NumpyDocString.__init__(self, doc)
        except ValueError, e:
            print '*'*78
            print "ERROR: '%s' while parsing `%s`" % (e, self._f)
            print '*'*78
            #print "Docstring follows:"
            #print doclines
            #print '='*78

        if not self['Signature']:
            func, func_name = self.get_func()
            try:
                # try to read signature
                argspec = inspect.getargspec(func)
                argspec = inspect.formatargspec(*argspec)
                argspec = argspec.replace('*','\*')
                signature = '%s%s' % (func_name, argspec)
            except TypeError, e:
                signature = '%s()' % func_name
            self['Signature'] = signature

    def get_func(self):
        func_name = getattr(self._f, '__name__', self.__class__.__name__)
        if inspect.isclass(self._f):
            func = getattr(self._f, '__call__', self._f.__init__)
        else:
            func = self._f
        return func, func_name

    def __str__(self):
        out = ''

        func, func_name = self.get_func()
        signature = self['Signature'].replace('*', '\*')

        roles = {'func': 'function',
                 'meth': 'method'}

        if self._role:
            if not roles.has_key(self._role):
                print "Warning: invalid role %s" % self._role
            out += '.. %s:: %s\n    \n\n' % (roles.get(self._role,''),
                                             func_name)

        out += super(FunctionDoc, self).__str__(func_role=self._role)
        return out

class ClassDoc(NumpyDocString):
    def __init__(self,cls,modulename='',func_doc=FunctionDoc,doc=None):
        if not inspect.isclass(cls):
            raise ValueError("Initialise using a class. Got %r" % cls)
        self._cls = cls

        if modulename and not modulename.endswith('.'):
            modulename += '.'
        self._mod = modulename
        self._name = cls.__name__
        self._func_doc = func_doc

        if doc is None:
            doc = pydoc.getdoc(cls)

        NumpyDocString.__init__(self, doc)

    @property
    def methods(self):
        return [name for name,func in inspect.getmembers(self._cls)
                if not name.startswith('_') and callable(func)]

    def __str__(self):
        out = ''
        out += super(ClassDoc, self).__str__()
        out += "\n\n"

        #for m in self.methods:
        #    print "Parsing `%s`" % m
        #    out += str(self._func_doc(getattr(self._cls,m), 'meth')) + '\n\n'
        #    out += '.. index::\n   single: %s; %s\n\n' % (self._name, m)

        return out