# SPD-License-Identifier: MIT

function load_filelist(uttids, feadir, ext)
    filelist= []
    open(uttids, "r") do f
        for line in eachline(f)
            push!(filelist, joinpath(feadir, line*ext))
        end
    end
    filelist
end

function makemodel(T, modelconf, μ, Σ)
    C = modelconf["ncomponents"]
    pstrength = modelconf["pstrength"]
    σ = modelconf["noise_init"]
    D = length(μ)
    μ₀ = Array{T}(μ)
    W₀ = Array{T}(Hermitian(inv(Σ)))
    diagΣ₀ = Array{T}(diag(Σ))

    @debug "creating model with $C components (covtype = $(modelconf["covtype"]))"
    components = []
    for c in 1:C
        if modelconf["covtype"] == "full"
            push!(components, Normal(T, D; μ₀, W₀, pstrength, σ))
        elseif modelconf["covtype"] == "diagonal"
            push!(components, NormalDiag(T, D; μ₀, diagΣ₀, pstrength, σ))
        else
            error("unknown cov type $(modelconf["covtype"])")
        end
    end
    Mixture(T, components = Tuple(components))
end

function savemodel(dir, model, modelconf, inferenceconf, step = -1)
    mtype = typeof(model)
    data = Dict(
        "modelconf" => modelconf,
        "inferenceconf" => inferenceconf,
        "mtype" => mtype,
        "mstate" => todict(model)
    )
    modelname = step > 0 ? "$step.jld2" : "final.jld2"

    save(joinpath(dir, modelname), data)
end

function loadmodel(path)
    dict = load(path)
    mtype = dict["mtype"]
    mstate = dict["mstate"]
    model = fromdict(mtype, mstate)
    modelconf = dict["modelconf"]
    model, modelconf
end


module GLRenderer

import PoseComposition: Pose
import Rotations
import Images as I
import Images: Color, RGBA
import Parameters: @with_kw
import GLFW
import PyCall
import FileIO
import MeshIO
import GeometryBasics as GB
import ColorSchemes
using ModernGL

@with_kw struct CameraIntrinsics
    width::Int = 640
    height::Int = 480
    fx::Float64 = width
    fy::Float64 = width
    cx::Float64 = width/2
    cy::Float64 = height/2
    near::Float64 = 0.001
    far::Float64 = 100.0
end

@with_kw mutable struct Mesh
    vertices::Any = nothing
    indices::Any = nothing
    normals::Any = nothing
    tex_coords::Any = nothing
    tex_path::Any = nothing
end

copy_mesh(m::Mesh) = Mesh(
                        vertices=m.vertices,
                        indices=m.indices,
                        normals=m.normals,
                        tex_coords=m.tex_coords,
                        tex_path=m.tex_path,
                    )

include("shaders.jl")
include("utils.jl")
include("mesh.jl")
include("point_cloud.jl")

abstract type RenderMode end
struct DepthMode <: RenderMode end
struct RGBBasicMode <: RenderMode end
struct RGBMode <: RenderMode end
struct TextureMode <: RenderMode end
struct TextureMixedMode <: RenderMode end

mutable struct Renderer{T <: RenderMode}
    window::Any
    camera_intrinsics::CameraIntrinsics
    shader_program::Any
    mesh_pointers::Any
    mesh_sizes::Any
    textures::Any
    perspective_matrix::Matrix
end

function setup_renderer(camera_intrinsics::CameraIntrinsics, mode::RenderMode; name="GLRenderer", gl_version=nothing)::Renderer
    if isnothing(gl_version)
        window = GLFW.CreateWindow(1,1, "dummy")
        GLFW.MakeContextCurrent(window)
        version_string = unsafe_string(glGetString(GL_VERSION))
        gl_version = (i -> parse(Int,i)).(split(split(version_string," ")[1],"."))
        GLFW.DestroyWindow(window)
    end

    perspective_matrix = get_perspective_matrix(
        camera_intrinsics.width, camera_intrinsics.height,
        camera_intrinsics.fx,
        camera_intrinsics.fy,
        camera_intrinsics.cx,
        camera_intrinsics.cy,
        camera_intrinsics.near,
        camera_intrinsics.far,
    )

    window_hint = [
        # (GLFW.SAMPLES,      0),
        # (GLFW.DEPTH_BITS,   24),
        # (GLFW.ALPHA_BITS,   8),
        # (GLFW.RED_BITS,     8),
        # (GLFW.GREEN_BITS,   8),
        # (GLFW.BLUE_BITS,    8),
        # (GLFW.STENCIL_BITS, 0),
        # (GLFW.AUX_BUFFERS,  0),
        (GLFW.CONTEXT_VERSION_MAJOR, gl_version[1]),
        (GLFW.CONTEXT_VERSION_MINOR, gl_version[2]),
        (GLFW.OPENGL_PROFILE, GLFW.OPENGL_CORE_PROFILE),
        (GLFW.OPENGL_FORWARD_COMPAT, GL_TRUE),
        (GLFW.VISIBLE, GL_FALSE)
    ]
    for (key, value) in window_hint
        GLFW.WindowHint(key, value)
    end

    window = GLFW.CreateWindow(camera_intrinsics.width, camera_intrinsics.height, "GLRenderer")
    GLFW.MakeContextCurrent(window)
    glEnable(GL_DEPTH_TEST)
    glClear(GL_DEPTH_BUFFER_BIT)

    gl_version_for_shaders = "$(gl_version[1])$(gl_version[2])0"
    @show gl_version_for_shaders
    if typeof(mode) == DepthMode
        vertex_shader = createShader(vertex_source_depth(gl_version_for_shaders), GL_VERTEX_SHADER)
        fragment_shader = createShader(fragment_source_depth(gl_version_for_shaders), GL_FRAGMENT_SHADER)
    elseif typeof(mode) == RGBBasicMode
        vertex_shader = createShader(vertexShader_rgb_basic(gl_version_for_shaders), GL_VERTEX_SHADER)
        fragment_shader = createShader(fragmentShader_rgb_basic(gl_version_for_shaders), GL_FRAGMENT_SHADER)
    elseif typeof(mode) == RGBMode
        vertex_shader = createShader(vertexShader_rgb(gl_version_for_shaders), GL_VERTEX_SHADER)
        fragment_shader = createShader(fragmentShader_rgb(gl_version_for_shaders), GL_FRAGMENT_SHADER)
    elseif typeof(mode) == TextureMode
        vertex_shader = createShader(vertexShader_texture(gl_version_for_shaders), GL_VERTEX_SHADER)
        fragment_shader = createShader(fragmentShader_texture(gl_version_for_shaders), GL_FRAGMENT_SHADER)
    elseif typeof(mode) == TextureMixedMode
        vertex_shader = createShader(vertexShader_texture_mixed(gl_version_for_shaders), GL_VERTEX_SHADER)
        fragment_shader = createShader(fragmentShader_texture_mixed(gl_version_for_shaders), GL_FRAGMENT_SHADER)
    else
        error("unsupported mode $(mode)")
    end

    shader_program = glCreateProgram()
    glAttachShader(shader_program, vertex_shader)
    glAttachShader(shader_program, fragment_shader)
    glLinkProgram(shader_program)

    fbo = Ref(GLuint(0))
    glGenFramebuffers(1, fbo)

    depth_tex = Ref(GLuint(0))
    glGenTextures(1, depth_tex)
    color_tex = Ref(GLuint(0))
    glGenTextures(1, color_tex)

    glBindFramebuffer(GL_FRAMEBUFFER, fbo[])

    glBindTexture(GL_TEXTURE_2D, depth_tex[])
    glTexImage2D(
      GL_TEXTURE_2D, 0, GL_DEPTH24_STENCIL8, camera_intrinsics.width, camera_intrinsics.height, 0,
      GL_DEPTH_STENCIL, GL_UNSIGNED_INT_24_8, C_NULL
    );
    glFramebufferTexture2D(GL_FRAMEBUFFER, GL_DEPTH_STENCIL_ATTACHMENT, GL_TEXTURE_2D, depth_tex[], 0);

    glBindTexture(GL_TEXTURE_2D, color_tex[])
    glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA32F, camera_intrinsics.width, camera_intrinsics.height, 0,
                 GL_RGBA, GL_UNSIGNED_BYTE, C_NULL)
    glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, color_tex[], 0)

    glViewport(0, 0, camera_intrinsics.width, camera_intrinsics.height)
    glDrawBuffers(2, [GL_DEPTH_STENCIL_ATTACHMENT, GL_COLOR_ATTACHMENT0])
    println(glCheckFramebufferStatus(GL_FRAMEBUFFER) == GL_FRAMEBUFFER_COMPLETE)

    Renderer{typeof(mode)}(window, camera_intrinsics, shader_program, [], [], [], perspective_matrix)
end

function activate_renderer(renderer::Renderer)
    GLFW.MakeContextCurrent(renderer.window)
end

function set_intrinsics!(renderer::Renderer, camera_intrinsics::CameraIntrinsics)
    renderer.camera_intrinsics = camera_intrinsics
    renderer.perspective_matrix = get_perspective_matrix(
        camera_intrinsics.width, camera_intrinsics.height,
        camera_intrinsics.fx,
        camera_intrinsics.fy,
        camera_intrinsics.cx,
        camera_intrinsics.cy,
        camera_intrinsics.near,
        camera_intrinsics.far,
    )

    fbo = Ref(GLuint(0))
    glGenFramebuffers(1, fbo)

    depth_tex = Ref(GLuint(0))
    glGenTextures(1, depth_tex)
    color_tex = Ref(GLuint(0))
    glGenTextures(1, color_tex)

    glBindFramebuffer(GL_FRAMEBUFFER, fbo[])

    glBindTexture(GL_TEXTURE_2D, depth_tex[])
    glTexImage2D(
      GL_TEXTURE_2D, 0, GL_DEPTH24_STENCIL8, camera_intrinsics.width, camera_intrinsics.height, 0,
      GL_DEPTH_STENCIL, GL_UNSIGNED_INT_24_8, C_NULL
    );
    glFramebufferTexture2D(GL_FRAMEBUFFER, GL_DEPTH_STENCIL_ATTACHMENT, GL_TEXTURE_2D, depth_tex[], 0);

    glBindTexture(GL_TEXTURE_2D, color_tex[])
    glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA32F, camera_intrinsics.width, camera_intrinsics.height, 0,
                 GL_RGBA, GL_UNSIGNED_BYTE, C_NULL)
    glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, color_tex[], 0)

    glViewport(0, 0, camera_intrinsics.width, camera_intrinsics.height)
    glDrawBuffers(2, [GL_DEPTH_STENCIL_ATTACHMENT, GL_COLOR_ATTACHMENT0])
    println(glCheckFramebufferStatus(GL_FRAMEBUFFER) == GL_FRAMEBUFFER_COMPLETE)
end

function load_object!(renderer::Renderer{T}, mesh) where T <: RenderMode
    vao = Ref(GLuint(0))
    glGenVertexArrays(1, vao)
    glBindVertexArray(vao[])

    if T == DepthMode || T == RGBBasicMode
        @assert size(mesh.vertices)[1] == 3
        vertex_data = mesh.vertices
    elseif T == RGBMode
        @assert size(mesh.vertices)[1] == 3
        @assert size(mesh.normals)[1] == 3
        @assert size(mesh.vertices)[2] == size(mesh.normals)[2]
        vertex_data = vcat(mesh.vertices, mesh.normals)
    else
        @assert size(mesh.vertices)[1] == 3
        @assert size(mesh.normals)[1] == 3
        if T == TextureMixedMode && isnothing(mesh.tex_coords)
            tex_coords = zeros(2, size(mesh.vertices)[2])
        else
            @assert size(mesh.tex_coords)[1] == 2
            tex_coords = mesh.tex_coords
        end
        @assert size(mesh.vertices)[2] == size(mesh.normals)[2] == size(tex_coords)[2]
        vertex_data = vcat(mesh.vertices, mesh.normals, tex_coords)
    end

    vertex_data = Matrix{Float32}(vertex_data)
    indices = Matrix{UInt32}(mesh.indices)

    # copy vertex data into an OpenGL buffer
    vbo = Ref(GLuint(0))
    glGenBuffers(1, vbo)
    glBindBuffer(GL_ARRAY_BUFFER, vbo[])
    glBufferData(GL_ARRAY_BUFFER, sizeof(vertex_data), Ref(vertex_data, 1), GL_STATIC_DRAW)

    # element buffer object for indices
    ebo = Ref(GLuint(0))
    glGenBuffers(1, ebo)
    glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, ebo[])
    glBufferData(GL_ELEMENT_ARRAY_BUFFER, sizeof(indices), Ref(indices, 1), GL_STATIC_DRAW)

    if T == DepthMode || T == RGBBasicMode
        glEnableVertexAttribArray(glGetAttribLocation(renderer.shader_program, "position"))

        # set vertex attribute pointers
        glVertexAttribPointer(glGetAttribLocation(renderer.shader_program, "position"),
            3, GL_FLOAT, GL_FALSE, 3 * sizeof(Float32), C_NULL)
        glEnableVertexAttribArray(glGetAttribLocation(renderer.shader_program, "position"))
    elseif T == RGBMode
        # set vertex attribute pointers
        glEnableVertexAttribArray(glGetAttribLocation(renderer.shader_program, "position"))
        glEnableVertexAttribArray(glGetAttribLocation(renderer.shader_program, "normal"))

        glVertexAttribPointer(
            glGetAttribLocation(renderer.shader_program, "position"),
            3, GL_FLOAT, GL_FALSE, 3 * sizeof(Float32) * 2, C_NULL)
        glVertexAttribPointer(
            glGetAttribLocation(renderer.shader_program, "normal"),
            3, GL_FLOAT, GL_FALSE, 3 * sizeof(Float32) * 2, C_NULL + (3 * sizeof(Float32)))
    else
        # set vertex attribute pointers
        glEnableVertexAttribArray(glGetAttribLocation(renderer.shader_program, "position"))
        glEnableVertexAttribArray(glGetAttribLocation(renderer.shader_program, "normal"))
        glEnableVertexAttribArray(glGetAttribLocation(renderer.shader_program, "vertTexCoord"))

        glVertexAttribPointer(
            glGetAttribLocation(renderer.shader_program, "position"),
            3, GL_FLOAT, GL_FALSE, 32, C_NULL)
        glVertexAttribPointer(
            glGetAttribLocation(renderer.shader_program, "normal"),
            3, GL_FLOAT, GL_FALSE, 32, C_NULL + 12)
        glVertexAttribPointer(
            glGetAttribLocation(renderer.shader_program, "vertTexCoord"),
            2, GL_FLOAT, GL_FALSE, 32, C_NULL + 24)

        PyCall.py"""
        from PIL import Image
        import numpy as np
        def load_texture_bytes(path):
            img = Image.open(path).transpose(Image.FLIP_TOP_BOTTOM)
            img_data = np.fromstring(img.tobytes(), np.uint8)
            return img_data, img.width, img.height
        """
        if isnothing(mesh.tex_path)
            push!(renderer.textures, nothing)
        else
            img_data, width, height = PyCall.py"load_texture_bytes"(mesh.tex_path);

            texture = Ref(GLuint(0))
            glGenTextures(1, texture)
            glPixelStorei(GL_UNPACK_ALIGNMENT, 1)
            glBindTexture(GL_TEXTURE_2D, texture[])
            glTexParameterf(
                GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR)
            glTexParameterf(
                GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR_MIPMAP_LINEAR)
            glTexParameterf(
                GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE)
            glTexParameterf(
                GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE)
            glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, width, height, 0,
                            GL_RGB, GL_UNSIGNED_BYTE, img_data)
            glGenerateMipmap(GL_TEXTURE_2D)

            push!(renderer.textures, texture)
        end

    end

    # unbind it
    glBindBuffer(GL_ARRAY_BUFFER, 0)
    glBindVertexArray(0)

    push!(renderer.mesh_pointers, vao[])
    push!(renderer.mesh_sizes, size(mesh.indices)[2] * 3)
    return true
end

function gl_render(
        renderer::Union{Renderer{T}}, mesh_ids::Vector{Int},
        poses::Vector{Pose}, camera_pose::Pose; colors=nothing
) where T <: RenderMode

    if isnothing(colors)
        colors = fill(I.colorant"black", length(poses))
    end
    colors = map(x -> convert(I.RGBA,x), colors)

    @assert length(mesh_ids) == length(poses)

    glClearColor(1.0, 1.0, 1.0, 1)
    glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
    glEnable(GL_DEPTH_TEST)

    camera_pose_mat = pose_to_matrix(inv(camera_pose * Pose(zeros(3),Rotations.RotX(pi))))

    for (id, p, c) in zip(mesh_ids, poses, colors)
        if id < 1
            continue
        end
        vao = renderer.mesh_pointers[id]

        glUseProgram(renderer.shader_program)

        glUniformMatrix4fv(glGetUniformLocation(
                renderer.shader_program,"P"), 1, GL_FALSE, Ref(renderer.perspective_matrix, 1))
        glUniformMatrix4fv(glGetUniformLocation(
                renderer.shader_program,"V"), 1, GL_FALSE, Ref(camera_pose_mat, 1))
        glUniformMatrix4fv(glGetUniformLocation(
                renderer.shader_program,"pose_mat"), 1, GL_FALSE, Ref(pose_to_matrix(p), 1))

        if T == RGBBasicMode || T == RGBMode
            glUniform4fv(glGetUniformLocation(
                    renderer.shader_program,"color"), 1, Float32[c.r, c.g, c.b, c.alpha])
        elseif T == TextureMode || T == TextureMixedMode
            glUniformMatrix4fv(glGetUniformLocation(
                    renderer.shader_program,"pose_rot"), 1, GL_FALSE, Ref(pose_to_matrix(Pose(zeros(3),p.orientation)), 1))

            if isnothing(renderer.textures[id])
                glUniform1f(glGetUniformLocation(
                    renderer.shader_program,"textured"), 0.0)
                glUniform4fv(glGetUniformLocation(
                        renderer.shader_program,"color"), 1, Float32[c.r, c.g, c.b, c.alpha])
            else
                glUniform1f(glGetUniformLocation(
                    renderer.shader_program,"textured"), 1.0)
                glActiveTexture(GL_TEXTURE0)
                glBindTexture(GL_TEXTURE_2D, renderer.textures[id][])
                glUniform1i(glGetUniformLocation(
                        renderer.shader_program,"tex"), 0)
            end
        end

        glBindVertexArray(vao)
        glDrawElements(GL_TRIANGLES, renderer.mesh_sizes[id], GL_UNSIGNED_INT, C_NULL)
        glBindVertexArray(0)
    end

    cam = renderer.camera_intrinsics
    data = Matrix{Float32}(undef, cam.width, cam.height)
    glReadPixels(0, 0, cam.width, cam.height, GL_DEPTH_COMPONENT, GL_FLOAT, Ref(data, 1))
    near,far = cam.near, cam.far
    depth_image = far .* near ./ (far .- (far .- near) .* data)
    depth_image = permutedims(depth_image[:,end:-1:1])

    if T == DepthMode
        return depth_image
    end

    glReadBuffer(GL_COLOR_ATTACHMENT0)
    data1 = zeros(Float32, 4,cam.width, cam.height)
    glReadPixels(0, 0, cam.width, cam.height, GL_BGRA, GL_FLOAT, Ref(data1, 1))
    rgb = permutedims(data1,(3,2,1))[end:-1:1,:,:]
    rgb = cat(rgb[:,:,3],rgb[:,:,2],rgb[:,:,1],rgb[:,:,4],dims=3)
    return rgb, depth_image
end

# Viewing images
function view_depth_image(depth_image; scheme=nothing)
    max_val = maximum(depth_image)
    min_val = minimum(depth_image)
    # vals = depth_image[(depth_image .> (min_val + 1e-3)) .&  (depth_image .< (max_val - 1e-3))]
    # vals = depth_image[(depth_image .> (min_val + 1e-3)) .&  (depth_image .< (max_val - 1e-3))]
    # max_val = maximum(vals)
    # min_val = minimum(vals)
    d = clamp.(depth_image, min_val, max_val)
    img = get(ColorSchemes.hot, d, (min_val, max_val))
    I.convert.(I.RGBA, img)
end

function view_rgb_image(rgb_image; in_255=false)
    if in_255
        rgb_image = Float64.(rgb_image)./255.0
    end
    if size(rgb_image)[3] == 3
        img = I.colorview(I.RGB, permutedims(rgb_image,(3,1,2)))
        img = I.convert.(I.RGBA, img)
    else
        img = I.colorview(I.RGBA, permutedims(rgb_image,(3,1,2)))
    end
    img
end

end


SEED_Char(io::IO, BUF::SeisIOBuf, nb::UInt16) = replace(String(read(io, nb)),
                                            ['\r', '\0'] =>"")

# This gets special handling as Float64 arrays are assumed in S.x[c]
function SEED_Float64!(io::IO, S::SeisData, c::Int64, xi::Int64, nb::UInt16)
  buf = getfield(BUF, :buf)
  x = getindex(getfield(S, :x), c)
  readbytes!(io, buf, nb)
  nx = div(nb, 8)
  if getfield(BUF, :xs) == true
    j = xi
    @inbounds for i = 1:nx
      j += 1
      y  = UInt32(buf[8*i-7]) << 56
      y |= UInt32(buf[8*i-6]) << 48
      y |= UInt32(buf[8*i-5]) << 40
      y |= UInt32(buf[8*i-4]) << 32
      y |= UInt32(buf[8*i-3]) << 24
      y |= UInt32(buf[8*i-2]) << 16
      y |= UInt32(buf[8*i-1]) << 8
      y |= UInt32(buf[8*i])
      x[j] = y
    end
  else
    xr = reinterpret(Float64, buf)
    copyto!(x, xi+1, xr, 1, nx)
  end
  setfield!(BUF, :k, Int64(nx))
  return nothing
end

function SEED_Unenc!(io::IO, S::SeisData, c::Int64, xi::Int64, nb::UInt16, nx::UInt16)
  buf = getfield(BUF, :buf)
  checkbuf_8!(buf, xi)
  x = getindex(getfield(S, :x), c)
  readbytes!(io, buf, nb)
  T::Type = if BUF.fmt == 0x01
      Int16
    elseif BUF.fmt == 0x03
      Int32
    elseif BUF.fmt == 0x04
      Float32
    end
  if getfield(BUF, :swap) == true
    xr = bswap.(reinterpret(T, buf))
  else
    xr = reinterpret(T, buf)
  end
  copyto!(x, xi+1, xr, 1, nx)
  setfield!(BUF, :k, Int64(nx))
  return nothing
end

function SEED_Geoscope!(io::IO, BUF::SeisIOBuf)
  mm = 0x0fff
  gm = BUF.fmt == 0x0d ? 0x7000 : 0xf000
  for i = 0x0001:BUF.n
    x = BUF.swap ? bswap(read(io, UInt16)) : read(io, UInt16)
    m = Int32(x & mm)
    g = Int32((x & gm) >> 12)
    ex = -1*g
    setindex!(BUF.x, ldexp(Float64(m-2048), ex), i)
  end
  BUF.k = BUF.n
  return nothing
end

function SEED_CDSN!(io::IO, BUF::SeisIOBuf)
  for i = 0x0001:BUF.n
    x = BUF.swap ? bswap(read(io, UInt16)) : read(io, UInt16)
    m = Int32(x & 0x3fff)
    g = Int32((x & 0xc000) >> 14)
    mult = 4^g * g==3 ? 2 : 1
    m -= 0x1fff
    setindex!(BUF.x, m*mult, i)
  end
  BUF.k = BUF.n
  return nothing
end

function SEED_SRO!(io::IO, BUF::SeisIOBuf)
  for i = 0x0001:BUF.n
    x = BUF.swap ? bswap(read(io, UInt16)) : read(io, UInt16)
    m = Int32(x & 0x0fff)
    g = Int32((x & 0xf000) >> 12)
    if m > 0x07ff
      m -= 0x1000
    end
    ex = -1*g + 10
    setindex!(BUF.x, ldexp(Float64(m), ex), i)
  end
  BUF.k = BUF.n
  return nothing
end

function SEED_DWWSSN!(io::IO, BUF::SeisIOBuf)
  for i = 0x0001:BUF.n
    x = signed(UInt32(BUF.swap ? bswap(read(io, UInt16)) : read(io, UInt16)))
    BUF.x[i] = x > 32767 ? x - 65536 : x
  end
  BUF.k = BUF.n
  return nothing
end

# Steim1 or Steim2
function SEED_Steim!(io::IO, BUF::SeisIOBuf, nb::UInt16)
  x = getfield(BUF, :x)
  buf = getfield(BUF, :buf)
  ff = getfield(BUF, :x32)
  nc = Int64(div(nb, 0x0040))
  ni = div(nb, 0x0004)
  readbytes!(io, buf, nb)

  # Parse buf as UInt32s
  if ni > lastindex(ff)
    resize!(ff, ni)
  end
  yy = zero(UInt32)
  if getfield(BUF, :xs) == true
    @inbounds for ib = 1:ni
      yy  = UInt32(buf[4*ib-3]) << 24
      yy |= UInt32(buf[4*ib-2]) << 16
      yy |= UInt32(buf[4*ib-1]) << 8
      yy |= UInt32(buf[4*ib])
      ff[ib] = yy
    end
  else
    @inbounds for il = 1:ni
      yy  = UInt32(buf[4*il]) << 24
      yy |= UInt32(buf[4*il-1]) << 16
      yy |= UInt32(buf[4*il-2]) << 8
      yy |= UInt32(buf[4*il-3])
      ff[il] = yy
    end
  end

  k = zero(Int64)
  x0 = zero(Float32)
  xn = zero(Float32)
  a = zero(UInt8)
  b = zero(UInt8)
  c = zero(UInt8)
  d = zero(UInt8)
  fq = zero(Float32)
  m = zero(UInt8)
  p = zero(UInt32)
  q = zero(Int32)
  u = zero(UInt32)
  y = zero(UInt32)
  z = zero(UInt32)
  χ = zero(UInt32)
  r = zero(Int64)
  for i = 1:nc
    z = getindex(ff, 1+r)
    for j = 1:16
      χ = getindex(ff, j+r)
      y = (z >> steim[j]) & 0x00000003
      if y == 0x00000001
        a = 0x00
        b = 0x08
        c = 0x04
      elseif BUF.fmt == 0x0a
        a = 0x00
        if y == 0x00000002
          b = 0x10
          c = 0x02
        elseif y == 0x00000003
          b = 0x20
          c = 0x01
        end
      else
        p = χ >> 0x0000001e
        if y == 0x00000002
          a = 0x02
          if p == 0x00000001
            b = 0x1e
            c = 0x01
          elseif p == 0x00000002
            b = 0x0f
            c = 0x02
          elseif p == 0x00000003
            b = 0x0a
            c = 0x03
          end
        elseif y == 0x00000003
          if p == 0x00000000
            a = 0x02
            b = 0x06
            c = 0x05
          elseif p == 0x00000001
            a = 0x02
            b = 0x05
            c = 0x06
          else
            a = 0x04
            b = 0x04
            c = 0x07
          end
        end
      end
      if y != 0x00000000
        u = χ << a
        m = zero(UInt8)
        d = 0x20 - b
        while m < c
          k = k + 1
          q = signed(u)
          q >>= d
          fq = Float32(q)
          setindex!(x, fq, k)
          m = m + 0x01
          u <<= b
        end
      end
      if i == 1
        if j == 2
          x0 = Float32(signed(χ))
        elseif j == 3
          xn = Float32(signed(χ))
        end
      end
    end
    r = r+16
  end

  if BUF.wo != 0x01
    vx = view(getfield(BUF, :x), 1:k)
    reverse!(vx)
  end
  setindex!(x, x0, 1)

  # Cumsum by hand
  xa = copy(x0)
  @inbounds for i1 = 2:k
    xa = xa + getindex(x, i1)
    setindex!(x, xa, i1)
  end

  # Check data values
  if isapprox(getindex(x, k), xn) == false
    println(stdout, string("RDMSEED: data integrity -- Steim-",
                            getfield(BUF, :fmt) - 0x09, " sequence #",
                            String(copy(getfield(BUF, :seq))),
                            " integrity check failed, last_data=",
                            getindex(getfield(BUF, :x), k),
                            ", should be xn=", xn))
  end
  setfield!(BUF, :k, k)
  return nothing
end


# load required tensor flow functionality

ENV["PYTHON"] = "/usr/bin/python3.6"

using Pkg
Pkg.build("PyCall")

using PyCall

py"""
import sys
sys.path.insert(0, "/home/singhk/learning-odometry/")
"""
# pushfirst!(PyVector(pyimport("sys")."path"), "/home/singhk/learning-odometry/" )

tf = pyimport("tensorflow")

tf.keras.models.load_model

Base.cd(ENV["USER"]*"/learning-odometry/")
mmodel = tf.keras.models.load_model("original_network0.h5")

mmodel.predict

A = [rand(4) for i in 1:25]

mmodel.predict([[A]])
# 1×2 Array{Float32,2}:
#  0.0169379  1.58055

models = []
# 0-element Array{Any,1}

using ProgressMeter

@showprogress "loading models" for i in 0:9
  push!(models, tf.keras.models.load_model("original_network$i.h5"))
end

models[1].predict([[A]])

preds = map(x->x.predict([[A]]), models)

## Using aggregate python functions

py"""
import sys
sys.path.insert(0, "/home/singhk/learning-odometry/")
"""

predsingle = pyimport("queryAllModels")["predictSinglePose"]

predsingle(A)
# returns  10×3 Array{Real,2}:


# Full Training Test (needs GPU for reasonable training time)using MuZero
using MuZero
import Zygote
using Grids
include("../../examples/tictactoe/state.jl")
include("../../examples/tictactoe/game.jl")
include("./old.jl")
include("./inference_tests.jl")
using Base.Threads: @spawn, @threads
config = make_tictactoe_config()
config.batch_size = 16
config.num_actors = 3
config.weight_decay = 1e-4
config.lr_init=0.002f0
config.save_interval = 10000
config.num_simulations = 100
config.num_unroll_steps = 1
losses, dynamics_stats = [], []

p_tests = getpolicytests(TicTacToe)
p_tests.period = 1000

policy_perfs = [];
function calc_policy_perf(l)
    push!(policy_perfs, policy_perf(l.test))
end

test_cb = LearnerCallback(
    "test_cb",
    calc_policy_perf,
    period=1000
)

learner = Learner(
    TicTacToe;
    config=config,
    dir="output",
    cb=[test_cb],
    tests=[p_tests]
)

#selfplay!(learner; limit=config.batch_size)
# @spawn run_selfplay!(config, storage, replay_buffer; limit=33)
# @spawn run_selfplay!(config, storage, replay_buffer; limit=33)
# @spawn run_selfplay!(config, storage, replay_buffer; limit=34)
# rand(1:length(replay_buffer))

for i = 1:(config.num_actors)
    @spawn selfplay!(learner)
end
learner.replay_buffer |> length

#cb = [(1000, 1, [calc_policy_perf]),]

#config.training_steps = 3
#train_network!(learner, losses=losses, callbacks=cb)

config.training_steps = 2000
@show @time fit!(learner; steps=3)
@spawn fit!(learner)
#@spawn train_network!(learner, losses=losses, callbacks=cb)
#@spawn train_network!(config, storage, replay_buffer, losses=losses, callbacks=cb)

@show 1
learner.stats.losses |> length
@show 2


module InternetTime

TIMEZONE_OFFSETS = Dict("UT" => Dates.Hour(0),
                        "GMT" => Dates.Hour(0),
                        "EST" => Dates.Hour(-5),
                        "EDT" => Dates.Hour(-4),
                        "CST" => Dates.Hour(-6),
                        "CDT" => Dates.Hour(-5),
                        "MST" => Dates.Hour(-7),
                        "MDT" => Dates.Hour(-6),
                        "PST" => Dates.Hour(-8),
                        "PDT" => Dates.Hour(-7),
                        "Z" => Dates.Hour(0),
                        "A" => Dates.Hour(-1),
                        "M" => Dates.Hour(-12),
                        "N" => Dates.Hour(1),
                        "Y" => Dates.Hour(+12),
                        )

function __init__()
    # Base extension needs to happen everytime the module is loaded (issue #24)
    Dates.CONVERSION_SPECIFIERS['I'] = RFC822TimeZone
    Dates.CONVERSION_SPECIFIERS['i'] = RFC822TimeZone
    Dates.CONVERSION_DEFAULTS[RFC822TimeZone] = ""
    Dates.CONVERSION_TRANSLATIONS[RFC822DateTime] = (
        Year, Month, Day, Hour, Minute, Second, RFC822DateTime
    )

    global ISOZonedDateTimeFormat = DateFormat("yyyy-mm-ddTHH:MM:SS.ssszzz")
end

# No parsing of MIME type headers, so add a helper function or two
# Dates.RFC1123Format is invalid, as it doesn't account for timezone, so try to handle that.
function get_response_timestamp(headers)
    if haskey(headers, "Date")
        header_value = headers["Date"]
        timezone = nothing
        datetime = nothing
        while length(header_value) > 0 && datetime == nothing
            try
                datetime = DateTime(header_value, Dates.RFC1123Format)
                if timezone != nothing
                    # convert timezone to offset
                    if timezone in keys(TIMEZONE_OFFSETS)
                        offset = TIMEZONE_OFFSETS[timezone]
                        datetime = datetime + offset
                    elseif (m = match(r"^(\+|\-)([0-9]{2})([0-9]{2})$", timezone)) != nothing
                        hours = parse(Int64, m.captures[2])
                        minutes = parse(Int64, m.captures[3])
                        offset = Dates.Hour(hours) + Dates.Minute(minutes)
                        if m.captures[1] == "-"
                            datetime = datetime - offset
                        else
                            datetime = datetime + offset
                        end
                    else
                        println("unknown timezone: ", timezone)
                    end
                end
                return datetime
            catch x
                if isa(x, ArgumentError)
                    parts = split(header_value, " ")
                    if timezone == nothing && length(parts) > 1
                        timezone = parts[end]
                        header_value = join(parts[1:end-1], " ")
                    else
                        # only allowed 1 retry
                        rethrow()
                    end
                end
            end
        end
    end
    return nothing
end

end # module


###################################################################################
# Antony Schutz 2015, ANR - POLCA - 2016
###################################################################################
## Data Type
###################################################################################
# PAINTER Data Type
#
# Structure containing all user data and oifits data
# OIDATA::PAINTER_Input
#
###################################################################################
# Initialise structure with nothing
###################################################################################
type PAINTER_Input
    PlotFct::Function               # user defined plot function
    Folder::String             # Folder of OIFITS/FITS Files
    FilesName::Array{String}   # name of the files present in Folder
    indfile::Array{Int,1}           # index of file used
    indwvl::Array{Int,1}            # index of wavelength used
    wvl::Array{Float64}             # mean value of anayzed wavelength
    U::Array{Float64}               # U spatial frequencies
    V::Array{Float64}               # V spatial frequencies
    P::Array{Float64}               # Matrix of V2
    W::Array{Float64}               # Matrix of V2err
    Xi::Dict                        # Dictionary of Vector of phases difference
    K::Dict                         # Dictionary of Vector of phases difference error
    Closure_index::Array{Int}       # index of phases closure
    nb::Int                         # number of bases (spatiale frequencies per wavelength)
    nw::Int                         # number of wavelength
    nx::Int                         # side size of image in pixels
    FOV::Real                       # Field Of View in arcsecond
    lambda_spat::Real               # From here to end: admm variables
    lambda_spec::Real
    lambda_L1::Real
    rho_y::Real

    rho_y_gamma::Real
    rho_y_xi::Real

    rho_spat::Real
    rho_spec::Real
    rho_ps::Real
    alpha::Real
    beta::Real
    eps1::Real
    eps2::Real
    epsilon::Real
    mask3D::Array                   # support constraint array
    xinit3D::Array                  # initial estimate array
    Wvlt::Array                     # list of wavelets basis
    T3::Array{Float64}              # Matrix of phases closure
    T3err::Array{Float64}           # Matrix of phases closure error
    DP::Array{Float64}              # Matrix of Differential phases
    DPerr::Array{Float64}           # Matrix of Differential phases error
    isDP::Int                       # To know if there is DP in data
    dptype::String             # way to construct diff phases
    dpprm::Int                      # horizon for sliding dp or index of reference
    baseNb::Dict
    orderedCluster::Dict
    flux::Array
    norm::Real
    # Kbefore::Dict
end
# Structure containing all data which are modified during admm
# PDATA::PAINTER_Data
type PAINTER_Data
    eta::Real
    plan::Array
    F3D::Array
    H::Dict
    M::Array
    x::Array
    z::Array
    tau_s::Array
    w::Array
    v::Array
    r::Array
    tau_w::Array
    tau_v::Array
    tau_r::Array
    Fx::Array
    tau_xc::Array
    tau_pwc::Array
    tau_xic::Array
    yc::Array
    y_v2::Array
    y_phi::Array
    crit1::Array
    crit2::Array
    ind::Int
    CountPlot::Int
    count::Int

end

function painterinputinit()
    return PAINTER_Input( painterplotfct, "", [], [], [], [], [], [], []
                         , [], Dict{}(), Dict{}(), [], 0., 0., 0., 0., 0., 0., 0., 0.
                         , 0., 0.
                         , 0., 0., 0., 0., 0., 0., 0., 0., [], [], []
                         , [], [], [], [], 0, "", 0, Dict{}(), Dict{}(),[],0) # [],Dict{}())
end
function painterdatainit()

  return PAINTER_Data(0., [], [], Dict{}(), [], [], [], [], [], []
                      , [], [], [], [], [], [], [], [], [], []
                      , [], Float64[], Float64[], 0, 0, 1)
end


# wrappers of the fathom c library
# currently only Linux support for Astellarn

const TB_RESULT_FAILED = 0xFFFFFFFF

# constants defined in fathom
const TB_LOSS = 0
const TB_BLESSED_LOSS = 1
const TB_DRAW = 2
const TB_CURSED_WIN = 3
const TB_WIN = 4

const TB_PROMOTES_NONE = 0
const TB_PROMOTES_QUEEN = 1
const TB_PROMOTES_ROOK = 2
const TB_PROMOTES_BISHOP = 3
const TB_PROMOTES_KNIGHT = 4

const TB_RESULT_WDL_MASK = 0x0000000F
const TB_RESULT_TO_MASK = 0x000003F0
const TB_RESULT_FROM_MASK = 0x0000FC00
const TB_RESULT_PROMOTES_MASK = 0x00070000
const TB_RESULT_EP_MASK = 0x00080000
const TB_RESULT_DTZ_MASK = 0xFFF00000
const TB_RESULT_WDL_SHIFT = 0
const TB_RESULT_TO_SHIFT = 4
const TB_RESULT_FROM_SHIFT = 10
const TB_RESULT_PROMOTES_SHIFT = 16
const TB_RESULT_EP_SHIFT = 19
const TB_RESULT_DTZ_SHIFT = 20

function TB_GET_WDL(res::UInt32)
    Int((res & TB_RESULT_WDL_MASK) >> TB_RESULT_WDL_SHIFT)
end

function TB_GET_TO(res::UInt32)
    Int(((res & TB_RESULT_TO_MASK) >> TB_RESULT_TO_SHIFT) + 1)
end

function TB_GET_FROM(res::UInt32)
    Int(((res & TB_RESULT_FROM_MASK) >> TB_RESULT_FROM_SHIFT) + 1)
end

function TB_GET_PROMOTES(res::UInt32)
    Int((res & TB_RESULT_PROMOTES_MASK) >> TB_RESULT_PROMOTES_SHIFT)
end

# check output of this function
function TB_GET_EP(res::UInt32)
    Int(((res & TB_RESULT_EP_MASK) >> TB_RESULT_EP_SHIFT) + 1)
end

function TB_GET_DTZ(res::UInt32)
    Int((res & TB_RESULT_DTZ_MASK) >> TB_RESULT_DTZ_SHIFT)
end

function tb_init(syzygypath::S)::Bool where S <: Union{String, SubString}
    return ccall((:tb_init, FATHOM_PATH), UInt8, (Ptr{UInt8},), syzygypath)
end

function tb_free()
    return ccall((:tb_free, FATHOM_PATH), Cvoid, ())
end

# We can't call static inline function :tb_probe_wdl, so we use :tb_probe_wdl_impl as a workaround.
# Because of the above, care is taken to ensure we conform to syzygy rules.
function tb_probe_wdl(board::Board)::UInt32
    if iszero(board.enpass) === false
        return TB_RESULT_FAILED
    elseif iszero(board.castling) === false
        return TB_RESULT_FAILED
    elseif iszero(board.halfmovecount) === false
        return TB_RESULT_FAILED
    else
        return ccall((:tb_probe_wdl_impl, FATHOM_PATH), Cuint, (UInt64, UInt64, UInt64, UInt64, UInt64, UInt64, UInt64, UInt64, UInt32, UInt8,),
            board[WHITE].val, board[BLACK].val, board[KING].val, board[QUEEN].val, board[ROOK].val, board[BISHOP].val, board[KNIGHT].val, board[PAWN].val,
            0, board.turn == WHITE ? 1 : 0)
    end
end

# We can't call static inline function :tb_probe_root, so we use :tb_probe_root_impl as a workaround.
# Because of the above, care is taken to ensure we conform to syzygy rules.
function tb_probe_root(board::Board)::UInt32
    if iszero(board.castling) === false
        return TB_RESULT_FAILED
    else
        return ccall((:tb_probe_root_impl, FATHOM_PATH), Cuint, (UInt64, UInt64, UInt64, UInt64, UInt64, UInt64, UInt64, UInt64, UInt32, UInt32, UInt8, UInt32, ),
            board[WHITE].val, board[BLACK].val, board[KING].val, board[QUEEN].val, board[ROOK].val, board[BISHOP].val, board[KNIGHT].val, board[PAWN].val,
            board.halfmovecount, board.enpass, board.turn == WHITE ? 1 : 0, 0)
    end
end

# Called at the root node if a TB result is returned
function interpret_syzygy(thread::Thread, res::UInt32)
    thread.ss.tbhits += 1
    _eval = TB_GET_WDL(res)
    if iszero(_eval)
        eval = -MATE
    elseif 1 <= _eval <= 3 # blessed / cursed loss and wins are draws
        eval = 0
    else
        eval = MATE
    end
    move_from = TB_GET_FROM(res)
    move_to = TB_GET_TO(res)
    promotion = TB_GET_PROMOTES(res)
    if promotion !== TB_PROMOTES_NONE
        if promotion == TB_PROMOTES_QUEEN
            push!(thread.pv[1], Move(move_from, move_to, __QUEEN_PROMO))
            return eval
        elseif promotion == TB_PROMOTES_ROOK
            push!(thread.pv[1], Move(move_from, move_to, __ROOK_PROMO))
            return eval
        elseif promotion == TB_PROMOTES_BISHOP
            push!(thread.pv[1], Move(move_from, move_to, __BISHOP_PROMO))
            return eval
        elseif promotion == TB_PROMOTES_KNIGHT
            push!(thread.pv[1], Move(move_from, move_to, __KNIGHT_PROMO))
            return eval
        end
    else
        clear!(thread.pv[1])
        push!(thread.pv[1], Move(move_from, move_to, __NORMAL_MOVE))
        return eval
    end
end


function construct_network!(
    optimization_container::OptimizationContainer,
    sys::PSY.System,
    ::Type{CopperPlatePowerModel},
    template::OperationsProblemTemplate,
)
    buses = PSY.get_components(PSY.Bus, sys)
    bus_count = length(buses)

    if get_balance_slack_variables(optimization_container.settings)
        add_slacks!(optimization_container, CopperPlatePowerModel)
    end
    copper_plate(optimization_container, :nodal_balance_active, bus_count)
    return
end

function construct_network!(
    optimization_container::OptimizationContainer,
    sys::PSY.System,
    ::Type{AreaBalancePowerModel},
    template::OperationsProblemTemplate,
)
    area_mapping = PSY.get_aggregation_topology_mapping(PSY.Area, sys)
    branches = get_available_components(PSY.Branch, sys)
    if get_balance_slack_variables(optimization_container.settings)
        throw(
            IS.ConflictingInputsError(
                "Slack Variables are not compatible with AreaBalancePowerModel",
            ),
        )
    end

    area_balance(optimization_container, :nodal_balance_active, area_mapping, branches)
    return
end

function construct_network!(
    optimization_container::OptimizationContainer,
    sys::PSY.System,
    ::Type{StandardPTDFModel},
    template::OperationsProblemTemplate,
)
    buses = PSY.get_components(PSY.Bus, sys)
    ptdf = get_PTDF(optimization_container)

    if ptdf === nothing
        throw(ArgumentError("no PTDF matrix supplied"))
    end

    if get_balance_slack_variables(optimization_container.settings)
        add_slacks!(optimization_container, StandardPTDFModel)
    end

    copper_plate(optimization_container, :nodal_balance_active, length(buses))
    return
end

function construct_network!(
    optimization_container::OptimizationContainer,
    sys::PSY.System,
    ::Type{T},
    template::OperationsProblemTemplate,
) where {T <: PTDFPowerModel}
    construct_network!(
        optimization_container,
        sys,
        T,
        template;
        instantiate_model = instantiate_nip_ptdf_expr_model,
    )

    add_pm_expr_refs!(optimization_container, T, sys)
    copper_plate(
        optimization_container,
        :nodal_balance_active,
        length(PSY.get_components(PSY.Bus, sys)),
    )

    return
end

function construct_network!(
    optimization_container::OptimizationContainer,
    sys::PSY.System,
    ::Type{T},
    template::OperationsProblemTemplate;
    instantiate_model = instantiate_nip_expr_model,
) where {T <: PM.AbstractPowerModel}
    if T in UNSUPPORTED_POWERMODELS
        throw(
            ArgumentError(
                "$(T) formulation is not currently supported in PowerSimulations",
            ),
        )
    end

    if get_balance_slack_variables(optimization_container.settings)
        add_slacks!(optimization_container, T)
    end

    @debug "Building the $T network with $instantiate_model method"
    powermodels_network!(optimization_container, T, sys, template, instantiate_model)
    add_pm_var_refs!(optimization_container, T, sys)
    add_pm_con_refs!(optimization_container, T, sys)

    return
end

function construct_network!(
    optimization_container::OptimizationContainer,
    sys::PSY.System,
    ::Type{T},
    template::OperationsProblemTemplate;
    instantiate_model = instantiate_bfp_expr_model,
) where {T <: PM.AbstractBFModel}
    if T in UNSUPPORTED_POWERMODELS
        throw(
            ArgumentError(
                "$(T) formulation is not currently supported in PowerSimulations",
            ),
        )
    end

    get_balance_slack_variables(optimization_container.settings) &&
        add_slacks!(optimization_container, T)

    @debug "Building the $T network with $instantiate_model method"
    powermodels_network!(optimization_container, T, sys, template, instantiate_model)
    add_pm_var_refs!(optimization_container, T, sys)
    add_pm_con_refs!(optimization_container, T, sys)
    return
end

function construct_network!(
    optimization_container::OptimizationContainer,
    sys::PSY.System,
    ::Type{T},
    template::OperationsProblemTemplate;
    instantiate_model = instantiate_vip_expr_model,
) where {T <: PM.AbstractIVRModel}
    if T in UNSUPPORTED_POWERMODELS
        throw(
            ArgumentError(
                "$(T) formulation is not currently supported in PowerSimulations",
            ),
        )
    end

    if get_balance_slack_variables(optimization_container.settings)
        add_slacks!(optimization_container, T)
    end

    @debug "Building the $T network with $instantiate_model method"
    powermodels_network!(optimization_container, T, sys, template, instantiate_model)
    add_pm_var_refs!(optimization_container, T, sys)
    add_pm_con_refs!(optimization_container, T, sys)
    return
end


"""
Validation PSSE/TGOV1:
This case study defines a three bus system with an infinite bus, GENROU+AC1A+TGOV1 and a load.
The fault drop the line connecting the infinite bus and GENROU.
"""

##################################################
############### SOLVE PROBLEM ####################
##################################################

raw_file = joinpath(dirname(@__FILE__), "benchmarks/psse/TGOV1/ThreeBusMulti.raw")
dyr_file = joinpath(dirname(@__FILE__), "benchmarks/psse/TGOV1/ThreeBus_TGOV1.dyr")
csv_file = joinpath(dirname(@__FILE__), "benchmarks/psse/TGOV1/TEST_TGOV1.csv")

#Construct system
sys = System(raw_file, dyr_file);

path = (joinpath(pwd(), "test-psse-tgov1"))
!isdir(path) && mkdir(path)
try
    sim = Simulation!(
        ImplicitModel,
        sys, #system
        path,
        (0.0, 20.0), #time span
        BranchTrip(1.0, "BUS 1-BUS 2-i_1"), #Type of Fault
    )

    #Solve problem in equilibrium
    execute!(sim, IDA(), dtmax = 0.005, saveat = 0.005)

    #Obtain small signal results for initial conditions
    #NOT WORKING DUE TO TYPES ON EXECUTE
    #small_sig = small_signal_analysis(sim)

    series = get_state_series(sim, ("generator-102-1", :δ))
    t = series[1]
    δ = series[2]

    #Obtain PSSE results
    t_psse, δ_psse = get_csv_delta(csv_file)

    diff = [0.0]
    res = get_init_values_for_comparison(sim)
    for (k, v) in test_psse_tgov1_init
        diff[1] += LinearAlgebra.norm(res[k] - v)
    end
    #Test Initial Condition
    @test (diff[1] < 1e-3)
    #Test Solution DiffEq
    @test sim.solution.retcode == :Success
    #Test Small Signal
    #@test small_sig.stable
    #Test Transient Simulation Results
    # PSSE results are in Degrees
    @test LinearAlgebra.norm(δ - (δ_psse .* pi / 180), Inf) <= 1e-2
    @test LinearAlgebra.norm(t - round.(t_psse, digits = 3)) == 0.0
finally
    @info("removing test files")
    rm(path, force = true, recursive = true)
end


# interoperability with Distributions

import Distributions: Distribution,Univariate,Continuous,Triweight,AbstractMixtureModel,
                      cdf,pdf,quantile,logcdf,logpdf,ccdf,cquantile,invlogcdf,invlogccdf

typealias TriDist Triweight
typealias MixDist AbstractMixtureModel{Univariate,Continuous}
typealias UniDist Distribution{Univariate,Continuous}

if (isdefined(Main,:PropagateState) && Main.PropagateState==true)

pdf(d::MixDist, x::Float65) =
    getstate(x) ? setstate(pdf(d, convert(Float64,x))) : pdf(d, convert(Float64,x))
logpdf(d::MixDist, x::Float65) =
    getstate(x) ? setstate(logpdf(d, convert(Float64,x))) : logpdf(d, convert(Float64,x))

for op in (:cdf,:pdf,:quantile,:logcdf,:logpdf,:ccdf,:cquantile,:invlogcdf,:invlogccdf)
    @eval ($op)(d::TriDist, x::Float65) =
        getstate(x) ? setstate(($op)(d, convert(Float64,x))) : ($op)(d, convert(Float64,x))
    @eval ($op)(d::UniDist, x::Float65) =
        getstate(x) ? setstate(($op)(d, convert(Float64,x))) : ($op)(d, convert(Float64,x))
end

else # do not propagate the set state

pdf(d::MixDist, x::Float65) = pdf(d, convert(Float64,x))
logpdf(d::MixDist, x::Float65) = logpdf(d, convert(Float64,x))

for op in (:cdf,:pdf,:quantile,:logcdf,:logpdf,:ccdf,:cquantile,:invlogcdf,:invlogccdf)
    @eval ($op)(d::TriDist, x::Float65) = ($op)(d, convert(Float64,x))
    @eval ($op)(d::UniDist, x::Float65) = ($op)(d, convert(Float64,x))
end

end # PropagateState


const SuffixVector = Vector{Vector{DNA}}

mutable struct RNASuffixArray{A<:BioSequences.Alphabet,I<:Integer,K<:Integer}
   sai::Vector{I}
   meta::Vector{K}
   depth::SuffixVector
   species::Vector{SuffixVector}
   length::Int
   isphylo::Bool

   function RNASuffixArray( seq::BioSequences.Sequence, len::Int, metadata::K=one(K))
      sai   = SuffixArrays.suffixsort( String(seq) ).index + one(I)
      meta  = K[metadata for i in 1:length(sai)]
      depth = SuffixVector(len)
      for i in 1:len
         depth[i] = map(x->x+i-1 > length(seq) ? gap(A) : seq[x+i-1], sai)
      end
      empty = Vector{SuffixVector}()
      new( sai, meta, depth, empty, len, false )
   end

   function RNASuffixArray( maf::MAFRecord, tree::PhyloTree, len::Int, metadata::K=one(K))
      deletegaps!( maf )
      for i in 2:length(maf.species)
         @assert length(maf.species[i].sequence) == length(maf.species[1].sequence)
      end
      sai   = SuffixArrays.suffixsort( String(maf[1].sequence) ).index + one(I)
      meta  = K[metadata for i in 1:length(sai)]
      depth = SuffixVector(len)
      for i in 1:len
         depth[i] = map(x->x+i-1 > length(maf[1].sequence) ? gap(A) : maf.species[1].sequence[x+i-1], sai)
      end
      species = Vector{SuffixVector}(length(tree.order)-1)
      m = 2
      for i in 1:length(species)
         curind = tree.index[maf[m].name] - 1
         species[i] = SuffixVector(len)
         if i != curind
            for j in 1:len
               species[i][j] = [gap(A) for i in 1:length(sai)]
            end
         else
           for j in 1:len
              species[i][j] = map(x->x+j-1 > length(maf[m].sequence) ? gap(A) : maf[m].sequence[x+j-1], sai)
           end
           m < length(maf) && (m += 1)
         end
      end
      new( sai, meta, depth, species, len, true )
   end
end

function Base.:(<=)( a::SuffixVector, ai, b::SuffixVector, bi )
   for i in 1:length(a)
      if a[i][ai] < b[i][bi]
         return true
      elseif a[i][ai] > b[i][bi]
         return false
      end
   end
   true
end

function Base.push!{A,I,K}( suf::RNASuffixArray{A,I,K}, seq::BioSequences.Sequence;
                            metadata::K=convert(K,maximum(suf.meta)+1) )

   suf.isphylo && error("Cannot push! single BioSequence when isphylo is enabled!")

   # suffix sort
   const suftmp  = RNASuffixArray{A,I,K}( seq, suf.length, metadata )

   # initialize
   const newsai    = zeros(I, length(suf.sai) + length(suftmp.sai) )
   const newmeta   = zeros(K, length(suf.sai) + length(suftmp.sai) )
   const newdepth  = SuffixVector(suf.length)
   for i in 1:suf.length
      newdepth[i] = Vector{DNA}(length(suf.sai) + length(suftmp.sai))
   end

   # merge
   i,j,k = 1,1,1
   while i <= length(suf.sai) || j <= length(suftmp.sai)
      if j > length(suftmp.sai) || (i <= length(suf.sai) && # does suf have next least suffix?
                                    <=( suf.depth, i, suftmp.depth, j ))
         newsai[k]  = suf.sai[i]
         newmeta[k] = suf.meta[i]
         for d in 1:suf.length
            newdepth[d][k] = suf.depth[d][i]
         end
         i += 1
      else # suftmp has least suffix
         newsai[k]  = suftmp.sai[j]
         newmeta[k] = suftmp.meta[j]
         for d in 1:suf.length
            newdepth[d][k] = suftmp.depth[d][j]
         end
         j += 1
      end
      k += 1
   end

   # set new values of suf
   suf.sai     = newsai
   suf.meta    = newmeta
   suf.depth   = newdepth
   suf
end

function Base.push!{A,I,K}( suf::RNASuffixArray{A,I,K}, maf::MAFRecord, tree::PhyloTree;
                            metadata::K=convert(K,maximum(suf.meta)+1) )

   suf.isphylo || error("Cannot push! MAF block when isphylo isn't enabled!")
   length(suf.species) == length(tree.order)-1 || error("# species in tree not equal to suffix array!")

   # suffix sort
   const suftmp  = RNASuffixArray{A,I,K}( maf, tree, suf.length, metadata )

   # initialize
   const newsai     = zeros(I, length(suf.sai) + length(suftmp.sai) )
   const newmeta    = zeros(K, length(suf.sai) + length(suftmp.sai) )
   const newdepth   = SuffixVector(suf.length)
   const newspecies = Vector{SuffixVector}(length(tree.order)-1)

   for i in 1:suf.length
      newdepth[i] = Vector{DNA}(length(suf.sai) + length(suftmp.sai))
   end
   for i in 1:length(newspecies)
      newspecies[i] = SuffixVector(suf.length)
      for j in 1:suf.length
         newspecies[i][j] = Vector{DNA}(length(suf.sai) + length(suftmp.sai))
      end
   end

   # merge
   i,j,k = 1,1,1
   while i <= length(suf.sai) || j <= length(suftmp.sai)
      if j > length(suftmp.sai) || (i <= length(suf.sai) && # does suf have next least suffix?
                                    <=( suf.depth, i, suftmp.depth, j ))
         newsai[k]  = suf.sai[i]
         newmeta[k] = suf.meta[i]
         for d in 1:suf.length
            newdepth[d][k] = suf.depth[d][i]
         end
         for s in 1:length(suf.species)
            for d in 1:suf.length
               newspecies[s][d][k] = suf.species[s][d][i]
            end
         end
         i += 1
      else # suftmp has least suffix
         newsai[k]  = suftmp.sai[j]
         newmeta[k] = suftmp.meta[j]
         for d in 1:suf.length
            newdepth[d][k] = suftmp.depth[d][j]
         end
         for s in 1:length(suf.species)
            for d in 1:suf.length
               newspecies[s][d][k] = suftmp.species[s][d][j]
            end
         end
         j += 1
      end
      k += 1
   end

   # set new values of suf
   suf.sai     = newsai
   suf.meta    = newmeta
   suf.depth   = newdepth
   suf.species = newspecies
   suf
end

struct RNAGenomeCoord
   name::String
   offset::Int
   strand::Bool
   length::Int
end

mutable struct RNADuplexArray{A,I,K}
   fwd::RNASuffixArray{A,I,K}
   rev::RNASuffixArray{A,I,K}
   seqs::Vector{BioSequences.Sequence}
   coords::Vector{RNAGenomeCoord}
   depth::Int
   isphylo::Bool

   function RNADuplexArray( seq::BioSequences.Sequence, depth::Int;
                            seqname::String="chr", genomepos::Int=1, strand::Bool=true )
      fwd    = RNASuffixArray{A,I,K}( seq, depth )
      rev    = RNASuffixArray{A,I,K}( reverse(seq), depth )
      seqs   = BioSequences.Sequence[seq]
      coords = RNAGenomeCoord[RNAGenomeCoord(seqname, genomepos, strand, length(seq))]
      return new( fwd, rev, seqs, coords, depth, false )
   end

   function RNADuplexArray( left::BioSequences.Sequence, right::BioSequences.Sequence, depth::Int;
                            left_seqname::String="chr", right_seqname::String="chr",
                            left_genomepos::Int=1, right_genomepos::Int=1,
                            left_strand::Bool=true, right_strand::Bool=true )
      fwd    = RNASuffixArray{A,I,K}( left,  depth, convert(K, 1) )
      rev    = RNASuffixArray{A,I,K}( reverse(right), depth, convert(K, 2) )
      seqs   = BioSequences.Sequence[left, right]

      coords = RNAGenomeCoord[RNAGenomeCoord(left_seqname, left_genomepos, left_strand, length(left)),
                              RNAGenomeCoord(right_seqname, right_genomepos, right_strand, length(right))]
      return new( fwd, rev, seqs, coords, depth, false )
   end

   function RNADuplexArray( seqs::Vector{BioSequences.Sequence}, depth::Int;
                            seqnames::Vector{String}=String["chr" for i in 1:length(seqs)],
                            genomepos::Vector{Int}=ones(Int, length(seqs)),
                            strands::Vector{Bool}=trues(length(seqs)))
      length(seqs) >= 1 || error("Cannot build RNADuplexArray from seq vector of length 0!")
      length(seqs) == length(seqnames) == length(genomepos) == length(strands) || error("all RNADuplexArray input vectors must be equal size!")
      fwd    = RNASuffixArray{A,I,K}( seqs[1], depth )
      rev    = RNASuffixArray{A,I,K}( reverse(seqs[1]), depth )
      coords = RNAGenomeCoord[RNAGenomeCoord(seqnames[i], genomepos[i], strands[i], length(seqs[i])) for i in 1:length(seqs)]
      for i in 2:length(seqs)
         push!( fwd, seqs[i] )
         push!( rev, reverse(seqs[i]) )
      end
      return new( fwd, rev, seqs, coords, depth, false )
   end

   function RNADuplexArray( maf::MAFRecord, tree::PhyloTree, depth::Int )
      refmaf = maf[1]
      fwd    = RNASuffixArray{A,I,K}( maf, tree, depth )
      reverse!( maf )
      rev    = RNASuffixArray{A,I,K}( maf, tree, depth )
      seqs   = BioSequences.Sequence[refmaf.sequence]
      coords = RNAGenomeCoord[RNAGenomeCoord(refmaf.chr, refmaf.position, refmaf.strand, length(refmaf.sequence))]
      return new( fwd, rev, seqs, coords, depth, true )
   end
end

function Base.push!{A,I,K}( rda::RNADuplexArray{A,I,K}, seq::BioSequences.Sequence;
                            seqname::String="chr", genomepos::Int=1, strand::Bool=true )
   rda.isphylo && error("Cannot push single sequence to RNADuplexArray with isphylo=true!")
   push!( rda.fwd, seq )
   push!( rda.rev, reverse(seq) )
   push!( rda.seqs, seq )
   push!( rda.coords, RNAGenomeCoord(seqname, genomepos, strand, length(seq)) )
end

function Base.push!{A,I,K}( rda::RNADuplexArray{A,I,K}, maf::MAFRecord, tree::PhyloTree )
   refmaf = maf[1]
   push!( rda.fwd, maf, tree )
   reverse!( maf )
   push!( rda.rev, maf, tree )
   push!( rda.seqs, refmaf.sequence )
   push!( rda.coords, RNAGenomeCoord(refmaf.chr, refmaf.position, refmaf.strand, length(refmaf.sequence)) )
end

tochar( strand::Bool ) = strand ? '+' : '-'

function positions{N <: NucleicAcid}( nucs::Vector{N}, lo::Int, hi::Int, alpha::Tuple{Vararg{N}} )
   map( x->searchsorted(nucs, x, lo, hi, Base.Order.ForwardOrdering()), alpha )
end

function traverse{A,I,K}( dsa::RNADuplexArray{A,I,K}, foldrange::UnitRange=3:typemax(Int);
                          bulge_max::Int=zero(Int), mismatch_max::Int=zero(Int),
                          raw_output::String="", minfold::Float64=-5.0,
                          single::PhyloTree=EMPTY_TREE, paired::PhyloTree=EMPTY_TREE)

   const pairs_idx      = index(A, pairs)
   const bulges_idx     = index(A, gaps)
   const mismatches_idx = index(A, mismatches)

   const alpha          = alphabet(A)

   const duplex         = RNADuplex()
   const deprange       = 1:dsa.depth
   const energy_max     = 0.01

   const intervals      = DuplexCollection{K}()

   if raw_output != ""
      const io = open( raw_output, "w" )
      const stream = ZlibDeflateOutputStream( io )
      const output = true
   else
      const output = false
   end

   @inline function _traverse( fdepth::Int64, rdepth::Int64,
                               frange::UnitRange{Int}, rrange::UnitRange{Int},
                               bulge_n::Int64, mismatch_n::Int64,
                               from_bulge::Bool, bulge_left::Bool,
                               from_mismatch::Bool )

      # obtain position range for current depth
      if fdepth <= dsa.fwd.length && rdepth <= dsa.rev.length
         const franges = positions( dsa.fwd.depth[fdepth], first(frange), last(frange), alpha )
         const rranges = positions( dsa.rev.depth[rdepth], first(rrange), last(rrange), alpha )
      else
         pop!(duplex)
         return
      end

      # recurse through valid watson-crick pairs
      for (l,r) in pairs_idx
         if length(franges[l]) >= 1 && length(rranges[r]) >= 1
            push!( duplex, convert(RNAPair, onehot(l), onehot(r)) )
            if output
               tab_write( stream, (depth, energy(duplex)))
            end
            if fdepth in deprange && rdepth in deprange && energy(duplex) < minfold
               @inbounds for ix in franges[l], jx in rranges[r]
                     const i = dsa.fwd.sai[ix]
                     const j = dsa.rev.sai[jx]
                     const fwd_entry = dsa.coords[ dsa.fwd.meta[ix] ]
                     const rev_entry = dsa.coords[ dsa.rev.meta[jx] ]
                     const ik = revoffset(i, fwd_entry.length)
                     const jk = revoffset(j, rev_entry.length)
                     const ifirst,ilast = fwd_entry.strand ? (i,i+fdepth-1) : (ik-fdepth+1,ik)
                     const jfirst,jlast = rev_entry.strand ? (jk-rdepth+1,jk) : (j,j+rdepth-1)
                     const fg = fwd_entry.offset-1
                     const rg = rev_entry.offset-1
                     fwd_entry.name == rev_entry.name && !(((rg+jfirst) - (fg+ilast)) - 1 in foldrange) && continue
                     if dsa.isphylo
                        evo = paired == EMPTY_TREE ? EvoDuplex(duplex, single, dsa, ix, jx) :
                                                     EvoDuplex(duplex, single, paired, dsa, ix, jx)
                        score(evo) < 0.0 && continue
                        push!( intervals, DuplexInterval( Interval(fwd_entry.name, fg+ifirst, fg+ilast, tochar(fwd_entry.strand), dsa.fwd.meta[ix]),
                                                          Interval(rev_entry.name, rg+jfirst, rg+jlast, tochar(rev_entry.strand), dsa.rev.meta[jx]),
                                                          evo ) )
                     else
                        push!( intervals, DuplexInterval( Interval(fwd_entry.name, fg+ifirst, fg+ilast, tochar(fwd_entry.strand), dsa.fwd.meta[ix]),
                                                          Interval(rev_entry.name, rg+jfirst, rg+jlast, tochar(rev_entry.strand), dsa.rev.meta[jx]),
                                                          deepcopy(duplex) ) )
                     end
               end
            end
            _traverse(  fdepth + 1, rdepth + 1,
                        franges[l], rranges[r],
                        bulge_n, mismatch_n,
                        false, false, false )
         end
      end

      # recurse through bulges
      if bulge_n < bulge_max &&
         (bulge_n >= 1 ? from_bulge : !from_bulge) &&
         fdepth > 1 && rdepth > 1 && duplex.energy[end] < energy_max
         for (l,r) in bulges_idx
            if l == 0
               (from_bulge && !bulge_left) && continue # only bulge one way
               if length(rranges[r]) >= 1 && rdepth + 1 <= dsa.rev.length
                  push!( duplex, convert(RNABulge, zero(UInt8), onehot(r)) )
                  _traverse( fdepth, rdepth + 1,
                             frange, rranges[r],
                             bulge_n + 1, mismatch_n,
                             true, true, from_mismatch )
               end
            elseif r == 0
               (from_bulge && bulge_left) && continue
               if length(franges[l]) >= 1 && fdepth + 1 <= dsa.fwd.length
                   push!( duplex, convert(RNABulge, onehot(l), zero(UInt8)) )
                   _traverse( fdepth + 1, rdepth,
                              franges[l], rrange,
                              bulge_n + 1, mismatch_n,
                              true, false, from_mismatch )
               end
            end
         end
      end

      # recurse through mismatches
      if mismatch_n < mismatch_max &&
         (mismatch_n >= 1 ? from_mismatch : !from_mismatch) &&
         fdepth > 1 && rdepth > 1 &&
         !from_bulge && duplex.energy[end] < energy_max &&
         fdepth < dsa.fwd.length && rdepth < dsa.rev.length
         for (l,r) in mismatches_idx
            if length(franges[l]) >= 1 && length(franges[r]) >= 1
                push!( duplex, convert(RNAMismatch, onehot(l), onehot(r)) )
                _traverse( fdepth + 1, rdepth + 1,
                           franges[l], rranges[r],
                           bulge_n, mismatch_n + 1,
                           from_bulge, bulge_left, true )
            end
         end
      end

      pop!( duplex ) # clean up this level
      return
   end

   _traverse( one(Int), one(Int),
              1:length(dsa.fwd.sai), 1:length(dsa.rev.sai),
              zero(Int), zero(Int),
              false, false, false )

   if output
      close(stream)
      close(io)
   end

   intervals
end


function sample_data_and_build_tree!(trnidx::Vector{Int}, validx::Vector{Int}, fitted_values_all_data_this_vector_is_modified_by_build_tree::Vector{Float64}, candMatWOMaxValues::Array{Array{Float64,1},1}, mappings::Array{Array{String,1},1}, settings::ModelSettings, numerator::Array{Float64,1}, denominator::Array{Float64,1}, weight::Array{Float64,1}, features, sampleSizeCanBeNEGATIVE, abssampleSize, sampleVector, T_Uint8_or_UInt16)
# NOTE: this function will call build_tree!()
	# fitted_values_all_data_this_vector_is_modified_by_build_tree=zeros(numerator)
	if abs(settings.subsampling_prop) >= 1.0
		n = build_tree!(trnidx, validx, candMatWOMaxValues, mappings, settings, numerator, denominator, weight, features, fitted_values_all_data_this_vector_is_modified_by_build_tree, T_Uint8_or_UInt16)
		return n
	else
        reused_fitted_leafnr_vector = zeros(Int, length(weight))
		# do subsampling
		# sampleSizeCanBeNEGATIVE=convert(Int,round(settings.subsampling_prop*length(denominator)))
		# note: this can probably be done more efficiently (see also bagging)
		# abssampleSize,sampleVector,num,denom,w,numf,charf,ooBagnum,ooBagdenom,ooBagw,ooBagnumf,ooBagcharf,ooBagsize=initBoostrapSample(sampleSizeCanBeNEGATIVE,numerator,denominator,weight,charfeatures,numfeatures)
		unusedSamplePart = sampleData!(trnidx, sampleSizeCanBeNEGATIVE, sampleVector)
		# build tree	on subsample of data
			thistree = build_tree!(sampleVector, validx, candMatWOMaxValues, mappings, deepcopy(settings), numerator, denominator, weight, features, fitted_values_all_data_this_vector_is_modified_by_build_tree, T_Uint8_or_UInt16)
	# apply tree to ooBagSample
			leaves_of_tree = create_leaves_array(thistree.rootnode)
        apply_tree_by_leaf!(fitted_values_all_data_this_vector_is_modified_by_build_tree, reused_fitted_leafnr_vector, unusedSamplePart, thistree.rootnode, features)
		# Determine Goodness of Fit
			# I think it is best to consider the goodness of fit per leaf here
			# out of bag performance should be measured here, however this is not coded yet (see also bagging!)
				# ooBagEstNumerator=ooBagEstimates.*ooBagdenom
				# fittedPerLeaf=Float64[x.fitted for x in leaves_of_tree]
				# ooBagcntperLeaf,ooBagsumnumeratorEST,ooBagsumnumerator,ooBagsumdenominator,ooBagsumweight=aggregateByLeafNr(leafNrooBag,ooBagEstNumerator,ooBagnum,ooBagdenom,ooBagw)
				# todo tbd check this!
				# thisError=calcErrorStats(fittedPerLeaf,ooBagsumnumerator./ooBagsumdenominator,ooBagsumweight)
				# thisError=goodnessOfFit(ooBagEstimates,ooBagnum,ooBagdenom,ooBagw,meanobservedvalue,ngroupsInput)
		thisError = ErrorStats() # Empty/Default Error Stats .
		return thistree
	end
end

function build_tree!(trnidx::Vector{Int}, validx::Vector{Int}, candMatWOMaxValues::Array{Array{Float64,1},1}, mappings::Array{Array{String,1},1}, settings::ModelSettings, numerator::Array{Float64,1}, denominator::Array{Float64,1}, weight::Array{Float64,1}, features, fitted_values::Vector{Float64}, T_Uint8_or_UInt16)
	intVarsUsed, inds, minweightcalculated = some_tree_settings(trnidx, validx, settings.fixedinds, candMatWOMaxValues, mappings, settings.minWeight, weight, settings.subsampling_features_prop, size(features, 2))
	settings.minWeight = minweightcalculated # update minWeight
	if !(length(inds) > 0)
        throw(ErrorException(string("Error: no features were selected length(inds)=", length(inds))))
    end
    # if (typeof(settings.crit) == mseSplit)||(typeof(settings.crit) == NormalDevianceDifferenceToMeanFitWEIGHTEDSplit)
    #    pointwiseRatio=numerator./denominator
    # else
    #   pointwiseRatio=zeros(length(numerator)) #this vector should not be used in this case (also we want to allow for zero deonominators in certain cases and thus refrain from calcuating the pointwise ratio)
    # end
    pointwiseRatio = Vector{Float64}(undef, length(numerator)) # this should not be used by any of the algorithms, hence it is not computed (it is filled with trash(undefined data))

	empty_xl_data = ExcelData(Array{ExcelSheet}(undef, 0), Array{Chart}(undef, 0))
	fp = get_feature_pools(features)
    thisEmptyNode = T_Uint8_or_UInt16 == UInt8 ? UInt8emptyNode : UInt16emptyNode
	resultingTree = Tree(thisEmptyNode, intVarsUsed, candMatWOMaxValues, mappings, inds, settings, empty_xl_data, fp)
	resultingTree.rootnode = build_tree_iteration!(trnidx, validx, settings, resultingTree, numerator, denominator, pointwiseRatio, weight, features, 0, settings.randomw, Array{Rulepath{T_Uint8_or_UInt16}}(undef, 0), Distributed.myid(), fitted_values, T_Uint8_or_UInt16)
	# set Leaf Numbers
	set_leaf_numbers!(resultingTree)
	return resultingTree
end

function build_tree_iteration!(trnidx::Vector{Int},validx::Vector{Int},settings::ModelSettings,thisTree::Tree,numerator::Array{Float64,1},denominator::Array{Float64,1},pointwiseRatio::Array{Float64,1},weight::Array{Float64,1},features::DataFrame,
									depth::Int,randomweight::Float64,parent_rp::Array{Rulepath{RPType},1},parentid::Int,fitted_values::Vector{Float64},T_Uint8_or_UInt16) where RPType
	# !!!! the current concept foresees that features is always the FULL DataFrame
	boolRandomizeOnlySplitAtTopNode = settings.boolRandomizeOnlySplitAtTopNode
	local inds
	# if fixedinds are provided we never change the selection of features for the whole tree! (bagged boosting model)
	if boolRandomizeOnlySplitAtTopNode || (length(settings.fixedinds) > 0)
		# these settings are randomly chosen by some_tree_settings in this case
		inds = thisTree.inds_considered
	else
		# inds need to be randomly picked for each iteration
		inds = randomFeatureSelection(size(features, 2), settings.subsampling_features_prop)
	end
	intVarsUsed = thisTree.intVarsUsed

	minweight = settings.minWeight
	crit = settings.crit
	spawnsmaller = settings.spawnsmaller
	catSortByThreshold = settings.catSortByThreshold
	catSortBy = settings.catSortBy

	nobs = size(numerator, 1)
	fnames = propertynames(features)
	# @code_warntype _split(settings.number_of_num_features,trnidx,validx,numerator,denominator,pointwiseRatio,weight,fnames,features, minweight, depth,randomweight,crit,parallel_level_threshold,parallel_weight_threshold,inds,catSortByThreshold,catSortBy)
	# T_Uint8_or_UInt16=T_Uint8_or_UInt16 #T_Uint8_or_UInt16=find_max_type(features)::DataType #Union{UInt8,UInt16}
	best_split = _split(one(T_Uint8_or_UInt16), settings.number_of_num_features, trnidx, validx, numerator, denominator, pointwiseRatio, weight, fnames, features, minweight, depth, randomweight, crit, inds, catSortByThreshold, catSortBy)
	id = best_split.featid
	subset = best_split.subset
	fname = best_split.featurename
	id2 = best_split.featid_new_positive

	tmpsn = sum(view(numerator, trnidx))
    tmpsd = sum(view(denominator, trnidx))
    # todo/tbd check if this "sort!(subset)" is necessary and find out why.... is it intended?
	sort!(subset)

	# check if no split was found
  # this can happen (EVEN AT THE TOP NODE) if subsampling (of data and or features) is enabled: it may be that there is in fact no split possible (e.g. if all variables are constant)
  # @show id,nobs,size(trnidx),subset
  # @show sum(view(weight,trnidx))
  # @show sum(view(weight,validx))
    if id == 0
	if (depth == 0)
		@warn("DTM: No split was found at the top node. This is generally not expected.")
	end
		trnsumn = sum(view(numerator, trnidx))
		trnsumd = sum(view(denominator, trnidx))
        mean_observed = trnsumn / trnsumd;this_leaf_size = sum(view(weight, trnidx))
		fitted_increment = mean_observed
        return Leaf(length(trnidx), mean_observed, fitted_increment, this_leaf_size, depth, parent_rp, trnsumn, trnsumd, -1)::Leaf
    end;
# Rulepath
		this_left_rp = deepcopy(parent_rp);
		this_right_rp = deepcopy(parent_rp);
		push!(this_left_rp, Rulepath(id, subset, true));
		push!(this_right_rp, Rulepath(id, subset, false));
		if id < 0 # (id<0 == !isa(features[!,id2].pool[1],Number)) #pool cannot be empty here, thus accessing the first element should be fine alternatively eltype(x)<:Number might be faster..
			# split by character variable
        for u in subset
            intVarsUsed[-id + settings.number_of_num_features][u] += 1
        end
    else
            # split by numeric variable
            # todo check performance of this and improve
        intVarsUsed[id][subset[end]] += 1 # set this value to true to indicate, that is is used by the tree
    end
    l, r = lrIndices(trnidx, features[!,id2], subset)

        # needs review: are these checks meaningful (June18 2020 )
    length(r), length(l), length(trnidx)
    @assert length(l) < length(trnidx)
    @assert length(r) < length(trnidx)

    countl = size(l, 1)
    countr = size(r, 1)
    sumwl = sum(view(weight, l))
    sumwr = sum(view(weight, r))

	leftchildwillbefurthersplit = sumwl < 2 * minweight
	rightchildwillbefurthersplit = sumwr < 2 * minweight
	(sumwr > sumwl) ? boolSpawnLeft = false : boolSpawnLeft = true
    if spawnsmaller;boolSpawnLeft = !boolSpawnLeft;end;

    boolRandomizeOnlySplitAtTopNode ? newrandomweight = copy(randomweight) : newrandomweight = 0.0

    # todo/tbd check the different if/then here, I think a few things are not quite accurate
	# Also, we probably do not need any parallelization for the construction of a single tree-> we should disable this functionality
	trnsumnl = sum(view(numerator, l))
	trnsumdl = sum(view(denominator, l))
	trnsumnr = sum(view(numerator, r))
	trnsumdr = sum(view(denominator, r))
	mean_observedl = trnsumnl / trnsumdl
	mean_observedr = trnsumnr / trnsumdr
	fitted_labelsl = mean_observedl;
	fitted_labelsr = mean_observedr;
	if (max(sumwr, sumwl) < minweight)
					if leftchildwillbefurthersplit
						fill_some_elements!(fitted_values, l, mean_observedl)
							remote_ref_build_tree_leftchild = Leaf(length(l), mean_observedl, fitted_labelsl, sumwl, depth + 1, this_left_rp, trnsumnl, trnsumdl, -1)
        else
            remote_ref_build_tree_leftchild = build_tree_iteration!(l, validx, settings, thisTree, numerator, denominator, pointwiseRatio, weight, features, depth + 1, newrandomweight, this_left_rp, Distributed.myid(), fitted_values, T_Uint8_or_UInt16)
		      end
					if rightchildwillbefurthersplit
						fill_some_elements!(fitted_values, r, mean_observedr)
            remote_ref_build_tree_rightchild = Leaf(length(r), mean_observedr, fitted_labelsr, sumwr, depth + 1, this_right_rp, trnsumnr, trnsumdr, -1)
        else
            remote_ref_build_tree_rightchild = build_tree_iteration!(r, validx, settings, thisTree, numerator, denominator, pointwiseRatio, weight, features, depth + 1, newrandomweight, this_right_rp, Distributed.myid(), fitted_values, T_Uint8_or_UInt16)
        end
    else
       # here we spawn a process for the smaller "child"
        if (!boolSpawnLeft)
        # here it is important that the @spawn is executed before the build_tree_iteration! call!
						if rightchildwillbefurthersplit
							fill_some_elements!(fitted_values, r, mean_observedr)
             	  remote_ref_build_tree_rightchild = Leaf(length(r), mean_observedr, fitted_labelsr, sumwr, depth + 1, this_right_rp, trnsumnr, trnsumdr, -1)
            else
                remote_ref_build_tree_rightchild = Distributed.@spawn build_tree_iteration!(r, validx, settings, thisTree, numerator, denominator, pointwiseRatio, weight, features, depth + 1, newrandomweight, this_right_rp, Distributed.myid(), fitted_values, T_Uint8_or_UInt16)
            end
						if leftchildwillbefurthersplit
							fill_some_elements!(fitted_values, l, mean_observedl)
						  remote_ref_build_tree_leftchild = Leaf(length(l), mean_observedl, fitted_labelsl, sumwl, depth + 1, this_left_rp, trnsumnl, trnsumdl, -1)
            else
                remote_ref_build_tree_leftchild = build_tree_iteration!(l, validx, settings, thisTree, numerator, denominator, pointwiseRatio, weight, features, depth + 1, newrandomweight, this_left_rp, Distributed.myid(), fitted_values, T_Uint8_or_UInt16)
            end
        else
						if leftchildwillbefurthersplit
							fill_some_elements!(fitted_values, l, mean_observedl)
						  remote_ref_build_tree_leftchild = Leaf(length(l), mean_observedl, fitted_labelsl, sumwl, depth + 1, this_left_rp, trnsumnl, trnsumdl, -1)
            else
                remote_ref_build_tree_leftchild = Distributed.@spawn build_tree_iteration!(l, validx, settings, thisTree, numerator, denominator, pointwiseRatio, weight, features, depth + 1, newrandomweight, this_left_rp, Distributed.myid(), fitted_values, T_Uint8_or_UInt16)
            end
						if rightchildwillbefurthersplit
							fill_some_elements!(fitted_values, r, mean_observedr)
                remote_ref_build_tree_rightchild = Leaf(length(r), mean_observedr, fitted_labelsr, sumwr, depth + 1, this_right_rp, trnsumnr, trnsumdr, -1)
            else
                remote_ref_build_tree_rightchild = build_tree_iteration!(r, validx, settings, thisTree, numerator, denominator, pointwiseRatio, weight, features, depth + 1, newrandomweight, this_right_rp, Distributed.myid(), fitted_values, T_Uint8_or_UInt16)
            end
        end
    end

    fetched_left = fetch(remote_ref_build_tree_leftchild)
    fetched_right = fetch(remote_ref_build_tree_rightchild)
  # fitted_values=mergeLeftAndRightFittedValues(fitted_valuesl,fitted_valuesr,left)
	if !((id < 0) || (maximum(subset) == subset[end])) # for numeric variables subset should generally be of the form [1:n], if not this means that the data was split in a way such that several candidates collapsed as there was no more data between to (or more) candidates.
        throw(ErrorException(string("Internal DTM Error. Subset had unexpected layout. You may want to report this as an issue")))
    end
    id2 = id < 0 ? abs(id) + settings.number_of_num_features : id
    return Node(id, id2, subset, fetched_left, fetched_right, parent_rp)::Node
end

function _split(val_of_some_UInt_type::T, number_of_num_features::Int, trnidx::Vector{Int}, validx::Vector{Int}, numerator::Array{Float64,1}, denominator::Array{Float64,1}, pointwiseRatio::Array{Float64,1}, weight::Array{Float64,1}, fnames::Vector{Symbol}, features, minweight::Float64, depth::Int, randomweight::Float64, crit::SplittingCriterion, inds::Array{Int,1}=Array{Int}(undef, 0), catSortByThreshold::Int=8, catSortBy::SortBy=SORTBYMEAN) where T <: Unsigned # ,RT<:Union{Splitdef{UInt16},Splitdef{UInt8}}
	# This function selects the maximal possible split defined by crit (thus depending on the impurity function, we need to put a minus sign in front of it)
		tmpsz::Int = 0
		if sum(view(weight, trnidx)) < 2 * minweight;
        if T == UInt8
            return UInt8emptySplitDef
        else
            return UInt16emptySplitDef
        end
		end
		tmp_splitlist = Vector{Splitdef{T}}(undef, 0)
		for i in inds
			# ATTENTION: for char variables we pass the variable i with a negative sing!!
			# this allows us to distinguish whether we are working on a char or num variable later on
			modified_i = eltype(features[!,i]) <: AbstractString ? number_of_num_features - i  :  i
			tmplist = _split_feature(val_of_some_UInt_type, number_of_num_features, trnidx, validx, numerator, denominator, pointwiseRatio, weight, fnames[i], features[!,i], minweight, crit, modified_i, randomweight, catSortByThreshold, catSortBy)::Vector{Splitdef{T}}
			append!(tmp_splitlist, tmplist)
		end

		# pick best "valid" (minweight feasible) split
		if !(size(tmp_splitlist, 1) > 0)
			if T == UInt8
            return UInt8emptySplitDef
        else
            return UInt16emptySplitDef
        end
		else
				tmp_splitlist = subset_splitlist(tmp_splitlist, minweight)
				if !(size(tmp_splitlist, 1) > 0)
					if T == UInt8
                return UInt8emptySplitDef
            else
                return UInt16emptySplitDef
            end
				else
            splitlist_sorted::Vector{Splitdef{T}} = sort_splitlist(tmp_splitlist)::Vector{Splitdef{T}}
						tmpsz = size(splitlist_sorted, 1)::Int
						if tmpsz > 0 # can it be that splitlist_sorted is empty even though tmp_splitlist was not? I think so!
							# randomize choice
							spl::Splitdef{T} = splitlist_sorted[1]
							if randomweight > 0 # ((depth==0) && (randomweight>0))
								rnd = Int(max(1, min(tmpsz, ceil(rand() * randomweight * tmpsz)))) # I am not sure if it can happen that rnd becomes 0 if we do not impose the max(1,...) condition. But it seems safe to enforce it in case an incredibly small random number is generated
								spl = splitlist_sorted[rnd]::Splitdef{T}
							else # randomweight<=0
								# deterministic choice (greedy)
								spl = splitlist_sorted[1]::Splitdef{T}
							end # "if condition randomweight>0"
							if isfinite(spl.splitvalue) && (spl.featid < 0)
								# For Character variables: The split can be defined by the subset or its complement, We choose to define it via the subset which defines the smaller child node such that when new values arrive (in out of sample testing) they will "go with the larger child node"
								if spl.weightl > spl.weightr
										lvls = unique(view(features[!,spl.featid_new_positive].refs, trnidx))
										spl = Splitdef(spl.featid, spl.featid_new_positive, spl.featurename, convert(Vector{T}, setdiff(lvls, spl.subset)), spl.splitvalue, spl.weightl, spl.weightr)::Splitdef{T}
								end
							end # isfinite(best_value_split) && (best[1]<0)
						end # size(splitlist_sorted,1)>0
        end # size(tmp_splitlist,1)>0
		end # size(tmp_splitlist,1)>0
    return spl::Splitdef{T}
end

# new approach; first summarize by label, then iterate over the gray code such that only one category needs to switch classes and "online" update the metrics
function _split_feature(ONE_return_type::T, number_of_num_features::Int, trnidx::Vector{Int}, validx::Vector{Int}, numerator::Array{Float64,1}, denominator::Array{Float64,1}, pointwiseRatio::Array{Float64,1}, weight::Array{Float64,1}, fname::Symbol, features, minweight::Float64, crit::SplittingCriterion, feature_column_id::Int, randomweight::Float64, catSortByThreshold::Int, catSortBy::SortBy) where T <: Unsigned
    crit_type = typeof(crit)
# This function is now for numeric and character variables!
# feature_column_id is negative in case of character variables
    best_value = -Inf
    best_thresh = best_wl = best_wr = NaN
    best_subset = Array{UInt8}(undef, 0)
    trnfeatures = view(features, trnidx)
    elt = T # eltype(trnfeatures.parent.refs) #not sure if this was really helping, let us determine elt through T
    labellist_sorted = collect(one(elt):convert(elt, length(trnfeatures.parent.pool))) # this used to be levels(features) #this also contains the val feature levels here! It is considerably faster than levels(view) \factor 100 or so

    case1a = (crit_type == DifferenceSplit || crit_type == MaxValueSplit || crit_type == MaxMinusValueSplit || crit_type == MaxAbsValueSplit)
    case1b = (crit_type == GammaDevianceSplit) || (crit_type == PoissonDevianceSplit)
    case1 = (crit_type == DifferenceSplit || crit_type == GammaDevianceSplit || crit_type == PoissonDevianceSplit || crit_type == MaxValueSplit || crit_type == MaxMinusValueSplit || crit_type == MaxAbsValueSplit)
    case2 = (crit_type == msePointwiseSplit) || (crit_type == mseSplit) || (crit_type == sseSplit)
	# THIS IS CRITICAL
	# THE WAY A PooledArray IS CONSTRUCTED, WE WILL ALWAYS HAVE
	# pda.pool[2] == "some string" -> any pda.refs[x].==0x02 will be "some string"
	# todo/tbd countsort here might be obsolete: we should check if levels is always sorted by construction
  # also we will later sort the labels in a different order anyway (then again the list probably needs to be sorted in the natural manner such that build_listOfMeanResponse is working properly)
    if size(labellist_sorted, 1) <= 1
        if T == UInt8
            return UInt8VECTORemptySplitDef
        else
            return UInt16VECTORemptySplitDef # collect(Vector{Splitdef{T}}(undef,0))::Vector{Splitdef{T}}
        end
    else
    # this may need improvement:
	if case1
    		labellist, sumnumerator, sumdenominator, sumweight, countlistfloat = build_listOfMeanResponse(crit, trnidx, validx, numerator, denominator, weight, trnfeatures, labellist_sorted, minweight)
	    elseif case2
		    labellist, sumnumerator, sumdenominator, sumweight, countlistfloat, moments_per_pdaclass = build_listOfMeanResponse(crit, numerator, denominator, weight, trnfeatures, labellist_sorted, minweight)
        # else #we catch this possibility earlier when checking the settings
        #	throw(ErrorException(string("Invalid Splitting criterion $(crit)")))
	end
  # todo/tbd
  # here we can introduce the possiblity to sort the labellist (e.g. by meanobserved or median (to be calculated)).
  # then we could only loop through the "increasing list" of sorted labels (instead of doing the 2^ncategories exhaustive search (bitflip_graycode_subsets))
    if feature_column_id > 0 # id>0 -> we are working on a numeric column
		# only consider to split at the candidate split points
		subs = increasing_subsets(labellist)
    else # id<0 -> we are working on a character column
		# distinguish between exhaustive and "increasing" search for split point
	    if size(labellist_sorted, 1) > catSortByThreshold
		    if case1
                sortlists!(catSortBy, labellist, sumnumerator, sumdenominator, sumweight, countlistfloat) # catSortBy::SortBy=SORTBYMEAN
            elseif case2
			    sortlists!(catSortBy, labellist, sumnumerator, sumdenominator, sumweight, countlistfloat, moments_per_pdaclass)
		    end
		subs = increasing_subsets(labellist)
	else
	    # perform exhaustive search
            subs = bitflip_graycode_subsetsHALF(labellist)
        end
	end

	if randomweight == 0.0
        if case1a
            tmp_result = calculateSplitValue(crit, fname, number_of_num_features, labellist, sumnumerator, sumdenominator, sumweight, countlistfloat, minweight, subs)
        elseif case2
            tmp_result = calculateSplitValue(crit, fname, number_of_num_features, labellist, sumnumerator, sumdenominator, sumweight, countlistfloat, minweight, subs, moments_per_pdaclass)
        elseif case1b
            tmp_result = calculateSplitValue(crit, fname, number_of_num_features, labellist, sumnumerator, sumdenominator, sumweight, countlistfloat, minweight, subs, numerator, denominator, pointwiseRatio, weight, trnfeatures)
        end
        if isfinite(tmp_result[1])
            feature_column_id2 = feature_column_id < 0 ? abs(feature_column_id) + number_of_num_features : feature_column_id
            return [Splitdef(feature_column_id, feature_column_id2, fname, Vector{T}(tmp_result[2]), tmp_result[1], tmp_result[3], tmp_result[4])]::Vector{Splitdef{T}}
        else
            if T == UInt8
                return UInt8VECTORemptySplitDef
            else
                return UInt16VECTORemptySplitDef # collect(Vector{Splitdef{T}}(undef,0))::Vector{Splitdef{T}}
            end
        end
    else
  # randomweight>0
        if case1a
            tmpres = calculateSplitValue(crit, fname, number_of_num_features, labellist, sumnumerator, sumdenominator, sumweight, countlistfloat, minweight, subs, feature_column_id)
        elseif case2
            tmpres = calculateSplitValue(crit, fname, number_of_num_features, labellist, sumnumerator, sumdenominator, sumweight, countlistfloat, minweight, subs, feature_column_id, moments_per_pdaclass)
        elseif case1b
            tmpres = calculateSplitValue(crit, fname, number_of_num_features, labellist, sumnumerator, sumdenominator, sumweight, countlistfloat, minweight, subs, numerator, denominator, pointwiseRatio, weight, trnfeatures, feature_column_id)
        end
            return tmpres::Vector{Splitdef{T}}
        end
    end
end


# sudo apt-get install python-dev
# sudo apt-get install python-pip
# sudo pip install biopython
# sudo apt-get install numpy | unless you want to compile it..
# julia: Pkg.add("PyCall")
module PDB
export get_structure, get_chains, structure_to_matrix, export_to_pdb, get_remote_pdb

using PyCall
using Formatting
@pyimport Bio.PDB as pdb

# get a structure from a PDB File
function get_structure(filename::AbstractString)
	# using QUIET to supress discontinous chains warning
	pdbparser = pdb.PDBParser(QUIET = 1)

	# parse the file
	structure = pdbparser[:get_structure](filename, filename)
	return structure
end

# get chains from structure
function get_chains(structure::PyObject)
	chains = structure[:get_chains]()

	chainMatrices = Any[]

	for c in chains
		push!(chainMatrices,structure_to_matrix(c))
	end

	return chainMatrices
end

# Read in a PDB file and return a matrix of C_Alpha atom coordinates
function structure_to_matrix(structure::PyObject)
	atoms = structure[:get_atoms]()
	# Filter out C_Alpha atoms
	atoms = filter(x -> x[:get_name]() == "CA", atoms)
	# Get the coordinates
	atoms = map(x -> x[:get_coord](), atoms)

	n = length(atoms)

	# initialize an empty Nx3 matrix
	matrix = zeros(n,3)

	# fill it
	for i in 1:n
		matrix[i,:] = [atoms[i][1] atoms[i][2] atoms[i][3]]
	end

	matrix = reshape(matrix,n,3)

	return matrix
end

# Export a matrix of C_alpha atom coordinates to a PDB file
function export_to_pdb(residueName::AbstractString,chainID::AbstractString,matrix::Array{Float64,2}, filename::AbstractString)
	lines = AbstractString[]

	atomExpr = FormatExpr("{: <6}{: >5} {: >4}{: <1}{: >3} {: <1}{: >4}{: <1}   {: >8}{: >8}{: >8}{: >6}{: >6}      {: <4}{: >2}{: <2}\n")

	for i in 1:size(matrix)[1]
		line = format(atomExpr, "ATOM", i, "CA", " ", residueName, chainID, "1", " ", matrix[i,:][1], matrix[i,:][2], matrix[i,:][3], 1.0, 0.0, "A1", "C", " ")

		push!(lines, line)
	end

	f = open(filename, "w")
	write(f,lines)
	close(f)

	return lines
end

# Get a PDB file by ID from rcsb.org
function get_remote_pdb(id::AbstractString)
	# create cache if not present
	if !isdir(joinpath(dirname(@__FILE__), "..", "..", ".cache"))
		mkdir(joinpath(dirname(@__FILE__), "..", "..", ".cache"))
	end
	filename = joinpath(dirname(@__FILE__), "..", "..", ".cache", id)
	# check if PDB isn't already cached
	if !isreadable(filename)
		data = get("http://www.rcsb.org/pdb/files/" * id * ".pdb")
		f = open(filename, "w")
		write(f, data.data)
	end
	return filename
end

end


# Single RefTx entry
struct RefTx
   info::TxInfo
   don::CoordTuple
   acc::CoordTuple
   length::Float64
end

# Single RefGene entry
struct RefGene
   info::GeneInfo
   don::CoordTuple
   acc::CoordTuple
   txst::CoordTuple
   txen::CoordTuple
   exons::CoordTree
   noveldon::CoordTuple
   novelacc::CoordTuple
   exonexpr::Float64
   length::Float64
   reftx::Vector{RefTx}
end

const RefSet = Dict{GeneName,RefGene}

# this guy maps an array of strings or substrings to Int parsing and then into a tuple
# which is then sliced based on the l (left) and r (right) adjusters, c is the mathmatical adjuster to the data
parse_splice( subarray; l=0, r=0, c=0 ) = tuple(map( x->convert(CoordInt,parse(Int,x)+c), subarray )...)[(1+l):(end-r)]
parse_coordint( i ) = convert(CoordInt, parse(Int, i))
parse_coordtup( i; c=0 ) = tuple(parse_coordint(i)+CoordInt(c))

unique_tuple( tup1::Tuple{}, tup2::Tuple{} ) = ()
unique_tuple( tup1::Tuple{Vararg{T}}, tup2::Tuple{} ) where T = unique_tuple(tup1, tup1)
unique_tuple( tup1::Tuple{}, tup2::Tuple{Vararg{T}} ) where T = unique_tuple(tup2, tup2)

function unique_tuple( tup1::Tuple{Vararg{T}}, tup2::Tuple{Vararg{T}}) where T
   uniq = SortedSet{T,Base.Order.ForwardOrdering}()
   for i in tup1, j in tup2
     push!(uniq, i)
     push!(uniq, j)
   end
   tuple(uniq...)
end

function minimum_threshold!( dict::Dict{K,V}, limit::V, range::UnitRange ) where {K, V <: Number}
   for k in collect(keys(dict))
      if dict[k] < limit || !(k in range)
         delete!( dict, k )
      end
   end
end

function load_refflat( fh; txbool=true )
   error("ERROR: Annotation files in refflat format have been deprecated as of Whippet v0.11, please use --gtf!")
end

function fetch_meta( var::String, meta; off=1 )
   val = "_" # empty
   for i in off:2:length(meta)
      if meta[i] == var && i <= length(meta)
         return string(meta[i+1]),i+1
      end
   end
   val,0
end

function load_gtf( fh; txbool=true, suppress=false, usebam=false, bamreader=Nullable{BAM.Reader}(), bamreads=2, bamoneknown=false )

   # Temporary variables
   gninfo   = Dict{GeneName,GeneInfo}()
   gndon    = Dict{GeneName,CoordTuple}()
   gnacc    = Dict{GeneName,CoordTuple}()
   gntxst   = Dict{GeneName,CoordTuple}()
   gntxen   = Dict{GeneName,CoordTuple}()
   gnlens   = Dict{GeneName,CoordTuple}()
   gnexons  = Dict{GeneName,CoordTree}()
   gnreftx  = Dict{GeneName,Vector{RefTx}}()
   gnlen    = Dict{GeneName,Float64}()
   gncnt    = Dict{GeneName,Int}()

   # RefSet variables
   geneset  = Dict{GeneName,RefGene}()

   used_txn = Dict{GeneName,Bool}()
   warning  = false
   warning_num = 0

   txnum = 75000
   sizehint!(geneset, txnum >> 1)

   curtran = ""
   curgene = ""
   curchrom = ""
   curstran = '+'

   trandon = Vector{CoordInt}()
   tranacc = Vector{CoordInt}()
   txlen   = 0

   function private_add_transcript!( curtran, curgene, curchrom, curstran, trandon, tranacc, txlen )
      sort!(trandon)
      sort!(tranacc)

      txS = minimum(tranacc)
      txE = maximum(trandon)

      txinfo = TxInfo(curtran, txS, txE, UInt16(length(tranacc)))

      don = tuple(trandon[1:(end-1)]...)
      acc = tuple(tranacc[2:end]...)

      if haskey(gninfo, curgene)
         curchrom == gninfo[curgene].name || return # can have only one chrom
         gndon[curgene]  = unique_tuple(gndon[curgene], don)
         gnacc[curgene]  = unique_tuple(gnacc[curgene], acc)
         gntxst[curgene] = unique_tuple(gntxst[curgene], tuple(txS))
         gntxen[curgene] = unique_tuple(gntxen[curgene], tuple(txE))
         gnlen[curgene] += txlen
         gncnt[curgene] += 1
         push!( gnreftx[curgene], RefTx( txinfo, tuple(trandon...), tuple(tranacc...), txlen ) )
      else
         gndon[curgene]  = don
         gnacc[curgene]  = acc
         gninfo[curgene] = GeneInfo(curgene, string(curchrom), curstran)
         gntxst[curgene] = tuple(txS)
         gntxen[curgene] = tuple(txE)
         gnlen[curgene]  = txlen
         gncnt[curgene]  = 1
         gnreftx[curgene]  = RefTx[ RefTx( txinfo, tuple(trandon...), tuple(tranacc...), txlen ) ]
      end
   end

   for l in eachline(fh)

      (l[1] == '#') && continue # ignore comment lines

      (chrom, src, entrytype,
       st, en, _,
       strand, _,
       meta) = split(chomp(l), '\t')

      (entrytype == "exon") || continue

      metaspl = split(meta, [' ',';','"'], keep=false)

      geneid,pos  = fetch_meta( "gene_id", metaspl )
      tranid,pos  = fetch_meta( "transcript_id", metaspl )
      genesym     = fetch_meta( "gene_name", metaspl )
      support,val = fetch_meta( "transcript_support_level", metaspl )

      # if we observe low transcript support levels, we will warn the user
      if (support != "_" && support != "1" && support != "2" &&  support != "NA")

         if suppress
            continue
         elseif !warning
            println(STDERR, "")
            warn("Using low quality Transcript Support Levels (TSL 3+) in your GTF file is not recommended!\nFor more information on TSL, see: http://www.ensembl.org/Help/Glossary?id=492\n\nIf you would like Whippet to ignore these when building its index, use `--suppress-low-tsl` option!\n\n")

            warning=true
         end

      elseif haskey(used_txn, tranid)
         error("ERROR: GTF file is not in valid GTF2.2 format!\n\nERROR: Annotation entries for 'transcript_id' $tranid has already been fully processed and closed.\nHINT: All GTF lines with the same 'transcript_id' must be adjacent in the GTF file and referring to the same transcript and gene!")
      elseif tranid == geneid && tranid != curtran
         if warning_num < 25
            warn("Generally 'transcript_id' should not equal 'gene_id' but does at $tranid == $geneid;")
         elseif warning_num == 25
            warn("... NOTE: 'transcript_id' == 'gene_id' will work OK for single isoform genes, but will not produce expected behavior for multi-isoform genes!\n... similar warnings will be suppressed; disregard if 'transcript_id' == 'gene_id' is intentional\n")
         end
         warning_num += 1
      end

      if tranid != curtran
         # add transcript and clean up
         curtran != "" && private_add_transcript!( curtran, curgene, curchrom, curstran, trandon, tranacc, txlen )

         used_txn[curtran] = true

         curtran = tranid
         curgene = geneid
         curchrom = chrom
         curstran = strand[1]

         empty!(trandon)
         empty!(tranacc)
         txlen = 0

      end

      sval = parse_coordint(st)
      eval = parse_coordint(en)

      push!(tranacc, sval)
      push!(trandon, eval)
      txlen += eval - sval + 1

      insval = Interval{CoordInt}(sval, eval)
      if haskey(gnexons, curgene)
         # make sure we are adding a unique value
         if !haskey(gnexons[curgene], (sval, eval))
            push!(gnexons[curgene], insval)
         end
      else
         gnexons[curgene] = CoordTree()
         push!(gnexons[curgene], insval)
      end
   end

   private_add_transcript!( curtran, curgene, curchrom, curstran, trandon, tranacc, txlen )

   if usebam
      bamnames = map(x->values(x)[1], find(BAM.header(get(bamreader)), "SQ"))
   end

   novel_ss = 0
   annot_ss = 0
   # iterate through genes, if usebam, add novel splice sites and calculate relative exonexpr
   # set RefGene at the end of each iteration
   for gene in keys(gninfo)

      novelacc = Dict{CoordInt,Int}()
      noveldon = Dict{CoordInt,Int}()
      exonexpr = 0.0
      meanleng = gnlen[gene] / gncnt[gene]
      seqname  = gninfo[gene].name
      strand   = gninfo[gene].strand
      bamwasused = false

      annot_ss += length(gndon[gene]) + length(gnacc[gene])

      if usebam && seqname in bamnames
         bamwasused = true
         range   = Int64(minimum(gntxst[gene])):Int64(maximum(gntxen[gene]))
         exoncount = process_records!( get(bamreader), seqname, range, strand,
                                       gnexons[gene], union(gnacc[gene], gndon[gene]), bamoneknown,
                                       novelacc, noveldon )
         exonexpr  = exoncount / (meanleng - 100)

         # clean up cryptic splice-sites
         minimum_threshold!( novelacc, bamreads, range )
         minimum_threshold!( noveldon, bamreads, range )
         novelacctup = map(CoordInt, Tuple(collect(keys(novelacc))))
         noveldontup = map(CoordInt, Tuple(collect(keys(noveldon))))

         # add splice-sites from bam
         annoacc     = gnacc[gene]
         annodon     = gndon[gene]
         gnacc[gene] = unique_tuple( gnacc[gene], novelacctup )
         gndon[gene] = unique_tuple( gndon[gene], noveldontup )
         novel_ss += length(gnacc[gene]) + length(gndon[gene]) - (length(annoacc) + length(annodon))
      end

      # clean up any txst or txen that match a splice site
      gntxst[gene] = Tuple( setdiff( gntxst[gene], gnacc[gene] ) )
      gntxen[gene] = Tuple( setdiff( gntxen[gene], gndon[gene] ) )

      geneset[gene] = RefGene( gninfo[gene],
                               gndon[gene],
                               gnacc[gene],
                               gntxst[gene],
                               gntxen[gene],
                               gnexons[gene],
                               bamwasused ? Tuple( setdiff(noveldontup, annodon) ) : Tuple{}(),
                               bamwasused ? Tuple( setdiff(novelacctup, annoacc) ) : Tuple{}(),
                               exonexpr,
                               meanleng,
                               gnreftx[gene] )
   end

   if usebam
      println(STDERR, "Found $novel_ss new splice-sites from BAM file..")
   end
   println(STDERR, "Loaded $annot_ss annotated splice-sites from GTF file..")

   return geneset
end