module layout

import Term: int, get_last_valid_str_idx
import ..renderables: RenderablesUnion, Renderable, AbstractRenderable
import ..measure: Measure
import ..segment: Segment
using ..box
import ..consoles: console_width, console_height

export Padding, vstack, hstack
export Spacer, vLine, hLine

# ---------------------------------------------------------------------------- #
#                                    PADDING                                   #
# ---------------------------------------------------------------------------- #
"""
    Padding

Stores empty string to pad a string to the desired with.
"""
struct Padding
    left::AbstractString
    right::AbstractString
end

"""
    Padding(text::AbstractString, target_width::Int, method::Symbol)::Padding

Create `Padding` for a string given a justification `method` ∈ (:left, :center, :right).
"""
function Padding(text::AbstractString, target_width::Int, method::Symbol)::Padding
    # get total padding size
    lw = Measure(text).w
    if lw == target_width
        return Padding("", "")
    else
        if lw < target_width
            @debug "\e[31mTarget width is $target_width but the text has width $lw: \e[0m\n   $text\n     Cleaned: '$(remove_ansi(remove_markup(text)))'"
        end
    end

    if target_width < lw
        @debug "While creating padding, target width < lw" target_width lw method text
        return Padding("", "")
    end
    padding = " "^(target_width - lw - 1)
    pad_width = Measure(padding).w

    # split left/right padding for left/right justify
    if pad_width > 0
        cut = pad_width % 2 == 0 ? Int(pad_width / 2) : (Int ∘ floor)(pad_width / 2)
        left, right = padding[1:cut], padding[(cut + 1):end]
        @assert length(left * right) <= length(padding) "\e[31m$(length(left * right)) instead of $(length(padding))"
    else
        left, right = "", ""
    end

    # craete padding
    if method == :center
        padding = Padding(" " * left, right)
    elseif method == :left
        padding = Padding(" ", padding)
    elseif method == :right
        padding = Padding(padding, " ")
    end

    # @info "made padding" text target_width method padding lw pad_width left right
    @assert Measure(padding.left * text * padding.right).w <= target_width "\e[31mPadded width $(Measure(padding.left * text * padding.right).w), target: $target_width $padding"

    return padding
end

function Base.show(io::IO, padding::Padding)
    return print(
        io,
        "$(typeof(padding))  \e[2m(left: $(length(padding.left)), right: $(length(padding.right)))\e[0m",
    )
end

# ---------------------------------------------------------------------------- #
#                                   STACKING                                   #
# ---------------------------------------------------------------------------- #
"""
    vstack(r1::RenderablesUnion, r2::RenderablesUnion)

Vertically stack two renderables to give a new renderable.
"""
function vstack(r1::RenderablesUnion, r2::RenderablesUnion)
    r1 = Renderable(r1)
    r2 = Renderable(r2)

    # get dimensions of final renderable
    w1 = r1.measure.w
    w2 = r2.measure.w

    # create segments stack
    segments::Vector{Segment} = vcat(r1.segments, r2.segments)
    measure = Measure(max(w1, w2), length(segments))

    return Renderable(segments, measure)
end

"""
    vstack(renderables...)

Vertically stack a variable number of renderables
"""
function vstack(renderables...)
    renderable = Renderable()

    for ren in renderables
        renderable = vstack(renderable, ren)
    end
    return renderable
end

"""
    hstack(r1::RenderablesUnion, r2::RenderablesUnion)

Horizontally stack two renderables to give a new renderable.
"""
function hstack(r1::RenderablesUnion, r2::RenderablesUnion)
    r1 = Renderable(r1)
    r2 = Renderable(r2)

    # get dimensions of final renderable
    h1 = r1.measure.h
    h2 = r2.measure.h

    # make sure both renderables have the same number of segments
    Δh = abs(h2 - h1)
    if h1 > h2
        r2.segments = vcat(r2.segments, [Segment(" "^r2.measure.w) for i in 1:Δh])
    elseif h1 < h2
        r1.segments = vcat(r1.segments, [Segment(" "^r1.measure.w) for i in 1:Δh])
    end

    # combine segments
    segments = [Segment(s1.text * s2.text) for (s1, s2) in zip(r1.segments, r2.segments)]

    return Renderable(segments, Measure(segments))
end

"""
    hstack(renderables...)

Horizonatlly stack a variable number of renderables
"""
function hstack(renderables...)
    renderable = Renderable()

    for ren in renderables
        renderable = hstack(renderable, ren)
    end
    return renderable
end

# --------------------------------- operators -------------------------------- #

"""
    Operators for more coincise layout of renderables
"""

Base.:/(r1::RenderablesUnion, r2::RenderablesUnion) = vstack(r1, r2)

Base.:*(r1::AbstractRenderable, r2::AbstractRenderable) = hstack(r1, r2)
Base.:*(r1::AbstractString, r2::AbstractRenderable) = hstack(r1, r2)
Base.:*(r1::AbstractRenderable, r2::AbstractString) = hstack(r1, r2)

# ---------------------------------------------------------------------------- #
#                                LINES & SPACER                                #
# ---------------------------------------------------------------------------- #
abstract type AbstractLayoutElement <: AbstractRenderable end

# ---------------------------------- spacer ---------------------------------- #
"""
        Spacer

A box of empty text with given width and height.
"""
mutable struct Spacer <: AbstractLayoutElement
    segments::Vector{Segment}
    measure::Measure
end

function Spacer(width::Number, height::Number; char::Char = ' ')
    width = int(width)
    height = int(height)

    line = char^width
    segments = [Segment(line) for i in 1:height]
    return Spacer(segments, Measure(segments))
end

# ----------------------------------- vline ---------------------------------- #
"""
    vLine

A multi-line renderable with each line made of a | to create a vertical line
"""
mutable struct vLine <: AbstractLayoutElement
    segments::Vector{Segment}
    measure::Measure
    height::Int
end

"""
    vLine(height::Number, style::Union{String, Nothing}; box::Symbol=:ROUNDED)

Create a `vLine` given a height and, optionally, style information.
"""
function vLine(
    height::Number; style::Union{String,Nothing} = nothing, box::Symbol = :ROUNDED
)
    height = int(height)
    char = string(eval(box).head.left)
    segments = [Segment(char, style) for i in 1:height]
    return vLine(segments, Measure(segments), height)
end

"""
    vLine(; style::Union{String, Nothing}=nothing, box::Symbol=:ROUNDED)

Create a `vLine` as tall as the `stdout` console
"""
function vLine(; style::Union{String,Nothing} = nothing, box::Symbol = :ROUNDED)
    return vLine(console_height(); style = style, box = box)
end

"""
    hLine

A 1-line renderable made of repeated character from a Box.
"""
mutable struct hLine <: AbstractLayoutElement
    segments::Vector{Segment}
    measure::Measure
    width::Int
end

"""
    hLine(width::Number, style::Union{String, Nothing}; box::Symbol=:ROUNDED)

Create a styled `hLine` of given width.
"""
function hLine(
    width::Number; style::Union{String,Nothing} = nothing, box::Symbol = :ROUNDED
)
    width = int(width)
    char = eval(box).row.mid
    segments = [Segment(char^width, style)]
    return hLine(segments, Measure(segments), width)
end

"""
    hLine(width::Number, text::String; style::Union{String, Nothing}=nothing, box::Symbol=:ROUNDED)

Creates an hLine object with texte centered horizontally.
"""
function hLine(
    width::Number,
    text::String;
    style::Union{String,Nothing} = nothing,
    box::Symbol = :ROUNDED,
)
    box = eval(box)

    initial_line = box.top.mid^width

    cutval = int(ncodeunits(initial_line) / 2 - ncodeunits(text) - 5)
    cut_start = get_last_valid_str_idx(initial_line, cutval)

    pre = Segment(
        Segment(initial_line[1:cut_start], style) *
        "\e[0m" *
        " " *
        Segment(text, style) *
        " ",
    )

    post = Segment(box.top.mid^(length(initial_line) - pre.measure.w - 1), style)

    segments = [Segment(pre * (post), style)]
    return hLine(segments, Measure(segments), width)
end

"""
    hLine(; style::Union{String, Nothing}=nothing, box::Symbol=:ROUNDED)

Construct an `hLine` as wide as the `stdout`
"""
function hLine(; style::Union{String,Nothing} = nothing, box::Symbol = :ROUNDED)
    return hLine(console_width(); style = style, box = box)
end

"""
    hLine(text::AbstractString; style::Union{String, Nothing}=nothing, box::Symbol=:ROUNDED)

Construct an `hLine` as wide as the `stdout` with centered text.
"""
function hLine(
    text::AbstractString; style::Union{String,Nothing} = nothing, box::Symbol = :ROUNDED
)
    return hLine(console_width(), text; style = style, box = box)
end
end


export
    Query,
    Queryable,
    Marginal,
    Joint,
    ProbValue,
    ProbFunction,
    ProbabilityBounds,
    Expectation,
    Mean,
    Variance,
    answer,
    marginal,
    joint,
    probability,
    probability_bounds,
    expectation,
    mean

"""
    abstract type Query end

General type of query that can be answered after running an algorithm.
"""
abstract type Query end

"""
A query target is either a variable instance or a variable.
Allowing queries to be defined in terms of instances rather than variables makes it possible
to ask queries across multiple instances of a variable at different times.
However, in many cases the current instance of the variable(s) is required and then it is easier
to use variables.
"""
Queryable{O} = Union{VariableInstance{O}, Variable{I,J,O}} where {I,J}

function _resolve(r::Runtime, q::Queryable{O})::VariableInstance{O} where O
    if q isa Instance
        return q
    else
        return current_instance(r, q)
    end
end

"""
    answer(::Query, ::Algorithm, ::Runtime, ::VariableInstance)
    answer(::Query, ::Algorithm, ::Runtime, ::Vector{VariableInstance})
    answer(::Query, ::Algorithm, ::Runtime, ::Queryable)
    answer(::Query, ::Algorithm, ::Runtime, ::Vector{Queryable})

    Answer the query.

    An implementation of an algorithm should implement an `answer` method for any queries
    it can handle. The type hierarchies of `Query` and `Algorithm` will enable
    query answering methods to be used wherever appropriate with the right specialization.
    The implementations of `answer` are differentiated along two dimensions:
    - single or multiple items
    - queryable items in general or specifically instances

    It is expected that an algorithm will implement one of the first two methods for queries it
    can handle. I.e., an algorithm is expected to handle a single instance or a vector of instances.
    If it can handle multiple instances, it should implement a second method and a single instance implementation
    is derived by default using a singleton vector. An algorithm can still override this default
    method if it handles single instances differently from multiple.

    Algorithms will generally not implement the latter two methods, which are provide for convenience.
    Default implementations are provided that delegate to the instance-specific methods.

    Defining a very high-level default implementation that throws an error enables implementations
    to go through sequences of preferences.
"""
function answer(q::Query, a::Algorithm, r::Runtime, i::VariableInstance)
    is = VariableInstance[i]
    answer(q, a, r, is)
end

function answer(q::Query, a::Algorithm, r::Runtime, item::Queryable)
    answer(q, a, r, _resolve(r,item))
end

function answer(q::Query, a::Algorithm, r::Runtime, items::Vector{Queryable})
    insts = VariableInstance[]
    for i in items
        inst::VariableInstance = _resolve(r,i)
        push!(insts, inst)
    end
    answer(q, a, r, insts)
end

answer(::Any, ::Any, ::Any, ::Any) = error("_answer_")

""

struct Marginal <: Query end

"""
    marginal(alg::Algorithm, runtime::Runtime, item::Queryable{O})::Union{Dist{O}, Tuple{Dist{O}, Dist{O}}} where O

Return the marginal distribution over `item`, or lower and upper marginals,
depending on the algorithm.

The returned `Score` assigns a score to each value of `item`.
"""
function marginal(alg::Algorithm, run::Runtime, item::Queryable{O})::Union{Dist{O}, Tuple{Dist{O}, Dist{O}}} where O
    mg = answer(Marginal(), alg, run, item)
    return mg
end

struct Joint <: Query end

"""
    joint(alg::Algorithm, run::Runtime, items::Vector{Queryable})::Union{Score{O}, Tuple{Score{O}, Score{O}}}

Return the joint distribution over `items`, or lower and upper distributions,
depending on the algorithm.

The returned `Score` assigns a score for each Vector of values of the items.
"""
function joint(alg::Algorithm, run::Runtime, items::Vector{Queryable})::Union{Score, Tuple{Score, Score}}
    answer(Joint(), alg, run, items)
end

struct ProbValue{O} <: Query
    value :: O
end

struct ProbFunction <: Query
    fn :: Function
end

"""
    probability(alg::Algorithm, run::Runtime, items::Vector{Queryable}, predicate::Function)::Union{Float64, Tuple{Float64, Float64}}

Return the probability that `items` satisfy `query` or lower and upper probabilities.

`predicate` is a function from Vector{Any} to `Bool`.
"""
function probability(alg::Algorithm, run::Runtime, items::Vector{Queryable}, predicate::Function)::Union{Float64, Tuple{Float64, Float64}}
    answer(ProbFunction(predicate), alg, run, items)
end

function probability(alg::Algorithm, run::Runtime, item::Queryable, predicate::Function)::Union{Float64, Tuple{Float64, Float64}}
    f(vec) = predicate(vec[1])
    insts = VariableInstance[_resolve(run,item)]
    answer(ProbFunction(f), alg, run, insts)
end

"""
    probability(alg::Algorithm, run::Runtime, item::Queryable{O}, value::O)::Union{Float64, Tuple{Float64, Float64}} where O

Return the probability that `item` has `value` or lower and upper probabilities.

The default implementation tries to use the more general probability of a query.
If that fails, it uses the `cpdf` operation on the marginal of `item`.
"""
function probability(alg::Algorithm, run::Runtime, item::Queryable{O}, value::O)::Union{Float64, Tuple{Float64, Float64}} where O
    inst::Instance = _resolve(run, item)
    try
        answer(ProbValue(value), alg, run, inst)
    catch e
        if e == ErrorException("_answer_")
            try
                probability(alg, run, item, x -> x == value)
            catch e
                if e == ErrorException("_answer_")
                    try
                        m = marginal(alg, run, inst)
                        return cpdf(m, (), value)
                    catch e
                        if e == ErrorException("_answer_")
                            error("None of the methods to compute the probability of a value are implemented")
                        else
                            rethrow(e)
                        end
                    end
                else
                    rethrow(e)
                end
            end
        else
            rethrow(e)
        end
    end
end

struct ProbabilityBounds{O} <: Query
    range :: Vector{O}
end

"""
    probability_bounds(alg::Algorithm, run::Runtime, item::Queryable, range::Vector)::Tuple{Vector{Float64}, Vector{Float64}}

    For an algorithm that produces lower and upper bounds, return vectors of lower and upper bounds on probabilities for values in the range.

    The range is important for computing the bounds, because it is assumed that values outside the range have probability zero.
"""
function probability_bounds(alg::Algorithm, run::Runtime, item::Queryable, range::Vector)::Tuple{Vector{Float64}, Vector{Float64}}
    return answer(ProbabilityBounds(range), alg, run, item)
end

struct Expectation <: Query
    fn :: Function
end

struct Mean <: Query end

"""
    expectation(alg::Algorithm, runtime::Runtime, item::Queryable, f::Function)::Float64

Return the expectation of the function `f` over the marginal distribution of `item`.

The default implementation uses the expectation operation on the SFunc representing the
marginal distribution.
"""
function expectation(alg::Algorithm, run::Runtime, item::Queryable, fn::Function)::Float64
    try
        return answer(Expectation(fn), alg, run, item)
    catch e
        if e == ErrorException("_answer_")
            m = marginal(alg, run, item)
            if m isa Tuple
                # marginal produced bounds; support is same for lower and upper bounds
                m = m[1]
            end
            tot = 0.0
            O = output_type(item)
            sup = support(m, (), 1000, O[])
            for x in unique(sup)
                tot += cpdf(m, (), x) * fn(x)
            end
            return tot
            # return Operators.f_expectation(marginal(alg, run, item), (), fn)
        else
            rethrow(e)
        end
    end
end

"""
    mean(alg::Algorithm, runtime::Runtime, item::Queryable)

Return the mean of `item`.
"""
function mean(alg::Algorithm, runtime::Runtime, item::Queryable{O})::Float64 where {O <: Number}
    try
        answer(Mean(), alg, runtime, item)
    catch e
        if e == ErrorException("_answer_")
            return expectation(alg, runtime, item, x -> x)
        else
            rethrow(e)
        end
    end
        return
end

struct Variance <: Query end

"""
    variance(alg::Algorithm, runtime::Runtime, item::Queryable)::Float64

Return the variance of `item`.
"""
function variance(alg::Algorithm, run::Runtime, item::Queryable)::Float64
    #answer(Variance(), alg, run, item)
    try
        return answer(Variance(), alg, run, item)
    catch e
        if e == ErrorException("_answer_")
            exp_x = expectation(alg, run, item, x->x)
            exp_xsq = expectation(alg, run, item, x->x^2)
            variance_x = exp_xsq - exp_x^2
            #return Operators.f_expectation(marginal(alg, run, item), (), fn)
            return variance_x
        else
            rethrow(e)
        end
    end
end


module Spectra

export power_spectrum

using MPI
using ClimateMachine
using ..BalanceLaws
using ..ConfigTypes
using ..DGMethods
using ..Mesh.Interpolation
using ..Mesh.Grids
using ..MPIStateArrays
using ..VariableTemplates
using ..Writers

include("power_spectrum.jl")

end


export OfflinePolicy, JuliaRLTransition, gen_JuliaRL_dataset

export calculate_CQL_loss, maximum_mean_discrepancy_loss

struct JuliaRLTransition
    state
    action
    reward
    terminal
    next_state
end

Base.@kwdef struct OfflinePolicy{L,T} <: AbstractPolicy
    learner::L
    dataset::T
    continuous::Bool
    batch_size::Int
end

(π::OfflinePolicy)(env) = π(env, ActionStyle(env), action_space(env))

function (π::OfflinePolicy)(env, ::MinimalActionSet, ::Base.OneTo)
    if π.continuous
        π.learner(env)
    else
        findmax(π.learner(env))[2]
    end
end
(π::OfflinePolicy)(env, ::FullActionSet, ::Base.OneTo) = findmax(π.learner(env), legal_action_space_mask(env))[2]

function (π::OfflinePolicy)(env, ::MinimalActionSet, A)
    if π.continuous
        π.learner(env)
    else
        A[findmax(π.learner(env))[2]]
    end
end
(π::OfflinePolicy)(env, ::FullActionSet, A) = A[findmax(π.learner(env), legal_action_space_mask(env))[2]]

function RLBase.update!(
    p::OfflinePolicy,
    traj::AbstractTrajectory,
    ::AbstractEnv,
    ::PreExperimentStage,
)
    l = p.learner
    if in(:pretrain_step, fieldnames(typeof(l)))
        println("Pretrain...")
        for _ in 1:l.pretrain_step
            inds, batch = sample(l.rng, p.dataset, p.batch_size)
            update!(l, batch)
        end
    end
end

function RLBase.update!(
    p::OfflinePolicy,
    traj::AbstractTrajectory,
    ::AbstractEnv,
    ::PreActStage,
)
    l = p.learner
    l.update_step += 1

    if in(:target_update_freq, fieldnames(typeof(l))) && l.update_step % l.target_update_freq == 0
        copyto!(l.target_approximator, l.approximator)
    end

    l.update_step % l.update_freq == 0 || return

    inds, batch = sample(l.rng, p.dataset, p.batch_size)

    update!(l, batch)
end

"""
    gen_JuliaRL_dataset(alg::Symbol, env::Symbol, type::AbstractString; dataset_size)

Generate the dataset by trajectory from the trajectory obtained from the experiment (`alg` + `env`). `type` represents the method of collecting data. Possible values: random/medium/expert. `dataset_size` is the size of the generated dataset.
"""
function gen_JuliaRL_dataset(alg::Symbol, env::Symbol, type::AbstractString; dataset_size::Int)
    dataset_ex = Experiment(
            Val(:GenDataset),
            Val(alg),
            Val(env),
            type;
            dataset_size = dataset_size)

    run(dataset_ex)

    dataset = []
    s, a, r, t = dataset_ex.policy.trajectory.traces
    for i in 1:dataset_size
        push!(dataset, JuliaRLTransition(s[:, i], a[i], r[i], t[i], s[:, i+1]))
    end
    dataset
end

function StatsBase.sample(rng::AbstractRNG, dataset::Vector{T}, batch_size::Int) where {T}
    valid_range = 1:length(dataset)
    inds = rand(rng, valid_range, batch_size)
    batch_data = dataset[inds]
    s_length = size(batch_data[1].state)[1]
    a_type = typeof(batch_data[1].action)

    s = Array{Float32}(undef, s_length, batch_size)
    s′ = Array{Float32}(undef, s_length, batch_size)
    a = Array{a_type}(undef, batch_size)
    r = Array{Float32}(undef, batch_size)
    t = Array{Float32}(undef, batch_size)
    for (i, data) in enumerate(batch_data)
        s[:, i] = data.state
        a[i] = data.action
        s′[:, i] = data.next_state
        r[i] = data.reward
        t[i] = data.terminal
    end
    batch = NamedTuple{SARTS}((s, a, r, t, s′))
    inds, batch
end

"""
    calculate_CQL_loss(q_value, action; method)
See paper: [Conservative Q-Learning for Offline Reinforcement Learning](https://arxiv.org/abs/2006.04779)
"""
function calculate_CQL_loss(q_value::Matrix{T}, action::Vector{R}; method = "CQL(H)") where {T, R}
    if method == "CQL(H)"
        cql_loss = mean(log.(sum(exp.(q_value), dims=1)) .- q_value[action])
    else
        @error Wrong method parameter
    end
    return cql_loss
end

function maximum_mean_discrepancy_loss(raw_sample_action, raw_actor_action, type::Symbol, mmd_σ::Float32=10.0f0)
    A, B, N = size(raw_sample_action)
    diff_xx = reshape(raw_sample_action, A, B, N, 1) .- reshape(raw_sample_action, A, B, 1, N)
    diff_xy = reshape(raw_sample_action, A, B, N, 1) .- reshape(raw_actor_action, A, B, 1, N)
    diff_yy = reshape(raw_actor_action, A, B, N, 1) .- reshape(raw_actor_action, A, B, 1, N)
    diff_xx = calculate_sample_distance(diff_xx, type, mmd_σ)
    diff_xy = calculate_sample_distance(diff_xy, type, mmd_σ)
    diff_yy = calculate_sample_distance(diff_yy, type, mmd_σ)
    mmd_loss = sqrt.(diff_xx .+ diff_yy .- 2.0f0 .* diff_xy .+ 1.0f-6)
end

function calculate_sample_distance(diff, type::Symbol, mmd_σ::Float32)
    if type == :gaussian
        diff = diff .^ 2
    elseif type == :laplacian
        diff = abs.(diff)
    else
        error("Wrong parameter.")
    end
    return vec(mean(exp.(-sum(diff, dims=1) ./ (2.0f0 * mmd_σ)), dims=(3, 4)))
end


module Gantner

using Dates

export gantnerread, gantnerinfo, gantnermask
include("read_exact.jl")

#=
This function implements the read_exact.mex functionality to read a Gantner created .dat file.
It utilizes the Julia ccall command to use the functionality of the giutility.dll file.  As this
is a windows specific file, this utility only works for Windows operating system.

Additional read_exact.mex functionality can be added as required
=#

"""
    gantnerread(filename :: String; scale :: Real = 1.0, lazytime :: Bool = true)
Read all data channels of data in a Gantner *.dat file.

The first channel of the .dat file is time data. This is ignored by default and returned in ti if lazytime=false.
scale - convert data in volts in .dat file to EU. If it is a scaler then use for all channels, otherwise if vector
            it must be the length of the number of channels
If lazytime = false this data will be read and returned as ti.
If lazytime = true this data will be reconstructed  (The default)

Program returns
ti - time of sample;
data - Array of all the data channels;
fs - sampling rate [Hz]
chanlegendtext - the legend text associated with each channel Vector{String}

This is a subset of the read_exact.c which is the base of the read_exact mex
file used for matlab.  This subset is only for reading data from a file.
"""
function gantnerread(filename::String; scale::Union{Vector{<:Float64},Float64} = 1.0, lazytime::Bool = true)
    gClient = gConnection = 0
    try
        gClient, gConnection = gantOpenFile(filename); # open file
        #read number of channels
        numchannels = gantChanNumRead(gConnection) - 1;
        numvalues = gantNumSamples(gClient, gConnection)

        # If typeof scale is Float64 or length is 1 then apply this factor to all channels
        if typeof(scale) == Float64  || length(scale) == 1
            scale = scale * ones(Float64, numchannels)
        end

        # check to see is scale is the same length at the number of channels
        if length(scale) != numchannels
            error("Data has $numchannels channels, ensure scale is a $numchannels vector")
        end

        global fs = gantSampleRate(gConnection);
        global chanlegendtext = Vector{String}(undef, numchannels)
        # global data = zeros(Float64, numvalues, numchannels-1)
        global data = Array{Float64}(undef, numvalues, numchannels)
        if lazytime
            global ti = range(0, step = 1.0/fs, length = numvalues)
        else
            global ti = gantChanDataRead(gClient, gConnection, 1) # time data assumed in column 1
        end

        for i=1:numchannels
            #read channel name
            chanlegendtext[i] = gantChanName(gConnection, i+1)

            #read channel data
            datachuck = gantChanDataRead(gClient, gConnection, i+1)
            for (j, value) in enumerate(datachuck)
                global data[j, i] = scale[i] * value;
            end
        end

    finally
        #close file
        gantCloseFile(gClient, gConnection)
    end
    return (ti, data, fs, chanlegendtext)
end

"""
    gantnerread(filename :: String; channel:: AbstractVector{Int}, scale :: Real = 1.0, lazytime :: Bool = true)

Read specified data channels of data in a Gantner *.dat file.

The first channel of the .dat file is time data. This is ignored by default and returned in ti if lazytime=false.
channel - the data channels to read
scale - convert data in volts in .dat file to EU. If it is a scaler then use for all channels, otherwise if vector
            it must be the length(channel)
If lazytime = false this data will be read and returned as ti.
If lazytime = true this data will be reconstructed  (The default)

Program returns
ti - time of sample;
data - Array of all the data channels;
fs - sampling rate [Hz]
chanlegendtext - the legend text associated with each channel Vector{String}

This is a subset of the read_exact.c which is the base of the read_exact mex
file used for matlab.  This subset is only for reading data from a file.
"""
function gantnerread(filename::String, channel::AbstractVector{Int}; scale::Union{Vector{<:Float64},Float64} = 1.0, lazytime::Bool = true)
    gClient = gConnection = 0
    try
        gClient, gConnection = gantOpenFile(filename); # open file
        #read number of channels
        numchannels = length(channel)
        numchanneldat = gantChanNumRead(gConnection) - 1;
        numvalues = gantNumSamples(gClient, gConnection)

        # ensure the specified channels are in the data file
        if !issubset(channel, 1:numchanneldat)
            error("Specified channel(s) not in data file")
        end

        # If typeof scale is Float64 or length is 1 then apply this factor to all channels
        if typeof(scale) == Float64  || length(scale) == 1
            scale = scale * ones(Float64, numchannels)
        end

        # check to see is scale is the same length at the number of channels
        if length(scale) != numchannels
            error("Data has $numchannels channels, ensure scale is a $numchannels vector")
        end

        global fs = gantSampleRate(gConnection);
        global chanlegendtext = Vector{String}(undef, numchannels)
        # global data = zeros(Float64, numvalues, numchannels-1)
        global data = Array{Float64}(undef, numvalues, numchannels)
        if lazytime
            global ti = range(0, step = 1.0/fs, length = numvalues)
        else
            global ti = gantChanDataRead(gClient, gConnection, 1) # time data assumed in column 1
        end

        for (i, ii) in enumerate(channel)
            #read channel name
            chanlegendtext[i] = gantChanName(gConnection, ii+1)

            #read channel data
            datachuck = gantChanDataRead(gClient, gConnection, ii+1)
            for (j, value) in enumerate(datachuck)
                global data[j, i] = scale[i] * value;
            end
        end

    finally
        #close file
        gantCloseFile(gClient, gConnection)
    end
    return (ti, data, fs, chanlegendtext)
end

"""
    gantnerread(filename :: String, channel :: Integer; scale :: Real = 1.0, lazytime :: Bool = true)
Read specified data channel (one channel) of data in a Gantner *.dat file.

channel - is the channel number to read the data from.  When channel is 0 the gantner time data is returned in data.
The first channel of the .dat file is time data. This is ignored by default and returned in ti if lazytime=false.
scale - convert data in volts in .dat file to EU
If lazytime = false this data will be read and returned as ti.
If lazytime = true this data will be reconstructed.

Program returns
ti - time of sample;
data - data in specified channel;
fs - sampling rate [Hz]
chanlegendtext - the legend text associated with each channel Vector{String}

This is a subset of the read_exact.c which is the base of the read_exact mex
file used for matlab.  This subset is only for reading data from a file.
"""
function gantnerread(filename::String, channel::Integer; scale::AbstractFloat = 1.0, lazytime::Bool = true)
    gClient = gConnection = 0
    try
        gClient, gConnection = gantOpenFile(filename); # open file

        #read number of channels
        numchanneldat = gantChanNumRead(gConnection) - 1;
        numvalues = gantNumSamples(gClient, gConnection)

        # ensure the specified channels are in the data file
        if !issubset(channel, 1:numchanneldat)
            error("Specified channel not in data file")
        end

        global fs = gantSampleRate(gConnection);
        global chanlegendtext = Vector{String}(undef, 1)
        global data = Vector{Float64}(undef, numvalues)
        if lazytime
            global ti = range(0, step = 1.0/fs, length = numvalues)
        else
            global ti = gantChanDataRead(gClient, gConnection, 1) # time data assumed in column 1
        end

        #read channel name
        chanlegendtext = gantChanName(gConnection, channel + 1)

        #read channel data
        datachuck = gantChanDataRead(gClient, gConnection, channel + 1)
        for (j, value) in enumerate(datachuck)
            global data[j] = scale * value;
        end

    finally
        #close file
        gantCloseFile(gClient, gConnection)
    end
    return (ti, data, fs, chanlegendtext)
end

"""
    gantnerinfo(filename)
Read the information in a Gantner *.dat file.

Returns
numchannels (We only count the data not the time Vector in the .dat file);
numvalues - per channel;
fs - sampling rate;
chanlegendtext - legend text associated with each channel
starttime - finish time - length of recording
finishtime - the timestamp when the file was created (last written)
"""
function gantnerinfo(filename :: String)
    gClient = gConnection = 0
    try
        gClient, gConnection = gantOpenFile(filename); # open file

        #read number of channels
        global numchannels = gantChanNumRead(gConnection) - 1;
        global numvalues = gantNumSamples(gClient, gConnection)

        global fs = gantSampleRate(gConnection);
        global chanlegendtext = Vector{String}(undef, numchannels)
        for i=1:numchannels
            #read channel name
            chanlegendtext[i] = gantChanName(gConnection, i + 1)
        end

        # obtain file start & finish time
        # note that ctime is the local time the file was created on the local file system
        global finishtime = unix2datetime(ctime(filename) + (datetime2unix(now()) - datetime2unix(now(UTC))))
        global starttime = finishtime - Dates.Second(round(Int64, numvalues/fs))

    finally
        #close file
        gantCloseFile(gClient, gConnection)
    end
    return (numchannels, numvalues, fs, chanlegendtext, starttime, finishtime)
end

"""
    gantnermask(data, bit::Integer)
take the input data which has a digitial tach channel embedded within it.  This will extract the information from the specified bit and return this information.
Assume data is a Vector{Float64} or Array{Float64,1}
bit - the bit number from the least significant bit that contains the data
"""
function gantnermask(data::Vector{Float64}, bit::Integer)
    mask = one(Int) << (bit - 1)
    # @show(mask)
    datanew = typeof(data)(undef,size(data))
    for (i, x) in enumerate(data)
        # @show(i, x, bitstring(Int(x)))
        x = Int(x) & mask
        x = x >> (bit - 1)
        datanew[i] = x
        # @show(x, Float64(x), datanew[i])
    end
    return datanew
end

"""
    gantnermask(data, bit::Integer, pprdivide::Integer)
take the input data which has a digitial tach channel embedded within it.  This will extract the information from the specified bit and return this information.
Assume data is a Vector{Float64} or Array{Float64,1}
bit - the bit number from the least significant bit that contains the data
pprdivide - the tach pulse divide by ratio
"""
function gantnermask(data::Vector{Float64}, bit::Integer, pprdivide::Integer)
    mask = one(Int) << (bit - 1)
    datanew = typeof(data)(undef,size(data))
    datanew[1] = xnew = Bool(0)
    count = 0

    x = Int(data[1]) & mask   # perform masking
    xi = Bool(x >> (bit - 1))             # shift bit to LSB
    # println(xi)

    for (i, x) in enumerate(data[begin+1:end])
        xi_1 = xi
        x = Int(x) & mask   # perform masking
        xi = Bool(x >> (bit - 1))       # shift bit to LSB
        # println(i, "\t", x, "\t", xi, "\t", xi_1)
        # check for trigger
        if xor(xi, xi_1)
            count += 1
            # @show(count)
            if count == pprdivide
                xnew = !xnew
                count = 0
            end
        end
        datanew[i+1] = xnew
    end
    return datanew
end

end # module


module RTLSDR

import Base.open, Base.close

export
    RtlSdr,
    open,
    close,
    read_bytes,
    read_samples,
    packed_bytes_to_iq

export
    get_rate,
    set_rate,
    get_freq,
    set_freq,
    get_gains,
    get_gain,
    set_gain,
    set_agc_mode,
    set_tuner_gain_mode

include("c_interface.jl")

mutable struct RtlSdr
    valid_ptr::Bool
    dongle_ptr::Ptr{rtlsdr_dev}

    function RtlSdr(index::Int64=0)
        dp = rtlsdr_open(index)

        r = new(true, dp)

        # default sample rate and center freq
        set_rate(r, 2.0e6)
        set_freq(r, 88.5e6)

        return r
    end
end

function open(r::RtlSdr, device_index::Int)
end

function close(r::RtlSdr)
    rtlsdr_close(r.dongle_ptr)
    r.valid_ptr = false
end

"""
`set_rate(r::RtlSdr, sample_rate_Hz)
"""
function set_rate(r::RtlSdr, sample_rate_Hz)
    @assert r.valid_ptr
    rtlsdr_set_sample_rate(r.dongle_ptr, sample_rate_Hz)
end
function get_rate(r::RtlSdr)
    @assert r.valid_ptr
    rate = rtlsdr_get_sample_rate(r.dongle_ptr)
    return Int(rate)
end

"""
`set_freq(r::RtlSdr, freq_Hz)`

Interface for `rtlsdr_set_center_freq`.
"""
function set_freq(r::RtlSdr, freq_Hz)
    @assert r.valid_ptr
    rtlsdr_set_center_freq(r.dongle_ptr, freq_Hz)
end
function get_freq(r::RtlSdr)
    @assert r.valid_ptr
    freq = rtlsdr_get_center_freq(r.dongle_ptr)
    return Int(freq)
end

"""
`get_gains(r::RtlSdr)`

Interface for `rtlsdr_get_tuner_gains`.
Gains are in tenths of a dB.
"""
function get_gains(r::RtlSdr)
    @assert r.valid_ptr
    return rtlsdr_get_tuner_gains(r.dongle_ptr)
end

"""
`set_gain(r::RtlSdr, gain_db)`

Interface for `rtlsdr_set_tuner_gain`.
"""
function set_gain(r::RtlSdr, gain_db)
    @assert r.valid_ptr
    # Implement rounding, taking after roger-'s pyrtlsdr
    avail_gains = get_gains(r)
    errors = [abs((10 * gain_db) - avail_gain) for avail_gain in get_gains(r)]
    closest_gain_idx = findall(errors .== minimum(errors))[1] # broadcast find
    rtlsdr_set_tuner_gain(r.dongle_ptr, avail_gains[closest_gain_idx])
end
function get_gain(r::RtlSdr)
    @assert r.valid_ptr
    return rtlsdr_get_tuner_gain(r.dongle_ptr) / 10
end

"""
Interface for `rtlsdr_set_agc_mode`
"""
function set_agc_mode(r::RtlSdr, mode)
    @assert r.valid_ptr
    rtlsdr_set_agc_mode(r.dongle_ptr, mode)
end

"""
Interface for `rtlsdr_set_tuner_gain_mode`
"""
function set_tuner_gain_mode(r::RtlSdr, manual)
    @assert r.valid_ptr
    rtlsdr_set_tuner_gain_mode(r.dongle_ptr, manual)
end

"""
`read_bytes(r::RtlSdr, num_bytes)`

High-level interface for `rtlsdr_read_sync`.

`num_bytes` must be a multiple of 512.

Returns a vector of length `num_bytes` of Uint8 (bytes).
"""
function read_bytes(r::RtlSdr, num_bytes)
    @assert r.valid_ptr
    return read_bytes(r.dongle_ptr, num_bytes)
end

"""
`read_samples(r::RtlSdr, num_samples)`

Returns a vector of length `num_samples` with complex numbers.
"""
function read_samples(r::RtlSdr, num_samples)
    @assert r.valid_ptr
    num_bytes = 2num_samples
    raw_data = read_bytes(r.dongle_ptr, num_bytes)
    return packed_bytes_to_iq(raw_data)
end

"""
`packed_bytes_to_iq(bytes)`
"""
function packed_bytes_to_iq(bytes)
    num_bytes = length(bytes)
    num_iq = round(Int, num_bytes/2.0, RoundDown)
    iq_vals = zeros(Complex{Float64}, num_iq)
    den = 255.0/2.0

    for i = 1:num_iq
        iq_vals[i] = bytes[2i-1]/den - 1.0 + im*(bytes[2i]/den - 1.0)
    end

    return iq_vals
end

end # module


# /Users/kartikeygupta/Workspace/Julia/Gym/NES/deps/roms/smb.nes
import Base: parse
MAGIC = (0x4e, 0x45, 0x53, 0x1a)

struct ROM
    raw_data::Array{UInt8, 1}
    header::Array{UInt8, 1}
end

function ROM(rom_path::AbstractString)
    io = open(rom_path, "r")
    buffer_size = 10
    buffer = zeros(UInt8, buffer_size)
    raw_data = UInt8[]
    while !eof(io)
        readbytes!(io, buffer)
        push!(raw_data, buffer...)
    end
    header = raw_data[1:16]
    !all(MAGIC .== header[1:4]) && error("ROM missing magic number in the header.")
    sum(header[12:end]) != 0 && error("ROM Header zero fill bytes are not zero.")

    ROM(raw_data, header)
end

parse(chr::Char) = Base.parse(UInt8, chr)
parse(str::AbstractString) = Base.parse(UInt8, str)

function Base.getproperty(rom::ROM, sym::Symbol)

    """Return the header of the ROM file as bytes."""
    if sym == :header
        return rom.raw_data[1:16]

    """Return the size of the PRG ROM in KB"""
    elseif sym == :prg_rom_size
        return 16 * rom.header[5]

    """Return the size of the CHR ROM in KB"""
    elseif sym == :chr_rom_size
        return 8 * rom.header[6]

    """Return the flags at the 6th byte of the header."""
    elseif sym == :flags_6
        return string(rom.header[7], base=2, pad=8)

    """Return the flags at the 7th byte of the header."""
    elseif sym == :flags_7
        return string(rom.header[8], base=2, pad=8)

    """Return the size of the PRG RAM in KB."""
    elseif sym == :prg_ram_size
        return 8 * (rom.header[9] == 0 ? 1 : rom.header[9])

    """Return the flags at the 9th byte of the header."""
    elseif sym == :flags_9
        return string(rom.header[10], base=2, pad=8)

    """
    Return the flags at the 10th byte of the header.

    Notes:
        - these flags are not part of official specification.
        - ignored in this emulator

    """
    elseif sym == :flags_10
        return string(rom.header[11], base=2, pad=8)

#-------------------------HEADER FLAGS-----------------------------------------#

    """Return the mapper number this ROM uses."""
    elseif sym == :mapper
        return parse(rom.flags_7[1:4] * rom.flags_6[1:4])

    """Return a boolean determining if the ROM ignores mirroring."""
    elseif sym == :is_ignore_mirroring
        return rom.flags_6[5] |> parse |> Bool

    """Return a boolean determining if the ROM has a trainer block."""
    elseif sym == :has_trainer
        return rom.flags_6[6] |> parse |> Bool

    """Return a boolean determining if the ROM has a battery-backed RAM."""
    elseif sym == :has_battery_backed_ram
        return rom.flags_6[7] |> parse |> Bool

    """Return the mirroring mode this ROM uses."""
    elseif sym == :is_vertical_mirroring
        return rom.flags_6[8] |> parse |> Bool

    """
    Return whether this cartridge uses PlayChoice-10.

    Note:
        - Play-Choice 10 uses different color palettes for a different PPU
        - ignored in this emulator

    """
    elseif sym == :has_play_choice_10
        return rom.flags_7[7] |> parse |> Bool

    """
    Return whether this cartridge has VS Uni-system.

    Note:
        VS Uni-system is for ROMs that have a coin slot (Arcades).
        - ignored in this emulator

    """
    elseif sym == :has_vs_unisystem
        return rom.flags_7[8] |> parse |> Bool

    """Returns the TV System this ROM supports"""
    elseif sym == :is_pal
        return rom.flags_9[8] |> parse |> Bool

#----------------------------------ROM------------------------------------------

    """The inclusive starting index of the trainer ROM."""
    elseif sym == :trainer_rom_start
        return 17

    """The inclusive stopping index of the trainer ROM."""
    elseif sym == :trainer_rom_stop
        return rom.has_trainer ? rom.trainer_rom_start + 512 : rom.trainer_rom_start - 1

    """Return the trainer ROM of the ROM file"""
    elseif sym == :trainer_rom
        return rom.raw_data[rom.trainer_rom_start:rom.trainer_rom_stop]

    """The inclusive starting index of the PRG ROM."""
    elseif sym == :prg_rom_start
        return rom.trainer_rom_stop + 1

    """The inclusive stopping index of the PRG ROM."""
    elseif sym == :prg_rom_stop
        return rom.prg_rom_start + rom.prg_rom_size * 2^10 - 1

    """Return the PRG ROM of the ROM file."""
    elseif sym == :prg_rom
        try
            return rom.raw_data[rom.prg_rom_start : rom.prg_rom_stop]
        catch BoundsError
            error("Failed to read PRG-ROM on ROM")
        end

    """The inclusive starting index of the CHR ROM."""
    elseif sym == :chr_rom_start
        return rom.prg_rom_stop + 1

    """The inclusive stopping index of the CHR ROM."""
    elseif sym == :chr_rom_stop
        return rom.chr_rom_start + rom.chr_rom_size * 2^10 - 1

    """Return the CHR ROM of the ROM file."""
    elseif sym == :chr_rom
        try
            return rom.raw_data[rom.chr_rom_start:rom.chr_rom_stop]
        catch BoundsError
            error("Failed to read CHR-ROM on ROM")
        end
    else
        return getfield(rom, sym)
    end
end


using MathProgBase
export CplexSolver

mutable struct CplexMathProgModel <: AbstractLinearQuadraticModel
    inner::Model
    lazycb
    cutcb
    heuristiccb
    branchcb
    incumbentcb
    infocb
    solvetime::Float64
    mipstart_effortlevel::Cint
    heuristic_buffer::Vector{Float64}
end

function CplexMathProgModel(;mipstart_effortlevel::Cint = CPX_MIPSTART_AUTO, options...)
    env = Env()
    # set_param!(env, "CPX_PARAM_MIPCBREDLP", 0) # access variables in original problem, not presolved
    # set_param!(env, "CPX_PARAM_PRELINEAR", 0) # MAY NOT BE NECESSARY, only performs linear presolving so can recover original variables
    set_param!(env, "CPX_PARAM_SCRIND", 1) # output logs to stdout by default
    for (name,value) in options
        set_param!(env, string(name), value)
    end

    m = CplexMathProgModel(Model(env), nothing, nothing, nothing, nothing, nothing, nothing, NaN, mipstart_effortlevel, [])
    return m
end

mutable struct CplexSolver <: AbstractMathProgSolver
    options
end
CplexSolver(;kwargs...) = CplexSolver(kwargs)
MathProgBase.LinearQuadraticModel(s::CplexSolver) = CplexMathProgModel(;s.options...)

MathProgBase.ConicModel(s::CplexSolver) = LPQPtoConicBridge(LinearQuadraticModel(s))
MathProgBase.supportedcones(::CplexSolver) = [:Free,:Zero,:NonNeg,:NonPos,:SOC]

function MathProgBase.setparameters!(s::CplexSolver; mpboptions...)
    opts = collect(Any,s.options)
    for (optname, optval) in mpboptions
        if optname == :TimeLimit
            push!(opts, (:CPX_PARAM_TILIM, optval))
        elseif optname == :Silent
            if optval == true
                push!(opts, (:CPX_PARAM_SCRIND, 0))
            end
        else
            error("Unrecognized parameter $optname")
        end
    end
    s.options = opts
    return
end

function MathProgBase.setparameters!(s::CplexMathProgModel; mpboptions...)
    for (optname, optval) in mpboptions
        if optname == :TimeLimit
            setparam!(m.inner, "CPX_PARAM_TILIM", optval)
        elseif optname == :Silent
            if optval == true
                setparam!(m.inner,"CPX_PARAM_SCRIND",0)
            end
        else
            error("Unrecognized parameter $optname")
        end
    end
end

function MathProgBase.loadproblem!(m::CplexMathProgModel, filename::String)
   read_model(m.inner, filename)
end

function MathProgBase.loadproblem!(m::CplexMathProgModel, A, collb, colub, obj, rowlb, rowub, sense)
  # throw away old model but keep env
  m.inner = Model(m.inner.env)
  add_vars!(m.inner, float(obj), float(collb), float(colub))

  neginf = typemin(eltype(rowlb))
  posinf = typemax(eltype(rowub))

  rangeconstrs = any((rowlb .!= rowub) .& (rowlb .> neginf) .& (rowub .< posinf))
  if rangeconstrs
    warn("Julia Cplex interface doesn't properly support range (two-sided) constraints.")
    add_rangeconstrs!(m.inner, float(A), float(rowlb), float(rowub))
  else
    b = Vector{Float64}(undef, length(rowlb))
    senses = Vector{Cchar}(undef, length(rowlb))
    for i in 1:length(rowlb)
      if rowlb[i] == rowub[i]
        senses[i] = 'E'
        b[i] = rowlb[i]
      elseif rowlb[i] > neginf
        senses[i] = 'G'
        b[i] = rowlb[i]
      else
        @assert rowub[i] < posinf
        senses[i] = 'L'
        b[i] = rowub[i]
      end
    end
    add_constrs!(m.inner, float(A), senses, b)
  end

  set_sense!(m.inner, sense)
end

MathProgBase.writeproblem(m::CplexMathProgModel, filename::String) = write_model(m.inner, filename)

MathProgBase.getvarLB(m::CplexMathProgModel) = get_varLB(m.inner)
MathProgBase.setvarLB!(m::CplexMathProgModel, l) = set_varLB!(m.inner, l)
MathProgBase.getvarUB(m::CplexMathProgModel) = get_varUB(m.inner)
MathProgBase.setvarUB!(m::CplexMathProgModel, u) = set_varUB!(m.inner, u)

# CPXchgcoef
MathProgBase.getconstrLB(m::CplexMathProgModel) = get_constrLB(m.inner)
MathProgBase.setconstrLB!(m::CplexMathProgModel, lb) = set_constrLB!(m.inner, lb)
MathProgBase.getconstrUB(m::CplexMathProgModel) = get_constrUB(m.inner)
MathProgBase.setconstrUB!(m::CplexMathProgModel, ub) = set_constrUB!(m.inner, ub)

MathProgBase.getobj(m::CplexMathProgModel) = get_obj(m.inner)
MathProgBase.setobj!(m::CplexMathProgModel, c) = set_obj!(m.inner, c)

MathProgBase.addvar!(m::CplexMathProgModel, l, u, coeff) = add_var!(m.inner, [], [], l, u, coeff)
MathProgBase.addvar!(m::CplexMathProgModel, constridx, constrcoef, l, u, coeff) = add_var!(m.inner, constridx, constrcoef, l, u, coeff)

function MathProgBase.addconstr!(m::CplexMathProgModel, varidx, coef, lb, ub)
  neginf = typemin(eltype(lb))
  posinf = typemax(eltype(ub))

  rangeconstrs = any((lb .!= ub) & (lb .> neginf) & (ub .< posinf))
  if rangeconstrs
    warn("Julia Cplex interface doesn't properly support range (two-sided) constraints.")
    add_rangeconstrs!(m.inner, [0], varidx, float(coef), float(lb), float(ub))
  else
    if lb == ub
      rel = 'E'
      rhs = lb
    elseif lb > neginf
      rel = 'G'
      rhs = lb
    else
      @assert ub < posinf
      rel = 'L'
      rhs = ub
    end
    add_constrs!(m.inner, ivec([1]), ivec(varidx), fvec(coef), convert(Vector{Cchar},[rel]), fvec(vec(collect(rhs))))
  end
end

MathProgBase.getconstrmatrix(m::CplexMathProgModel) = get_constr_matrix(m.inner)

MathProgBase.setsense!(m::CplexMathProgModel, sense) = set_sense!(m.inner, sense)

MathProgBase.getsense(m::CplexMathProgModel) = get_sense(m.inner)

MathProgBase.numvar(m::CplexMathProgModel) = num_var(m.inner)
MathProgBase.numconstr(m::CplexMathProgModel) = num_constr(m.inner) + num_qconstr(m.inner)
MathProgBase.numlinconstr(m::CplexMathProgModel) = num_constr(m.inner)
MathProgBase.numquadconstr(m::CplexMathProgModel) = num_qconstr(m.inner)

# MathProgBase.optimize!(m::CplexMathProgModel) = optimize!(m.inner)

function MathProgBase.optimize!(m::CplexMathProgModel)
    # set callbacks if present
    if m.lazycb != nothing
        setmathproglazycallback!(m)
    end
    if m.cutcb != nothing
        setmathprogcutcallback!(m)
    end
    if m.heuristiccb != nothing
        setmathprogheuristiccallback!(m)
    end
    if m.branchcb != nothing
        setmathprogbranchcallback!(m)
    end
    if m.incumbentcb != nothing
        setmathprogincumbentcallback!(m)
    end
    if m.infocb != nothing
        setmathproginfocallback!(m)
    end
    start = time()
    optimize!(m.inner)
    m.solvetime = time() - start
end

function MathProgBase.status(m::CplexMathProgModel)
    ret = get_status(m.inner)
    return (if ret in [:CPX_STAT_OPTIMAL, :CPXMIP_OPTIMAL, :CPXMIP_OPTIMAL_TOL]
        :Optimal
    elseif ret in [:CPX_STAT_UNBOUNDED, :CPXMIP_UNBOUNDED]
        :Unbounded
    elseif ret in [:CPX_STAT_INFEASIBLE, :CPXMIP_INFEASIBLE]
        :Infeasible
    elseif ret in [:CPX_STAT_INForUNBD, :CPXMIP_INForUNBD]
        Base.warn_once("CPLEX reported infeasible or unbounded. Set CPX_PARAM_REDUCE=1 to check
                        infeasibility or CPX_PARAM_REDUCE=2 to check unboundedness.")
        :InfeasibleOrUnbounded
    elseif occursin("TIME_LIM", string(ret)) || occursin("MIP_ABORT", string(ret))
        :UserLimit
    else
        ret
    end)
end

MathProgBase.getobjval(m::CplexMathProgModel)   = get_objval(m.inner)
MathProgBase.getobjbound(m::CplexMathProgModel) = get_best_bound(m.inner)
MathProgBase.getsolution(m::CplexMathProgModel) = get_solution(m.inner)
MathProgBase.getconstrsolution(m::CplexMathProgModel) = get_constr_solution(m.inner)
MathProgBase.getreducedcosts(m::CplexMathProgModel) = get_reduced_costs(m.inner)
MathProgBase.getconstrduals(m::CplexMathProgModel) = get_constr_duals(m.inner)
MathProgBase.getrawsolver(m::CplexMathProgModel) = m.inner
MathProgBase.getnodecount(m::CplexMathProgModel) = get_node_count(m.inner)

const var_type_map = Dict(
    'C' => :Cont,
    'B' => :Bin,
    'I' => :Int,
    'S' => :SemiCont,
    'N' => :SemiInt
)

const rev_var_type_map = Dict(
    :Cont     => 'C',
    :Bin      => 'B',
    :Int      => 'I',
    :SemiCont => 'S',
    :SemiInt  => 'N'
)

function MathProgBase.setvartype!(m::CplexMathProgModel, v::Vector{Symbol})
    target_int = all(x->isequal(x,:Cont), v)
    prob_type = get_prob_type(m.inner)
    if target_int
        if m.inner.has_sos # if it has sos we need to keep has_int==true and the MI(prob_type) version.
            set_vartype!(m.inner, map(x->rev_var_type_map[x], v))
        else
            m.inner.has_int = false
            if !(prob_type in [:LP,:QP,:QCP])
                toggleproblemtype!(m)
            end
        end
    else
        if prob_type in [:LP,:QP,:QCP]
            toggleproblemtype!(m)
        end
        set_vartype!(m.inner, map(x->rev_var_type_map[x], v))
    end
    return nothing
end

const prob_type_toggle_map = Dict(
    :LP    => :MILP,
    :MILP  => :LP,
    :QP    => :MIQP,
    :MIQP  => :QP,
    :QCP   => :MIQCP,
    :MIQCP => :QCP
)

function toggleproblemtype!(m::CplexMathProgModel)
    prob_type = get_prob_type(m.inner)
    set_prob_type!(m.inner, prob_type_toggle_map[prob_type])
end

function MathProgBase.getvartype(m::CplexMathProgModel)
    if m.inner.has_int
        return map(x->var_type_map[x], get_vartype(m.inner))
    else
        return fill(:Cont, num_var(m.inner))
    end
end

function MathProgBase.getsolvetime(m::CplexMathProgModel)
    return m.solvetime
end

MathProgBase.getinfeasibilityray(m::CplexMathProgModel) = get_infeasibility_ray(m.inner)
MathProgBase.getunboundedray(m::CplexMathProgModel) = get_unbounded_ray(m.inner)

MathProgBase.getbasis(m::CplexMathProgModel) = get_basis(m.inner)

function MathProgBase.setwarmstart!(m::CplexMathProgModel, v)
    # This means that warm starts are ignored if you haven't called setvartype! first
    if m.inner.has_int
        set_warm_start!(m.inner, v, m.mipstart_effortlevel)
    end
end

MathProgBase.addsos1!(m::CplexMathProgModel, idx, weight) = add_sos!(m.inner, :SOS1, idx, weight)
MathProgBase.addsos2!(m::CplexMathProgModel, idx, weight) = add_sos!(m.inner, :SOS2, idx, weight)

######
# QCQP
######
MathProgBase.addquadconstr!(m::CplexMathProgModel, linearidx, linearval, quadrowidx, quadcolidx, quadval, sense, rhs) =
    add_qconstr!(m.inner,linearidx,linearval,quadrowidx,quadcolidx,quadval,sense,rhs)
MathProgBase.setquadobj!(m::CplexMathProgModel,rowidx,colidx,quadval) = add_qpterms!(m.inner,rowidx,colidx,quadval)

######
# Data
######
function getdettime(m::CplexMathProgModel)
    tim = Vector{Cdouble}(undef, 1)
    stat = @cpx_ccall(getdettime, Cint, (Ptr{Cvoid},Ptr{Cdouble}), m.inner.env.ptr, tim)
    if stat != 0
        error(CplexError(m.inner.env, stat).msg)
    end
    return tim[1]
end

MathProgBase.getobjgap(m::CplexMathProgModel) = get_rel_gap(m.inner)

###########
# Callbacks
###########
export cbaddboundbranchup!,
       cbaddboundbranchdown!,
       setmathprogbranchcallback!,
       cbgetnodelb,
       cbgetnodeub,
       cbgetnodeobjval,
       cbgetnodesleft,
       cbgetmipiterations,
       cbgetfeasibility,
       cbgetgap,
       cbgetstarttime,
       cbgetdetstarttime,
       cbgettimestamp,
       cbgetdettimestamp,
       cbgetintfeas

abstract type CplexCallbackData <: MathProgCallbackData end

# set to nothing to clear callback
MathProgBase.setlazycallback!(m::CplexMathProgModel,f) = (m.lazycb = f)
MathProgBase.setcutcallback!(m::CplexMathProgModel,f) = (m.cutcb = f)
MathProgBase.setheuristiccallback!(m::CplexMathProgModel,f) = (m.heuristiccb = f)
setbranchcallback!(m::CplexMathProgModel,f) = (m.branchcb = f)
setincumbentcallback!(m::CplexMathProgModel,f) = (m.incumbentcb = f)
MathProgBase.setinfocallback!(m::CplexMathProgModel,f) = (m.infocb = f)

function MathProgBase.cbgetmipsolution(d::CplexCallbackData)
    @assert d.state == :MIPSol || d.state == :MIPIncumbent
    n = num_var(d.cbdata.model)
    sol = Vector{Cdouble}(undef, n)
    stat = @cpx_ccall(getcallbacknodex, Cint, (Ptr{Cvoid},Ptr{Cvoid},Cint,Ptr{Cdouble},Cint,Cint),
                      d.cbdata.model.env.ptr, d.cbdata.cbdata, d.where, sol, 0, n-1)
    if stat != 0
        error(CplexError(d.cbdata.model.env, stat).msg)
    end
    return sol
end

function MathProgBase.cbgetmipsolution(d::CplexCallbackData, sol::Vector{Cdouble})
    @assert d.state == :MIPSol
    stat = @cpx_ccall(getcallbacknodex, Cint, (Ptr{Cvoid},Ptr{Cvoid},Cint,Ptr{Cdouble},Cint,Cint),
                      d.cbdata.model.env.ptr, d.cbdata.cbdata, d.where, sol, 0, length(sol)-1)
    if stat != 0
        error(CplexError(d.cbdata.model.env, stat).msg)
    end
    return nothing
end

function MathProgBase.cbgetlpsolution(d::CplexCallbackData)
    @assert d.state == :MIPNode || d.state == :MIPBranch
    n = num_var(d.cbdata.model)
    sol = Vector{Cdouble}(undef, n)
    stat = @cpx_ccall(getcallbacknodex, Cint, (Ptr{Cvoid},Ptr{Cvoid},Cint,Ptr{Cdouble},Cint,Cint),
                      d.cbdata.model.env.ptr, d.cbdata.cbdata, d.where, sol, 0, n-1)
    if stat != 0
        error(CplexError(d.cbdata.model.env, stat).msg)
    end
    return sol
end

function MathProgBase.cbgetlpsolution(d::CplexCallbackData, sol::Vector{Cdouble})
    @assert d.state == :MIPNode || d.state == :MIPIncumbent || d.state == :MIPBranch
    stat = @cpx_ccall(getcallbacknodex, Cint, (Ptr{Cvoid},Ptr{Cvoid},Cint,Ptr{Cdouble},Cint,Cint),
                      d.cbdata.model.env.ptr, d.cbdata.cbdata, d.where, sol, 0, length(sol)-1)
    if stat != 0
        error(CplexError(d.cbdata.model.env, stat).msg)
    end
    return nothing
end

for (func,param,typ) in ((:cbgetexplorednodes,CPX_CALLBACK_INFO_NODE_COUNT_LONG,:Int64),
                         (:cbgetobj,CPX_CALLBACK_INFO_BEST_INTEGER,:Cdouble),
                         (:cbgetbestbound,CPX_CALLBACK_INFO_BEST_REMAINING,:Cdouble))

    @eval begin
        function (MathProgBase.$func)(d::CplexCallbackData)
            val = Vector{$(typ)}(undef, 1)
            ret = @cpx_ccall(getcallbackinfo, Cint, (Ptr{Cvoid},Ptr{Cvoid},Cint,Cint,Ptr{Cvoid}),
                              d.cbdata.model.env.ptr, d.cbdata.cbdata, d.where, $(convert(Cint,param)), val)
            if ret != 0
                error(CplexError(d.cbdata.model.env, stat).msg)
            end
            return val[1]
        end
    end
end

for (func,param,typ) in ((:cbgetnodesleft,CPX_CALLBACK_INFO_NODES_LEFT_LONG,:Int64),
                         (:cbgetmipiterations,CPX_CALLBACK_INFO_MIP_ITERATIONS_LONG,:Int64),
                         (:cbgetgap,CPX_CALLBACK_INFO_MIP_REL_GAP,:Cdouble),
                         (:cbgetfeasibility,CPX_CALLBACK_INFO_MIP_FEAS,:Cint),
                         (:cbgetstarttime,CPX_CALLBACK_INFO_STARTTIME,:Cdouble),
                         (:cbgetdetstarttime,CPX_CALLBACK_INFO_STARTDETTIME,:Cdouble),
                         (:cbgettimestamp,CPX_CALLBACK_INFO_ENDTIME,:Cdouble),
                         (:cbgetdettimestamp,CPX_CALLBACK_INFO_ENDDETTIME,:Cdouble))

    @eval begin
        function ($func)(d::CplexCallbackData)
            val = Vector{$(typ)}(undef, 1)
            ret = @cpx_ccall(getcallbackinfo, Cint, (Ptr{Cvoid},Ptr{Cvoid},Cint,Cint,Ptr{Cvoid}),
                              d.cbdata.model.env.ptr, d.cbdata.cbdata, d.where, $(convert(Cint,param)), val)
            if ret != 0
                error(CplexError(d.cbdata.model.env, stat).msg)
            end
            return val[1]
        end
    end
end

# returns :MIPNode :MIPSol :Intermediate
MathProgBase.cbgetstate(d::CplexCallbackData) = d.state

#const sensemap = Dict('=' => 'E', '<' => 'L', '>' => 'G')
function MathProgBase.cbaddcut!(d::CplexCallbackData,varidx,varcoef,sense,rhs)
    @assert d.state == :MIPNode
    cbcut(d.cbdata, d.where, convert(Vector{Cint}, varidx), float(varcoef), sensemap[sense], float(rhs))
    unsafe_store!(d.userinteraction_p, convert(Cint,CPX_CALLBACK_ABORT_CUT_LOOP), 1)
end

function MathProgBase.cbaddcutlocal!(d::CplexCallbackData,varidx,varcoef,sense,rhs)
    @assert d.state == :MIPNode
    cbcutlocal(d.cbdata, d.where, convert(Vector{Cint}, varidx), float(varcoef), sensemap[sense], float(rhs))
    unsafe_store!(d.userinteraction_p, convert(Cint,CPX_CALLBACK_ABORT_CUT_LOOP), 1)
end

function MathProgBase.cbaddlazy!(d::CplexCallbackData,varidx,varcoef,sense,rhs)
    @assert d.state == :MIPNode || d.state == :MIPSol
    cblazy(d.cbdata, d.where, convert(Vector{Cint}, varidx), float(varcoef), sensemap[sense], float(rhs))
end

function MathProgBase.cbaddlazylocal!(d::CplexCallbackData,varidx,varcoef,sense,rhs)
    @assert d.state == :MIPNode || d.state == :MIPSol
    cblazylocal(d.cbdata, d.where, convert(Vector{Cint}, varidx), float(varcoef), sensemap[sense], float(rhs))
end

function MathProgBase.cbaddsolution!(d::CplexCallbackData)
    val = unsafe_wrap(Array, d.userinteraction_p, 1)
    if val[1] == CPX_CALLBACK_SET
        error("CPLEX only allows one heuristic solution for each call to the callback")
    end
    unsafe_store!(d.userinteraction_p, convert(Cint,CPX_CALLBACK_SET), 1)
end

function MathProgBase.cbsetsolutionvalue!(d::CplexCallbackData,varidx,value)
    @assert 1 <= varidx <= num_var(d.cbdata.model)
    d.heur_x[varidx] = value
    d.user_solution = true
    nothing
end

function cbaddboundbranchup!(d::CplexCallbackData,idx,bd,nodeest)
    seqnum = cbbranch(d.cbdata, d.where,convert(Cint,idx-1),convert(Cchar,'L'),bd,nodeest)
    unsafe_store!(d.userinteraction_p, convert(Cint,CPX_CALLBACK_SET), 1)
    seqnum
end

function cbaddboundbranchdown!(d::CplexCallbackData,idx,bd,nodeest)
    seqnum = cbbranch(d.cbdata, d.where,convert(Cint,idx-1),convert(Cchar,'U'),bd,nodeest)
    unsafe_store!(d.userinteraction_p, convert(Cint,CPX_CALLBACK_SET), 1)
    seqnum
end

function cbaddconstrbranch!(d::CplexCallbackData, indices, coeffs, rhs, sense, nodeest)
    seqnum = cbbranchconstr(d.cbdata,
                   d.where,
                   Cint[idx-1 for idx in indices],
                   Cdouble[c for c in coeffs],
                   convert(Cdouble, rhs),
                   convert(Cchar, sense),
                   nodeest)
    unsafe_store!(d.userinteraction_p, convert(Cint,CPX_CALLBACK_SET), 1)
    seqnum
end

function cbprocessincumbent!(d::CplexCallbackData,accept::Bool)
    if accept
        unsafe_store!(d.isfeas_p, convert(Cint, 1), 1)
    else
        unsafe_store!(d.isfeas_p, convert(Cint, 0), 1)
    end
    nothing
end

mutable struct CplexLazyCallbackData <: CplexCallbackData
    cbdata::CallbackData
    state::Symbol
    where::Cint
    userinteraction_p::Ptr{Cint}
end

mutable struct CplexCutCallbackData <: CplexCallbackData
    cbdata::CallbackData
    state::Symbol
    where::Cint
    userinteraction_p::Ptr{Cint}
end

terminate(model::CplexMathProgModel) = terminate(model.inner)

# breaking abstraction, define our low-level callback to eliminate
# a level of indirection
function mastercallback(env::Ptr{Cvoid}, cbdata::Ptr{Cvoid}, wherefrom::Cint, userdata::Ptr{Cvoid}, userinteraction_p::Ptr{Cint})
    model = MathProgBase.unsafe_pointer_to_objref(userdata)::CplexMathProgModel
    cpxrawcb = CallbackData(cbdata, model.inner)
    if wherefrom == CPX_CALLBACK_MIP_CUT_FEAS || wherefrom == CPX_CALLBACK_MIP_CUT_UNBD
        state = :MIPSol
    # elseif wherefrom == CPX_CALLBACK_MIP_CUT_LOOP || wherefrom == CPX_CALLBACK_MIP_CUT_LAST
    elseif wherefrom == CPX_CALLBACK_MIP_CUT_LAST
        state = :MIPNode
    else
        state = :Intermediate
    end

    if model.infocb != nothing
        cpxcb = CplexInfoCallbackData(cpxrawcb, state, wherefrom)
        stat = model.infocb(cpxcb)
        if stat == :Exit
            terminate(model.inner)
        end
    end
    if model.lazycb != nothing && state == :MIPSol
        cpxcb = CplexLazyCallbackData(cpxrawcb, state, wherefrom, userinteraction_p)
        stat = model.lazycb(cpxcb)
        if stat == :Exit
            terminate(model.inner)
        end
    end
    if model.cutcb != nothing && state == :MIPNode
        cpxcb = CplexCutCallbackData(cpxrawcb, state, wherefrom, userinteraction_p)
        stat = model.cutcb(cpxcb)
        if stat == :Exit
            terminate(model.inner)
        end
    end
    return convert(Cint, 0)
end

mutable struct CplexHeuristicCallbackData <: CplexCallbackData
    cbdata::CallbackData
    state::Symbol
    where::Cint
    sol::Vector{Float64}
    heur_x::Vector{Float64}
    isfeas_p::Ptr{Cint}
    userinteraction_p::Ptr{Cint}
    user_solution::Bool # true if user has set any solution values
end

function MathProgBase.cbgetlpsolution(d::CplexHeuristicCallbackData)
    return copy(d.sol)
end

function MathProgBase.cbgetlpsolution(d::CplexHeuristicCallbackData, sol::Vector{Cdouble})
    copyto!(sol,d.sol)
end

function masterheuristiccallback(env::Ptr{Cvoid},
                                 cbdata::Ptr{Cvoid},
                                 wherefrom::Cint,
                                 userdata::Ptr{Cvoid},
                                 objval_p::Ptr{Cdouble},
                                 xx::Ptr{Cdouble},
                                 isfeas_p::Ptr{Cint},
                                 userinteraction_p::Ptr{Cint})
    model = MathProgBase.unsafe_pointer_to_objref(userdata)::CplexMathProgModel
    cpxrawcb = CallbackData(cbdata, model.inner)
    if wherefrom == CPX_CALLBACK_MIP_HEURISTIC
        state = :MIPNode
    else
        state = :Intermediate
    end

    if model.infocb != nothing
        cpxcb = CplexInfoCallbackData(cpxrawcb, state, wherefrom)
        stat = model.infocb(cpxcb)
        if stat == :Exit
            terminate(model.inner)
        end
    end
    if model.heuristiccb != nothing && state == :MIPNode
        sol = unsafe_wrap(Array, xx, numvar(model))
        cpxcb = CplexHeuristicCallbackData(cpxrawcb, state, wherefrom, sol, model.heuristic_buffer, isfeas_p, userinteraction_p, false)
        stat = model.heuristiccb(cpxcb)
        if stat == :Exit
            terminate(model.inner)
        end
        if cpxcb.user_solution # we filled in some solution values
            unsafe_store!(objval_p, dot(get_obj(model.inner), cpxcb.heur_x), 1)
            for i in 1:numvar(model)
                unsafe_store!(xx, cpxcb.heur_x[i], i)
            end
            if any(x->isnan(x), cpxcb.heur_x) # we have a partial solution
                unsafe_store!(isfeas_p, convert(Cint,CPX_ON),  1)
            else
                unsafe_store!(isfeas_p, convert(Cint,CPX_OFF), 1)
            end
            fill!(model.heuristic_buffer, NaN)
        end
    end
    return convert(Cint, 0)
end

function setmathproglazycallback!(model::CplexMathProgModel)
    set_param!(model.inner.env, "CPX_PARAM_MIPCBREDLP", 0)
    set_param!(model.inner.env, "CPX_PARAM_PRELINEAR", 0)
    set_param!(model.inner.env, "CPX_PARAM_REDUCE", CPX_PREREDUCE_PRIMALONLY)
    cpxcallback = @cfunction(mastercallback, Cint, (Ptr{Cvoid}, Ptr{Cvoid}, Cint, Ptr{Cvoid}, Ptr{Cint}))
    stat = @cpx_ccall(setlazyconstraintcallbackfunc, Cint, (
                      Ptr{Cvoid},
                      Ptr{Cvoid},
                      Any,
                      ),
                      model.inner.env.ptr, cpxcallback, model)
    if stat != 0
        throw(CplexError(model.env, stat))
    end
    nothing
end

function setmathprogcutcallback!(model::CplexMathProgModel)
    set_param!(model.inner.env, "CPX_PARAM_MIPCBREDLP", 0)
    set_param!(model.inner.env, "CPX_PARAM_PRELINEAR", 0)
    cpxcallback = @cfunction(mastercallback, Cint, (Ptr{Cvoid}, Ptr{Cvoid}, Cint, Ptr{Cvoid}, Ptr{Cint}))
    stat = @cpx_ccall(setusercutcallbackfunc, Cint, (
                      Ptr{Cvoid},
                      Ptr{Cvoid},
                      Any,
                      ),
                      model.inner.env.ptr, cpxcallback, model)
    if stat != 0
        throw(CplexError(model.env, stat))
    end
    nothing
end

function setmathprogheuristiccallback!(model::CplexMathProgModel)
    set_param!(model.inner.env, "CPX_PARAM_MIPCBREDLP", 0)
    set_param!(model.inner.env, "CPX_PARAM_PRELINEAR", 0)
    cpxcallback = @cfunction(masterheuristiccallback, Cint, (Ptr{Cvoid}, Ptr{Cvoid}, Cint, Ptr{Cvoid}, Ptr{Cdouble}, Ptr{Cdouble}, Ptr{Cint}, Ptr{Cint}))
    stat = @cpx_ccall(setheuristiccallbackfunc, Cint, (
                      Ptr{Cvoid},
                      Ptr{Cvoid},
                      Any,
                      ),
                      model.inner.env.ptr, cpxcallback, model)
    if stat != 0
        throw(CplexError(model.env, stat))
    end
    model.heuristic_buffer = fill(NaN, numvar(model))
    nothing
end

struct BranchingChoice
    indices::Vector{Cint}
    bounds::Vector{Cdouble}
    lu::Vector{Cchar}
end
export BranchingChoice

struct CplexBranchCallbackData <: CplexCallbackData
    cbdata::CallbackData
    state::Symbol
    where::Cint
    userinteraction_p::Ptr{Cint}
    nodes::Vector{BranchingChoice}
end

function masterbranchcallback(env::Ptr{Cvoid},
                              cbdata::Ptr{Cvoid},
                              wherefrom::Cint,
                              userdata::Ptr{Cvoid},
                              typ::Cint,
                              sos::Cint,
                              nodecnt::Cint,
                              bdcnt::Cint,
                              nodebeg::Ptr{Cint},
                              indices::Ptr{Cint},
                              lu::Ptr{Cchar},
                              bd::Ptr{Cdouble},
                              nodeest::Ptr{Cdouble},
                              userinteraction_p::Ptr{Cint})
    model = unsafe_pointer_to_objref(userdata)::CplexMathProgModel
    cpxrawcb = CallbackData(cbdata, model.inner)
    if wherefrom == CPX_CALLBACK_MIP_BRANCH
        @assert 0 <= nodecnt <= 2
        state = :MIPBranch
    else
        state = :Intermediate
    end

    if model.infocb != nothing
        cpxcb = CplexInfoCallbackData(cpxrawcb, state, wherefrom)
        stat = model.infocb(cpxcb)
        if stat == :Exit
            terminate(model.inner)
        end
    end
    if model.branchcb != nothing && state == :MIPBranch
        numbranchingvars = unsafe_wrap(Array, nodebeg, convert(Cint,nodecnt))::Vector{Cint} .+ 1
        idxs = unsafe_wrap(Array, indices, sum(numbranchingvars))::Vector{Cint}
        vals = unsafe_wrap(Array, bd, sum(numbranchingvars))::Vector{Cdouble}
        dirs = unsafe_wrap(Array, lu, sum(numbranchingvars))::Vector{Cchar}
        nodes = Vector{BranchingChoice}(undef, nodecnt)
        if nodecnt >= 1
            subidx = 1 : (numbranchingvars[1])
            nodes[1] = BranchingChoice(idxs[subidx], vals[subidx], dirs[subidx])
        end
        if nodecnt == 2
            subidx = (numbranchingvars[1]+1) : (numbranchingvars[2])
            nodes[2] = BranchingChoice(idxs[subidx], vals[subidx], dirs[subidx])
        end
        cpxcb = CplexBranchCallbackData(cpxrawcb, state, wherefrom, userinteraction_p, nodes)
        stat = model.branchcb(cpxcb)
        if stat == :Exit
            terminate(model.inner)
        end
    end
    return convert(Cint, 0)
end

function setmathprogbranchcallback!(model::CplexMathProgModel)
    set_param!(model.inner.env, "CPX_PARAM_MIPCBREDLP", 0)
    set_param!(model.inner.env, "CPX_PARAM_PRELINEAR", 0)
    set_param!(model.inner.env, "CPX_PARAM_REDUCE", CPX_PREREDUCE_PRIMALONLY)
    cpxcallback = @cfunction(masterbranchcallback, Cint, (Ptr{Cvoid},
                                                         Ptr{Cvoid},
                                                         Cint,
                                                         Ptr{Cvoid},
                                                         Cint,
                                                         Cint,
                                                         Cint,
                                                         Cint,
                                                         Ptr{Cint},
                                                         Ptr{Cint},
                                                         Ptr{Cchar},
                                                         Ptr{Cdouble},
                                                         Ptr{Cdouble},
                                                         Ptr{Cint}))
    stat = @cpx_ccall(setbranchcallbackfunc, Cint, (
                      Ptr{Cvoid},
                      Ptr{Cvoid},
                      Any,
                      ),
                      model.inner.env.ptr, cpxcallback, model)
    if stat != 0
        throw(CplexError(model.env, stat))
    end
    nothing
end

mutable struct CplexIncumbentCallbackData <: CplexCallbackData
    cbdata::CallbackData
    state::Symbol
    where::Cint
    sol::Vector{Float64}
    isfeas_p::Ptr{Cint}
    userinteraction_p::Ptr{Cint}
    nodes::Vector{BranchingChoice}
end

function masterincumbentcallback(env::Ptr{Cvoid},
                                 cbdata::Ptr{Cvoid},
                                 wherefrom::Cint,
                                 userdata::Ptr{Cvoid},
                                 objval::Cdouble,
                                 xx::Ptr{Cdouble},
                                 isfeas_p::Ptr{Cint},
                                 useraction_p::Ptr{Cint})
    model = unsafe_pointer_to_objref(userdata)::CplexMathProgModel
    cpxrawcb = CallbackData(cbdata, model.inner)
    if wherefrom == CPX_CALLBACK_MIP_INCUMBENT_NODESOLN ||
       wherefrom == CPX_CALLBACK_MIP_INCUMBENT_HEURSOLN ||
       wherefrom == CPX_CALLBACK_MIP_INCUMBENT_USERSOLN
        state = :MIPIncumbent
    else
        state = :Intermediate
    end

    if model.infocb != nothing
        cpxcb = CplexInfoCallbackData(cpxrawcb, state, wherefrom)
        stat = model.infocb(cpxcb)
        if stat == :Exit
            terminate(model.inner)
        end
    end
    if model.incumbentcb != nothing && state == :MIPIncumbent
        sol = unsafe_wrap(Array, xx, numvar(model))
        cpxcb = CplexIncumbentCallbackData(cpxrawcb, state, wherefrom, sol, isfeas_p, useraction_p, BranchingChoice[])
        stat = model.incumbentcb(cpxcb)
        if stat == :Exit
            terminate(model.inner)
        end
    end
    return convert(Cint, 0)
end

function setmathprogincumbentcallback!(model::CplexMathProgModel)
    set_param!(model.inner.env, "CPX_PARAM_MIPCBREDLP", 0)
    set_param!(model.inner.env, "CPX_PARAM_PRELINEAR", 0)
    set_param!(model.inner.env, "CPX_PARAM_REDUCE", CPX_PREREDUCE_PRIMALONLY)
    cpxcallback = @cfunction(masterincumbentcallback, Cint, (Ptr{Cvoid},
                                                            Ptr{Cvoid},
                                                            Cint,
                                                            Ptr{Cvoid},
                                                            Cdouble,
                                                            Ptr{Cdouble},
                                                            Ptr{Cint},
                                                            Ptr{Cint}))
    stat = @cpx_ccall(setincumbentcallbackfunc, Cint, (
                      Ptr{Cvoid},
                      Ptr{Cvoid},
                      Any,
                      ),
                      model.inner.env.ptr, cpxcallback, model)
    if stat != 0
        throw(CplexError(model.env, stat))
    end
    nothing
end

mutable struct CplexInfoCallbackData <: CplexCallbackData
    cbdata::CallbackData
    state::Symbol
    where::Cint
end

function masterinfocallback(env::Ptr{Cvoid},
                            cbdata::Ptr{Cvoid},
                            wherefrom::Cint,
                            userdata::Ptr{Cvoid})
    model = unsafe_pointer_to_objref(userdata)::CplexMathProgModel
    if model.infocb != nothing
        state = :Intermediate
        cpxrawcb = CallbackData(cbdata, model.inner)
        cpxcb = CplexInfoCallbackData(cpxrawcb, state, wherefrom)
        stat = model.infocb(cpxcb)
        if stat == :Exit
            terminate(model.inner)
        end
    end
    return convert(Cint, 0)
end

function setmathproginfocallback!(model::CplexMathProgModel)
    cpxcallback = @cfunction(masterinfocallback, Cint, (Ptr{Cvoid}, Ptr{Cvoid}, Cint, Ptr{Cvoid}))
    stat = @cpx_ccall(setinfocallbackfunc, Cint, (
                      Ptr{Cvoid},
                      Ptr{Cvoid},
                      Any,
                      ),
                      model.inner.env.ptr, cpxcallback, model)
    if stat != 0
        throw(CplexError(model.env, stat))
    end
    nothing
end

function cbgetnodelb(d::CplexCallbackData)
    n = num_var(d.cbdata.model)
    lb = Vector{Cdouble}(undef, n)
    stat = @cpx_ccall(getcallbacknodelb, Cint, (Ptr{Cvoid},Ptr{Cvoid},Cint,Ptr{Cdouble},Cint,Cint),
                      d.cbdata.model.env.ptr, d.cbdata.cbdata, d.where, lb, 0, n-1)
    if stat != 0
        throw(CplexError(model.env, stat))
    end
    return lb
end

function cbgetnodeub(d::CplexCallbackData)
    n = num_var(d.cbdata.model)
    ub = Vector{Cdouble}(undef, n)
    stat = @cpx_ccall(getcallbacknodeub, Cint, (Ptr{Cvoid},Ptr{Cvoid},Cint,Ptr{Cdouble},Cint,Cint),
                      d.cbdata.model.env.ptr, d.cbdata.cbdata, d.where, ub, 0, n-1)
    if stat != 0
        throw(CplexError(model.env, stat))
    end
    return ub
end

function cbgetnodeobjval(d::CplexCallbackData)
    val = Vector{Cdouble}(undef, 1)
    stat = @cpx_ccall(getcallbacknodeobjval, Cint, (Ptr{Cvoid},Ptr{Cvoid},Cint,Ptr{Cdouble}),
                      d.cbdata.model.env.ptr, d.cbdata.cbdata, d.where, val)
    if stat != 0
        throw(CplexError(model.env, stat))
    end
    return val[1]
end

function cbgetintfeas(d::CplexCallbackData)
    n = num_var(d.cbdata.model)
    feas = Vector{Cint}(undef, n)
    stat = @cpx_ccall(getcallbacknodeintfeas, Cint, (Ptr{Cvoid},Ptr{Cvoid},Cint,Ptr{Cint},Cint,Cint),
                      d.cbdata.model.env.ptr, d.cbdata.cbdata, d.where, feas, 0, n-1)
    if stat != 0
        throw(CplexError(model.env, stat))
    end
    return convert(Vector{Int64},feas)
end


struct DNAdataset{S <: Real}
    #=
    data can be
        1. simulated data: Vector{sim_dna_str_w_motif}
        2. fasta that has labeled ground truth
            (those contigugous substring with CAP latters):
            NamedTuple{(:str, :labeled_ground_truth),
                Tuple{Vector{String}, Vector{String}}}
        3. fasta file without labeled ground tuth:
                Vector{String}
    =#
    raw_data::Union{Vector{sim_dna_str_w_motif},
                Vector{String},
                Vector{public_dna_str_w_motif},
                Nothing
                }

    # motif:
    # - if it's a simulated data, then it has a motif type
    # - if data is loaded from fasta as in case 2 or 3 above,
    #   then it is a Nothing type
    motif::motif_type

    # number of data points
    N::Int

    # number of rows is the length of each transformed data_pt
    # number of columns is the total number of data entries
    # currently work for data of the same length only
    dat::Matrix{S}

    """
    Input:
        motif: a motif type that generates the data
        num_data_pts: how many data points to generate
        GPU: (true/false) use GPU to store the wyk-data
            (so can use GPU for training)
    """
    function DNAdataset{S}(motif::motif_type,
                        num_data_pts::Int
        ) where {S <: Real}
        raw_data = sample_backgound_with_motif_multiple(motif, num_data_pts);
            new(
                raw_data,
                motif,
                num_data_pts,
                sim_data_2_dummy(raw_data;F=S)
            )
    end

    """
    Input:
        filepath: fasta filepath

        has_ground_truth: (true/false) the input fasta
            file has each string contained capped-letter
            DNA-alphabets that are labeled as motifs.

        same_len: (true/false) each string in the input
            fasta file has the same length.

        GPU: (true/false) use GPU to store the wyk-data
            (so can use GPU for training)
    """
    function DNAdataset{S}(dna_read::Vector{String},
                        motif::motif_type,
                        raw_data::Vector{public_dna_str_w_motif}
        ) where {S <: Real}
        new(raw_data,
            motif,
            length(dna_read),
            DNA_strings_2_dummy_vectors_same_length(dna_read;F=S)
        );
    end
    function DNAdataset{S}(dna_read::Vector{String}) where {S <: Real}
        new(nothing,
        nothing,
        length(dna_read),
        DNA_strings_2_dummy_vectors_same_length(dna_read; F=S)
        );
    end
end