module GlobalSearchRegressionGUI
using GlobalSearchRegression, HTTP, WebSockets, DataStructures, Mux, JSON, CSV, Pkg, Distributed, UUIDs, Base64

const SERVER_BASE_DIR = "../front/dist"
const INSTALLED_PACKAGES = Pkg.installed()
const GSREG_VERSION = ( haskey(INSTALLED_PACKAGES, "GlobalSearchRegression") ) ? INSTALLED_PACKAGES["GlobalSearchRegression"] : v"1.0.3"

mutable struct GSRegJob
    file # tempfile of data
    hash # hash id of user
    options # options for calculation
    id # unique identifier for this job
    time_enqueued # time enqueued
    time_started
    time_finished
    res::GlobalSearchRegression.GSRegResult # results array
    GSRegJob(file, hash, options) = new(file, hash, options, string(uuid4()), time())
end

"""
    Enqueue the job and notify worker
"""
function enqueue_job(job::GSRegJob)
    global job_queue
    global job_queue_cond
    enqueue!(job_queue, job)
    notify(job_queue_cond)
    job
end

function sendMessage(id, data)
    global connections
    if haskey(connections, string(id))
        ws = connections[id]
        sendMessage(ws, data)
    end
end

function sendMessage(ws::WebSocket, data)
    write(ws, JSON.json(data))
end

function gsreg(job::GSRegJob)
    global jobs_finished
    try
        sendMessage(job.hash, Dict("operation_id" => job.id, "message" => "Reading data"))
        data = CSV.read(job.file)
        sendMessage(job.hash, Dict("operation_id" => job.id, "message" => "Executing GSReg"))

        opt = job.options["options"]
        opt[:parallel] = job.options["paraprocs"]

        job.time_started = time()
        job.res = GlobalSearchRegression.gsreg(job.options["depvar"] * " " * join(job.options["expvars"], " "), data; opt...,
                    onmessage = message -> sendMessage(job.hash, Dict("operation_id" => job.id, "message" => message)) )
        job.time_finished = time()
        push!(jobs_finished, Pair(job.id, job))

        sendMessage(job.hash, Dict("operation_id" => job.id, "done" => true, "message" => "Successful operation", "result" => GlobalSearchRegression.to_dict(job.res)))
    catch e
        io = IOBuffer()
        showerror(io, e)
        sendMessage(job.hash, Dict("operation_id" => job.id, "done" => false, "message" => String(take!(io))))
    end
end

"""
    Log request for analytics and debugging
"""
function logRequest(app, req)
#     log = string(
#         get(req[:headers], "Host", ""), " ",
#         get(req[:headers], "Origin", ""), " ",
#         Libc.strftime("%d/%b/%Y:%H:%M:%S %z", time()), " ",
#         "\"", req[:method], " ", req[:resource], "\"", " ",
#         size(req[:data],1), "B",
#         "\n")
#
#     open(joinpath(dirname(@__FILE__), "..", "access.log"), "a") do fp
#         write(fp, log)
#     end
    app(req)
end

"""
    Try to execute controller and if there is any exception, show exception message
"""
function errorCatch(app, req)
    try
        app(req)
    catch e
        io = IOBuffer()
        showerror(io, e)
        err_text = String(take!(io))
        toJsonWithCors(Dict("message" => err_text, "error" => true), req)
    end
end

"""
    Setting CORS headers and parsing it to JSON
"""
function toJsonWithCors(res, req)
    headers = []

    push!(headers, "Server" => "Julia/$VERSION")
    push!(headers, "Content-Type" => "text/html; charset=utf-8")

    if( req[:method] != "OPTIONS" )
        push!(headers, "Content-Type" => "application/json; charset=utf-8")
    end
    push!(headers, "Access-Control-Allow-Headers" => "X-User-Token, Content-Type")
    push!(headers, "Access-Control-Allow-Origin" => "*")

    Dict(
        :headers => headers,
        :body => codeunits((req[:method] == "OPTIONS") ? "" : JSON.json(res))
    )
end

"""
    Get Auth from header, and reply an error when it's not present
"""
function authHeader(app, req)
    headers = Dict(req[:headers])
    if(haskey(headers, "X-User-Token"))
        req[:token] = get(headers, "X-User-Token", "")
    end
    app(req)
end

function constructCommand(options)
    command = """Pkg.add("GlobalSearchRegression")
using GlobalSearchRegression, CSV
data = CSV.read("yourcsvfile.csv")
res = GlobalSearchRegression.gsreg("", data; opts...)"""
end

"""
    TODO: doc and d better validation, more secure for cloud environments
"""
function validateInput!(opt)
    options = Dict{Symbol,Any}()
    opt_types = Dict(
        "intercept" => Bool,
        "time" => String,
        "residualtest" => Bool,
        "ttest" => Bool,
        "orderresults" => Bool,
        "modelavg" => Bool,
        "outsample" => Int,
        "method" => String,
        "criteria" => Array{Symbol}
    )
    for (name, value) in opt["options"]
        if value != nothing && name == "criteria"
            push!(options, Pair(:criteria, map(Symbol, value)))
        elseif value != nothing && name == "time"
            push!(options, Pair(:time, Symbol(value)))
        elseif value != nothing && name != "csv" && name != "resultscsv"
            push!(options, Pair(Symbol(name), value))
        end
    end
    opt["options"] = options
    opt
end

"""
    Receives base64 encoded data in path with regression
    options and a content body with CSV file for processing

    req[:data] -> Array{UInt8,1}
    req[:params][:b64] -> String
"""
function upload(req)
    """
    Save the file to tmp folder, if there is any exception, it should be returned to the user for
    reuse. It must be available until the user explicit deletes it, or its deleted from
    operating system. If it is deleted, the file must be uploaded again.
    """
    tempfile = try
        temp = tempname()
        write(temp, IOBuffer(req[:data]))
        temp
    catch
        error("We can't save that file, try again later.")
    end

    """
    Check csv parsing & get data
    """
    data = try
        CSV.read(IOBuffer(req[:data]))
    catch
        error("The file must be a valid CSV")
    end

    global files_dict
    id = string(uuid4())
    push!(files_dict, Pair(id, tempfile))

    Dict(
        "filename" => id,
        "datanames" => names(data),
        "nobs" => size(data, 1)
    )
end

function server_info(req)
    global job_queue
    Dict(
        "ncores" => Sys.CPU_THREADS,
        "nworkers" => nworkers(),
        "gsreg_version" => string(GSREG_VERSION),
        "julia_version" => string(VERSION),
        "job_queue" => Dict(
            "length" => length(job_queue)
        )
    )
end

"""
    Enqueue the execution of regressions, expecting a confirmation. If any there is any error, should be a detailed report.
"""
function solve(req)
    """
    Try to get the filename from params
    """
    global files_dict

    if haskey(files_dict, req[:params][:hash])
        tempfile = files_dict[req[:params][:hash]]
        if (!isfile(tempfile))
            error("File was deleted")
        end
    else
        error("Filekey inexistent")
    end

    """
    Input options should be a valid base64:json encoded String.
    """
    options = try
        b64 = convert(String, req[:params][:options])
        JSON.parse(String(base64decode(b64)))
    catch
        error("Bad format in options")
    end

    # validate correct use of options
    validateInput!(options)

    # Enqueue the job
    job = GSRegJob(tempfile, req[:token], options)
    enqueue_job(job)

    Dict(
        "ok" => true,
        "operation_id" => job.id,
        "in_queue" => length(job_queue)
    )
end

function validpath(path; dirs = true)
    return true # fallback index.html will manage this
    (isfile(path) || (dirs && isdir(path))) || isfile(joinpath(path,"index.html"))
end

ormatch(r::RegexMatch, x) = r.match
ormatch(nothing, x) = x

extension(f) = ormatch(match(r"(?<=\.)[^\.\\/]*$", f), "")

mimetypes = Dict{AbstractString,AbstractString}([
    ("jpg", "image/jpeg"),
    ("png", "image/png"),
    ("html", "text/html"),
    ("csv", "text/csv"),
    ("css", "text/css"),
    ("js", "application/javascript"),
    ("ttf", "application/x-font-ttf"),
    ("svg", "image/svg+xml"),
    ("mp4", "video/mp4"),
    ("ico", "image/x-icon"),
    ("map", "text/plain")
])

fileheaders(f) = Dict("Content-Type" => get(mimetypes, extension(f), "application/octet-stream"))

fileresponse(f) = Dict(
                    :body => read(f),
                    :headers => fileheaders(f)
                    )

fresp(f) = isfile(f) ? fileresponse(f) : fileresponse(joinpath(dirname(@__FILE__), SERVER_BASE_DIR,"index.html"))

"""
    operation_id -> csv
"""
function result_file(req)
    global jobs_finished
    id = req[:params][:id]
    if haskey(jobs_finished, id)
        csv = IOBuffer()
        job = jobs_finished[id]
        export_csv(csv, job.res)
        delete!(jobs_finished, id)
        if job.options["csv"] != nothing
            filename = endswith(job.options["csv"], ".csv") ? job.options["csv"] : job.options["csv"] * ".csv"
        else
            filename = "gsreg" * Libc.strftime("%d/%b/%Y%H%M%S", time()) * ".csv"
        end
        Dict(
            :body => String(take!(csv)),
            :headers => Dict("Content-Type" => "application/octet-stream", "Content-Disposition" => "attachment; filename=$filename")
        )
    else
        Dict( :status => 404, :body => "Not found" )
    end
end

files_dict = Dict{String,String}()
jobs_finished = Dict{String,GSRegJob}()

job_queue = Queue{GSRegJob}()

job_queue_cond = Condition()

# Array of connections based in user token
connections = Dict{String,WebSocket}()

noop(app, req) = app(req)

function gui(;openbrowser=true, port=45872, cloud=false, log=false)
    global job_queue_cond
    global job_queue

    #Consume job_queue after notification
    function consume_job_queue()
        while true
            wait(job_queue_cond)
            while !(isempty(job_queue))
#                 try
                    gsreg(dequeue!(job_queue))
#                 catch
#                     println("Error executing gsreg job")
#                 end
            end
        end
    end

    schedule(Task(consume_job_queue))

    @app app = (
        Mux.stack(Mux.todict, logRequest, Mux.splitquery, errorCatch, authHeader, Mux.toresponse),
        page("/info", respond("
info
")),
        page("/upload", req -> toJsonWithCors(upload(req), req)),
        page("/server-info", req -> toJsonWithCors(server_info(req), req)),
        page("/result/:id", req -> result_file(req)),
        page("/solve/:hash/:options", req -> toJsonWithCors(solve(req), req)),
        branch( req -> validpath(joinpath(SERVER_BASE_DIR, req[:path]...)), req -> fresp(joinpath(dirname(@__FILE__), SERVER_BASE_DIR, req[:path]...))),
        Mux.notfound
    )

    """
     This WebSocket handler mantain a collection of ws opened with the user id.
    """
    function wshandler(ws)
        global connections
        while isopen(ws)
            msg, = readguarded(ws)
            try
                msg = JSON.parse(String(copy(msg)))
                id = msg["user-token"]
                connections[id] = ws
                sendMessage(ws, Dict("ok" => true, "message" => "Waiting in queue"))
            catch
                sendMessage(ws, Dict("ok" => false, "message" => "The next message must be in JSON format"))
            end
        end
        # TODO: find some way for delete connection with any reference
        # delete!(connections, id)
        # close(ws)
    end

    wsapp = Mux.App(Mux.mux(
        Mux.wdefaults,
        Mux.route("/ws", req -> wshandler(req[:socket])),
        Mux.wclose,
        Mux.notfound(),
    ))

    @async Mux.serve(app, wsapp, port)

    if(openbrowser)
        url = "http://127.0.0.1:" * string(port)
        sleep(3)
        if Sys.iswindows()
            run(`cmd /c start $url`)
        else
            start = ( Sys.isapple() ? "open" : "xdg-open" )
            run(Cmd([start,url]))
        end
    end
end

export gui

end # module GlobalSearchRegressionGUI


module BARON

using MathProgBase
importall MathProgBase.SolverInterface

using Compat

export BaronSolver
immutable BaronSolver <: AbstractMathProgSolver
    options
end
BaronSolver(;kwargs...) = BaronSolver(kwargs)

const baron_exec = ENV["BARON_EXEC"]

type BaronMathProgModel <: AbstractNonlinearModel
    options

    xˡ::Vector{Float64}
    xᵘ::Vector{Float64}
    gˡ::Vector{Float64}
    gᵘ::Vector{Float64}

    nvar::Int
    ncon::Int

    obj
    constrs

    vartypes::Vector{Symbol}

    v_names::Vector{Compat.UTF8String}
    c_names::Vector{Compat.UTF8String}

    sense::Symbol

    x₀::Vector{Float64}

    probfile::Compat.UTF8String
    sumfile::Compat.UTF8String
    resfile::Compat.UTF8String

    objval::Float64
    solution::Vector{Float64}
    status::Symbol

    d::AbstractNLPEvaluator

    function BaronMathProgModel(;options...)
        dir = tempdir()
	    push!(options, (:ResName, joinpath(dir, "res.lst")))
	    push!(options, (:TimName, joinpath(dir, "tim.lst")))
	    push!(options, (:SumName, joinpath(dir, "sum.lst")))
        new(options,
	    zeros(0),
	    zeros(0),
	    zeros(0),
        zeros(0),
	    0,
	    0,
	    :(0),
	    Expr[],
	    Symbol[],
	    Compat.UTF8String[],
	    Compat.UTF8String[],
	    :Min,
	    zeros(0),
        "",
	    "",
	    "",
	    NaN,
	    zeros(0),
        :NotSolved)
    end
end

MathProgBase.NonlinearModel(s::BaronSolver) = BaronMathProgModel(;s.options...)
MathProgBase.LinearQuadraticModel(s::BaronSolver) = MathProgBase.NonlinearToLPQPBridge(MathProgBase.NonlinearModel(s))

verify_support(c) = c

function verify_support(c::Expr)
    if c.head == :comparison
        map(verify_support, c.args)
        return c
    end
    if c.head == :call
        if c.args[1] in (:+, :-, :*, :/, :exp, :log)
            return c
        elseif c.args[1] in (:<=, :>=, :(==))
            map(verify_support, c.args[2:end])
            return c
        elseif c.args[1] == :^
            @assert isa(c.args[2], Real) || isa(c.args[3], Real)
            return c
        else # TODO: do automatic transformation for x^y, |x|
            error("Unsupported expression $c")
        end
    end
    return c
end

function MathProgBase.loadproblem!(m::BaronMathProgModel,
    nvar, ncon, xˡ, xᵘ, gˡ, gᵘ, sense,
    d::MathProgBase.AbstractNLPEvaluator)

    @assert nvar == length(xˡ) == length(xᵘ)
    @assert ncon == length(gˡ) == length(gᵘ)

    m.xˡ, m.xᵘ = xˡ, xᵘ
    m.gˡ, m.gᵘ = gˡ, gᵘ
    m.sense = sense
    m.nvar, m.ncon = nvar, ncon
    m.d = d

    m.v_names = ["x$i" for i in 1:nvar]
    m.c_names = ["e$i" for i in 1:ncon]

    MathProgBase.initialize(d, [:ExprGraph])

    m.obj = verify_support(MathProgBase.obj_expr(d))
    m.vartypes = fill(:Cont, nvar)

    m.constrs = map(1:ncon) do c
        verify_support(MathProgBase.constr_expr(d,c))
    end

    dir = tempdir()
    m.probfile = joinpath(dir, "baron_problem.bar")
    m.sumfile  = joinpath(dir, "sum.lst")
    m.resfile  = joinpath(dir, "res.lst")
    m
end

function MathProgBase.setvartype!(m::BaronMathProgModel, cat::Vector{Symbol})
    @assert all(x-> (x in (:Cont,:Bin,:Int)), cat)
    m.vartypes = cat
end

function print_var_definitions(m, fp, header, condition)
    idx = filter(condition, 1:m.nvar)
    if !isempty(idx)
        println(fp, header, join([m.v_names[i] for i in idx], ", "), ";")
    end
end

# to_str(x::Int) = string(x)
# to_str(x) = (@show(x); string(float(x)))
to_str(x) = string(x)

function to_str(c::Expr)
    if c.head == :comparison
        if length(c.args) == 3
            return join([to_str(c.args[1]), c.args[2], c.args[3]], " ")
        elseif length(c.args) == 5
            return join([c.args[1], c.args[2], to_str(c.args[3]),
                         c.args[4], c.args[5]], " ")
        end
    elseif c.head == :call
        if c.args[1] in (:<=,:>=,:(==))
            if length(c.args) == 3
                return join([to_str(c.args[2]), to_str(c.args[1]), to_str(c.args[3])], " ")
            elseif length(c.args) == 5
                return join([to_str(c.args[1]), to_str(c.args[2]), to_str(c.args[3]), to_str(c.args[4]), to_str(c.args[5])], " ")
            end
        elseif c.args[1] in (:+,:-,:*,:/,:^)
            if all(d->isa(d, Real), c.args[2:end]) # handle unary case
                return string(eval(c))
            elseif c.args[1] == :- && length(c.args) == 2
                return string("(-$(to_str(c.args[2])))")
            else
                return string("(", join([to_str(d) for d in c.args[2:end]], string(c.args[1])), ")")
            end
        elseif c.args[1] in (exp,:log)
            if isa(c.args[2], Real)
                return string(eval(c))
            else
                return string(c.args[1], "( ", to_str(c.args[2]), " )")
            end
        end
    elseif c.head == :ref
        if c.args[1] == :x
            @assert isa(c.args[2], Int)
            return "x$(c.args[2])"
        else
            error("Unrecognized reference expression $c")
        end
    end
end

function write_bar_file(m::BaronMathProgModel)
    fp = open(m.probfile, "w")

    # First: process any options
    println(fp, "OPTIONS{")
    for (opt,setting) in m.options
        if isa(setting, AbstractString) # wrap it in quotes
            println(fp, unescape_string("$opt: $('"')$setting$('"');"))
        else
            println(fp, "$opt: $setting;")
        end
    end
    println(fp, "}")
    println(fp)

    # Next, define variables
    print_var_definitions(m, fp, "BINARY_VARIABLES ",   v->(m.vartypes[v]==:Bin))
    print_var_definitions(m, fp, "INTEGER_VARIABLES ",  v->(m.vartypes[v]==:Int))
    print_var_definitions(m, fp, "POSITIVE_VARIABLES ", v->(m.vartypes[v]==:Cont && m.xˡ[v]==0))
    print_var_definitions(m, fp, "VARIABLE ",           v->(m.vartypes[v]==:Cont && m.xˡ[v]!=0))
    println(fp)

    # Print variable bounds
    if any(c->!isinf(c), m.xˡ)
        println(fp, "LOWER_BOUNDS{")
        for (i,l) in enumerate(m.xˡ)
            if !isinf(l)
                println(fp, "$(m.v_names[i]): $l;")
            end
        end
        println(fp, "}")
        println(fp)
    end
    if any(c->!isinf(c), m.xᵘ)
        println(fp, "UPPER_BOUNDS{")
        for (i,u) in enumerate(m.xᵘ)
            if !isinf(u)
                println(fp, "$(m.v_names[i]): $u;")
            end
        end
        println(fp, "}")
        println(fp)
    end

    # Now let's declare the equations
    if !isempty(m.constrs)
        println(fp, "EQUATIONS ", join(m.c_names, ", "), ";")
        for (i,c) in enumerate(m.constrs)
            str = to_str(c)
            println(fp, "$(m.c_names[i]): $str;")
        end
        println(fp)
    end

    # Now let's do the objective
    print(fp, "OBJ: ")
    print(fp, m.sense == :Min ? "minimize " : "maximize ")
    print(fp, to_str(m.obj))
    println(fp, ";")
    println(fp)

    if !isempty(m.x₀)
        println(fp, "STARTING_POINT{")
        for (i,v) in enumerate(m.x₀)
            println(fp, "$(m.v_names[i]): $v;")
        end
        println(fp, "}")
    end
    close(fp)
end

const user_limits = (
    "Max. allowable nodes in memory reached",
    "Max. allowable BaR iterations reached",
    "Max. allowable CPU time exceeded",
    "Problem is numerically sensitive",
    "Insufficient Memory for Data structures"
)

function read_results(m::BaronMathProgModel)
    # First, we read the summary file to get the solution status
    fp = open(m.sumfile, "r")
    stat = :Undefined
    while true
        line = readline(fp)
        spl = split(chomp(line))
        if !isempty(spl) && spl[1] == "***"
            if spl[2] in user_limits
                stat = :UserLimit
            end
            break
        end
        eof(fp) && error("Reached EOF while searching for termination notice")
    end
    while true
        line = readline(fp)
        spl = split(chomp(line))
        if !isempty(spl) && spl[1:3] == ["Best","solution","found"]
            node = parse(Int,match(r"\d+", line).match)
            if node == -3
                stat = :Infeasible
            end
            break
        elseif spl == ["***", "Problem", "is", "infeasible", "***"]
            stat = :Infeasible
            break
        elseif spl == ["***", "Problem", "is", "unbounded", "***"]
            stat = :Unbounded
            break
        else
            stat = :Optimal
            break
        end
        eof(fp) && error("Reached OEF while looking for node with best solution")
    end
    m.status = stat
    close(fp)

    # Next, we read the results file to get the solution
    x = fill(NaN, m.nvar)
    m.objval = NaN
    if stat == :Optimal
        fp = open(m.resfile, "r")
        while true
            startswith(readline(fp), "The best solution found") && break
            eof(fp) && error()
        end
        readline(fp)
        readline(fp)

        while true
            line = chomp(readline(fp))
            parts = split(line)
            isempty(parts) && break
            mt = match(r"\d+", parts[1])
            mt == nothing && error("Cannot find appropriate variable index from $(parts[1])")
            v_idx = parse(Int,mt.match)
            v_val = float(parts[3])
            x[v_idx] = v_val
        end
        m.solution = x
        line = readline(fp)
        val = match(r"[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?", chomp(line))
        if val != nothing
            m.objval = float(val.match)
        end
    end
    nothing
end

MathProgBase.setwarmstart!(m::BaronMathProgModel, v::Vector{Float64}) = (m.x₀ = v)

function MathProgBase.optimize!(m::BaronMathProgModel)
    write_bar_file(m)
    run(`$baron_exec $(m.probfile)`)

    read_results(m)
end

MathProgBase.status(m::BaronMathProgModel) = m.status

MathProgBase.numvar(m::BaronMathProgModel) = m.nvar
MathProgBase.getsolution(m::BaronMathProgModel) = m.solution
MathProgBase.getobjval(m::BaronMathProgModel) = m.objval

end


abstract type AbstractJammer{T} <: AbstractEmitter{T} end

struct CWJammer{
    T <: AbstractFloat,
    D <: Union{SVector{3}, AbstractDOA},
    E <: Union{Bool, AbstractExistence},
    C <: Union{Unitful.Gain, AbstractAmplitude},
} <: AbstractJammer{T}
    id::Int
    doppler::typeof(1.0Hz)
    phase::Float64
    jnr::C
    exists::E
    doa::D
    signal::StructArray{Complex{T},1,NamedTuple{(:re, :im),Tuple{Array{T,1},Array{T,1}}},Int}
end

struct NoiseJammer{
    T <: AbstractFloat,
    D <: Union{SVector{3}, AbstractDOA},
    E <: Union{Bool, AbstractExistence},
    C <: Union{Unitful.Gain, AbstractAmplitude},
} <: AbstractJammer{T}
    id::Int
    jnr::C
    exists::E
    doa::D
    signal::StructArray{Complex{T},1,NamedTuple{(:re, :im),Tuple{Array{T,1},Array{T,1}}},Int}
end

function CWJammer(
    ::Type{T},
    id::Integer,
    jnr::J;
    doppler = 0.0Hz,
    phase = 0.0,
    exists::E = true,
    doa::D = SVector(0, 0, 1)
) where {
    T <: AbstractFloat,
    D <: Union{SVector{3}, AbstractDOA},
    E <: Union{Bool, AbstractExistence},
    J <: Union{Unitful.Gain{Unitful.Decibel, :?, <:Real}, AbstractAmplitude}
}
    phase = phase / 2π
    signal = StructArray{Complex{T}}(undef, 0)
    CWJammer{T, D, E, J}(
        id,
        float(doppler),
        float(phase),
        jnr,
        exists,
        doa,
        signal
    )
end

function CWJammer(
    id::Integer,
    jnr::Union{Unitful.Gain{Unitful.Decibel, :?, <:Real}, AbstractAmplitude};
    doppler = 0.0Hz,
    phase = 0.0,
    exists = true,
    doa = SVector(0, 0, 1)
)
    CWJammer(
        Float32,
        id,
        jnr,
        doppler = doppler,
        phase = phase,
        exists = exists,
        doa = doa
    )
end

function gen_signal!(
    jammer::CWJammer{T},
    sampling_frequency,
    intermediate_frequency,
    n0,
    num_samples::Integer,
    rng
) where T
    resize!(jammer.signal, num_samples)
    gen_carrier!(
        jammer.signal,
        intermediate_frequency + get_carrier_doppler(jammer),
        sampling_frequency,
        get_carrier_phase_2pi(jammer),
        T(get_amplitude(jammer.jnr, n0, sampling_frequency))
    )
end

function NoiseJammer(
    ::Type{T},
    id,
    jnr::J;
    exists::E = true,
    doa::D = SVector(0, 0, 1)
) where {
    T <: AbstractFloat,
    D <: Union{SVector{3}, AbstractDOA},
    E <: Union{Bool, AbstractExistence},
    J <: Union{Unitful.Gain{Unitful.Decibel, :?, <:Real}, AbstractAmplitude}
}
    signal = StructArray{Complex{T}}(undef, 0)
    NoiseJammer{T, D, E, J}(
        id,
        jnr,
        exists,
        doa,
        signal
    )
end

function NoiseJammer(
    id,
    jnr;
    exists = true,
    doa = SVector(0, 0, 1)
)
    NoiseJammer(
        Float32,
        id,
        jnr,
        exists = exists,
        doa = doa
    )
end

function gen_signal!(
    jammer::NoiseJammer{T},
    sample_frequency,
    intermediate_frequency,
    n0,
    num_samples::Integer,
    rng
) where T
    resize!(jammer.signal, num_samples)
    signal = randn!(rng, jammer.signal)
    jammer.signal .*= T(get_amplitude(jammer.jnr, n0, sample_frequency))
    jammer.signal
end

function propagate(
    jammer::CWJammer,
    num_samples,
    intermediate_frequency,
    sample_frequency,
    n0,
    rng
)
    carrier_phase = (intermediate_frequency + get_carrier_doppler(jammer)) * num_samples /
        sample_frequency + get_carrier_phase_2pi(jammer)
    modded_carrier_phase = mod(carrier_phase + 0.5, 1) - 0.5
    Δt = num_samples / sample_frequency
    exists = propagate(jammer.exists, Δt, rng)
    doa = propagate(jammer.doa, Δt, rng)
    jnr = propagate(jammer.jnr, n0, sample_frequency, Δt, rng)
    CWJammer(
        jammer.id,
        get_carrier_doppler(jammer),
        modded_carrier_phase,
        jnr,
        exists,
        doa,
        jammer.signal
    )
end

function propagate(
    jammer::NoiseJammer,
    num_samples,
    intermediate_frequency,
    sample_frequency,
    n0,
    rng
)
    Δt = num_samples / sample_frequency
    exists = propagate(jammer.exists, Δt, rng)
    doa = propagate(jammer.doa, Δt, rng)
    jnr = propagate(jammer.jnr, n0, sample_frequency, Δt, rng)
    NoiseJammer(jammer.id, jnr, exists, doa, jammer.signal)
end

@inline get_carrier_phase(jammer::CWJammer) = 2π * jammer.phase
@inline get_carrier_phase_2pi(jammer::CWJammer) = jammer.phase
@inline get_carrier_doppler(jammer::CWJammer) = jammer.doppler
@inline get_id(jammer::AbstractJammer) = jammer.id
@inline get_jammer_to_noise_ratio(jammer::AbstractJammer) = get_jammer_to_noise_ratio(jammer.jnr)


using PyCall

pyinitialize()
@pyimport modeller
@pyimport _modeller
mod_lib = _modeller.mod_libdir_get()
modeller.log[:verbose]()
env = modeller.environ()

#-- Prepare the input files

#-- Read in the sequence database
sdb = modeller.sequence_db(env)
sdb[:read](seq_database_file="pdb_95.pir", seq_database_format="PIR",
         chains_list="ALL", minmax_db_seq_len=(30, 4000), clean_sequences=true)

#-- Write the sequence database in binary form
sdb[:write](seq_database_file="pdb_95.bin", seq_database_format="BINARY",
          chains_list="ALL")

#-- Now, read in the binary database
sdb[:read](seq_database_file="pdb_95.bin", seq_database_format="BINARY",
         chains_list="ALL")

#-- Read in the target sequence/alignment
aln = modeller.alignment(env)
aln[:append](file="TvLDH.ali", alignment_format="PIR", align_codes="ALL")

#-- Convert the input sequence/alignment into
#   profile format
prf = aln[:to_profile]()

#-- Scan sequence database to pick up homologous sequences
prf[:build](sdb, matrix_offset=-450, rr_file=string(mod_lib,"/blosum62.sim.mat"),
          gap_penalties_1d=(-500, -50), n_prof_iterations=1,
          check_profile=false, max_aln_evalue=0.01)

#-- Write out the profile in text format
prf[:write](file="build_profile.prf", profile_format="TEXT")

#-- Convert the profile back to alignment format
aln = prf[:to_alignment]()

#-- Write out the alignment file
aln[:write](file="build_profile.ali", alignment_format="PIR")


#!/usr/bin/env julia
using CSV, DataFrames, PyCall, PyPlot
PyDict(matplotlib."rcParams")["axes.formatter.use_locale"] = true
infb_sv_1019 = CSV.File("data/infb_sv_1019.csv") |> DataFrame
infb_sv_1019[:b_prop] = infb_sv_1019[:b]./infb_sv_1019[:samples]
scatter(infb_sv_1019[:b_prop_vic],
    infb_sv_1019[:median_age_b], s = infb_sv_1019[:b_prop]*5000,
    color="lightblue")
grid(1)
for (i, season) in enumerate(infb_sv_1019[:season])
    annotate(season, (infb_sv_1019[:b_prop_vic][i],
        infb_sv_1019[:median_age_b][i]))
end
title("Laboratorierapporterad influensa B Sverige 2010–19")
xlabel("Andel B Victoralinjen")
ylabel("Medianålder fall")
savefig("../../images/infb_sv_1019.svg")


# Note: we use the 100s place of the enum to represent the number of dimensions
# And the 10s place to represent which dimension is the observation (N), 0 for no observation dimension
@enum ImageFormat begin
    L=100;
    CL=200; LC; HW; WH; NL=210; LN=220;
    HWC=300; WHC; CHW; CWH; NCL=310; NHW; NWH; NLC; CLN=330; LCN; HWN; WHN;
    NHWC=410; NWHC; NCHW; NCWH; HWCN=440; WHCN; CHWN; CWHN;
end

expected_ndims(x::ImageFormat) = Int(x) ÷ 100 # get the 100's digit
obs_dim(x::ImageFormat) = (Int(x) % 100) ÷ 10 # get the 10's digit
const strip_obs = Dict(
    NL=>L, LN=>L, NCL=>CL, CLN=>CL, NLC=>LC, LCN=>LC,
    NHW=>HW, HWN=>HW, NWH=>WH, WHN=>WH,
    NHWC=>HWC, NWHC=>WHC, NCHW=>CHW, NCWH=>CWH, HWCN=>HWC, WHCN=>WHC, CHWN=>CHW, CWHN=>CWH
)

"""
    log_images(logger::TBLogger, name::AbstractString, imgArrays::AbstractArray, format::ImageFormat; step = nothing)

Log multiple images using `Array` of images and format
- imgArrays: `Array` of images, e.g. Array{Array{Float64, 3}, 1}. `Array` of images can be multidimensional.
- format: format which applies to each image in the `Array` of images. It can be one of
  {L, LN, NL,
   CL, LC, NCL, NLC, CLN, LCN,
   HW, WH, HWC, WHC, CHW, CWH,
   HWN, WHN, NHW, NWH,
   HWCN, WHCN, CHWN, CWHN, NHWC, NWHC, NCHW, NCWH}
"""
function log_images(logger::TBLogger, name::AbstractString, imgArrays::AbstractArray; step = nothing)
    @assert isa(first(imgArrays), AbstractArray{<:Colorant}) "Please specify format"
    for (n, imgArray) in enumerate(imgArrays)
        log_image(logger, name*"/$n", imgArray, step = step)
    end
end

function log_images(logger::TBLogger, name::AbstractString, imgArrays::AbstractArray, format::ImageFormat; step = nothing)
    for (n, imgArray) in enumerate(imgArrays)
        log_image(logger, name*"/$n", imgArray, format, step = step)
    end
end
"""
    log_image(logger::TBLogger, name::AbstractString, imgArray::AbstractArray, format::ImageFormat, step = nothing)

Log an image using image data and format
- imgArray: image data. A 1-D, 2-D or 3-D `Array` of pixel values. pixel values can be Real [0, 1] or Integer[0, 255]
- format: format of the image. It can be one of
  {L, LN, NL,
   CL, LC, NCL, NLC, CLN, LCN,
   HW, WH, HWC, WHC, CHW, CWH,
   HWN, WHN, NHW, NWH,
   HWCN, WHCN, CHWN, CWHN, NHWC, NWHC, NCHW, NCWH}
  - L: Length
  - C: Channel/Color
  - H: Height
  - W: Width
  - N: Observation
"""
function log_image(logger::TBLogger, name::AbstractString, img::AbstractArray{<:Colorant}; step = nothing)
    dimensions = ndims(img)
    #if it has 3 dimensions, eg MRI image, log each channel vth same name
    #so that one can slide through channels
    if dimensions == 3
        #3rd d is channel dim as observed in testimages
        channels = size(img, 3)
        for c in 1:channels-1
            log_image(logger, name, img[:, :, c], step = step)
        end
        img = img[:, :, channels]
    end
    summ = SummaryCollection(image_summary(name, img))
    write_event(logger.file, make_event(logger, summ, step=step))
end

function log_image(logger::TBLogger, name::AbstractString, imgArray::AbstractArray, format::ImageFormat; step=nothing)
    imgArray = channelview(imgArray)
    dims = ndims(imgArray)
    @assert dims == expected_ndims(format)
    obsdim = obs_dim(format)
    if iszero(obsdim)
        summ = SummaryCollection()
        push!(summ.value, image_summary(name, imgArray, format))
        write_event(logger.file, make_event(logger, summ, step=step))
    else
        format = strip_obs[format]
        index = collect("[:"* ",:"^(dims-1) *"]")
        index[2*obsdim] = 'g'
        index = join(index)
        global gimgArray = imgArray
        nth_img = Meta.parse("gimgArray$(index)")
        for n in 1:size(imgArray, obsdim)
            global g = n
            log_image(logger, name*"/$n", eval(nth_img), format, step = step)
        end
    end
end

function image_summary(name::AbstractString, img::AbstractArray{<:Colorant})
    #image is of type AbstractArray{<:Colorant}
    getcolorspaceid(::Type{<:Gray}) = 1
    getcolorspaceid(::Type{<:GrayA}) = 2
    getcolorspaceid(::Type{<:RGB}) = 3
    getcolorspaceid(::Type{<:RGBA}) = 4
    getcolorspaceid(::Type{<:BGRA}) = 6
    colorspace = getcolorspaceid(eltype(img))
    dimensions = ndims(img)
    if dimensions == 1
        #Grayscale/color array
        height = 1
        width = size(img, 1)
    elseif dimensions == 2
        #Grayscale/color matrix
        height, width = size(img)
    else
        throw("Too many dimensions")
    end
    #save image in a buffer
    io = IOBuffer()
    save(Stream(format"PNG", io), img)
    #read from buffer to obtain encoded string of the image
    eis = io.data
    imgsumm = Summary_Image(height = height, width = width, colorspace = colorspace, encoded_image_string = eis)
    Summary_Value(tag = name, image = imgsumm)
end

function image_summary(name::AbstractString, imgArray::AbstractArray, format::ImageFormat)
    #logger and step are only relevant when using explicit function `log_image` and format contains N
    #unpack RGB, RGBA value to channels using channelview
    imgArray = channelview(imgArray)
    #if data contains integer numbers, scale them to 0-255
    if isa(first(imgArray), Integer)
        imgArray = (imgArray./255)
    end
    #convert all values to `Float64` for uniformity
    imgArray = Float64.(imgArray)
    #scale all values to 0-1
    imgArray = (imgArray./(max(maximum(imgArray), 1)))
    #convert any format to CHW
    imgArray =
    format == L   ? reshape(imgArray, (1, 1, size(imgArray, 1))) :
    format == CL  ? reshape(imgArray, (size(imgArray, 1), 1, size(imgArray, 2))) :
    format == LC  ? reshape(transpose(imgArray), (size(imgArray, 2), 1, size(imgArray, 1))) :
    format == HW  ? reshape(imgArray, (1, size(imgArray, 1), size(imgArray, 2))) :
    format == WH  ? reshape(transpose(imgArray), (1, size(imgArray, 2), size(imgArray, 1))) :
    format == HWC ? permutedims(imgArray, (3, 1, 2)) :
    format == WHC ? permutedims(imgArray, (3, 2, 1)) :
    format == CHW ? imgArray :
    format == CWH ? permutedims(imgArray, (1, 3, 2)) :
    #== else ==#    throw("Invalid format")
    channels, height, width = size(imgArray)
    color =
    channels == 1 ? Gray :
    channels == 2 ? GrayA :
    channels == 3 ? RGB :
    channels == 4 ? RGBA :
    #== else ==#    throw("Too many channels")
    #if it is a single channel Array, convert it to HW
    if color == Gray
        imgArray = imgArray[1, :, :]
    end
    #convert Array to PNG pass it to image_summary
    img = colorview(color, imgArray)
    image_summary(name, img)
end


"""
    run_omniscape("path/to/config.ini")

Run the Omniscape algorithm using the files and options specified in a .ini file

Visit https://circuitscape.github.io/Omniscape.jl/stable for detailed
documentation
"""
function run_omniscape(path::String)
    start_time = time()
    n_threads = nthreads()
    cfg_user = parse_cfg(path)

    check_missing_args(cfg_user) && return

    cfg = init_cfg()
    update_cfg!(cfg, cfg_user)

    ## Parse commonly called integer arguments
    int_arguments = Dict{String, Int64}()

    int_arguments["block_size"] = Int64(round(parse(Float64,
                                                    cfg["block_size"])))

    check_block_size(int_arguments["block_size"]) &&
        (int_arguments["block_size"] = int_arguments["block_size"] + 1)

    int_arguments["block_radius"] = Int64((int_arguments["block_size"] - 1) / 2)
    int_arguments["radius"] = Int64(round(parse(Float64, cfg["radius"])))
    int_arguments["buffer"] = Int64(round(parse(Float64, cfg["buffer"])))
    int_arguments["n_conditions"] = Int64(round(parse(Float64, cfg["n_conditions"])))

    ## Parse other arguments
    compare_to_future = lowercase(cfg["compare_to_future"])
    precision = cfg["precision"] in SINGLE ? Float32 : Float64

    # flags
    calc_flow_potential = cfg["calc_flow_potential"] in TRUELIST
    calc_normalized_current = cfg["calc_normalized_current"] in TRUELIST
    write_raw_currmap = cfg["write_raw_currmap"] in TRUELIST
    parallelize = cfg["parallelize"] in TRUELIST
    correct_artifacts = cfg["correct_artifacts"] in TRUELIST
    source_from_resistance = cfg["source_from_resistance"] in TRUELIST
    conditional = cfg["conditional"] in TRUELIST
    mask_nodata = cfg["mask_nodata"] in TRUELIST
    resistance_file_is_conductance = cfg["resistance_file_is_conductance"] in TRUELIST
    write_as_tif = cfg["write_as_tif"] in TRUELIST
    allow_different_projections = cfg["allow_different_projections"] in TRUELIST
    reclassify = cfg["reclassify_resistance"] in TRUELIST
    write_reclassified_resistance = cfg["write_reclassified_resistance"] in TRUELIST

    if int_arguments["block_size"] == 1
        correct_artifacts = false
    end

    # other
    source_threshold = parse(Float64, cfg["source_threshold"])
    project_name = cfg["project_name"]
    r_cutoff = parse(Float64, cfg["r_cutoff"])
    file_format = write_as_tif ? "tif" : "asc"

    ## Set number of BLAS threads to 1
    BLAS.set_num_threads(1)

    ## Import sources and resistances
    resistance_raster = read_raster("$(cfg["resistance_file"])", precision)

    resistance = resistance_raster[1]
    wkt = resistance_raster[2]
    transform = resistance_raster[3]

    check_resistance_values(resistance) && return

    # Reclassify resistance layer
    if reclassify
        resistance_old = deepcopy(resistance)
        reclass_table = convert.(precision, readdlm("$(cfg["reclass_table"])"))

        for i in 1:(size(reclass_table)[1])
            resistance[coalesce.(resistance_old .== reclass_table[i, 1], false)] .= reclass_table[i, 2]
        end

        resistance_old = nothing # remove from memory
    end

    # Compute source strengths from resistance if needed
    if source_from_resistance
        source_strength = deepcopy(resistance)
        if !resistance_file_is_conductance
            source_strength = Array{Union{precision, Missing}, 2}(1 ./ source_strength)
        end
        source_strength[coalesce.(source_strength .< 1/r_cutoff, true)] .= 0.0 # handles replacing NoData with 0 as well
    else
        sources_raster = read_raster("$(cfg["source_file"])", precision)
        source_strength = sources_raster[1]

        # Check for raster alignment
        check_raster_alignment(resistance_raster, sources_raster,
                               "resistance_file", "sources_file",
                               allow_different_projections) && return

        # get rid of unneeded raster to save memory
        sources_raster = nothing

        # overwrite nodata with 0
        source_strength[ismissing.(source_strength)] .= 0.0

        # Set values < user-specified threshold to 0
        source_strength[source_strength .< source_threshold] .= 0.0
    end

    int_arguments["nrows"] = size(source_strength, 1)
    int_arguments["ncols"] = size(source_strength, 2)

    # Import conditional rasters and other conditional connectivity stuff
    if conditional
        condition1_raster = read_raster("$(cfg["condition1_file"])", precision)
        condition1 = condition1_raster[1]

        # Check for raster alignment
        check_raster_alignment(resistance_raster, condition1_raster,
                               "resistance_file", "condition1_file",
                               allow_different_projections) && return

        # get rid of unneedecheck_rasterd raster to save memory
        condition1_raster = nothing

        if compare_to_future == "1" || compare_to_future == "both"
            condition1_future_raster = read_raster("$(cfg["condition1_future_file"])", precision)
            condition1_future = condition1_future_raster[1]

            # Check for raster alignment
            check_raster_alignment(resistance_raster, condition1_future_raster,
                                   "resistance_file", "condition1_future_file",
                                   allow_different_projections) && return

            # get rid of unneeded raster to save memory
            condition1_future_raster = nothing
        else
            condition1_future = condition1
        end

        if int_arguments["n_conditions"] == 2
            condition2_raster = read_raster("$(cfg["condition2_file"])", precision)
            condition2 = condition2_raster[1]

            # Check for raster alignment
            check_raster_alignment(resistance_raster, condition2_raster,
                                   "resistance_file", "condition2_file",
                                   allow_different_projections) && return

            # get rid of unneeded raster to save memory
            condition2_raster = nothing

            if compare_to_future == "2" || compare_to_future == "both"
                condition2_future_raster = read_raster("$(cfg["condition2_future_file"])", precision)
                condition2_future = condition2_future_raster[1]

                # Check for raster alignment
                check_raster_alignment(resistance_raster, condition2_future_raster,
                                       "resistance_file", "condition2_future_file",
                                       allow_different_projections) && return

                # get rid of unneeded raster to save memory
                condition2_future_raster = nothing
            else
                condition2_future = condition2
            end

        else
            condition2 = Array{Union{Missing, precision}, 2}(undef, 1, 1)
            condition2_future = condition2
        end
    else
        condition1 = Array{Union{Missing, precision}, 2}(undef, 1, 1)
        condition2 = Array{Union{Missing, precision}, 2}(undef, 1, 1)
        condition1_future = condition1
        condition2_future = condition2
    end

    # get rid of unneeded raster to save memory
    resistance_raster = nothing

    comparison1 = cfg["comparison1"]
    comparison2 = cfg["comparison2"]
    condition1_lower = parse(Float64, cfg["condition1_lower"])
    condition2_lower = parse(Float64, cfg["condition2_lower"])
    condition1_upper = parse(Float64, cfg["condition1_upper"])
    condition2_upper = parse(Float64, cfg["condition2_upper"])

    ## Setup Circuitscape configuration
    cs_cfg_dict = init_csdict(cfg)
    cs_cfg = Circuitscape.init_config()
    Circuitscape.update!(cs_cfg, cs_cfg_dict)

    ## Calculate targets
    targets = get_targets(source_strength,
                          int_arguments,
                          precision)

    ## Circuitscape calls in loop over targets
    n_targets = size(targets, 1)

    ## Define parameters for cs
    # Get flags
    o = Circuitscape.OutputFlags(false, false,
                                 false, false,
                                 false, false,
                                 false, false)

    precision_name = precision == Float64 ? "double" : "single"
    ## Add parallel workers
    if parallelize
        println("Starting up Omniscape. Using $(n_threads) workers in parallel. Using $(precision_name) precision...")

        cum_currmap = fill(convert(precision, 0.),
                           int_arguments["nrows"],
                           int_arguments["ncols"],
                           n_threads)

        if calc_flow_potential || calc_normalized_current
            fp_cum_currmap = fill(convert(precision, 0.),
                                  int_arguments["nrows"],
                                  int_arguments["ncols"],
                                  n_threads)
        else
            # Hacky fix -- a later function needs fp_cum_currmap to be an array
            fp_cum_currmap = Array{precision, 3}(undef, 1, 1, 1)
        end
    else
        println("Starting up Omniscape. Running in serial using 1 worker. Using $(precision_name) precision...")
        cum_currmap = fill(convert(precision, 0.),
                          int_arguments["nrows"],
                          int_arguments["ncols"],
                          1)

        if calc_flow_potential || calc_normalized_current
            fp_cum_currmap = fill(convert(precision, 0.),
                                  int_arguments["nrows"],
                                  int_arguments["ncols"],
                                  1)
        else
            fp_cum_currmap = Array{precision, 3}(undef, 1, 1, 1)
        end
    end

    solver = "cg+amg"

    # n_cells = int_arguments["nrows"] * int_arguments["ncols"]
    # if n_cells <= 2000000
    #     solver = "cholmod" # TODO: "cholmod" not available in advanced mode
    # end

    flags = Circuitscape.RasterFlags(true, false, true,
                                     false, false,
                                     false, Symbol("rmvsrc"),
                                     cfg["connect_four_neighbors_only"] in TRUELIST,
                                     false, solver, o)

    if correct_artifacts
        println("Calculating block artifact correction array...")
        correction_array = calc_correction(int_arguments,
                                           cs_cfg,
                                           flags,
                                           o,
                                           conditional,
                                           condition1,
                                           condition1_future,
                                           condition2,
                                           condition2_future,
                                           comparison1,
                                           comparison2,
                                           condition1_lower,
                                           condition1_upper,
                                           condition2_lower,
                                           condition2_upper,
                                           precision)

    else
        correction_array = Array{precision, 2}(undef, 1, 1)
    end

    ## Calculate and accumulate currents on each worker
    println("Solving moving window targets...")

    ## Create progress object
    p = Progress(n_targets; dt = 0.25, barlen = 60)

    if parallelize
        parallel_batch_size = Int64(round(parse(Float64, cfg["parallel_batch_size"])))
        n_batches = Int(ceil(n_targets / parallel_batch_size))

        @threads for i in 0:(n_batches - 1)
            start_ind = parallel_batch_size * i + 1
            end_ind = min(n_targets, start_ind + parallel_batch_size - 1)

            for j in start_ind:end_ind
                solve_target!(j,
                              n_targets,
                              int_arguments,
                              targets,
                              source_strength,
                              resistance,
                              cs_cfg,
                              flags,
                              o,
                              calc_flow_potential || calc_normalized_current,
                              correct_artifacts,
                              conditional,
                              condition1,
                              condition1_future,
                              condition2,
                              condition2_future,
                              comparison1,
                              comparison2,
                              condition1_lower,
                              condition1_upper,
                              condition2_lower,
                              condition2_upper,
                              correction_array,
                              cum_currmap,
                              fp_cum_currmap,
                              precision,
                              resistance_file_is_conductance)
                next!(p)
            end
        end
    else
        for i in 1:n_targets
            solve_target!(i,
                          n_targets,
                          int_arguments,
                          targets,
                          source_strength,
                          resistance,
                          cs_cfg,
                          flags,
                          o,
                          calc_flow_potential || calc_normalized_current,
                          correct_artifacts,
                          conditional,
                          condition1,
                          condition1_future,
                          condition2,
                          condition2_future,
                          comparison1,
                          comparison2,
                          condition1_lower,
                          condition1_upper,
                          condition2_lower,
                          condition2_upper,
                          correction_array,
                          cum_currmap,
                          fp_cum_currmap,
                          precision,
                          resistance_file_is_conductance)
            next!(p)
        end
    end

    ## Set some objects to nothing to free up memory
    source_strength = nothing
    condition1 = nothing
    condition1_future = nothing
    condition2 = nothing
    condition2_future = nothing
    GC.gc()

    ## Collapse 3-dim cum current arrays to 2-dim via sum
    cum_currmap = dropdims(sum(cum_currmap, dims = 3), dims = 3)

    if calc_flow_potential || calc_normalized_current
        fp_cum_currmap = dropdims(sum(fp_cum_currmap, dims = 3), dims = 3)
    end

    ## Normalize by flow potential
    if calc_normalized_current
        normalized_cum_currmap = cum_currmap ./ fp_cum_currmap
        # replace NaNs with 0's
        normalized_cum_currmap[isnan.(normalized_cum_currmap)] .= 0
    end

    ## create new directory if project_name already exists
    dir_suffix = 1
    while isdir(string(project_name, "_$(dir_suffix)"))
        dir_suffix+=1
    end
    isdir(project_name) && (project_name = string(project_name, "_$(dir_suffix)"))

    mkpath(project_name)

    # Copy .ini file to output directory
    cp(path, "$(project_name)/config.ini")

    ## Overwrite no data
    if mask_nodata
        if calc_normalized_current
            normalized_cum_currmap[ismissing.(resistance)] .= -9999
        end
        if calc_flow_potential
            fp_cum_currmap[ismissing.(resistance)] .= -9999
        end
        cum_currmap[ismissing.(resistance)] .= -9999
    end

    # Get rid of resistance (save first if needed)
    if reclassify && write_reclassified_resistance
        resistance[ismissing.(resistance)] .= -9999
        write_raster("$(project_name)/classified_resistance",
                     convert(Array{precision, 2}, resistance),
                     wkt,
                     transform,
                     file_format)
    end
    resistance = nothing
    GC.gc()

    ## Write outputs
    if write_raw_currmap
        write_raster("$(project_name)/cum_currmap",
                     cum_currmap,
                     wkt,
                     transform,
                     file_format)
    end

    if calc_flow_potential
        write_raster("$(project_name)/flow_potential",
                     fp_cum_currmap,
                     wkt,
                     transform,
                     file_format)
    end

    if calc_normalized_current
        write_raster("$(project_name)/normalized_cum_currmap",
                     normalized_cum_currmap,
                     wkt,
                     transform,
                     file_format)
    end

    println("Done!")
    println("Time taken to complete job: $(round(time() - start_time; digits = 4)) seconds")

    println("Outputs written to $(string(pwd(),"/",project_name))")

    ## Return outputs, depending on user options
    if calc_normalized_current && !calc_flow_potential
        return cum_currmap, normalized_cum_currmap
    elseif !calc_normalized_current && calc_flow_potential
        return cum_currmap, fp_cum_currmap
    elseif calc_normalized_current && calc_flow_potential
        return cum_currmap, fp_cum_currmap, normalized_cum_currmap
    else
        return cum_currmap
    end
end


#!/bin/bash
# -*- mode: julia -*-
#=
exec julia --color=yes --startup-file=no -e "include(popfirst!(ARGS))" "${BASH_SOURCE[0]}" "$@"
=#

# dicom2nifti.jl
# Simple DICOM to NIfTI converter

# This script walks through the current working directory looking for DICOM
# files and sorts them into volumes. It then creates directories corresponding
# to each protocol, and saves nii files named by series number. It has
# (barely) been tested with DICOM files from a Siemens scanner. It may not
# work with other scanners. It may give incorrect information. Test thoroughly
# and use at your own risk.

using DICOM, NIfTI, ArgParse, LinearAlgebra

function parse_commandline()
    s = ArgParseSettings()
    s.description = "DICOM to NIfTI converter"

    @add_arg_table s begin
        "--extension"
            help = "assume files with given extension are DICOM files. Set to blank for no extension."
            default = ".dcm"
        "--sphinx"
            help = "correct for sphinx position (for monkeys)"
            action = :store_true
        "--axes"
            help = """orientation of tensor in the resulting volume. This is a three-character label
            that determines how the data is stored, and how it is displayed in applications that do
            not make use of the orientation matrix in the NIfTI header. Valid characters are:
            L (= left), R (= right), A (= anterior), P (= posterior), I (= inferior), and
            S (= superior). If not specified, the orientation in the DICOM file is preserved."""
            default = nothing
        "dicomdir"
            help = "directory of DICOM files"
            required = true
        "niftidir"
            help = "directory in which to put NIfTI files"
            required = true
    end

    parse_args(s)
end

function main()
    cmd = parse_commandline()

    if cmd["axes"] == nothing
        axes = nothing
    else
        axes = uppercase(cmd["axes"])
        if length(axes) != 3 || (!('R' in axes) && !('L' in axes)) ||
                (!('A' in axes) && !('P' in axes)) ||
                (!('S' in axes) && !('I' in axes))
            error("Invalid axes provided")
        end
    end

    transform = cmd["sphinx"] ? [1 0 0; 0 0 1; 0 -1 0] : [-1 0 0; 0 -1 0; 0 0 1]

    # Load all DICOMs into an array, indexed by series number
    dicoms = Dict{Int, Any}()
    for (root, dirs, files) in walkdir(cmd["dicomdir"])
        for file in files
            fpath = joinpath(root, file)
            if !endswith(fpath, cmd["extension"])
                return
            end

            local d
            println(fpath)
            d = DICOM.dcm_parse(fpath)

            if !(tag"Series Number" in keys(d))
                println("No series number found for $fpath; ignoring")
                return
            end
            series_number = d[tag"Series Number"]

            dicom_arr = get(dicoms, series_number, nothing)
            if dicom_arr == nothing
                dicoms[series_number] = dicom_arr = []
            end
            push!(dicom_arr, d)
        end
    end

    # Sort DICOMs
    for (series_number, slices) in dicoms
        d = slices[1]

        protocol_name = d[tag"Protocol Name"]
        orientation = d[tag"Image Orientation (Patient)"]
        pixel_spacing = d[tag"Pixel Spacing"]
        slice_spacing = slice_thickness = d[tag"Slice Thickness"]
        if tag"Spacing Between Slices" in keys(d)
            slice_spacing = d[tag"Spacing Between Slices"]
        end
        time_step = false
        if tag"Repetition Time" in keys(d)
            time_step = d[tag"Repetition Time"] # This is the TR, but may not be the time step
        end
        phase_encoding_direction = d[tag"In-plane Phase Encoding Direction"]
        pixel_rows = d[tag"Rows"]
        pixel_cols = d[tag"Columns"]

        # Convert orientation to RAS
        orientation = transform*reshape(orientation, (3, 2))

        # Determine Z as cross product of X and Y
        orientation = hcat(orientation, cross(orientation[:, 1], orientation[:, 2]))

        # Scale by pixel size
        orientation = orientation*[pixel_spacing[1] 0 0; 0 pixel_spacing[2] 0; 0 0 slice_spacing]

        # Find Z coordinates of each slice
        image_positions = transform * hcat([d[tag"Image Position (Patient)"] for d in slices]...)
        z_coords = [dot(image_positions[:, i], orientation[:, 3])
            for i = 1:size(image_positions, 2)]

        # Sort Z coordinates
        p = sortperm(z_coords)

        # Concatenate slices along Z axis according to increasing Z coordinate
        raw = cat([slices[i][tag"Pixel Data"] for i in p]..., dims = 3)

        orientation = hcat(orientation, image_positions[:, p[1]])

        dim_info = phase_encoding_direction == "ROW" ? (1, 2, 3) :
                phase_encoding_direction == "COL" ? (2, 1, 3) :
                (0, 0, 0)

        voxel_size = tuple(pixel_spacing..., slice_spacing)

        # Permute according to axes specified on command line
        if axes != nothing
            # Determine permutation of current volume to RAS
            ras = Array{Int,1}(undef, 3)
            sign = Array{Bool,1}(undef, 3)
            rg = [1:3;]
            abs_orientation = abs.(orientation)
            for i = 1:3
                idx = findmax(abs_orientation[i, rg])[2]
                ras[i] = splice!(rg, idx)
                sign[i] = orientation[i, ras[i]] >= 0
            end

            # Determine permutation of RAS to user-preferred orientation
            specified_axes = Int[(c in "RL" ? 1 :
                                  c in "AP" ? 2 : 3)
                                 for c in axes]
            specified_sign = Bool[c in "RAS" for c in axes]
            output_axes = ras[specified_axes]

            # Permute
            raw = permutedims(raw, output_axes)
            orientation[:, 1:3] = orientation[:, output_axes]
            voxel_size = voxel_size[output_axes]
            dim_info = dim_info[output_axes]

            # Flip signs and dimensions
            flip_sign = findall(sign[specified_axes] .!= specified_sign)
            orientation[:, flip_sign] = -orientation[:, flip_sign]
            for dim in flip_sign
                raw = reverse(raw, dims=dim)
                orientation[dim, 4] += (sign[specified_axes[dim]] ? 1 : -1) *
                    (size(raw, dim) - 1)*voxel_size[dim]
            end
        end

        # Create a directory for each protocol
        protocol_dir = joinpath(cmd["niftidir"], replace(protocol_name, "/" => ""))

        if !isdir(cmd["niftidir"])
            mkdir(cmd["niftidir"])
        end
        if !isdir(protocol_dir)
            mkdir(protocol_dir)
        end

        # Write NIfTI volumes
        # convert to Float32 as mricron and fsleyes apparently do not support UInt16
        ni = NIVolume(Array{Float32, 3}(raw); voxel_size=voxel_size,
            orientation=convert(Array{Float32, 2}, orientation), dim_info=dim_info,
            time_step=time_step != false ? time_step : 0f0)
        niwrite(joinpath(protocol_dir, "$(series_number).nii"), ni)
    end
end

main()


using Halite3
using Logging

# Initialize the game.
log, constants, game = init_game()
# log is a function which takes a string and allows you to save messages for yourself.
# log is not available to use inside /src/*jl instead use @info to log from inside Halite3 module.
# This is required because the regular STDOUT (print statements) are reserved for the engine-bot communication.

# At this point "game" variable is populated with initial map data.
# This is a good place to do computationally expensive start-up pre-processing.
# As soon as you call "start_game" function below, the 2 second per turn timer will start.

start_game("MyJuliaBot")
# Now that your bot is initialized, save a message to yourself in the log file with some important information.
#   Here, you log here your id, which you can always fetch from the game object by using game.id.
log("Successfully created bot! My Player ID is $(game.id)")

while true
    # This loop handles each turn of the game. The game object changes every turn, and you refresh that state by
    #   running update_frame!(game).
    update_frame!(game)
    log("=============== TURN $(game.turn_number) ================")
    # You extract player metadata and the updated map metadata here for convenience.
    me = game.me
    game_map = game.game_map

    # A command queue holds all the commands you will run this turn. You build this list up and submit it at the
    #   end of the turn.
    command_queue = Vector()

    for ship in get_ships(me)
        # For each of your ships, move randomly if the ship is on a low halite location or the ship is full.
        #   Else, collect halite.
        if game_map[ship.position].halite_amount < constants.MAX_HALITE / 10 || is_full(ship, constants.MAX_HALITE)
            push!(command_queue, move(ship, random_direction()))
        else
            push!(command_queue, stay_still(ship))
        end
    end

    # If the game is in the first 200 turns and you have enough halite, spawn a ship.
    # Don't spawn a ship if you currently have a ship at port, though - the ships will collide.
    if game.turn_number <= 200 && me.halite_amount >= constants.SHIP_COST && !is_occupied(game_map[me.shipyard.position])
        push!(command_queue, spawn(me.shipyard))
    end

    # Send your moves back to the game environment, ending this turn.
    end_turn(command_queue)
end


@everywhere include("self-phase.jl")
using YAML
using Iterators
#=
"""
To run, simply use:
```
julia multi-thread.jl <parameter file> <#saves>
```
e.g:
```
julia multi-thread.jl params.yaml 10
```
"""
"""
    unpack(p, numSaves)
given a dictionary of inital paremeters, and a number of times to save, performs
the entire simulation with those parameters.
"""
=#
@everywhere function unpack(p, numSaves)
    # Generate folder string
    fname = @sprintf("%.0fnm_%.0fμJ_%.2fbar_%.2fbar_%.0ffs_%.1fm_%.0ffs^2_%.0fμm",
                     p["λ"]*1E9, p["Energy"]*1E6, p["Pin"], p["Pout"], p["Tfwhm"]*1E15,
                     p["zmax"], p["Chirp"], p["fiberD"]*1E6)
    # Derive additional constants and spatial grids
    derive_constants(p)
    # Initialize electric field
    E, zinit = initialize(fname, p, "resume" in ARGS, "keep" in ARGS)
    # Save inital data
    saveData(fname, E, 0, zinit)
    # Simulate the whole thing
    simulate(E, p, zinit, fname, numSaves)
end

# Load paramters which are possible list
lists = YAML.load(open(ARGS[1]))
# Rename lambda to λ in dictionary.
if haskey(lists, "lambda")
    merge!(lists, Dict("λ" => pop!(lists, "lambda")))
end

# If paramter is a list of form [initial, final, stepsize] convert to list
# of all paramters in range. Otherwise keep it as a single value.
for key in keys(lists)
    if length(lists[key]) != 1
        lists[key] = collect(lists[key][1]:lists[key][3]:lists[key][2])
    else
        lists[key] = lists[key][1]
    end
end

# Generate all possible combinations of parameters.
param_tuples = collect(product(values(lists)...))

# Put all results into a single folder named data.
if "data" ∉ readdir()
    mkdir("data")
end
# Put all processes in same place.
@everywhere cd("data")

if length(ARGS) < 2
    println("No number of saves passed, assuming 2.")
    numSaves = 2
else
    numSaves = parse(Int, ARGS[2])
end

# Parallel map over all paramters to simulate!
pmap(jawn -> unpack(Dict{Any,Any}(zip(keys(lists), jawn)), numSaves), param_tuples)


abstract type AbstractHydroFormulation <: AbstractDeviceFormulation end
abstract type AbstractHydroDispatchFormulation <: AbstractHydroFormulation end
abstract type AbstractHydroUnitCommitment <: AbstractHydroFormulation end
abstract type AbstractHydroReservoirFormulation <: AbstractHydroDispatchFormulation end
struct HydroDispatchRunOfRiver <: AbstractHydroDispatchFormulation end
struct HydroDispatchReservoirFlow <: AbstractHydroReservoirFormulation end
struct HydroDispatchReservoirStorage <: AbstractHydroReservoirFormulation end
#=
# Commenting out all Unit Commitment formulations as all Hydro UC
# formulations are currently not supported
struct HydroCommitmentRunOfRiver <: AbstractHydroUnitCommitment end
struct HydroCommitmentReservoirFlow <: AbstractHydroUnitCommitment end
struct HydroCommitmentReservoirStorage <: AbstractHydroUnitCommitment end
=#
########################### Hydro generation variables #################################
function activepower_variables!(
    psi_container::PSIContainer,
    devices::IS.FlattenIteratorWrapper{H},
) where {H <: PSY.HydroGen}
    add_variable(
        psi_container,
        devices,
        variable_name(ACTIVE_POWER, H),
        false,
        :nodal_balance_active;
        lb_value = d -> PSY.get_activepowerlimits(d).min,
        ub_value = d -> PSY.get_activepowerlimits(d).max,
        init_value = d -> PSY.get_activepower(d),
    )

    return
end

function reactivepower_variables!(
    psi_container::PSIContainer,
    devices::IS.FlattenIteratorWrapper{H},
) where {H <: PSY.HydroGen}
    add_variable(
        psi_container,
        devices,
        variable_name(REACTIVE_POWER, H),
        false,
        :nodal_balance_reactive;
        ub_value = d -> PSY.get_reactivepowerlimits(d).max,
        lb_value = d -> PSY.get_reactivepowerlimits(d).min,
        init_value = d -> PSY.get_reactivepower(d),
    )

    return
end

function energy_variables!(
    psi_container::PSIContainer,
    devices::IS.FlattenIteratorWrapper{H},
) where {H <: PSY.HydroGen}
    add_variable(
        psi_container,
        devices,
        variable_name(ENERGY, H),
        false;
        ub_value = d -> PSY.get_storage_capacity(d),
        lb_value = d -> 0.0,
        init_value = d -> PSY.get_initial_storage(d),
    )

    return
end

#=
function inflow_variables!(
    psi_container::PSIContainer,
    devices::IS.FlattenIteratorWrapper{H},
) where {H <: PSY.HydroGen}
    add_variable(
        psi_container,
        devices,
        variable_name(INFLOW, H),
        false;
        ub_value = d -> PSY.get_inflow(d),
        lb_value = d -> 0.0,
    )

    return
end
=#

function spillage_variables!(
    psi_container::PSIContainer,
    devices::IS.FlattenIteratorWrapper{H},
) where {H <: PSY.HydroGen}
    add_variable(
        psi_container,
        devices,
        variable_name(SPILLAGE, H),
        false;
        lb_value = d -> 0.0,
    )

    return
end

"""
This function add the variables for power generation commitment to the model
"""
#=
function commitment_variables!(
    psi_container::PSIContainer,
    devices::IS.FlattenIteratorWrapper{H},
) where {H <: PSY.HydroGen}
    time_steps = model_time_steps(psi_container)
    var_names = [variable_name(ON, H), variable_name(START, H), variable_name(STOP, H)]

    for v in var_names
        add_variable(psi_container, devices, v, true)
    end

    return
end

# All Hydro UC formulations are currently not supported
### Constraints for Thermal Generation without commitment variables ####
"""
This function adds the Commitment Status constraint when there are CommitmentVariables
"""
function commitment_constraints!(
    psi_container::PSIContainer,
    devices::IS.FlattenIteratorWrapper{H},
    model::DeviceModel{H,D},
    system_formulation::Type{S},
    feedforward::Union{Nothing,AbstractAffectFeedForward},
) where {H<:PSY.HydroGen,D<:AbstractHydroUnitCommitment,S<:PM.AbstractPowerModel}
    device_commitment(
        psi_container,
        get_initial_conditions(psi_container, ICKey(DeviceStatus, H)),
        constraint_name(COMMITMENT, H),
        (variable_name(START, H), variable_name(STOP, H), variable_name(ON, H)),
    )

    return
end
=#
####################################### Reactive Power Constraints #########################
function reactivepower_constraints!(
    psi_container::PSIContainer,
    devices::IS.FlattenIteratorWrapper{H},
    model::DeviceModel{H, D},
    system_formulation::Type{<:PM.AbstractPowerModel},
    feedforward::Union{Nothing, AbstractAffectFeedForward},
) where {H <: PSY.HydroGen, D <: AbstractHydroDispatchFormulation}
    constraint_data = Vector{DeviceRange}(undef, length(devices))
    for (ix, d) in enumerate(devices)
        limits = PSY.get_reactivepowerlimits(d)
        name = PSY.get_name(d)
        range_data = DeviceRange(name, limits)
        constraint_data[ix] = range_data
    end

    device_range(
        psi_container,
        constraint_data,
        constraint_name(REACTIVE_RANGE, H),
        variable_name(REACTIVE_POWER, H),
    )
    return
end

function activepower_constraints!(
    psi_container::PSIContainer,
    devices::IS.FlattenIteratorWrapper{H},
    model::DeviceModel{H, <:AbstractHydroDispatchFormulation},
    system_formulation::Type{<:PM.AbstractPowerModel},
    feedforward::Union{Nothing, AbstractAffectFeedForward},
) where {H <: PSY.HydroGen}
    parameters = model_has_parameters(psi_container)
    use_forecast_data = model_uses_forecasts(psi_container)

    if !parameters && !use_forecast_data
        constraint_data = Vector{DeviceRange}(undef, length(devices))
        for (ix, d) in enumerate(devices)
            name = PSY.get_name(d)
            ub = PSY.get_activepower(d)
            limits = (min = 0.0, max = ub)
            range_data = DeviceRange(name, limits)
            add_device_services!(range_data, d, model)
            constraint_data[ix] = range_data
        end
        device_range(
            psi_container,
            constraint_data,
            constraint_name(ACTIVE_RANGE, H),
            variable_name(ACTIVE_POWER, H),
        )
        return
    end

    forecast_label = "get_rating"
    constraint_data = Vector{DeviceTimeSeries}(undef, length(devices))
    for (ix, d) in enumerate(devices)
        ts_vector = get_time_series(psi_container, d, forecast_label)
        timeseries_data = DeviceTimeSeries(d, x -> PSY.get_rating(x), ts_vector)
        add_device_services!(timeseries_data, d, model)
        constraint_data[ix] = timeseries_data
    end

    if parameters
        device_timeseries_param_ub(
            psi_container,
            constraint_data,
            constraint_name(ACTIVE, H),
            UpdateRef{H}(ACTIVE_POWER, forecast_label),
            variable_name(ACTIVE_POWER, H),
        )
    else
        device_timeseries_ub(
            psi_container,
            constraint_data,
            constraint_name(ACTIVE, H),
            variable_name(ACTIVE_POWER, H),
        )
    end
    return
end

function activepower_constraints!(
    psi_container::PSIContainer,
    devices::IS.FlattenIteratorWrapper{H},
    model::DeviceModel{H, <:AbstractHydroReservoirFormulation},
    system_formulation::Type{<:PM.AbstractPowerModel},
    feedforward::Union{Nothing, AbstractAffectFeedForward},
) where {H <: PSY.HydroGen}
    constraint_data = Vector{DeviceRange}(undef, length(devices))
    for (ix, d) in enumerate(devices)
        name = PSY.get_name(d)
        limits = PSY.get_activepowerlimits(d)
        range_data = DeviceRange(name, limits)
        add_device_services!(range_data, d, model)
        constraint_data[ix] = range_data
    end
    device_range(
        psi_container,
        constraint_data,
        constraint_name(ACTIVE_RANGE, H),
        variable_name(ACTIVE_POWER, H),
    )
    return
end

#=
# All Hydro UC formulations are currently not supported
function activepower_constraints!(
    psi_container::PSIContainer,
    devices::IS.FlattenIteratorWrapper{H},
    model::DeviceModel{H,<:AbstractHydroUnitCommitment},
    system_formulation::Type{<:PM.AbstractPowerModel},
    feedforward::Union{Nothing,AbstractAffectFeedForward},
) where {H<:PSY.HydroGen}
    parameters = model_has_parameters(psi_container)
    use_forecast_data = model_uses_forecasts(psi_container)

    ts_data_active, constraint_data = get_time_series(
        psi_container,
        devices,
        model,
        x -> PSY.get_activepowerlimits(x),
    )

    if !parameters && !use_forecast_data
        device_semicontinuousrange(
            psi_container,
            constraint_data,
            constraint_name(ACTIVE_RANGE, H),
            variable_name(ACTIVE_POWER, H),
            variable_name(ON, H),
        )
        return
    end

    if parameters
        device_timeseries_ub_bigM(
            psi_container,
            ts_data_active,
            constraint_name(ACTIVE_RANGE, H),
            variable_name(ACTIVE_POWER, H),
            UpdateRef{H}(ON, "get_rating"),
            variable_name(ON, H),
        )
    else
        device_timeseries_ub_bin(
            psi_container,
            ts_data_active,
            constraint_name(ACTIVE_RANGE, H),
            variable_name(ACTIVE_POWER, H),
            variable_name(ON, H),
        )
    end

    return
end
=#
######################## Inflow constraints ############################
#=
# TODO: Determine if this is useful for ROR formulation ?
function inflow_constraints!(
    psi_container::PSIContainer,
    devices::IS.FlattenIteratorWrapper{H},
    model::DeviceModel{H,<:AbstractHydroDispatchFormulation},
    system_formulation::Type{<:PM.AbstractPowerModel},
    feedforward::Union{Nothing,AbstractAffectFeedForward},
) where {H<:PSY.HydroGen}

    return
end

function inflow_constraints!(
    psi_container::PSIContainer,
    devices::IS.FlattenIteratorWrapper{H},
    model::DeviceModel{PSY.HydroEnergyReservoir,HydroDispatchReservoirStorage},
    system_formulation::Type{<:PM.AbstractPowerModel},
    feedforward::Union{Nothing,AbstractAffectFeedForward},
) where {H<:PSY.HydroGen}
    parameters = model_has_parameters(psi_container)
    use_forecast_data = model_uses_forecasts(psi_container)

    ts_data_inflow, constraint_data = _get_inflow_time_series(
        psi_container,
        devices,
        model,
        x -> (min = 0.0, max = PSY.get_inflow(x)),
    )

    if !parameters && !use_forecast_data
        device_range(
            psi_container,
            constraint_data,
            constraint_name(INFLOW_RANGE, H),
            variable_name(INFLOW, H),
        )
        return
    end

    if parameters
        device_timeseries_param_ub(psi_container,
                            ts_data_inflow,
                            constraint_name(INFLOW_RANGE, H),
                            UpdateRef{H}(INFLOW_RANGE, "get_inflow"),
                            variable_name(INFLOW, H))
    else
        device_timeseries_ub(
            psi_container,
            ts_data_inflow,
            constraint_name(INFLOW_RANGE, H),
            variable_name(INFLOW, H),
        )
    end

    return
end
=#
######################## Energy balance constraints ############################

function energy_balance_constraint!(
    psi_container::PSIContainer,
    devices::IS.FlattenIteratorWrapper{H},
    model::DeviceModel{H, HydroDispatchReservoirStorage},
    system_formulation::Type{<:PM.AbstractPowerModel},
    feedforward::Union{Nothing, AbstractAffectFeedForward},
) where {H <: PSY.HydroEnergyReservoir}
    key = ICKey(EnergyLevel, H)
    parameters = model_has_parameters(psi_container)
    use_forecast_data = model_uses_forecasts(psi_container)

    if !has_initial_conditions(psi_container.initial_conditions, key)
        throw(IS.DataFormatError("Initial Conditions for $(H) Energy Constraints not in the model"))
    end

    forecast_label = "get_inflow"
    constraint_data = Vector{DeviceTimeSeries}(undef, length(devices))
    for (ix, d) in enumerate(devices)
        ts_vector = get_time_series(psi_container, d, forecast_label)
        timeseries_data = DeviceTimeSeries(d, x -> PSY.get_rating(x), ts_vector)
        add_device_services!(timeseries_data, d, model)
        constraint_data[ix] = timeseries_data
    end

    if parameters
        energy_balance_external_input_param(
            psi_container,
            get_initial_conditions(psi_container, key),
            constraint_data,
            constraint_name(ENERGY_CAPACITY, H),
            (
                variable_name(SPILLAGE, H),
                variable_name(ACTIVE_POWER, H),
                variable_name(ENERGY, H),
            ),
            UpdateRef{H}(INFLOW, forecast_label),
        )
    else
        energy_balance_external_input(
            psi_container,
            get_initial_conditions(psi_container, key),
            constraint_data,
            constraint_name(ENERGY_CAPACITY, H),
            (
                variable_name(SPILLAGE, H),
                variable_name(ACTIVE_POWER, H),
                variable_name(ENERGY, H),
            ),
        )
    end
    return
end

########################## Make initial Conditions for a Model #############################
function initial_conditions!(
    psi_container::PSIContainer,
    devices::IS.FlattenIteratorWrapper{H},
    device_formulation::Type{<:AbstractHydroUnitCommitment},
) where {H <: PSY.HydroGen}
    status_init(psi_container.initial_conditions, devices)
    output_init(psi_container.initial_conditions, devices)
    duration_init(psi_container.initial_conditions, devices)

    return
end

function initial_conditions!(
    psi_container::PSIContainer,
    devices::IS.FlattenIteratorWrapper{H},
    device_formulation::Type{D},
) where {H <: PSY.HydroGen, D <: AbstractHydroDispatchFormulation}
    output_init.initial_conditions_container(psi_container, devices)

    return
end

########################## Addition of to the nodal balances ###############################
function nodal_expression!(
    psi_container::PSIContainer,
    devices::IS.FlattenIteratorWrapper{H},
    system_formulation::Type{<:PM.AbstractPowerModel},
) where {H <: PSY.HydroGen}
    nodal_expression!(psi_container, devices, PM.AbstractActivePowerModel)
    # Commented out since PF = 1.0 is the assumtion for RoR Hydro
    #=
     parameters = model_has_parameters(psi_container)
    use_forecast_data = model_uses_forecasts(psi_container)
    if parameters
     forecast_label = "get_rating"
     peak_value_function = x -> PSY.get_rating(x) * sin(acos(PSY.get_powerfactor(x)))
    else
     forecast_label = "get_rating"
     peak_value_function = x -> PSY.get_reactivepower(x)
    end
    constraint_data = Vector{DeviceTimeSeries}(undef, length(devices))
    for (ix, d) in enumerate(devices)
     ts_vector = get_time_series(psi_container, d, forecast_label)
     timeseries_data = DeviceTimeSeries(d, peak_value_function, ts_vector)
     constraint_data[ix] = timeseries_data
    end

    if parameters
     include_parameters(
         psi_container,
         constraint_data,
         UpdateRef{R}(REACTIVE_POWER, forecast_label),
         :nodal_balance_active,
     )
     return
    else
     for t in model_time_steps(psi_container)
         for device in constraint_data
             add_to_expression!(
                 psi_container.expressions[:nodal_balance_reactive],
                 device.bus_number,
                 t,
                 device.multiplier * device.timeseries[t],
             )
         end
     end
    end
    return
    =#
end

function nodal_expression!(
    psi_container::PSIContainer,
    devices::IS.FlattenIteratorWrapper{H},
    system_formulation::Type{<:PM.AbstractActivePowerModel},
) where {H <: PSY.HydroGen}
    parameters = model_has_parameters(psi_container)
    use_forecast_data = model_uses_forecasts(psi_container)
    if use_forecast_data
        forecast_label = "get_rating"
        peak_value_function = x -> PSY.get_rating(x)
    else
        forecast_label = ""
        peak_value_function = x -> PSY.get_activepower(x)
    end
    constraint_data = Vector{DeviceTimeSeries}(undef, length(devices))
    for (ix, d) in enumerate(devices)
        ts_vector = get_time_series(psi_container, d, forecast_label)
        timeseries_data = DeviceTimeSeries(d, peak_value_function, ts_vector)
        constraint_data[ix] = timeseries_data
    end

    if parameters
        include_parameters(
            psi_container,
            constraint_data,
            UpdateRef{H}(ACTIVE_POWER, forecast_label),
            :nodal_balance_active,
        )
        return
    else
        for t in model_time_steps(psi_container)
            for device in constraint_data
                add_to_expression!(
                    psi_container.expressions[:nodal_balance_active],
                    device.bus_number,
                    t,
                    device.multiplier * device.timeseries[t],
                )
            end
        end
    end
    return
end

##################################### Hydro generation cost ############################
function cost_function(
    psi_container::PSIContainer,
    devices::IS.FlattenIteratorWrapper{PSY.HydroEnergyReservoir},
    device_formulation::Type{D},
    system_formulation::Type{<:PM.AbstractPowerModel},
) where {D <: AbstractHydroFormulation}
    add_to_cost(
        psi_container,
        devices,
        variable_name(ACTIVE_POWER, PSY.HydroEnergyReservoir),
        :fixed,
        -1.0,
    )

    return
end

function cost_function(
    psi_container::PSIContainer,
    devices::IS.FlattenIteratorWrapper{H},
    device_formulation::Type{D},
    system_formulation::Type{<:PM.AbstractPowerModel},
) where {D <: AbstractHydroFormulation, H <: PSY.HydroGen}

    return
end

##################################### Water/Energy Limit Constraint ############################
function energy_limit_constraints!(
    psi_container::PSIContainer,
    devices::IS.FlattenIteratorWrapper{H},
    model::DeviceModel{H, <:AbstractHydroDispatchFormulation},
    system_formulation::Type{<:PM.AbstractPowerModel},
    feedforward::IntegralLimitFF,
) where {H <: PSY.HydroGen}
    return
end

function energy_limit_constraints!(
    psi_container::PSIContainer,
    devices::IS.FlattenIteratorWrapper{H},
    model::DeviceModel{H, <:AbstractHydroDispatchFormulation},
    system_formulation::Type{<:PM.AbstractPowerModel},
    feedforward::Union{Nothing, AbstractAffectFeedForward},
) where {H <: PSY.HydroGen}

    forecast_label = "get_storage_capacity"
    constraint_data = Vector{DeviceTimeSeries}(undef, length(devices))
    for (ix, d) in enumerate(devices)
        ts_vector = get_time_series(psi_container, d, forecast_label)
        timeseries_data = DeviceTimeSeries(d, x -> PSY.get_rating(x), ts_vector)
        add_device_services!(timeseries_data, d, model)
        constraint_data[ix] = timeseries_data
    end

    if model_has_parameters(psi_container)
        device_energy_limit_param_ub(
            psi_container,
            constraint_data,
            constraint_name(ENERGY_LIMIT, H),
            UpdateRef{H}(ENERGY_BUDGET, forecast_label),
            variable_name(ACTIVE_POWER, H),
        )
    else
        device_energy_limit_ub(
            psi_container,
            constraint_data,
            constraint_name(ENERGY_LIMIT),
            variable_name(ACTIVE_POWER, H),
        )
    end
end

function device_energy_limit_param_ub(
    psi_container::PSIContainer,
    energy_limit_data::Vector{DeviceTimeSeries},
    cons_name::Symbol,
    param_reference::UpdateRef,
    var_name::Symbol,
)
    time_steps = model_time_steps(psi_container)
    variable = get_variable(psi_container, var_name)
    set_name = (r.name for r in energy_limit_data)
    constraint = add_cons_container!(psi_container, cons_name, set_name)
    container = add_param_container!(psi_container, param_reference, set_name, 1)
    multiplier = get_multiplier_array(container)
    param = get_parameter_array(container)
    for data in energy_limit_data
        name = data.name
        multiplier[name, 1] = data.multiplier
        param[name, 1] = PJ.add_parameter(psi_container.JuMPmodel, sum(data.timeseries))
        constraint[name] = JuMP.@constraint(
            psi_container.JuMPmodel,
            sum([variable[name, t] for t in time_steps]) <= multiplier[name, 1] * param[name, 1]
        )
    end

    return
end

function device_energy_limit_ub(
    psi_container::PSIContainer,
    energy_limit_data::Vector{DeviceTimeSeries},
    cons_name::Symbol,
    var_name::Symbol,
)
    time_steps = model_time_steps(psi_container)
    variable = get_variable(psi_container, var_name)
    names = (r.name for r in energy_limit_data)
    constraint = add_cons_container!(psi_container, cons_name, names)

    for data in energy_limit_data
        name = data.name
        forecast = data.timeseries
        multiplier = data.multiplier
        constraint[name] = JuMP.@constraint(
            psi_container.JuMPmodel,
            sum([variable[name, t] for t in time_steps]) <= multiplier * sum(forecast)
        )
    end

    return
end


#######################################
#
# Read SSR
#
#######################################

"""
## Read SSR from file or IO stream
Read a file or IO stream and store the data in an `SSR` type.

Matching .mat files are read and modulation frequency information extracted.
Failing that, user passed arguments are used or the modulation frequency is extracted from the file name.

#### Arguments

* `fname`: Name of the file to be read
* `min_epoch_length`: Minimum epoch length in samples. Shorter epochs will be removed (0)
* `max_epoch_length`: Maximum epoch length in samples. Longer epochs will be removed (0 = all)
* `valid_triggers`: Triggers that are considered valid, others are removed ([1,2])
* `stimulation_amplitude`: Amplitude of stimulation (NaN)
* `modulationrate`: Modulation frequency of SSR stimulation (NaN)
* `carrier_frequency`: Carrier frequency (NaN)
* `participant_name`: Name of participant ("")
* `remove_first`: Number of epochs to be removed from start of recording (0)
* `max_epochs`: Maximum number of epochs to retain (0 = all)
* `env` (nothing)
* `bkt` ("")

#### Supported file formats

* BIOSEMI (.bdf)
"""
function read_SSR(
    fname::String;
    stimulation_amplitude::Number = NaN,   # User can set these
    modulationrate::Number = NaN,          # values, but if not
    carrier_frequency::Number = NaN,       # then attempt to read
    stimulation_side::AbstractString = "", # from file name or mat
    participant_name::AbstractString = "",
    valid_triggers::Array{Int} = [1, 2],
    min_epoch_length::Int = 0,
    max_epoch_length::Int = 0,
    remove_first::Int = 0,
    max_epochs::Int = 0,
    env = nothing,
    bkt = "",
    kwargs...,
)

    @info("Importing SSR from file: $fname")

    file_path, file_name, ext = fileparts(fname)

    if env != nothing

        @debug("File type is S3")
        fname = S3.get_object(env, bkt, fname).obj
    end

    #
    # Extract meta data
    #

    # Extract frequency from the file name if not set manually
    if occursin("Hz", file_name) && isnan(modulationrate)
        a = match(r"[-_](\d+[_.]?[\d+]?)Hz|Hz(\d+[_.]?[\d+]?)[-_]", file_name).captures
        modulationrate = parse(Float64, a[[i !== nothing for i in a]][1]) * 1.0u"Hz"
        @debug("Extracted modulation frequency from file name: $modulationrate")
    end

    # Or even better if there is a mat file read it
    mat_path = string(file_path, file_name, ".mat")
    if !isfile(mat_path)
        mat_path = string(file_path, file_name, "-properties.mat")
    end

    if isfile(mat_path)
        modulationrate,
        stimulation_side,
        participant_name,
        stimulation_amplitude,
        carrier_frequency = read_rba_mat(mat_path)
        valid_mat = true
    else
        valid_mat = false
    end

    #
    # Read file data
    #

    # Import raw data
    if ext == "bdf"
        data, triggers, system_codes, samplingrate, reference_channel, header =
            import_biosemi(fname; kwargs...)
    else
        warn("File type $ext is unknown")
    end

    # Create electrodes
    elecs = Electrode[]
    for e in header["chanLabels"]
        push!(elecs, Electrode(e, Talairach(NaN * u"m", NaN * u"m", NaN * u"m"), Dict()))
    end

    # Create SSR type
    if unit(modulationrate) == unit(1.0u"Hz")
        #nothing
    else
        modulationrate = modulationrate * 1.0u"Hz"
    end
    if unit(samplingrate) == unit(1.0u"Hz")
        #nothing
    else
        samplingrate = samplingrate * 1.0u"Hz"
    end

    a = SSR(
        data,
        elecs,
        triggers,
        system_codes,
        samplingrate,
        modulationrate,
        [reference_channel],
        file_path,
        file_name,
        Dict(),
        header,
    )

    # If a valid mat file was found then store that information with the raw header
    if valid_mat
        a.header["rbadata"] = matread(mat_path)
    end

    #
    # Store meta data in processing dictionary
    #

    if stimulation_side != ""
        a.processing["Side"] = stimulation_side
    end
    if participant_name != ""
        a.processing["Name"] = participant_name
    end
    if !isnan(stimulation_amplitude)
        a.processing["Amplitude"] = stimulation_amplitude
    end
    if !isnan(carrier_frequency)
        a.processing["Carrier_Frequency"] = carrier_frequency
    end

    #
    # Clean up
    #

    # Remove status channel information
    remove_channel!(a, "Status")

    # Clean epoch index
    a.triggers = clean_triggers(
        a.triggers,
        valid_triggers,
        min_epoch_length,
        max_epoch_length,
        remove_first,
        max_epochs,
    )

    return a
end

#######################################
#
# Read event files
#
#######################################

function read_evt(a::SSR, fname::AbstractString; kwargs...)
    d = read_evt(fname, a.samplingrate; kwargs...)
    validate_triggers(d)
    a.triggers = d
    return a
end

#######################################
#
# File IO
#
#######################################

function write_SSR(
    a::SSR,
    fname::S;
    chanLabels = channelnames(a),
    subjID = a.header["subjID"],
    startDate = a.header["startDate"],
    startTime = a.header["startTime"],
    kwargs...,
) where {S<:AbstractString}

    fname = convert(String, fname)

    @info("Saving $(size(a.data)[end]) channels to $fname")

    writeBDF(
        fname,
        a.data',
        vec(trigger_channel(a)),
        vec(system_code_channel(a)),
        samplingrate(Int, a),
        chanLabels = convert(Array{S,1}, chanLabels),
        startDate = startDate,
        startTime = startTime,
        subjID = subjID,
    )

end


function CartesianOrbit(arg0::Orbit)
    return CartesianOrbit((Orbit,), arg0)
end

function CartesianOrbit(arg0::PVCoordinates, arg1::Frame, arg2::AbsoluteDate, arg3::jdouble)
    return CartesianOrbit((PVCoordinates, Frame, AbsoluteDate, jdouble), arg0, arg1, arg2, arg3)
end

function CartesianOrbit(arg0::TimeStampedPVCoordinates, arg1::Frame, arg2::jdouble)
    return CartesianOrbit((TimeStampedPVCoordinates, Frame, jdouble), arg0, arg1, arg2)
end

function add_kepler_contribution(obj::CartesianOrbit, arg0::PositionAngle, arg1::jdouble, arg2::Vector{jdouble})
    return jcall(obj, "addKeplerContribution", void, (PositionAngle, jdouble, Vector{jdouble}), arg0, arg1, arg2)
end

function get_a(obj::CartesianOrbit)
    return jcall(obj, "getA", jdouble, ())
end

function get_a_dot(obj::CartesianOrbit)
    return jcall(obj, "getADot", jdouble, ())
end

function get_date(obj::Orbit)
    return jcall(obj, "getDate", AbsoluteDate, ())
end

function get_e(obj::CartesianOrbit)
    return jcall(obj, "getE", jdouble, ())
end

function get_e_dot(obj::CartesianOrbit)
    return jcall(obj, "getEDot", jdouble, ())
end

function get_equinoctial_ex(obj::CartesianOrbit)
    return jcall(obj, "getEquinoctialEx", jdouble, ())
end

function get_equinoctial_ex_dot(obj::CartesianOrbit)
    return jcall(obj, "getEquinoctialExDot", jdouble, ())
end

function get_equinoctial_ey(obj::CartesianOrbit)
    return jcall(obj, "getEquinoctialEy", jdouble, ())
end

function get_equinoctial_ey_dot(obj::CartesianOrbit)
    return jcall(obj, "getEquinoctialEyDot", jdouble, ())
end

function get_frame(obj::Orbit)
    return jcall(obj, "getFrame", Frame, ())
end

function get_hx(obj::CartesianOrbit)
    return jcall(obj, "getHx", jdouble, ())
end

function get_hx_dot(obj::CartesianOrbit)
    return jcall(obj, "getHxDot", jdouble, ())
end

function get_hy(obj::CartesianOrbit)
    return jcall(obj, "getHy", jdouble, ())
end

function get_hy_dot(obj::CartesianOrbit)
    return jcall(obj, "getHyDot", jdouble, ())
end

function get_i(obj::CartesianOrbit)
    return jcall(obj, "getI", jdouble, ())
end

function get_i_dot(obj::CartesianOrbit)
    return jcall(obj, "getIDot", jdouble, ())
end

function get_jacobian_wrt_cartesian(obj::Orbit, arg0::PositionAngle, arg1::Vector{Vector{jdouble}})
    return jcall(obj, "getJacobianWrtCartesian", void, (PositionAngle, Vector{Vector{jdouble}}), arg0, arg1)
end

function get_jacobian_wrt_parameters(obj::Orbit, arg0::PositionAngle, arg1::Vector{Vector{jdouble}})
    return jcall(obj, "getJacobianWrtParameters", void, (PositionAngle, Vector{Vector{jdouble}}), arg0, arg1)
end

function get_keplerian_mean_motion(obj::Orbit)
    return jcall(obj, "getKeplerianMeanMotion", jdouble, ())
end

function get_keplerian_period(obj::Orbit)
    return jcall(obj, "getKeplerianPeriod", jdouble, ())
end

function get_le(obj::CartesianOrbit)
    return jcall(obj, "getLE", jdouble, ())
end

function get_le_dot(obj::CartesianOrbit)
    return jcall(obj, "getLEDot", jdouble, ())
end

function get_lm(obj::CartesianOrbit)
    return jcall(obj, "getLM", jdouble, ())
end

function get_lm_dot(obj::CartesianOrbit)
    return jcall(obj, "getLMDot", jdouble, ())
end

function get_lv(obj::CartesianOrbit)
    return jcall(obj, "getLv", jdouble, ())
end

function get_lv_dot(obj::CartesianOrbit)
    return jcall(obj, "getLvDot", jdouble, ())
end

function get_mu(obj::Orbit)
    return jcall(obj, "getMu", jdouble, ())
end

function get_pv_coordinates(obj::Orbit)
    return jcall(obj, "getPVCoordinates", TimeStampedPVCoordinates, ())
end

function get_pv_coordinates(obj::Orbit, arg0::AbsoluteDate, arg1::Frame)
    return jcall(obj, "getPVCoordinates", TimeStampedPVCoordinates, (AbsoluteDate, Frame), arg0, arg1)
end

function get_pv_coordinates(obj::Orbit, arg0::Frame)
    return jcall(obj, "getPVCoordinates", TimeStampedPVCoordinates, (Frame,), arg0)
end

function get_type(obj::CartesianOrbit)
    return jcall(obj, "getType", OrbitType, ())
end

function has_derivatives(obj::CartesianOrbit)
    return jcall(obj, "hasDerivatives", jboolean, ())
end

function interpolate(obj::CartesianOrbit, arg0::AbsoluteDate, arg1::Stream)
    return jcall(obj, "interpolate", CartesianOrbit, (AbsoluteDate, Stream), arg0, arg1)
end

function interpolate(obj::TimeInterpolable, arg0::AbsoluteDate, arg1::Collection)
    return jcall(obj, "interpolate", TimeInterpolable, (AbsoluteDate, Collection), arg0, arg1)
end

function shifted_by(obj::CartesianOrbit, arg0::jdouble)
    return jcall(obj, "shiftedBy", CartesianOrbit, (jdouble,), arg0)
end

function to_string(obj::CartesianOrbit)
    return jcall(obj, "toString", JString, ())
end


import Base: +, -, *, &, |, xor, ==, ~

for op in (:+, :-, :&, :|, :xor, :(==))
    @eval begin
        function ($op)(a::PyenumValues{T}, b::PyenumValues{S}) where {T<:Integer,S<:Integer}
            N = promote_type(T, S)
            ($op)(N(a.value), N(b.value))
        end
        function ($op)(a::PyenumValues{T}, b::PyenumValues{S}) where {T<:String,S<:String}
            print("convert")
            N = promote_type(T, S)
            ($op)(N(a.value), N(b.value))
        end
        function ($op)(a::PyenumValues{T}, b::S) where {T<:Integer,S<:Integer}
            N = promote_type(T, S)
            ($op)(N(a.value), N(b))
        end
        function ($op)(a::T, b::PyenumValues{S}) where {T<:Integer,S<:Integer}
            N = promote_type(T, S)
            ($op)(N(a), N(b.value))
        end
    end
end

function ~(a::PyenumValues{T}) where {T<:Integer}
    ~(T(a.value))
end

function ~(a::PyenumValues{T}) where {T<:String}
    ~(T(a.value))
end

Base.convert(::Type{T1}, x::PyenumValues{T2}) where {T1<:Integer,T2<:Integer} = convert(T1, T2(x.value))
Base.convert(::Type{T1}, x::PyenumValues{T2}) where {T1<:String,T2<:String} = convert(T1, T2(x.value))