module CUBLAS

using ..APIUtils

using ..CUDA
using ..CUDA: CUstream, cuComplex, cuDoubleComplex, libraryPropertyType, cudaDataType
using ..CUDA: libcublas, unsafe_free!, @retry_reclaim, isdebug, @sync, @context!

using GPUArrays

using LinearAlgebra

using BFloat16s

using CEnum

using Memoization

using DataStructures

# core library
include("libcublas_common.jl")
include("error.jl")
include("libcublas.jl")
include("libcublas_deprecated.jl")

# low-level wrappers
include("util.jl")
include("wrappers.jl")

# high-level integrations
include("linalg.jl")

function math_mode!(handle, mode)
    flags = 0

    # https://github.com/facebookresearch/faiss/issues/1385
    if version() > v"11"
        flags = CUBLAS_MATH_DISALLOW_REDUCED_PRECISION_REDUCTION
    end

    flags |= if mode == CUDA.PEDANTIC_MATH
        # prevent use of tensor cores
        if version() < v"11"
            CUBLAS_DEFAULT_MATH
        else
            CUBLAS_PEDANTIC_MATH
        end
    elseif mode == CUDA.DEFAULT_MATH
        # use tensor cores, but don't reduce precision
        if version() < v"11"
            CUBLAS_TENSOR_OP_MATH
        else
            CUBLAS_DEFAULT_MATH
        end
    elseif mode == CUDA.FAST_MATH
        # we'll additionally select a compute-mode with reduced precision whenever possible
        if version() < v"11"
            CUBLAS_TENSOR_OP_MATH
        else
            CUBLAS_TF32_TENSOR_OP_MATH
        end
    end

    cublasSetMathMode(handle, flags)

    return
end

# cache for created, but unused handles
const idle_handles = HandleCache{CuContext,cublasHandle_t}()
const idle_xt_handles = HandleCache{Any,cublasXtHandle_t}()

function handle()
    state = CUDA.active_state()
    handle, stream, math_mode = get!(task_local_storage(), (:CUBLAS, state.context)) do
        new_handle = pop!(idle_handles, state.context) do
            cublasCreate()
        end

        finalizer(current_task()) do task
            push!(idle_handles, state.context, new_handle) do
                @context! skip_destroyed=true state.context cublasDestroy_v2(handle)
            end
        end

        cublasSetStream_v2(new_handle, state.stream)

        math_mode!(new_handle, state.math_mode)

        new_handle, state.stream, state.math_mode
    end::Tuple{cublasHandle_t,CuStream,CUDA.MathMode}

    if stream != state.stream
        cublasSetStream_v2(handle, state.stream)
        task_local_storage((:CUBLAS, state.context), (handle, state.stream, math_mode))
        stream = state.stream
    end

    if math_mode != state.math_mode
        math_mode!(handle, state.math_mode)
        task_local_storage((:CUBLAS, state.context), (handle, stream, state.math_mode))
        math_mode = state.math_mode
    end

    return handle
end

function xt_handle()
    ctxs = Tuple(context(dev) for dev in devices())
    get!(task_local_storage(), (:CUBLASxt, ctxs)) do
        handle = pop!(idle_xt_handles, ctxs) do
            cublasXtCreate()
        end

        finalizer(current_task()) do task
            push!(idle_xt_handles, ctxs, handle) do
                # TODO: which context do we need to destroy this on?
                cublasXtDestroy(handle)
            end
        end

        # select all devices
        devs = convert.(Cint, devices())
        cublasXtDeviceSelect(handle, length(devs), devs)

        handle
    end::cublasXtHandle_t
end

## logging

const MAX_LOG_BUFLEN = UInt(1024*1024)
const log_buffer = Vector{UInt8}(undef, MAX_LOG_BUFLEN)
const log_cursor = Threads.Atomic{UInt}(0)
const log_cond = Ref{Base.AsyncCondition}()    # root

function log_message(ptr)
    # NOTE: this function may be called from unmanaged threads (by cublasXt),
    #       so we can't even allocate, let alone perform I/O.
    len = @ccall strlen(ptr::Cstring)::Csize_t
    cursor = Threads.atomic_add!(log_cursor, len+1)
    if cursor+len+1 <= MAX_LOG_BUFLEN
        @ccall memmove((pointer(log_buffer)+cursor)::Ptr{Nothing},
                       pointer(ptr)::Ptr{Nothing}, (len+1)::Csize_t)::Nothing
        @ccall uv_async_send(log_cond[]::Ptr{Nothing})::Cint
    end

    return
end

function _log_message(blob)
    # the message format isn't documented, but it looks like a message starts with a capital
    # and the severity (e.g. `I!`), and subsequent lines start with a lowercase mark (`!i`)
    #
    # lines are separated by a \0 if they came in separately, but there may also be multiple
    # actual lines separated by \n in each message.
    for message in split(blob, r"[\0\n]+(?=[A-Z]!)")
        code = message[1]
        lines = split(message[3:end], r"[\0\n]+[a-z]!")
        submessage = join(lines, '\n')
        if code == 'I'
            @debug submessage
        elseif code == 'W'
            @warn submessage
        elseif code == 'E'
            @error submessage
        elseif code == 'F'
            error(submessage)
        else
            @info "Unknown log message, please file an issue.\n$message"
        end
    end
    return
end

function __runtime_init__()
    # register a log callback
    log_cond[] = Base.AsyncCondition() do async_cond
        blob = ""
        while true
            blob = unsafe_string(pointer(log_buffer), log_cursor[])
            if Threads.atomic_cas!(log_cursor, UInt(length(blob)), UInt(0)) == length(blob)
                break
            end
        end
        _log_message(blob)
        return
    end
    if !Sys.iswindows() # NVIDIA bug #3321130
        callback = @cfunction(log_message, Nothing, (Cstring,))
        cublasSetLoggerCallback(callback)
    end
end

end


#===============================================================================

    AmpOutput.jl

    amplification output formats per step or ramp:
    long (full) and short (for conversion to JSON)

    Author: Tom Price
    Date:   June 2019

===============================================================================#

import DataStructures.OrderedDict
import StaticArrays: SArray, SMatrix, SVector
import Ipopt: IpoptSolver #, NLoptSolver

#===============================================================================
    Field definitions >>
===============================================================================#

const AMPLONGOUTPUT_FIELD_DEFS = [
    Field(:raw_data,                     Array{<: Real,3}), ## fr_ary3
    ## computed in amp_analysis()
    Field(:background_subtracted_data,   Array{<: Real,3}), ## mw_ary3
    Field(:norm_data,                    SArray{S,<: Real} where {S}), ## wva_data
    Field(:k_deconv,                     DeconvolutionMatrices), ## k4dcv
    Field(:deconvoluted_data,            Array{<: Real,3}), ## dcvd_ary3
    Field(:rbbs_ary3,                    Array{Float_T,3}), ## calibrated_data
    # cq_method,                           Symbol
    ## for ct method
    Field(:ct_fluos,                     SVector{C,Float_T} where {C}),
    ## computed by fit_model!()
    Field(:bl_fit,                       SMatrix{W,C,Union{AmpModelFit,Symbol}} where {W,C}),
    Field(:bl_notes,                     SMatrix{W,C,Vector{String}} where {W,C}),
    Field(:blsub_fluos,                  Array{Float_T,3}),
    Field(:blsub_fluos_flb,                  Array{Float_T,3}),
    Field(:quant_fit,                    SMatrix{W,C,Union{AmpModelFit,Symbol}} where {W,C}),
    Field(:quant_status,                 SMatrix{W,C,Symbol} where {W,C}),
    Field(:coefs,                        Array{Float_T,3}),
    Field(:d0,                           SMatrix{W,C,Float_T} where {W,C}),
    Field(:quant_fluos,                  Array{Float_T,3}),
    Field(:dr1_pred,                     Array{Float_T,3}),
    Field(:dr2_pred,                     Array{Float_T,3}),
    Field(:dr1_pred1,                     Array{Float_T,3}),
    Field(:dr2_pred1,                     Array{Float_T,3}),
    Field(:max_dr1,                      SMatrix{W,C,Float_T} where {W,C}),
    Field(:max_dr2,                      SMatrix{W,C,Float_T} where {W,C}),
    Field(:cyc_vals_4cq,                 SMatrix{W,C,OrderedDict{Symbol,Float_T}} where {W,C}),
    Field(:eff_vals_4cq,                 SMatrix{W,C,OrderedDict{Symbol,Float_T}} where {W,C}),
    Field(:cq_raw,                       SMatrix{W,C,Float_T} where {W,C}),
    Field(:cq,                           SMatrix{W,C,Float_T} where {W,C}),
    Field(:eff,                          SMatrix{W,C,Float_T} where {W,C}),
    Field(:cq_fluo,                      SMatrix{W,C,Float_T} where {W,C}),
    ## computed in amp_analysis()
    Field(:qt_fluos,                     SMatrix{W,C,Float_T} where {W,C}),
    Field(:max_qt_fluo,                  Float_T),
    ## computed by report_cq!()
    Field(:max_bsf,                      SMatrix{W,C,Float_T} where {W,C}),
    Field(:scaled_max_bsf,               SMatrix{W,C,Float_T} where {W,C}),
    Field(:scaled_max_dr1,               SMatrix{W,C,Float_T} where {W,C}),
    Field(:scaled_max_dr2,               SMatrix{W,C,Float_T} where {W,C}),
    Field(:why_NaN,                      SMatrix{W,C,String} where {W,C}),
    ## allelic discrimination output
    Field(:assignments_adj_labels_dict,  OrderedDict{Symbol,Vector{String}}),
    Field(:agr_dict,                     OrderedDict{Symbol,AssignGenosResult})]

const AMPSHORTOUTPUT_FIELDNAMES = [
    :rbbs_ary3,
    :blsub_fluos,
    :blsub_fluos_flb,
    :dr1_pred,
    :dr2_pred,
    :dr1_pred1,
    :dr2_pred1,
    :quant_status,
    :coefs,
    :cq,
    :d0,
    :ct_fluos,
    :assignments_adj_labels_dict]

#===============================================================================
    struct generation >>
===============================================================================#

abstract type AmpOutput end

SCHEMA = AMPLONGOUTPUT_FIELD_DEFS
@make_struct_from_SCHEMA AmpLongOutput AmpOutput true
@make_constructor_from_SCHEMA AmpLongOutput

SCHEMA = subset_schema(
    AMPLONGOUTPUT_FIELD_DEFS,
    AMPSHORTOUTPUT_FIELDNAMES)
@make_struct_from_SCHEMA AmpShortOutput AmpOutput true
@make_constructor_from_SCHEMA AmpShortOutput

#===============================================================================
    constructors >>
===============================================================================#

function AmpOutput(
    ::Type{Val{long}},
    i                           ::AmpInput,
    background_subtracted_data  ::Array{<: Real,3},
    k_deconv                    ::DeconvolutionMatrices,
    deconvoluted_data           ::Array{<: Real,3},
    norm_data                   ::SArray{S,R} where {S <: Tuple, R <: Real},
    norm_wells                  ::AbstractVector{Symbol},
    calibrated_data             ::Array{<: Real,3},
    ct_fluos                    ::AbstractVector;
)
    NaN_array2        = amp_init(i, NaN_T)
    # fitted_init       = amp_init(i, FIT[i.amp_model]())
    empty_vals_4cq    = amp_init(i, OrderedDict{Symbol, Float_T}())
    AmpLongOutput(
        i.raw_data, ## formerly fr_ary3
        background_subtracted_data, ## formerly mw_ary3
        norm_data, ## formerly wva_data
        k_deconv, ## formerly k4dcv
        deconvoluted_data, ## formerly dcvd_ary3
        calibrated_data, ## formerly rbbs_ary3
        # cq_method,
        ## ct_fluos
        SVector{i.num_channels, Float_T}(ct_fluos), ## ct_fluos
        ## model fit
        fitted_init, ## bl_fit,
        amp_init(i, Vector{String}()), ## bl_notes
        calibrated_data, ## blsub_fluos
        calibrated_data,
        fitted_init, ## quant_fit,
        amp_init(i, :not_fitted), ## quant_status
        fill(NaN_T, 1, i.num_wells, i.num_channels), ## coefs ## NB size = 1 for 1st dimension may not be correct for the chosen model
        NaN_array2, ## d0
        calibrated_data, ## quant_fluos,
        calibrated_data, ## dr1_pred
        calibrated_data, ## dr2_pred
        calibrated_data, ## dr1_pred
        calibrated_data, ## dr2_pred
        NaN_array2, ## max_dr1
        NaN_array2, ## max_dr2
        empty_vals_4cq, ## cyc_vals_4cq
        empty_vals_4cq, ## eff_vals_4cq
        NaN_array2, ## cq_raw
        NaN_array2, ## cq
        NaN_array2, ## eff
        NaN_array2, ## cq_fluo
        ## set_qt_fluos!()
        NaN_array2, ## qt_fluos
        NaN_array2, ## max_qt_fluo
        ## report_cq!()
        NaN_array2, ## max_bsf
        NaN_array2, ## scaled_max_bsf
        NaN_array2, ## scaled_max_dr1
        NaN_array2, ## scaled_max_dr2
        amp_init(i, ""), ## why_NaN
        ## allelic discrimination
        OrderedDict{Symbol, Vector{String}}(), ## assignments_adj_labels_dict
        OrderedDict{Symbol, AssignGenosResult}()) ## agr_dict
end ## constructor

function AmpOutput(
    ::Type{Val{short}},
    i                           ::AmpInput,
    background_subtracted_data  ::Array{<: Real,3},
    k_deconv                    ::DeconvolutionMatrices,
    deconvoluted_data           ::Array{<: Real,3},
    norm_data                   ::SArray{S,R} where {S <: Tuple, R <: Real},
    norm_well_nums              ::AbstractVector,
    calibrated_data             ::Array{<: Real,3},
    ct_fluos                    ::AbstractVector;
    reporting                   ::Function = roundoff(JSON_DIGITS) ## reporting function
)
    NaN_array2 = amp_init(i, NaN_T)
    cd = reporting.(calibrated_data)
    AmpShortOutput(
        cd, ## formerly rbbs_ary3
        cd, ## blsub_fluos
        cd,
        cd, ## dr1_pred
        cd, ## dr2_pred
        cd, ## dr1_pred
        cd, ## dr2_pred
        amp_init(i, :not_fitted), ## quant_status
        fill(NaN_T, 1, i.num_wells, i.num_channels),
        NaN_array2, ## d0
        NaN_array2, ## cq
        ct_fluos, ## ct_fluos
        OrderedDict{Symbol, AssignGenosResult}()) ## agr_dict
end ## constructor


module trlib
    type trlib_data
        H
        invM
        grad          :: Array{Float64, 1}
        itmax         :: Int64
        equality      :: Int64
        itmax_lanczos :: Int64
        tol_rel_i     :: Float64
        tol_abs_i     :: Float64
        tol_rel_b     :: Float64
        tol_abs_b     :: Float64
        zero          :: Float64
        obj_lb        :: Float64
        ctl_invariant :: Int64
        convexify     :: Int64
        earlyterm     :: Int64
        refine        :: Int64
        verbose       :: Int64
        h_pointer     :: Int64
        iwork         :: Array{Int64, 1}
        fwork         :: Array{Float64, 1}
        timing        :: Array{Int64, 1}
        init          :: Int64
        sol           :: Array{Float64, 1}
        g             :: Array{Float64, 1}
        v             :: Array{Float64, 1}
        gm            :: Array{Float64, 1}
        p             :: Array{Float64, 1}
        Hp            :: Array{Float64, 1}
        Q             :: Array{Float64, 2}
        ret           :: Int64
        obj           :: Float64
        lam           :: Float64
        trlib_data(H, invM, grad) = (itmax = min(round(Int, 2e8/size(grad,1)), 2*size(grad, 1));
                               iwork_size = Array{Clong}(1);
                               fwork_size = Array{Clong}(1);
                               h_pointer = Array{Clong}(1);
                               ccall(
                                     (:trlib_krylov_memory_size, "libtrlib"),
                                     Clong,
                                     (Clong, Ptr{Clong}, Ptr{Clong}, Ptr{Clong}),
                                     itmax, iwork_size, fwork_size, h_pointer);
                               fwork = Array{Cdouble}(fwork_size[1]);
                               ccall(
                                     (:trlib_krylov_prepare_memory, "libtrlib"),
                                     Clong,
                                     (Clong, Ptr{Cdouble}),
                                     itmax, fwork);
                               new(H, invM, grad, itmax,
                                   0, 100, -2.0, 0.0, -3.0, 0.0, 2e-16, -1e20, 0, 1, 1, 1, 0,
                                   h_pointer[1], Array{Clong}(iwork_size[1]), fwork,
                                   Array{Clong}(ccall((:trlib_krylov_timing_size, "libtrlib"), Clong, ())),
                                   1, similar(grad), similar(grad), similar(grad),
                                   similar(grad), similar(grad), similar(grad),
                                   Array{Float64}(size(grad,1), itmax+1), 0, 0.0, 0.0 ) )
        trlib_data(H, grad) = (itmax = min(round(Int, 2e8/size(grad,1)), 2*size(grad, 1));
                               iwork_size = Array{Clong}(1);
                               fwork_size = Array{Clong}(1);
                               h_pointer = Array{Clong}(1);
                               ccall(
                                     (:trlib_krylov_memory_size, "libtrlib"),
                                     Clong,
                                     (Clong, Ptr{Clong}, Ptr{Clong}, Ptr{Clong}),
                                     itmax, iwork_size, fwork_size, h_pointer);
                               fwork = Array{Cdouble}(fwork_size[1]);
                               ccall(
                                     (:trlib_krylov_prepare_memory, "libtrlib"),
                                     Clong,
                                     (Clong, Ptr{Cdouble}),
                                     itmax, fwork);
                               new(H, speye(size(grad,1)), grad, itmax,
                                   0, 100, -2.0, 0.0, -3.0, 0.0, 2e-16, -1e20, 0, 1, 1, 1, 0,
                                   h_pointer[1], Array{Clong}(iwork_size[1]), fwork,
                                   Array{Clong}(ccall((:trlib_krylov_timing_size, "libtrlib"), Clong, ())),
                                   1, similar(grad), similar(grad), similar(grad),
                                   similar(grad), similar(grad), similar(grad),
                                   Array{Float64}(size(grad,1), itmax+1), 0, 0.0, 0.0 ) )
    end

    function trlib_solve(TR::trlib_data, radius::Float64)
        g_dot_g = 0.0
        v_dot_g = 0.0
        p_dot_Hp = 0.0

        action = Array{Clong}(1)
        it     = Array{Clong}(1)
        ityp   = Array{Clong}(1)
        flt1   = Array{Cdouble}(1)
        flt2   = Array{Cdouble}(1)
        flt3   = Array{Cdouble}(1)

        while true
            TR.ret = ccall(
                          (:trlib_krylov_min, "libtrlib"),
                          Clong,
                          (Clong, Cdouble, Clong, Clong, Clong, Cdouble, Cdouble,
                           Cdouble, Cdouble, Cdouble, Cdouble, Clong, Clong, Clong,
                           Cdouble, Cdouble, Cdouble, Ptr{Clong}, Ptr{Cdouble},
                           Clong, Clong, Clong, Cstring, Ptr{Clong}, Ptr{Clong},
                           Ptr{Clong}, Ptr{Clong}, Ptr{Clong}, Ptr{Cdouble},
                           Ptr{Cdouble}, Ptr{Cdouble}),
                          TR.init, radius, TR.equality, TR.itmax, TR.itmax_lanczos,
                          TR.tol_rel_i, TR.tol_abs_i, TR.tol_rel_b, TR.tol_abs_b,
                          TR.zero, TR.obj_lb, TR.ctl_invariant, TR.convexify,
                          TR.earlyterm, g_dot_g, v_dot_g, p_dot_Hp, TR.iwork,
                          TR.fwork, TR.refine, 0, 0, "", C_NULL,
                          TR.timing, action, it, ityp, flt1, flt2, flt3)
            TR.init = 0
            if action[1] == 1
                TR.sol[:] = 0.0
                TR.gm[:] = 0.0
                TR.g = TR.grad
                TR.v = TR.invM * TR.g
                g_dot_g = dot(TR.g, TR.g)
                v_dot_g = dot(TR.v, TR.g)
                TR.p = -TR.v
                TR.Hp = TR.H*TR.p
                p_dot_Hp = dot(TR.p, TR.Hp)
                TR.Q[:,1] = TR.v/sqrt(v_dot_g)
            end
            if action[1] == 2
                TR.sol = TR.Q[:,1:1+it[1]] * TR.fwork[1+TR.h_pointer:1+TR.h_pointer+it[1]]
            end
            if action[1] == 3
                if ityp[1] == 1
                    TR.sol += flt1[1] * TR.p
                end
            end
            if action[1] == 4
                if ityp[1] == 1
                    TR.Q[:,1+it] = flt2[1] * TR.v
                    TR.gm = TR.g
                    TR.g += flt1[1]*TR.Hp
                end
                if ityp[1] == 2
                    TR.sol = TR.Hp + flt1[1] * TR.g + flt2[1] * TR.gm
                    TR.gm = flt3[1] * TR.g
                    TR.g = TR.sol
                end
                TR.v = TR.invM * TR.g
                g_dot_g = dot(TR.g, TR.g)
                v_dot_g = dot(TR.v, TR.g)
            end
            if action[1] == 5
                TR.p = flt1[1]*TR.v + flt2[1]*TR.p
                TR.Hp = TR.H*TR.p
                p_dot_Hp = dot(TR.p, TR.Hp)
                if ityp[1] == 2
                    TR.Q[:,1+it] = TR.p
                end
            end
            if action[1] == 8
                g_dot_g = .5 * dot(TR.sol, TR.H*TR.sol) + dot(TR.sol, TR.grad)
            end
            if TR.ret < 10
                break
            end
        end
        TR.obj = TR.fwork[9]
        TR.lam = TR.fwork[8]
        TR.init = 2
    end
end


"""
	BafflingDriver
Driver that randomly changes lanes and speeds.

# Constructors
	BafflingDriver(timestep::Float64;mlon::LaneFollowingDriver=IntelligentDriverModel(), mlat::LateralDriverModel=ProportionalLaneTracker(), mlane::LaneChangeModel=RandLaneChanger(timestep),rec::SceneRecord = SceneRecord(1, timestep))

# Fields
- `rec::SceneRecord` A record that will hold the resulting simulation results
- `mlon::LaneFollowingDriver = IntelligentDriverModel()` Longitudinal driving model
- `mlat::LateralDriverModel = ProportionalLaneTracker()` Lateral driving model
- `mlane::LaneChangeModel =RandLaneChanger` Lane change model (randomly)
"""
mutable struct BafflingDriver <: DriverModel{LatLonAccel}
    rec::SceneRecord
    mlon::LaneFollowingDriver
    mlat::LateralDriverModel
    mlane::LaneChangeModel
    ΔT::Float64

    function BafflingDriver(
        timestep::Float64;
        mlon::LaneFollowingDriver=BafflingLongitudinalTracker(),
        mlat::LateralDriverModel=ProportionalLaneTracker(),
        mlane::LaneChangeModel=BafflingLaneChanger(timestep),
        rec::SceneRecord = SceneRecord(1, timestep)
        )

        retval = new()

        retval.rec = rec
        retval.mlon = mlon
        retval.mlat = mlat
        retval.mlane = mlane
        retval.ΔT = timestep

        retval
    end
end
get_name(::BafflingDriver) = "BafflingDriver"
function AutomotiveDrivingModels.set_desired_speed!(model::BafflingDriver, v_des::Float64)
    set_desired_speed!(model.mlon, v_des)
    set_desired_speed!(model.mlane, v_des)
    model
end
# function track_longitudinal!(driver::LaneFollowingDriver, scene::Scene, roadway::Roadway, vehicle_index::Int, fore::NeighborLongitudinalResult)
#     v_ego = scene[vehicle_index].state.v
#     if fore.ind != nothing
#         headway, v_oth = fore.Δs, scene[fore.ind].state.v
#     else
#         headway, v_oth = NaN, NaN
#     end
#     return track_longitudinal!(driver, v_ego, v_oth, headway)
# end
function AutomotiveDrivingModels.observe!(driver::BafflingDriver, scene::Scene, roadway::Roadway, egoid::Int)

    update!(driver.rec, scene)
    observe!(driver.mlane, scene, roadway, egoid) # receive action from the lane change controller

    vehicle_index = findfirst(egoid, scene)
    lane_change_action = rand(driver.mlane)
    laneoffset = get_lane_offset(lane_change_action, driver.rec, roadway, vehicle_index)
    lateral_speed = convert(Float64, get(VELFT, driver.rec, roadway, vehicle_index))

    if lane_change_action.dir == DIR_MIDDLE
        fore = get_neighbor_fore_along_lane(scene, vehicle_index, roadway, VehicleTargetPointFront(), VehicleTargetPointRear(), VehicleTargetPointFront())
    elseif lane_change_action.dir == DIR_LEFT
        fore = get_neighbor_fore_along_left_lane(scene, vehicle_index, roadway, VehicleTargetPointFront(), VehicleTargetPointRear(), VehicleTargetPointFront())
    else
        @assert(lane_change_action.dir == DIR_RIGHT)
        fore = get_neighbor_fore_along_right_lane(scene, vehicle_index, roadway, VehicleTargetPointFront(), VehicleTargetPointRear(), VehicleTargetPointFront())
    end

    track_longitudinal!(driver.mlon, scene, roadway, vehicle_index, fore) # receive acceleration from the longitudinal controller
    track_lateral!(driver.mlat, laneoffset, lateral_speed) # receive acceleration from the lateral controller

    driver
end
Base.rand(driver::BafflingDriver) = LatLonAccel(rand(driver.mlat), rand(driver.mlon).a)
Distributions.pdf(driver::BafflingDriver, a::LatLonAccel) = pdf(driver.mlat, a.a_lat) * pdf(driver.mlon, a.a_lon)
Distributions.logpdf(driver::BafflingDriver, a::LatLonAccel) = logpdf(driver.mlat, a.a_lat) * logpdf(driver.mlon, a.a_lon)


function main(args)
    return args
end
# Load inline arguments for worker unit
num_workers, this = main(ARGS)
num_workers = parse(Int64, num_workers)
this        = parse(Int64, this)

#
# Important directories
simdir          = "/remote/ceph2/group/gerda/data/mpik/gerda-simulations/gerda-gems-db/pss-dev"
simdir_old      = "/remote/ceph/group/gerda/data/simulation/gerda-mage-sim"
mapping_file    = simdir_old*"/UTILS/det-data/ged-mapping.json"
parameters_file = simdir_old*"/UTILS/det-data/ged-parameters.json"

#
# Load packages and functions
include("../src/init.jl")
include("../src/fct.jl")
include("../src/worker_fct.jl")

#
# Output paths and filter settings
plots_base_path = "../../waveforms/sim/plots/raw/"
base_path_raw   = "../../waveforms/sim/raw/"
base_path       = "../../waveforms/sim/processed/"
n_sim_events    = 5000
isTP = 0;
isBL = 0;
mult = 1;
sampling_time   = 1u"ns"
hits_threshold  = 0.005; # MeV
E = 400;

#
# Load workers log file
log_file = base_path_raw * "log-" * string(this) * ".json"
if !isdir(base_path_raw)
    mkpath(base_path_raw)
end
if !isfile(log_file)
    open(log_file, "w") do f
        JSON.print(f, Dict(), 4)
    end
    global log = Dict()
else
    global log = JSON.parsefile(log_file)
end;

#
# Collect simulation filepaths
current_dir = pwd()
cd(simdir)
filenames = glob("calib/*/*/*/*.root")
cd(current_dir)

filenames = get_share_for_worker(filenames, num_workers, this)

for filename in filenames
    E_lim = E
    output_path = base_path_raw*basename(filename)*"/";
    if !haskey(log, basename(filename))
        Base.run(`clear`)
        @info(string(findfirst(x->x == filename, filenames)) * " of " * string(length(filenames)) * " in progress!")
        file = TFile(joinpath(simdir, filename))
        tree = file["fTree"];

        tt = Table(eventnumber = tree.eventnumber[:],
            hits_iddet   = tree.hits_iddet[:],
            hits_edep    = tree.hits_edep[:],
            hits_xpos    = tree.hits_xpos[:] .* 10, # mm
            hits_ypos    = tree.hits_ypos[:] .* 10, # mm
            hits_zpos    = tree.hits_zpos[:] .* 10) # mm

        tt = tt |> @filter(length(_.hits_edep) != 0 && sum(_.hits_edep) >= E_lim/1000 && length(unique(_.hits_iddet)) == 1) |> Table;

        data = Table(
            evtno        = [],
            multiplicity = [],
            detno        = [],
            hits_totnum  = [],
            edep         = [],
            pos          = []
        )
        prog = Progress(size(tt,1), dt=0.5,
            barglyphs=BarGlyphs('|','█', ['▁' ,'▂' ,'▃' ,'▄' ,'▅' ,'▆', '▇'],' ','|',),
            barlen=10)
        for i in eachindex(tt)
            multi = 1
            det = tt[i].hits_iddet[1]
            parameters[sim_to_channel[det][2]]["upside_down"] == true ? upside_down = -1 : upside_down = 1
            append!(data, Table(
            evtno        = [tt[i].eventnumber],
            multiplicity = [multi],
            detno        = [Array{Int64,1}(zeros(length(tt[i].hits_iddet)) .+ sim_to_channel[det][1])],
            hits_totnum  = [length(tt[i].hits_edep)],
            edep         = [tt[i].hits_edep],
            pos          = [[ SVector{3}(([ tt[i].hits_xpos[k] .- parameters[sim_to_channel[det][2]]["detcenter_x"],
                                        tt[i].hits_ypos[k] .- parameters[sim_to_channel[det][2]]["detcenter_y"],
                                        upside_down .* (tt[i].hits_zpos[k] .- parameters[sim_to_channel[det][2]]["detcenter_z"] .+ upside_down * parameters[sim_to_channel[det][2]]["height"]/2)
                            ] * u"mm")...) for k in eachindex(tt[i].hits_xpos) ]]
                )
            )
            next!(prog)
        end
        tt = nothing
        dets = []
        for detno in data.detno
            push!(dets, unique(detno)[1])
        end
        dets = unique(dets)
        prog = Progress(length(dets), dt=0.5,
            barglyphs=BarGlyphs('|','█', ['▁' ,'▂' ,'▃' ,'▄' ,'▅' ,'▆', '▇'],' ','|',),
            barlen=10)
        @info("Creating files for each detector")
        for detno in dets
            det = channel_to_name[detno]
            t = data |> @filter(unique(_.detno)[1] == detno) |> Table;
            !isdir(output_path) ? mkpath(output_path) : ""
            HDF5.h5open(output_path * lpad(detno, 2, "0") * "-" * det * ".h5", "w") do h5f
                LegendHDF5IO.writedata( h5f, "data", Table(
                    evtno       = t.evtno,
                    event_mult  = t.multiplicity,
                    detno       = VectorOfArrays(t.detno),
                    hits_totnum = t.hits_totnum,
                    edep        = VectorOfArrays(t.edep),
                    pos         = VectorOfArrays(t.pos)))
            end
            next!(prog)
        end
        #
        # Log the progress
        log[basename(filename)] = "Done!"
        open(log_file, "w") do f
            JSON.print(f, log, 4)
        end
    end
end


using Unitful

export BrukerRobot, BrukerRobotParams

export moveCenter, movePark

Base.@kwdef struct BrukerRobotParams <: DeviceParams
  connectionName::String = "RobotServer"
  axisRange::Vector{Vector{typeof(1.0u"mm")}} = [[-85.0,225.0], [-Inf, Inf], [-Inf, Inf]]u"mm"
  namedPositions::Dict{String, Vector{typeof(1.0u"mm")}} = Dict("origin" => [0,0,0]u"mm")
end

BrukerRobotParams(dict::Dict) = params_from_dict(BrukerRobotParams, dict)

Base.@kwdef mutable struct BrukerRobot <: Robot
  "Unique device ID for this device as defined in the configuration."
  deviceID::String
  "Parameter struct for this devices read from the configuration."
  params::BrukerRobotParams
  "Flag if the device is optional."
  optional::Bool = false
  "Flag if the device is present."
  present::Bool = false
  "Vector of dependencies for this device."
  dependencies::Dict{String, Union{Device, Missing}}
  state::RobotState = INIT
end

""" `BrukerCommand(command::String)` """
struct BrukerCommand
  command::String
end

# Useful links
# http://unix.stackexchange.com/questions/87831/how-to-send-keystrokes-f5-from-terminal-to-a-process
#
# https://github.com/JuliaLang/julia/pull/6948
#
# Kind of deprecated:
# http://blog.leahhanson.us/post/julia/julia-commands.html

const center = "center\n"
const park = "park\n"
const pos = "pos\n"
const quit = "quit\n"
const exit = "exit\n"
const err = "err?\n"

dof(rob::BrukerRobot) = 3
axisRange(rob::BrukerRobot) = rob.params.axisRange
defaultVelocity(rob::BrukerRobot) = nothing
_getPosition(rob::BrukerRobot) = sendCommand(rob, BrukerCommand(pos))
_isReferenced(rob::BrukerRobot) = true
_enable(rob::BrukerRobot) = nothing
_disable(rob::BrukerRobot) = nothing
_reset(rob::BrukerRobot) = nothing
_setup(rob::BrukerRobot) = nothing
_doReferenceDrive(rob::BrukerRobot) = nothing

""" Move Bruker Robot to center"""
function moveCenter(sd::BrukerRobot)
    sendCommand(sd, BrukerCommand(center))
end

""" Move Bruker Robot to park"""
function movePark(sd::BrukerRobot)
    sendCommand(sd, BrukerCommand(park))
end

function _moveAbs(rob::BrukerRobot, pos::Vector{<:Unitful.Length}, speed::Union{Vector{<:Unitful.Velocity},Nothing})
    if speed !== nothing
        @warn "BrukerRobot does not support setting velocities!"
    end
    cmd = BrukerCommand("goto $(ustrip(Float64, u"mm", pos[1])),$(ustrip(Float64, u"mm", pos[2])),$(ustrip(Float64, u"mm", pos[3]))\n")
    res = sendCommand(rob, cmd)
end

""" Not Implemented """
function _moveRel(rob::BrukerRobot, dist::Vector{<:Unitful.Length}, speed::Union{Vector{<:Unitful.Velocity},Nothing})
    error("BrukerRobot does not support moveRel")
end

""" Send Command `sendCommand(sd::BrukerRobot, brukercmd::BrukerCommand)`"""
function sendCommand(sd::BrukerRobot, brukercmd::BrukerCommand)

    (result, startmovetime, endmovetime) = _sendCommand(sd, brukercmd)

    if result == "0\n"
        return true
    elseif length(split(result, ",")) == 3
        @info "$(brukercmd.command) returned position: $(result)"
        return parse.(Float64, split(result, ",")) * u"mm"
    elseif result == "?\n"
        @warn "$(brukercmd.command) is unknown! Try again..."
        return false
    elseif result == "!\n"
        throw(ErrorException("Error during command $(brukercmd.command) execution."))
    else
        throw(ErrorException("$(brukercmd.command) has unexpected result $(result)"))
    end
end

function _sendCommand(sd::BrukerRobot, brukercmd::BrukerCommand)
    p = open(`$(sd.params.connectionName)`, "r+");
  # p = open(`cat`,"r+");
    startmovetime = now(Dates.UTC)
    writetask = write(p.in, brukercmd.command)
    writetaskexit = write(p.in, exit)
    readtask = @async readavailable(p.out)
    wait(readtask)
    endmovetime = now(Dates.UTC)
    if readtask.state == :done
        return (ascii(String(readtask.result)), startmovetime, endmovetime);
    else
        @error "end" readTask.state readTask.exception
        return readtask.exception
    end
end


#=
This file is auto-generated. Do not edit.
=#
"""
    mutable struct ThreePartCost <: OperationalCost
        variable::VariableCost
        fixed::Float64
        startup::Float64
        shutdn::Float64
        forecasts::InfrastructureSystems.Forecasts
        internal::InfrastructureSystemsInternal
    end

Data Structure Operational Cost Data in Three parts fixed, variable cost and start - stop costs.

# Arguments
- `variable::VariableCost`: variable cost
- `fixed::Float64`: fixed cost
- `startup::Float64`: startup cost, validation range: (0, nothing), action if invalid: warn
- `shutdn::Float64`: shutdown cost, validation range: (0, nothing), action if invalid: warn
- `forecasts::InfrastructureSystems.Forecasts`: internal forecast storage
- `internal::InfrastructureSystemsInternal`: power system internal reference, do not modify
"""
mutable struct ThreePartCost <: OperationalCost
    "variable cost"
    variable::VariableCost
    "fixed cost"
    fixed::Float64
    "startup cost"
    startup::Float64
    "shutdown cost"
    shutdn::Float64
    "internal forecast storage"
    forecasts::InfrastructureSystems.Forecasts
    "power system internal reference, do not modify"
    internal::InfrastructureSystemsInternal
end

function ThreePartCost(variable, fixed, startup, shutdn, forecasts=InfrastructureSystems.Forecasts(), )
    ThreePartCost(variable, fixed, startup, shutdn, forecasts, InfrastructureSystemsInternal(), )
end

function ThreePartCost(; variable, fixed, startup, shutdn, forecasts=InfrastructureSystems.Forecasts(), )
    ThreePartCost(variable, fixed, startup, shutdn, forecasts, )
end

# Constructor for demo purposes; non-functional.
function ThreePartCost(::Nothing)
    ThreePartCost(;
        variable=VariableCost((0.0, 0.0)),
        fixed=0.0,
        startup=0.0,
        shutdn=0.0,
        forecasts=InfrastructureSystems.Forecasts(),
    )
end

"""Get ThreePartCost variable."""
get_variable(value::ThreePartCost) = value.variable
"""Get ThreePartCost fixed."""
get_fixed(value::ThreePartCost) = value.fixed
"""Get ThreePartCost startup."""
get_startup(value::ThreePartCost) = value.startup
"""Get ThreePartCost shutdn."""
get_shutdn(value::ThreePartCost) = value.shutdn

InfrastructureSystems.get_forecasts(value::ThreePartCost) = value.forecasts
"""Get ThreePartCost internal."""
get_internal(value::ThreePartCost) = value.internal

"""Set ThreePartCost variable."""
set_variable!(value::ThreePartCost, val::VariableCost) = value.variable = val
"""Set ThreePartCost fixed."""
set_fixed!(value::ThreePartCost, val::Float64) = value.fixed = val
"""Set ThreePartCost startup."""
set_startup!(value::ThreePartCost, val::Float64) = value.startup = val
"""Set ThreePartCost shutdn."""
set_shutdn!(value::ThreePartCost, val::Float64) = value.shutdn = val

InfrastructureSystems.set_forecasts!(value::ThreePartCost, val::InfrastructureSystems.Forecasts) = value.forecasts = val
"""Set ThreePartCost internal."""
set_internal!(value::ThreePartCost, val::InfrastructureSystemsInternal) = value.internal = val


__precompile__()
module st16SseqJuliaTools
    using Statistics
    using StatsBase
    using FastaIO
    using AutoHashEquals
    using DataStructures
    using DataFrames
    using Conda
    using BioSequences
    using FASTX
    using JLD2
    using Distributions
    using BioTools.BLAST
    using IntervalSets
    using ProgressMeter
    using Artifacts
    using BioAlignments

    #Alignment analysis stuff
    export Alignment
    export read_fasta
    export write_to_fasta
    export sub_alignment
    export vsearh_cluster!
    export get_max_cl_length
    export char_array_entropy
    export aln_conservatio
    export eval_recomb

    #Export BLAST parsing
    export split_by_query
    export add_new_interval
    export covered_length
    export unique_length_fraction
    export get_coverage_maximising_hits

    #Export TaxonKitWrap
    export taxid_to_species
    export taxid_to_genus

    #HMM parsing
    export extract_rRNA_names

    #export workflow specific staff
    export parse_mapping
    export parse_dada_results
    export check_pair
    export fasta_to_dict
    export dict_to_fasta
    export read_salmon

    #export clustering output parsing functinos
    export read_jc_clusters
    export uc2jc
    export mergejc
    export mark_proper_size_clusters

    include("utils.jl"	)
    include("BLASTparsing.jl")
    include("TaxonKitWrap.jl")
    include("HMMutils.jl")
    include("UCparsing.jl")
    include("AlignmentMatrix.jl")
end


function apply_boundary_conditions_on_φ_axis!(  rbpot::Array{T, 4}, iz::Int, ir::Int, rbi::Int, nrbi::Int, ax::DiscreteAxis{T, :periodic, :periodic}, int::Interval{:closed, :open, T},
                                                grid_boundary_factors::NTuple{2, T}, only2d::Val{only_2d})::Nothing where {T, only_2d}
    rbpot[iz,   1, ir, rbi] = rbpot[ iz, end - 1, ir, rbi] # cycling boundary
    rbpot[iz, end, ir, rbi] = rbpot[ iz,       2, ir, rbi] # cycling boundary
    nothing
end

function apply_boundary_conditions_on_φ_axis!(  rbpot::Array{T, 4}, iz::Int, ir::Int, rbi::Int, nrbi::Int, ax::DiscreteAxis{T, :reflecting, :reflecting}, int::Interval{:closed, :closed, T},
                                                grid_boundary_factors::NTuple{2, T}, only2d::Val{false})::Nothing where {T}
    rbpot[iz,   1, ir, rbi] = rbpot[ iz,       3, ir, rbi] # cycling boundary
    rbpot[iz, end, ir, rbi] = rbpot[ iz, end - 2, ir, rbi] # cycling boundary
    nothing
end
function apply_boundary_conditions_on_φ_axis!(  rbpot::Array{T, 4}, iz::Int, ir::Int, rbi::Int, nrbi::Int, ax::DiscreteAxis{T, :reflecting, :reflecting}, int::Interval{:closed, :closed, T},
                                                grid_boundary_factors::NTuple{2, T}, only2d::Val{true})::Nothing where {T}
    rbpot[iz,   1, ir, nrbi] = rbpot[ iz, 2, ir, rbi] # cycling boundary
    rbpot[iz, end, ir, nrbi] = rbpot[ iz, 2, ir, rbi] # cycling boundary
    nothing
end
function apply_boundary_conditions_on_r_axis!(  rbpot::Array{T, 4}, iz::Int, iφ::Int, rbi::Int, ax::DiscreteAxis{T, :r0, :infinite}, int::Interval{:closed, :closed, T},
                                                grid_boundary_factors::NTuple{2, T})::Nothing where {T}
    rbpot[iz, iφ, end, rbi] = grid_boundary_factors[2] * rbpot[iz, iφ, end - 2, rbi] # infinity boundaries in r
    nothing
end
function apply_boundary_conditions_on_r_axis!(  rbpot::Array{T, 4}, iz::Int, iφ::Int, rbi::Int, ax::DiscreteAxis{T, :r0, :reflecting}, int::Interval{:closed, :closed, T},
                                                grid_boundary_factors::NTuple{2, T})::Nothing where {T}
    rbpot[iz, iφ, end, rbi] = rbpot[iz, iφ, end - 2, rbi] # infinity boundaries in r
    nothing
end
function apply_boundary_conditions_on_r_axis!(  rbpot::Array{T, 4}, iz::Int, iφ::Int, rbi::Int, ax::DiscreteAxis{T, :r0, :fixed}, int::Interval{:closed, :closed, T},
                                                grid_boundary_factors::NTuple{2, T})::Nothing where {T}
    # rbpot[iz, iφ, end, rbi] = rbpot[iz, iφ, end - 2, rbi] # infinity boundaries in r
    nothing
end
function apply_boundary_conditions_on_cyl_z_axis!(  rbpot::Array{T, 4}, ir::Int, iφ::Int, rbi::Int, ax::DiscreteAxis{T, :infinite, :infinite}, int::Interval{:closed, :closed, T},
                                                    grid_boundary_factors::NTuple{2, T})::Nothing where {T}
    rbpot[  1, iφ, ir, rbi] = grid_boundary_factors[1] * rbpot[ 2, iφ, ir, rbi]  # infinity boundaries in z (only ±1 because this is the compressed (redblack) dimension)
    rbpot[end, iφ, ir, rbi] = grid_boundary_factors[2] * rbpot[ end - 1, iφ, ir, rbi]    # infinity boundaries in z (only ±1 because this is the compressed (redblack) dimension)
    nothing
end
function apply_boundary_conditions_on_cyl_z_axis!(  rbpot::Array{T, 4}, ir::Int, iφ::Int, rbi::Int, ax::DiscreteAxis{T, :infinite, :fixed}, int::Interval{:closed, :closed, T},
                                                    grid_boundary_factors::NTuple{2, T})::Nothing where {T}
    rbpot[  1, iφ, ir, rbi] = grid_boundary_factors[1] * rbpot[ 2, iφ, ir, rbi]  # infinity boundaries in z (only ±1 because this is the compressed (redblack) dimension)
    nothing
end
function apply_boundary_conditions_on_cyl_z_axis!(  rbpot::Array{T, 4}, ir::Int, iφ::Int, rbi::Int, ax::DiscreteAxis{T, :fixed, :infinite}, int::Interval{:closed, :closed, T},
                                                    grid_boundary_factors::NTuple{2, T})::Nothing where {T}
    rbpot[end, iφ, ir, rbi] = grid_boundary_factors[2] * rbpot[ end - 1, iφ, ir, rbi]    # infinity boundaries in z (only ±1 because this is the compressed (redblack) dimension)
    nothing
end
function apply_boundary_conditions_on_cyl_z_axis!(  rbpot::Array{T, 4}, ir::Int, iφ::Int, rbi::Int, ax::DiscreteAxis{T, :reflecting, :reflecting}, int::Interval{:closed, :closed, T},
                                                    grid_boundary_factors::NTuple{2, T})::Nothing where {T}
    rbpot[  1, iφ, ir, rbi] = rbpot[ 2, iφ, ir, rbi]  # infinity boundaries in z (only ±1 because this is the compressed (redblack) dimension)
    rbpot[end, iφ, ir, rbi] = rbpot[ end - 1, iφ, ir, rbi]    # infinity boundaries in z (only ±1 because this is the compressed (redblack) dimension)
    nothing
end
function apply_boundary_conditions_on_cyl_z_axis!(  rbpot::Array{T, 4}, ir::Int, iφ::Int, rbi::Int, ax::DiscreteAxis{T, :fixed, :reflecting}, int::Interval{:closed, :closed, T},
                                                    grid_boundary_factors::NTuple{2, T})::Nothing where {T}
    rbpot[end, iφ, ir, rbi] = rbpot[ end - 1, iφ, ir, rbi]    # infinity boundaries in z (only ±1 because this is the compressed (redblack) dimension)
    nothing
end
function apply_boundary_conditions_on_cyl_z_axis!(  rbpot::Array{T, 4}, ir::Int, iφ::Int, rbi::Int, ax::DiscreteAxis{T, :reflecting, :fixed}, int::Interval{:closed, :closed, T},
                                                    grid_boundary_factors::NTuple{2, T})::Nothing where {T}
    rbpot[  1, iφ, ir, rbi] = rbpot[ 2, iφ, ir, rbi]  # infinity boundaries in z (only ±1 because this is the compressed (redblack) dimension)
    nothing
end
function apply_boundary_conditions_on_cyl_z_axis!(  rbpot::Array{T, 4}, ir::Int, iφ::Int, rbi::Int, ax::DiscreteAxis{T, :fixed, :fixed}, int::Interval{:closed, :closed, T},
                                                    grid_boundary_factors::NTuple{2, T})::Nothing where {T}
    nothing
end

function apply_boundary_conditions!(fssrb::PotentialSimulationSetupRB{T, 3, 4, :cylindrical}, update_even_points::Val{even_points}, only2d::Val{only_2d}) where {T, even_points, only_2d}
    rbi::Int = even_points ? rb_even::Int : rb_odd::Int
    nrbi::Int = even_points ? rb_odd::Int : rb_even::Int
    ax1::typeof(fssrb.grid.axes[1]) = fssrb.grid.axes[1]
    ax2::typeof(fssrb.grid.axes[2]) = fssrb.grid.axes[2]
    ax3::typeof(fssrb.grid.axes[3]) = fssrb.grid.axes[3]
    int1::typeof(ax1.interval) = ax1.interval
    int2::typeof(ax2.interval) = ax2.interval
    int3::typeof(ax3.interval) = ax3.interval
    if only_2d
        iφ::Int = 2
        @inbounds for iz in axes(fssrb.potential, 1)
            # for ir in axes(fssrb.potential, 3)
            #     apply_boundary_conditions_on_φ_axis!( fssrb.potential, iz, ir, rbi, nrbi, ax2, int2, fssrb.grid_boundary_factors[2], only2d)
            # end
            apply_boundary_conditions_on_r_axis!( fssrb.potential, iz, iφ, rbi, ax1, int1, fssrb.grid_boundary_factors[1])
        end
        @inbounds for ir in axes(fssrb.potential, 3)
            apply_boundary_conditions_on_cyl_z_axis!( fssrb.potential, ir, iφ, rbi, ax3, int3, fssrb.grid_boundary_factors[3])
        end
    else
        @inbounds for iz in axes(fssrb.potential, 1)
            for iφ in axes(fssrb.potential, 2)
                apply_boundary_conditions_on_r_axis!( fssrb.potential, iz, iφ, rbi, ax1, int1, fssrb.grid_boundary_factors[1])
            end
            for ir in axes(fssrb.potential, 3)
                apply_boundary_conditions_on_φ_axis!( fssrb.potential, iz, ir, rbi, nrbi, ax2, int2, fssrb.grid_boundary_factors[2], only2d)
            end
        end
        @inbounds for iφ in axes(fssrb.potential, 2) # z boundaries
            for ir in axes(fssrb.potential, 3)
                apply_boundary_conditions_on_cyl_z_axis!( fssrb.potential, ir, iφ, rbi, ax3, int3, fssrb.grid_boundary_factors[3])
            end
        end
        begin # r = 0 handling
            nφ::Int = size(fssrb.potential, 2) - 1
            gw_φ::Array{T, 2} = fssrb.geom_weights[2].weights
            @inbounds for inz in 1:(size(fssrb.ϵ, 3) - 1)
                m::T = 0
                l::T = 0
                for inφ in 1:nφ
                    if even_points ? isodd(inz + inφ) : iseven(inz + inφ)
                        l += gw_φ[3, inφ]
                        m += fssrb.potential[rbidx(inz), inφ + 1, 2, rbi] * gw_φ[3, inφ]
                    end
                end
                m *= inv(l)
                for inφ in 1:nφ
                    if even_points ? isodd(inz + inφ) : iseven(inz + inφ)
                        fssrb.potential[rbidx(inz), inφ + 1, 2, rbi]::T = m
                    end
                end
            end
        end
    end
    nothing
end


prob(p::LogProb) = p
prob(s::Product) = first(s)

#####################################################
### Classes of Traversal, Completion, and ItemKey ###
#####################################################

struct Traversal{S}
    edgeid  :: Int
    consid  :: Int
    score   :: S
    inloop  :: Bool
end
make_accesses(Traversal)

Traversal(edgeid, consid, score) = Traversal(edgeid, consid, score, false)
prob(trav::Traversal) = prob(score(trav))

abstract type Completion{S} end
prob(comp::Completion) = prob(score(comp))

struct EdgeCompletion{CR,S} <: Completion{S}
    edgeid :: Int
    rule   :: CR
    score  :: S
    inloop :: Bool
end
make_accesses(EdgeCompletion)

EdgeCompletion(edgeid, rule, score) = EdgeCompletion(edgeid, rule, score, false)
EdgeCompletion{CR,S}(edgeid, rule, score) where {CR,S} = EdgeCompletion{CR,S}(edgeid, rule, score, false)

struct TerminalCompletion{T,TR,S} <: Completion{S}
    terminal :: T
    rule     :: TR
    score    :: S
end
make_accesses(TerminalCompletion)

# is not a bitstype
@auto_hash_equals struct EdgeKey{St}
    start :: Int
    tail  :: Int
    state :: St
end
make_accesses(EdgeKey)

# is a bitstype
struct ConsKey{C}
    start :: Int
    tail  :: Int
    cat   :: C
end
make_accesses(ConsKey)

function ==(k1::ConsKey{T}, k2::ConsKey{T}) where {T <: AbstractString}
    k1.start == k2.start && k1.tail == k2.tail && k1.cat == k2.cat
end

function Base.hash(k::ConsKey{T}) where {T <: AbstractString}
    hash(k.start, hash(k.tail, hash(k.cat)))
end

##################
### Item class ###
##################

abstract type Item end
Item(key::EdgeKey, trav, id, grammar) = Edge(key, trav, id, grammar)
Item(key::ConsKey, comp, id, grammar) = Constituent(key, comp, id, grammar)
start(item::Item) = item.start
tail(item::Item) = item.tail
length(item::Item) = item.tail - item.start
id(item::Item) = item.id
lastpopscore(item::Item) = item.lastpopscore
insidepopnumber(item::Item) = item.insidepopnumber
outsidepopnumber(item::Item) = item.outsidepopnumber

##################
### Edge class ###
##################

mutable struct Edge{St,S} <: Item
    start              :: Int
    tail               :: Int
    state              :: St
    score              :: S
    isfinished         :: Bool
    traversals         :: Vector{Traversal{S}}
    id                 :: Int # edges have positive ids
    lastpopscore       :: S
    lastoutsidepopscore:: S
    insidepopnumber    :: Int
    outsidepopnumber   :: Int
    outside_finished   :: Bool
    trav_outs_summands :: Dict{Tuple{Int, Int}, LogProb}
    comp_outs_summands :: Dict{Int, LogProb}
end
Edge(start, tail, state, traversals::Vector, id, score) =
    Edge(start, tail, state, score, false, traversals, id, score, score, 0, 0, false, Dict{Tuple{Int, Int}, LogProb}(), Dict{Int, LogProb}())
Edge(start, tail, state, traversal::Traversal{S}, id) where S =
    Edge(start, tail, state, [traversal], id, zero(S))
Edge(key::EdgeKey, traversal::Traversal, id, grammar) =
    Edge(start(key), tail(key), state(key), traversal, id)

score(edge::Edge) = if edge.isfinished
    edge.score
else
    sum(score(trav) for trav in traversals(edge))
end
prob(edge::Edge) = prob(score(edge))
outside(edge::Edge) = sum(values(edge.trav_outs_summands)) + sum(values(edge.comp_outs_summands))
state(edge::Edge) = edge.state
cat_or_state(edge::Edge) = edge.state
key(edge::Edge) = EdgeKey(edge.start, edge.tail, edge.state)
traversals(edge::Edge) = edge.traversals
show(io::IO, edge::Edge) = print(io, state(edge), "(", start(edge), ",", tail(edge), ")")

function add!(edge::Edge, traversal)
    for (i, trav) in enumerate(traversals(edge))
        if edgeid(trav) == edgeid(traversal) && consid(trav) == consid(traversal)
            traversals(edge)[i] = @set traversal.inloop = true
            return edge
        end
    end
    push!(traversals(edge), traversal)
    edge
end

#########################
### Constituent class ###
#########################

mutable struct Constituent{C,T,CR,TR,S} <: Item
    start               :: Int
    tail                :: Int
    cat                 :: C
    score               :: S
    isfinished          :: Bool
    completions         :: Vector{EdgeCompletion{CR,S}}
    terminal_completion :: Nullable{TerminalCompletion{T,TR,S}}
    id                  :: Int # constituents have negative ids
    lastpopscore        :: S
    lastoutsidepopscore :: S
    insidepopnumber     :: Int
    outsidepopnumber    :: Int
    outside_finished    :: Bool
    trav_outs_summands  :: Dict{Tuple{Int, Int}, LogProb}
end
Constituent(st,ta,ca,co,te,id,sc) =
    Constituent(st,ta,ca,sc,false,co,te,id,sc,sc,0,0,false,Dict{Tuple{Int, Int}, LogProb}())
function Constituent(start, tail, cat, comp::EdgeCompletion, id, grammar)
    T, TR, S = terminal_type(grammar), terminal_rule_type(grammar), score_type(grammar)
    Constituent(start, tail, cat, [comp], Nullable{TerminalCompletion{T,TR,S}}(), id, zero(S))
end
function Constituent(start, tail, cat, comp::TerminalCompletion, id, grammar)
    CR, S = category_rule_type(grammar), score_type(grammar)
    Constituent(start, tail, cat, Vector{EdgeCompletion{CR,S}}(), Nullable(comp), id, zero(S))
end
Constituent(key::ConsKey, comp, id, grammar) =
    Constituent(start(key), tail(key), cat(key), comp, id, grammar)

score(cons::Constituent) = if cons.isfinished
    cons.score
else
    if has_terminal(cons)
        if isempty(completions(cons))
            score(get(cons.terminal_completion)) # get(cons.terminal_completion))
        else
            sum(score(comp) for comp in completions(cons)) +
                score(get(cons.terminal_completion)) # get(cons.terminal_completion))
        end
    else
        sum(score(comp) for comp in completions(cons))
    end
end

prob(cons::Constituent) = prob(score(cons))
outside(cons::Constituent) = sum(values(cons.trav_outs_summands))
cat(cons::Constituent) = cons.cat
cat_or_state(cons::Constituent) = cons.cat
key(cons::Constituent) = ConsKey(cons.start, cons.tail, cons.cat)
completions(cons::Constituent) = cons.completions
has_terminal(cons::Constituent) = !isnull(cons.terminal_completion)
terminal_completion(cons::Constituent) = get(cons.terminal_completion)
terminal(cons::Constituent) = terminal(terminal_completion(cons))

function add!(cons::Constituent, completion::EdgeCompletion)
    for (i, comp) in enumerate(completions(cons))
        if edgeid(comp) == edgeid(completion)
            completions(cons)[i] = @set completion.inloop = true
            return cons
        end
    end
    push!(completions(cons), completion)
    nothing
end

function show(io::IO, c::Constituent)
    print(io, cat(c), "(", start(c), ",", tail(c), ")")
    if has_terminal(c)
        print(io, "[", terminal(c) ,"]")
    end
end

###########################
### ParserLogbook class ###
###########################

mutable struct ParserLogbook{C,T,CR,TR,St,S}
    edges :: Vector{Edge{St,S}}
    conss :: Vector{Constituent{C,T,CR,TR,S}}
    edgeids :: Dict{EdgeKey{St}, Int}
    consids :: Dict{ConsKey{C}, Int}
end
ParserLogbook(C, T, CR, TR, St, S) =
    ParserLogbook(Vector{Edge{St,S}}(),
                  Vector{Constituent{C,T,CR,TR,S}}(),
                  Dict{EdgeKey{St}, Int}(),
                  Dict{ConsKey{C}, Int}())

function discover!(logbook, edge::Edge)
    push!(logbook.edges, edge)
    id = length(logbook.edges)
    logbook.edgeids[key(edge)] = id
    edge.id = id
    nothing
end
function discover!(logbook, cons::Constituent)
    push!(logbook.conss, cons)
    id = -length(logbook.conss)
    logbook.consids[key(cons)] = id
    cons.id = id
    nothing
end
isdiscovered(logbook, key::EdgeKey) = haskey(logbook.edgeids, key)
isdiscovered(logbook, key::ConsKey) = haskey(logbook.consids, key)
get_edge(logbook, id::Int) = logbook.edges[id]
get_cons(logbook, id::Int) = logbook.conss[-id]
get_edge(logbook, key::EdgeKey) = get_edge(logbook, logbook.edgeids[key])
get_cons(logbook, key::ConsKey) = get_cons(logbook, logbook.consids[key])
get_item(logbook, key::EdgeKey) = get_edge(logbook, logbook.edgeids[key])
get_item(logbook, key::ConsKey) = get_cons(logbook, logbook.consids[key])

###################
### Chart class ###
###################

# maybe be used in a future implementation, maybe not
"like normal merge, but value vectors with the same key will be concatenated"
function Base.merge(dicts::Dict{K,Vector{V}}...) where {K,V}
    result = Dict{K,Vector{V}}()
    for d in dicts
        for (key,valvec) in d # key and value vector
            if haskey(result, key)
                append!(result[key], valvec)
            else
                result[key] = valvec
            end
        end
    end
    result
end

mutable struct ChartCell{C,St}
    edgeids :: Dict{St, Vector{Int}}
    consids :: Dict{C, Vector{Int}}
end
ChartCell(C, St) =
    ChartCell(Dict{St, Vector{Int}}(), Dict{C, Vector{Int}}())

mutable struct Chart{C,St}
    cells :: Vector{ChartCell{C,St}}
end
edgeids(chart, edge::Edge) = chart.cells[tail(edge)].edgeids
consids(chart, edge::Edge) = chart.cells[tail(edge)].consids
edgeids(chart, cons::Constituent) = chart.cells[start(cons)].edgeids
consids(chart, cons::Constituent) = chart.cells[start(cons)].consids
# dict(chart, edge::Edge) = chart.cells[tail(edge)].edgeids
# dict(chart, cons::Constituent) = chart.cells[start(cons)].consids
function insert!(chart::Chart, edge::Edge)
    dict = edgeids(chart, edge)
    if haskey(dict, state(edge))
        push!(dict[state(edge)], id(edge))
    else
        dict[state(edge)] = [id(edge)]
    end
    nothing
end
function insert!(chart::Chart, cons::Constituent)
    dict = consids(chart, cons)
    if haskey(dict, cat(cons))
        push!(dict[cat(cons)], id(cons))
    else
        dict[cat(cons)] = [id(cons)]
    end
    nothing
end

####################
### Agenda class ###
####################

# fast viterbi parsing does not work yet

mutable struct Agenda{T}
    pqueue :: PriorityQueue{Int, T}
end

function Agenda(parsing_method)
    @assert parsing_method in (:full, :viterbi)
    if parsing_method == :full
        Agenda(PriorityQueue{Int, Int}())
    else
        Agenda(PriorityQueue{Int, Float64}())
    end
end

function enqueue!(agenda::Agenda, item, just_used)
    agenda.pqueue[id(item)] = priority(agenda, item, just_used) ::Int
    nothing
end

dequeue!(agenda::Agenda) = dequeue!(agenda.pqueue)
isempty(agenda::Agenda) = isempty(agenda.pqueue)

priority(agenda::Agenda{Int}, edge::Edge, just_completed::Bool) =
    4 * length(edge) - 2*!just_completed - 1
priority(agenda::Agenda{Int}, cons::Constituent, just_introduced_edge::Bool) =
    4 * length(cons) - 2*!just_introduced_edge

priority(agenda::Agenda{Float64}, item, just_used) =
    - prob(item).value # negative logprob, agenda favours high probs but is min orientated

##########################
### ParseForest class ###
##########################

mutable struct ParseForest{C,T,CR,TR,St,S}
    heads :: Vector{Constituent{C,T,CR,TR,S}}
    logbook :: ParserLogbook{C,T,CR,TR,St,S}
    terminals :: Vector{T}
end

function ParseForest(chart::Chart, logbook::ParserLogbook{C,T,CR,TR,St,S}, terminals, grammar) where {C,T,CR,TR,St,S}
    heads = Constituent{C,T,CR,TR,S}[]
    cell = chart.cells[1]
    for cat in startsymbols(grammar)
        if haskey(cell.consids, cat)
            for consid in cell.consids[cat]
                cons = get_cons(logbook, consid)
                if tail(cons) == length(terminals)+1
                    push!(heads, cons)
                end
            end
        end
    end
    ParseForest(heads, logbook, terminals)
end

heads(f::ParseForest) = f.heads
is_complete(f::ParseForest) = !isempty(f.heads)
score(f::ParseForest) = sum(map(score, f.heads))

### OutsideAgenda ###

mutable struct OutsideAgenda
    pqueue :: PriorityQueue{Int, Int}
end

function OutsideAgenda()
    OutsideAgenda(PriorityQueue{Int, Int}())
end

function enqueue!(agenda::OutsideAgenda, item, just_used)
    agenda.pqueue[id(item)] = priority(agenda, item, just_used)
    agenda
end

dequeue!(agenda::OutsideAgenda) = dequeue!(agenda.pqueue)
isempty(agenda::OutsideAgenda) = isempty(agenda.pqueue)

priority(agenda::OutsideAgenda, edge::Edge, just_used::Bool) =
    - 4 * length(edge) - 2*!just_used -1
priority(agenda::OutsideAgenda, cons::Constituent, just_used::Bool) =
    - 4 * length(cons) - 2*!just_used

######################

inside = prob

@inline function acc_or_init!(d::Dict, k, v)
    if haskey(d, k)
        d[k] += v
    else
        d[k] = v
    end
end

function calculate_outside_probs_and_expected_rule_usages!(f::ParseForest{C,T,CR,TR,St,S}, grammar) where {C,T,CR,TR,St,S}
    @assert is_complete(f)
    outside_agenda = OutsideAgenda()
    # mapping categories to rules to (startindex,endindex) to probabilities
    rule_usage_probs = Dict{C, Dict{typejoin(CR,TR), Dict{Tuple{Int,Int}, LogProb}}}()
    for head in heads(f)
        # traversal outside score summands
        head.trav_outs_summands[0,0] = one(LogProb)
        enqueue!(outside_agenda, head, false)
    end
    while !isempty(outside_agenda)
        id = dequeue!(outside_agenda)
        if id > 0
            parent_edge = get_edge(f.logbook, id)
            parent_edge.outsidepopnumber += 1
            # @show parent_edge.id
            # println()
            o = outside(parent_edge)
            if !(outside(parent_edge) ≈ parent_edge.lastoutsidepopscore) &&
                outsidepopnumber(parent_edge) < MAX_POP_NUMBER
                for trav in traversals(parent_edge)
                    if trav.inloop
                        cons = get_cons(f.logbook, consid(trav))
                        if edgeid(trav) == 0
                            acc_or_init!(cons.trav_outs_summands, (id,0), o)
                            # cons.trav_outs_summands[id, 0] = o
                            enqueue!(outside_agenda, cons, false)
                        end
                    end
                end
                parent_edge.lastoutsidepopscore = o
                enqueue!(outside_agenda, parent_edge, true)
            elseif !(parent_edge.outside_finished)
                parent_edge.outside_finished = true
                for trav in traversals(parent_edge)
                    if true # !(trav.inloop)
                        # println("edge")
                        cons = get_cons(f.logbook, consid(trav))
                        if edgeid(trav) == 0
                            # acc_or_init!(cons.trav_outs_summands, (id,0), o)
                            cons.trav_outs_summands[id, 0] = o
                            enqueue!(outside_agenda, cons, false)
                        else
                            edge = get_edge(f.logbook, edgeid(trav))
                            edge.trav_outs_summands[id,cons.id] = o * inside(cons)
                            # acc_or_init!(edge.trav_outs_summands, (id, cons.id), o * inside(cons))
                            enqueue!(outside_agenda, edge, false)
                            cons.trav_outs_summands[id,edge.id] = o * inside(edge)
                            # acc_or_init!(cons.trav_outs_summands, (id, edge.id), o * inside(edge))
                            enqueue!(outside_agenda, cons, false)
                        end
                    end
                end
            end
        else
            parent_cons = get_cons(f.logbook, id)
            parent_cons.outsidepopnumber += 1
            # @show parent_cons
            # println(map(c->c.edgeid, parent_cons.completions))
            # println(exp(score(parent_cons)))
            # println(parent_cons.trav_outs_summands)
            # println(outside(parent_cons) ≈ parent_cons.lastoutsidepopscore)
            # println()
            o = outside(parent_cons)
            if !(outside(parent_cons) ≈ parent_cons.lastoutsidepopscore) &&
                outsidepopnumber(parent_cons) < MAX_POP_NUMBER
                for comp in completions(parent_cons)
                    if comp.inloop
                        edge = get_edge(f.logbook, edgeid(comp))
                        p = prob(grammar, cat(parent_cons), rule(comp))
                        acc_or_init!(edge.comp_outs_summands, id, o * p)
                        # edge.comp_outs_summands[id] = o * p
                        enqueue!(outside_agenda, edge, false)
                    end
                end
                parent_cons.lastoutsidepopscore = o
                enqueue!(outside_agenda, parent_cons, true)
            else#if !(parent_cons.outside_finished)
                parent_cons.outside_finished = true
                for comp in completions(parent_cons)
                    if true # !(comp.inloop)
                        # println("cons")
                        edge = get_edge(f.logbook, edgeid(comp))
                        p = prob(grammar, cat(parent_cons), rule(comp))
                        # acc_or_init!(edge.comp_outs_summands, id, o * p)
                        edge.comp_outs_summands[id] = o * p
                        enqueue!(outside_agenda, edge, false)
                        if haskey(rule_usage_probs, cat(parent_cons))
                            if haskey(rule_usage_probs[cat(parent_cons)], rule(comp))
                                # if haskey(rule_usage_probs[cat(parent_cons)][rule(comp)], (start(parent_cons),tail(parent_cons)))
                                #     rule_usage_probs[cat(parent_cons)][rule(comp)][(start(parent_cons),tail(parent_cons))] += o * p * inside(edge)
                                # else
                                #     rule_usage_probs[cat(parent_cons)][rule(comp)][(start(parent_cons),tail(parent_cons))] = o * p * inside(edge)
                                # end
                                rule_usage_probs[cat(parent_cons)][rule(comp)][(start(parent_cons),tail(parent_cons))] = o * p * inside(edge)
                            else
                                rule_usage_probs[cat(parent_cons)][rule(comp)] = Dict((start(parent_cons),tail(parent_cons)) => o * p * inside(edge))
                            end
                        else
                            rule_usage_probs[cat(parent_cons)] = Dict(rule(comp)=>Dict((start(parent_cons),tail(parent_cons)) => o * p * inside(edge)))
                        end
                    end
                end
                if has_terminal(parent_cons)
                    comp = terminal_completion(parent_cons)
                    p = prob(grammar, cat(parent_cons), rule(comp))
                    if haskey(rule_usage_probs, cat(parent_cons))
                        if haskey(rule_usage_probs[cat(parent_cons)], rule(comp))
                            # if haskey(rule_usage_probs[cat(parent_cons)][rule(comp)], (start(parent_cons),tail(parent_cons)))
                            #     rule_usage_probs[cat(parent_cons)][rule(comp)][(start(parent_cons),tail(parent_cons))] += o * p
                            # else
                            #     rule_usage_probs[cat(parent_cons)][rule(comp)][(start(parent_cons),tail(parent_cons))] = o * p
                            # end
                            rule_usage_probs[cat(parent_cons)][rule(comp)][(start(parent_cons),tail(parent_cons))] = o * p
                        else
                            rule_usage_probs[cat(parent_cons)][rule(comp)] = Dict((start(parent_cons),tail(parent_cons)) => o * p)
                        end
                    else
                        rule_usage_probs[cat(parent_cons)] = Dict(rule(comp)=>Dict((start(parent_cons),tail(parent_cons)) => o * p))
                    end
                end
            end
        end
    end

    sequence_prop = sum(inside(head) for head in f.heads)

    expected_rule_usages = Dict(
        cat => Dict(
            rule => exp(sum(values(rule_usage_probs[cat][rule])).value - sequence_prop.value)
            for rule in keys(rule_usage_probs[cat]))
        for cat in keys(rule_usage_probs)
    )
end

function calculate_outside_probs_and_expected_rule_usages!(sent :: Vector{T}, grammar :: Grammar{C,T,CR,TR,S,Cond}) where {C,T,CR,TR,S,Cond}
    forest = run_chartparser(sent, grammar)
    d = calculate_outside_probs_and_expected_rule_usages!(forest, grammar)
    (forest, d)
end

function best_tree(f::ParseForest)
    @assert is_complete(f)
    best_tree(f, f.heads[findmax(map(prob, f.heads))[2]])
end

function best_tree(f::ParseForest{C,T,CR,TR,St,S}, cons::Constituent) where {C,T,CR,TR,St,S}
    if has_terminal(cons)
        TreeNode((cons,rule(terminal_completion(cons))),
                 Tuple{Constituent{C,T,CR,TR,S}, Union{CR,TR}})
    else
        comp = completions(cons)[findmax(map(prob, completions(cons)))[2]]
        node = TreeNode((cons, rule(comp)),
                        Tuple{Constituent{C,T,CR,TR,S}, Union{CR,TR}})
        trees = best_trees(f, get_edge(f.logbook, edgeid(comp))) # children trees
        for t in trees
            insert_child!(node, t)
        end
        node
    end
end

function best_trees(f::ParseForest, edge::Edge)
    best_traversal = traversals(edge)[findmax(map(prob, traversals(edge)))[2]]
    if edgeid(best_traversal) == 0
        (best_tree(f, get_cons(f.logbook, consid(best_traversal))),)
    else
        (best_trees(f, get_edge(f.logbook, edgeid(best_traversal)))...,
         best_tree(f,  get_cons(f.logbook, consid(best_traversal))))
    end
end

function sample_tree(f::ParseForest)
    @assert is_complete(f)
    head = categorical_sample(heads(f), map(prob, heads(f)))
    sample_tree(f, head)
end

function sample_tree(f::ParseForest{C,T,CR,TR,St,S}, cons::Constituent) where{C,T,CR,TR,St,S}
    if has_terminal(cons)
        TreeNode((cons,rule(terminal_completion(cons))),
                 Tuple{Constituent{C,T,CR,TR,S}, Union{CR,TR}})
    else
        comp = categorical_sample(completions(cons),
                                           map(prob, completions(cons)))
        node = TreeNode((cons, rule(comp)),
                        Tuple{Constituent{C,T,CR,TR,S}, Union{CR,TR}})
        trees = sample_trees(f, get_edge(f.logbook, edgeid(comp))) # children trees
        for t in trees
            insert_child!(node, t)
        end
        node
    end
end

function sample_trees(f::ParseForest, edge::Edge)
    traversal = categorical_sample(traversals(edge),
                                            map(prob, traversals(edge)))
    if edgeid(traversal) == 0 # then this edge contains the start state
        (sample_tree(f, get_cons(f.logbook, consid(traversal))),)
    else
        (sample_trees(f, get_edge(f.logbook, edgeid(traversal)))...,
         sample_tree(f,  get_cons(f.logbook, consid(traversal))))
    end
end

# # all_trees no yet implemented
# function all_trees{C,T,CR,TR,St,S}(f::ParseForest{C,T,CR,TR,St,S})
#     if is_complete(f)
#         vcat([all_trees(f, cons) for cons in heads(f)]...)
#     else
#         Vector{TreeNode{Tuple{Constituent{C,T,CR,TR,S}, Union{CR,TR}}}}
#     end
# end
#
# function all_trees{C,T,CR,TR,St,S}(
#     f    :: ParseForest{C,T,CR,TR,St,S},
#     cons :: Constituent
#     )
#     if has_terminal(cons)
#         [TreeNode((cons,rule(terminal_completion(cons))),
#                   Tuple{Constituent{C,T,CR,TR,S}, Union{CR,TR}})]
#     else
#         for comp in completions(cons)
#             node = TreeNode((cons, rule(comp)),
#                             Tuple{Constituent{C,T,CR,TR,S}, Union{CR,TR}})
#         end
#     end
# end
#
# function all_trees(f::ParseForest, edge::Edge)
#
# end

#######################
### ParseTree class ###
#######################

ParseTree{Cons<:Constituent,R} = TreeNode{Tuple{Cons,R}}

function show(io::IO, tree::ParseTree)
    print(io, "[", tree.data[1])
    for child in tree.children
        print(io, child)
    end
    print(io, "]")
end

cats_and_rules(t::ParseTree) = ((node.data[1].cat, node.data[2]) for node in t)


# Parameters for Liquid Density
DataLiquidDensity = Dict("1,1,1-Trichloroethane"=>[[0.47476,0.27258,0.29333],[545.00,242.75,545.00]],
            "1,1,2-Trichloroethane"=>[[0.47455,0.25475,0.31000],[602.00,236.50,602.00]],
            "1,1-Dichloroethane"=>[[0.41231,0.26533,0.28700],[523.00,176.19,523.00]],
            "1,2-Dichloroethane"=>[[0.46501,0.28742,0.31041],[561.00,237.49,561.00]],
            "1,3-Butadiene"=>[[0.24597,0.27227,0.29074],[425.37,164.25,425.37]],
            "1,4-Dioxane"=>[[0.37018,0.28130,0.30470],[587.00,284.95,587.00]],
            "1-Butanol"=>[[0.26891,0.26674,0.24570],[562.93,183.85,562.93]],
            "1-Butene"=>[[0.23224,0.26630,0.28530],[419.59,87.80,419.59]],
            "1-Decene"=>[[0.23981,0.25776,0.28562],[617.05,206.89,617.05]],
            "1-Nonane"=>[[0.23364,0.25556,0.28571],[595.65,219.63,595.65]],
            "1-Octene"=>[[0.23682,0.25649,0.28571],[566.60,171.45,566.60]],
            "1-Propanol"=>[[0.27684,0.27200,0.24940],[536.71,146.95,536.71]],
            "Acetaldehyde"=>[[0.28207,0.26004,0.27760],[461.00,150.15,461.00]],
            "Acetic acid"=>[[0.35182,0.26954,0.26843],[592.71,289.81,592.71]],
            "Acetic anhydride"=>[[0.33578,0.24080,0.26990],[569.15,200.15,569.15]],
            "Acetone"=>[[0.27728,0.25760,0.29903],[508.20,178.45,508.20]],
            "Acrylic acid"=>[[0.34645,0.25822,0.30701],[615.00,286.65,615.00]],
            "Ammonia"=>[[0.23689,0.25471,0.28870],[405.65,195.41,405.65]],
            "Aniline"=>[[0.31190,0.25000,0.28571],[699.00,267.13,699.00]],
            "Benzene"=>[[0.30090,0.26770,0.28180],[562.16,278.68,562.16]],
            "Butyric acid"=>[[0.31132,0.26192,0.27997],[628.00,267.95,628.00]],
            "Carbon disulfide"=>[[0.47589,0.28749,0.32260],[552.00,161.58,552.00]],
            "Carbon dioxide"=>[[0.46382,0.26160,0.29030],[304.19,216.58,304.19]],
            "Carbon monoxide"=>[[0.29818,0.27655,0.29053],[132.92,68.15,132.92]],
            "Carbon tetrachloride"=>[[0.56607,0.27663,0.29000],[556.35,250.33,556.35]],
            "Chlorine"=>[[0.56600,0.27315,0.28830],[417.15,172.12,417.15]],
            "Chloroform"=>[[0.49807,0.25274,0.28766],[536.40,209.63,536.40,1.480]],
            "Cyclohexane"=>[[0.27376,0.27408,0.28511],[553.54,279.69,553.54]],
            "Cyclohexanol"=>[[0.29681,0.24340,0.28570],[625.15,296.60,625.15]],
            "Cyclopropane"=>[[0.25880,0.27400,0.28571],[397.91,145.73,397.91]],
            "Dichloromethane"=>[[0.45965,0.25678,0.29020],[510.00,178.01,510.00]],
            "Diethyl ether"=>[[0.27267,0.27608,0.29358],[466.70,156.85,466.70]],
            "Diethyl ketone"=>[[0.25635,0.24291,0.27364],[560.95,234.18,560.95]],
            "Dimethylamine"=>[[0.24110,0.26785,0.24800],[437.65,180.96,437.65]],
            "Ethane"=>[[0.20087,0.27330,0.28330],[305.42,90.35,305.42]],
            "Ethanol"=>[[0.26570,0.26395,0.23670],[516.25,159.05,516.25]],
            "Ethyl acetate"=>[[0.30654,0.25856,0.27800],[523.30,189.60,523.30]],
            "Ethyl chloride"=>[[0.32259,0.27464,0.23140],[460.35,136.75,460.35]],
            "Ethylbenzene"=>[[0.28889,0.26438,0.29210],[617.17,178.20,617.17]],
            "Ethylene"=>[[0.21428,0.28061,0.28571],[282.36,104.01,282.36]],
            "Ethylene glycol"=>[[0.32503,0.25499,0.17200],[645.00,260.15,645.00]],
            "Ethylene oxide"=>[[0.31402,0.26089,0.28253],[469.15,161.45,469.15]],
            "Fluorine"=>[[0.57092,0.28518,0.29000],[144.31,53.48,144.31]],
            "Glycerol"=>[[0.34908,0.24902,0.15410],[723.00,291.33,723.00]],
            "Hydrogen"=>[[0.03125,0.34730,0.27560],[33.18,13.95,33.18]],
            "Hydrogen chloride"=>[[0.44134,0.26957,0.31870],[324.65,159.97,324.65]],
            "Hydrogen cyanide"=>[[0.19501,0.18589,0.2820],[456.65,259.91,465.65]],
            "Hydrogen peroxide"=>[[0.43776,0.24982,0.28770],[730.15,272.74,730.15]],
            "i-Butane"=>[[0.22281,0.27294,0.27301],[408.14,113.54,408.14]],
            "Methane"=>[[0.15998,0.28810,0.27700],[190.58,90.67,190.58]],
            "Methanol"=>[[0.27197,0.27192,0.23310],[512.58,175.47,512.58]],
            "Methyl bromide"=>[[0.60859,0.26292,0.28030],[467.00,179.55,467.00]],
            "Methyl chloride"=>[[0.35821,0.26109,0.28690],[416.25,175.45,416.25]],
            "Methyl isobutyl ketone"=>[[0.26654,0.25887,0.28571],[571.40,189.15,571.40]],
            "Methylamine"=>[[0.20168,0.21405,0.22750],[430.05,179.69,430.05]],
            "m-Xylene"=>[[0.27866,0.25925,0.27243],[617.05,225.30,617.05]],
            "Naphthalene"=>[[0.30619,0.25037,0.28300],[748.35,353.43,748.35]],
            "n-Butane"=>[[0.22827,0.27240,0.28630],[425.18,134.86,425.18]],
            "n-Decane"=>[[0.23276,0.25240,0.28570],[618.45,243.49,618.45]],
            "n-Heptane"=>[[0.23237,0.26020,0.27910],[540.26,182.57,540.26]],
            "n-Hexane"=>[[0.23242,0.26500,0.27810],[507.43,177.84,507.43]],
            "Nitrobenzene"=>[[0.36140,0.24731,0.28570],[719.00,278.91,719.00]],
            "Nitrogen"=>[[0.31205,0.28479,0.29250],[126.10,63.15,126.10]],
            "n-Nonane"=>[[0.23364,0.25556,0.28571],[595.65,219.63,595.65]],
            "n-Octane"=>[[0.22807,0.25476,0.26940],[568.83,216.38,568.83]],
            "n-Pentane"=>[[0.23143,0.26923,0.28215],[469.65,143.42,469.65]],
            "Oxygen"=>[[0.43533,0.28772,0.29240],[154.58,54.35,154.58]],
            "o-Xylene"=>[[0.28381,0.26083,0.27410,630.37,247.98,630.37]],
            "Phenol"=>[[0.41476,0.32162,0.32120],[694.25,314.06,694.25]],
            "p-Xylene"=>[[0.27984,0.26003,0.27900],[616.26,286.41,616.26]],
            "Pyridine"=>[[0.30752,0.24333,0.30450],[619.95,231.53,619.95]],
            "Styrene"=>[[0.29383,0.26315,0.28570],[648.00,242.54,648.00]],
            "Toluene"=>[[0.29999,0.27108,0.29889],[591.79,178.18,591.79]],
            "Vinyl acetate"=>[[0.31843,0.25803,0.28270],[524.00,180.35,524.00]],
            "Water"=>[[0.34710,0.27400,0.28571],[647.13,273.16,647.13]])

# Parameters for Gas Viscosity
DataGasViscosity=Dict("1,1,1-Trichloroethane"=>[-19.216 4.0308E-01 -6.8618E-05 347.0 997.0],
            "1,1,2-Trichloroethane"=>[-8.293 3.3989E-01 -5.3678E-05 387.0 987.0],
            "1,1-Dichloroethane"=>[-12.991 4.0085E-01 -1.1779E-04 320.0 472.0],
            "1,2-Dichloroethane"=>[1.025 3.1792E-01 -4.1853E-05 322.0 561.0],
            "1,3-Butadiene"=>[10.256 2.6833E-01 -4.1148E-05 250.0 650.0],
            "1,4-Dioxane"=>[-16701 3.4988E-01 -5.3736E-05 374.0 994.0],
            "1-Butanol"=>[-11.144 2.8790E-01 -5.6275E-05 391.0 1000.0],
            "1-Butene"=>[-9.143 3.1562E-01 -8.4164E-05 175.0 800.0],
            "1-Decene"=>[4.015 1.8083E-01 -3.8216E-05 444.0 1000.0],
            "1-Nonane"=>[-6.802 1.8688E-01 3.4929E-07 273.0 773.0],
            "1-Octene"=>[2.722 2.0327E-01 -4.3879E-05 394.0 994.0],
            "1-Propanol"=>[-14.894 3.2171E-01 -5.8021E-05 200.0 1000.0],
            "Acetaldehyde"=>[0.069 3.0246E-01 -4.2372E-05 294.0 1000.0],
            "Acetic acid"=>[-28.660 2.3510E-01 2.2087E-04 366.0 523.0],
            "Acetic anhydride"=>[-1.485 2.8869E-01 -2.3391E-05 295.0 993.0],
            "Acetone"=>[-4.055 2.6655E-01 -5.6936E-06 300.0 650.0],
            "Acrylic acid"=>[-6.532 3.0600E-01 -4.6620E-05 287.0 1000.0],
            "Ammonia"=>[-7.874 3.6700E-01 -4.4700E-06 195.0 1000.0],
            "Aniline"=>[-6.918 2.5935E-01 -3.4348E-05 458.0 1000.0],
            "Benzene"=>[-0.151 2.5706E-01 -8.9797E-06 287.0 628.0],
            "Butyric acid"=>[-5.781 2.6159E-01 -3.4903E-05 268.0 1000.0],
            "Carbon disulfide"=>[-7.700 3.6594E-01 -2.5416E-05 273.0 583.0],
            "Carbon dioxide"=>[11.811 4.9838E-01 -1.0851E-04 195.0 1500.0],
            "Carbon monoxide"=>[23.811 5.3944E-01 -1.5411E-04 68.0 1250.0],
            "Carbon tetrachloride"=>[-7.745 3.9481E-01 -1.1150E-04 280.0 800.0],
            "Chloroform"=>[-4.392 3.7309E-01 -5.1696E-05 250.0 700.0],
            "Chlorine"=>[-3.571 4.8700E-01 -8.5300E-05 200.0 1000.0],
            "Cyclohexane"=>[1.190 2.4542E-01 -3.8334E-05 315.0 600.0],
            "Cyclohexanol"=>[-13.542 2.9086E-01 -3.9472E-05 400.0 1000.0],
            "Cyclopropane"=>[-9.521 3.7037E-01 -1.3138E-04 240.0 446.0],
            "Dichloromethane"=>[-20.372 4.3745E-01 -7.7549E-05 273.0 993.0],
            "Diethyl ether"=>[-7.932 3.0235E-01 -7.3858E-05 200.0 1000.0],
            "Diethyl ketone"=>[-3.593 2.4308E-01 -3.7613E-05 234.0 1000.0],
            "Dimethylamine"=>[-9.275 2.8958E-01 -1.3875E-06 250.0 450.0],
            "Ethane"=>[0.514 3.3449E-01 -7.1071E-05 150.0 1000.0],
            "Ethanol"=>[1.499 3.0741E-01 -4.4479E-05 200.0 1000.0],
            "Ethyl acetate"=>[-9.259 3.0725E-01 -7.1069E-05 190.0 1000.0],
            "Ethyl chloride"=>[0.458 3.2827E-01 -1.2467E-05 213.0 523.0],
            "Ethylbenzene"=>[-4.267 2.4735E-01 -5.4264E-05 409.0 1000.0],
            "Ethylene"=>[-3.985 3.8726E-01 -1.1227E-04 150.0 1000.0],
            "Ethylene glycol"=>[-7.178 3.1246E-01 -4.4028E-05 260.0 1000.0],
            "Ethylene oxide"=>[-12.180 3.7672E-01 -7.7599E-05 250.0 1000.0],
            "Fluorine"=>[-0.811 8.9800E-01 -3.9600E-04 90.0 500.0],
            "Glycerol"=>[-23.119 2.8879E-01 -3.4277E-05 563.0 993.0],
            "Hydrogen"=>[27.758 2.1200E-01 -3.2800E-05 150.0 1500.0],
            "Hydrogen chloride"=>[-9.118 5.5500E-01 -1.1100E-04 200.0 1000.0],
            "Hydrogen cyanide"=>[-8.486 9.0368E-02 7.9146E-05 300.0 425.0],
            "Hydrogen peroxide"=>[8.039 2.7000E-01 8.2900E-05 373.0 600.0],
            "i-Butane"=>[-4.731 2.9131E-01 -8.0995E-05 150.0 1000.0],
            "Methane"=>[3.844 4.0112E-01 -1.4303E-04 91.0 850.0],
            "Methanol"=>[-14.236 3.8935E-01 -6.2762E-05 240.0 1000.0],
            "Methyl bromide"=>[-27.740 5.5901E-01 -7.0942E-05 260.0 440.0],
            "Methyl chloride"=>[-1.374 3.8627E-01 -4.8650E-05 230.0 700.0],
            "Methyl isobutyl ketone"=>[-3.237 2.3310E-01 -3.5612E-05 189 1000],
            "Methylamine"=>[-5.334 3.4181E-01 -7.4297E-05 267.0 1000.0],
            "m-Xylene"=>[-21.620 2.7820E-01 -6.0531E-05 250.0 1000.0],
            "Naphthalene"=>[-16.789 2.5406E-01 -3.5495E-05 353.0 1000.0],
            "n-Butane"=>[-4.946 2.9001E-01 -6.9665E-05 150.0 1200.0],
            "n-Decane"=>[-7.297 1.8506E-01 -4.8008E-06 287.0 783.0],
            "n-Heptane"=>[-10.378 2.4401E-01 -5.4003E-05 338.0 700.0],
            "n-Hexane"=>[-8.222 2.6229E-01 -5.7366E-05 300.0 1000.0],
            "Nitrobenzene"=>[-16.569 2.9184E-01 -2.5523E-05 450.0 1000.0],
            "Nitrogen"=>[42.606 4.7500E-01 -9.8800E-05 150.0 1500.0],
            "n-Nonane"=>[-6.802 1.8688E-01 3.4929E-07 273.0 773.0],
            "n-Octane"=>[3.940 1.6640E-01 1.4470E-05 374.0 670.0],
            "n-Pentane"=>[-3.202 2.6746E-01 -6.6178E-05 303.0 900.0],
            "Oxygen"=>[44.224 5.6200E-01 -1.1300E-04 150.0 1500.0],
            "o-Xylene"=>[-19.763 2.8022E-01 -5.9293E-05 250.0 1000.0],
            "Phenol"=>[-7.185 2.7179E-01 -3.6205E-05 455.0 1000.0],
            "Propane"=>[-5.462 3.2722E-01 -1.0672E-04 193.0 750.0],
            "p-Xylene"=>[-17.226 2.5098E-01 -2.8232E-05 286.0 1000.0],
            "Pyridine"=>[-5.739 2.7135E-01 -1.7202E-05 369.0 998.0],
            "Styrene"=>[-10.035 2.5191E-01 -3.7932E-05 243.0 1000.0],
            "Sulfur dioxide"=>[-11.103 5.0200E-01 -1.0800E-04 200.0 1000.0],
            "Toluene"=>[1.787 2.3566E-01 -9.3508E-06 275.0 600.0],
            "Vinyl acetate"=>[-7.462 3.0466E-01 -5.7544E-05 346.0 1000.0],
            "Water"=>[-36.826 4.2900E-01 -1.6200E-05 280.0 1073.0])

# Parameters for Liquid Viscosity
DataLiquidViscosity=Dict("1,1,1-Trichloroethane"=>[-3.9096 7.0709E+02 7.5847E-03 -9.1662E-06 243 545],
            "1,1,2-Trichloroethane"=>[-3.2716 6.8810E+02 4.8932E-03 -5.4671E-06 237 602],
            "1,1-Dichloroethane"=>[-3.8388 5.9046E+02 8.0953E-03 -9.9210E-06 176 523],
            "1,2-Dichloroethane"=>[-0.1656 2.7576E+02 -3.3493E-03 1.4093E-06 245 561],
            "1,3-Butadiene"=>[0.3772 7.9658E+01 -5.8889E-03 2.9221E-06 250 425],
            "1,4-Dioxane"=>[-7.5724 1.3813E+03 1.3556E-02 -1.1464E-05 288 587],
            "1-Butanol (n-Butanol)"=>[-5.3970 1.3256E+03 6.2223E-03 -5.5062E-06 250 563],
            "1-Butene"=>[-4.9218 4.9503E+02 1.4390E-02 -2.0853E-05 160 420],
            "1-Decene"=>[-6.8845 1.1003E+03 1.4341E-02 -1.3520E-05 273 617],
            "1-Nonane (n-Nonane)"=>[-6.0742 9.6861E+02 1.2677E-02 -1.2675E-05 220 596],
            "1-Octene"=>[-5.6209 8.1305E+02 1.2523E-02 -1.3384E-05 250 567],
            "1-Propanol (n-Propanol)"=>[-3.7702 9.9151E+02 4.0836E-03 -5.4586E-06 220 537],
            "Acetaldehyde"=>[-6.6171 6.8123E+02 1.9979E-02 -2.5563E-05 260 461],
            "Acetic acid"=>[-3.8937 7.8482E+02 6.6650E-03 -7.5606E-06 290 593],
            "Acetic anhydride"=>[-17.3580 2.3611E+03 4.2734E-02 -3.8202E-05 265 569],
            "Acetone"=>[-7.2126 9.0305E+02 1.8385E-02 -2.0353E-05 223 508],
            "Acrylic acid"=>[5.9215 2.4408E+03 3.4383E-02 -2.7677E-05 293 615],
            "Ammonia"=>[-8.5910 8.7640E+02 2.6810E-02 -3.6120E-05 195 406],
            "Aniline"=>[-13.8625 2.5109E+03 2.5681E-02 -1.8281E-05 268 699],
            "Benzene"=>[-7.4005 1.1815E+03 1.4888E-02 -1.3713E-05 285 562],
            "Butyric acid"=>[-7.9846 1.3636E+03 1.6315E-02 -1.4511E-05 268 628],
            "carbon disulfide"=>[-9.1108 1.1216E+03 2.3216E-02 -2.2648E-05 235 552],
            "Carbon dioxide"=>[-19.4921 1.5948E+03 7.9274E-02 -1.2025E-04 219 304],
            "Carbon monoxide"=>[-1.1224 5.7858E+01 -4.9174E-03 8.2233E-06 69 133],
            "Carbon tetrachloride"=>[-6.4564 1.0379E+03 1.4021E-02 -1.4107E-05 265 556],
            "Chlorine"=>[-0.7681 1.5140E+02 -8.0650E-04 4.0750E-07 172 417],
            "Chloroform"=>[-47831 6.9902E+02 1.0929E-02 -1.2244E-05 210 536],
            "Cyclohexane"=>[.7423 -2.5322E+02 -1.6927E-02 1.2472E-05 285 554],
            "Cyclohexanol"=>[-5.3792 1.8793E+03 1.7011E-03 1.0187E-07 303 625],
            "Cyclopropane"=>[-3.2541 3.2192E+02 9.9766E-03 -1.8191E-05 146 398],
            "Dichloromethane"=>[5.1043 6.8653E+02 1.2459E-02 -1.4540E-05 208 510],
            "Diethyl ether"=>[-8.5060 1.0020E+03 2.2753E-02 -2.5780E-05 233 467],
            "Diethyl ketone"=>[-9.2905 1.2716E+03 2.1925E-02 -2.1036E-05 274 561],
            "Dimethylamine"=>[-11.5558 1.2126E+03 3.4999E-02 -4.1253E-05 240 438],
            "Ethane"=>[-4.2694 2.8954E+02 1.7111E-02 -3.6092E-05 98 305],
            "Ethanol"=>[-6.4406 1.1176E+03 1.3721E-02 -1.5465E-05 240 516],
            "Ethyl acetate"=>[-3.6861 5.5228E+02 8.0018E-03 -1.0439E-05 220 523],
            "Ethyl chloride"=>[-4.4279 5.1891E+02 1.2035E-02 -1.6620E-05 150 460],
            "Ethylbenzene"=>[-5.2585 8.3065E+02 1.0784E-02 -1.0618E-05 210 617],
            "Ethylene"=>[-4.5611 3.0811E+02 1.8030E-02 -3.8145E-05 105 282],
            "Ethylene glycol"=>[-16.9728 3.1886E+03 3.2537E-02 -2.4480E-05 261 645],
            "Ethylene oxide"=>[-5.7794 6.7020E+02 1.5686E-02 -1.9462E-05 190 469],
            "Fluorine"=>[-1.5760 8.5630E+01 -4.0730E-04 -2.7250E-06 54 145],
            "Glycerol"=>[-18.2152 4.2305E+03 2.8705E-02 -1.8648E-05 293 723],
            "Hydrogen"=>[-7.0154 4.0791E+01 2.3714E-01 -4.0830E-03 14 33],
            "Hydrogen chloride"=>[-1.5150 1.9460E+02 3.0670E-03 -1.3760E-05 159 325],
            "Hydrogen cyanide"=>[-12.0812 1.3183E+03 3.5234E-02 -4.0185E-05 260 457],
            "Hydrogen peroxide"=>[-1.6150 5.0380E+02 3.5010E-04 -1.1680E-06 273 728],
            "i-Butane (iso-Butane)"=>[-13.4207 1.3131E+03 4.4329E-02 -5.5793E-05 190 408],
            "Methane"=>[-7.3801 3.1925E+02 4.7934E-02 -1.4120E-04 91 191 - 0.020],
            "Methyl bromide"=>[-9.5533 1.0306E+03 2.8322E-02 -3.1920E-05 193 467],
            "Methyl chloride"=>[-7.3473 8.5395E+02 1.9485E-02 -2.3484E-05 249 416],
            "Methyl isobutyl ketone"=>[-3.0570 5.0050E+02 6.5038E-03 -8.8243E-06 246 571],
            "Methylamine"=>[-9.4670 9.8286E+02 2.8918E-02 -3.5672E-05 180 430],
            "m-Xylene"=>[-6.0517 9.2460E+02 1.2583E-02 -1.1850E-05 225 617],
            "Naphthalene"=>[-10.3716 1.8572E+03 1.9320E-02 -1.4012E-05 353 748],
            "n-Butane"=>[-6.8590 6.7393E+02 2.1973E-02 -3.0686E-05 180 425],
            "n-Decane"=>[-6.0716 1.0177E+03 1.2247E-02 -1.1892E-05 243 618],
            "n-Heptane"=>[-5.7782 8.0587E+02 1.3355E-02 -1.4794E-05 183 540],
            "n-Hexane"=>[-5.0715 6.5536E+02 1.2349E-02 -1.5042E-05 178 507],
            "Nitrobenzene"=>[-7.7710 1.4019E+03	1.4653E-02 -1.1512E-05 273 719],
            "Nitrogen"=>[-15.6104 4.6505E+02 1.6259E-01 -6.3353E-04 63 125],
            "n-Nonane"=>[-6.0742 9.6861E+02 1.2677E-02 -1.2675E-05 220 596],
            "n-Octane"=>[-5.9245 8.8809E+02 1.2955E-02 -1.3596E-05 216 569],
            "n-Pentane"=>[-7.1711 7.4736E+02 2.1697E-02 -2.7176E-05 143	470],
            "Oxygen"=>[-5.0957 1.7983E+02 3.9779E-02 -1.4664E-04 54 150],
            "o-Xylene"=>[-7.8805 1.2500E+03 1.6116E-02 -1.3993E-05 268 630],
            "Phenol"=>[.5349 4.2620E+02 -9.1577E-03 6.2322E-06 318 694],
            "p-Xylene"=>[-9.4655 1.4400E+03 1.9910E-02 -1.6994E-05 288 616],
            "Pyridine"=>[-6.8100 1.1496E+03 1.3229E-02 -1.1661E-05 232 620],
            "Styrene"=>[-8.0291 1.2666E+03 1.6127E-02 -1.3475E-05 243 648],
            "Sulfur dioxide"=>[-2.6700 4.0670E+02 6.1440E-03 -1.2540E-05 200 431],
            "Toluene"=>[-5.1649 8.1068E+02 1.0454E-02 -1.0488E-05 200 592],
            "Vinyl acetate"=>[-9.0671 1.1863E+03 2.2663E-02	-2.3208E-05	250 524],
            "Water"=>[-10.2158 1.7925E+03 1.7730E-02 -1.2631E-05 273 643])


#################################################################################
#Type Alias for long type signatures
const MinMax = NamedTuple{(:min, :max), NTuple{2, Float64}}
const NamedMinMax = Tuple{String, MinMax}
const UpDown = NamedTuple{(:up, :down), NTuple{2, Float64}}
const InOut = NamedTuple{(:in, :out), NTuple{2, Float64}}

# Type Alias From other Packages
const PM = PowerModels
const PSY = PowerSystems
const PSI = PowerSimulations
const IS = InfrastructureSystems
const MOI = MathOptInterface
const MOIU = MathOptInterface.Utilities
const PJ = ParameterJuMP
const MOPFM = MOI.FileFormats.Model
const TS = TimeSeries

const BUILD_SIMULATION_TIMER = TimerOutputs.TimerOutput()
const RUN_SIMULATION_TIMER = TimerOutputs.TimerOutput()

#Type Alias for JuMP and PJ containers
const JuMPExpressionMatrix = Matrix{<:JuMP.AbstractJuMPScalar}
const PGAE{V} =
    PJ.ParametrizedGenericAffExpr{Float64, V} where {V <: JuMP.AbstractVariableRef}
const GAE{V} = JuMP.GenericAffExpr{Float64, V} where {V <: JuMP.AbstractVariableRef}
const JuMPAffineExpressionArray = Matrix{GAE{V}} where {V <: JuMP.AbstractVariableRef}
const JuMPAffineExpressionVector = Vector{GAE{V}} where {V <: JuMP.AbstractVariableRef}
const JuMPConstraintArray = JuMP.Containers.DenseAxisArray{JuMP.ConstraintRef}
const JuMPParamArray = JuMP.Containers.DenseAxisArray{PJ.ParameterRef}
const DenseAxisArrayContainer = Dict{Symbol, JuMP.Containers.DenseAxisArray}

@enum BUILD_STATUS begin
    BUILT = 1
    IN_PROGRESS = -1
    EMPTY = 0
end

# Settings constants
const UNSET_HORIZON = 0
const UNSET_INI_TIME = Dates.DateTime(0)

# Tolerance of comparisons
# MIP gap tolerances in most solvers are set to 1e-4
const ABSOLUTE_TOLERANCE = 1.0e-3
const SLACK_COST = 1e6
const COST_EPSILON = 1e-3
const MISSING_INITIAL_CONDITIONS_TIME_COUNT = 999.0

const OPERATIONS_ACCEPTED_KWARGS = [
    :horizon,
    :initial_time,
    :use_forecast_data,
    :PTDF,
    :use_parameters,
    :optimizer,
    :warm_start,
    :slack_variables,
    :system_to_file,
    :constraint_duals,
]

const OPERATIONS_SOLVE_KWARGS = [:optimizer, :save_path]

const STAGE_ACCEPTED_KWARGS =
    [:PTDF, :warm_start, :slack_variables, :constraint_duals, :system_to_file]

const PSI_NAME_DELIMITER = "__"

# The constants below are strings instead of enums because there is a requirement that users
# should be able to define their own without changing PowerSimulations.

# Variables / Parameters
const ACTIVE_POWER = "P"
const ENERGY = "E"
const ENERGY_BUDGET = "energy_budget"
const FLOW_ACTIVE_POWER = "Fp"
const ON = "On"
const REACTIVE_POWER = "Q"
const ACTIVE_POWER_IN = "Pin"
const ACTIVE_POWER_OUT = "Pout"
const RESERVE = "R"
const SERVICE_REQUIREMENT = "service_requirement"
const START = "Start"
const STOP = "Stop"
const THETA = "theta"
const VM = "Vm"
const INFLOW = "In"
const SPILLAGE = "Sp"
const SLACK_UP = "γ⁺"
const SLACK_DN = "γ⁻"

# Constraints
const ACTIVE = "active"
const ACTIVE_RANGE = "activerange"
const ACTIVE_RANGE_LB = "activerange_lb"
const ACTIVE_RANGE_UB = "activerange_ub"
const COMMITMENT = "commitment"
const DURATION = "duration"
const DURATION_DOWN = "duration_dn"
const DURATION_UP = "duration_up"
const ENERGY_CAPACITY = "energy_capacity"
const ENERGY_LIMIT = "energy_limit"
const FEEDFORWARD = "FF"
const FEEDFORWARD_BIN = "FF_bin"
const FLOW_LIMIT = "FlowLimit"
const FLOW_LIMIT_FROM_TO = "FlowLimitFT"
const FLOW_LIMIT_TO_FROM = "FlowLimitTF"
const FLOW_REACTIVE_POWER_FROM_TO = "FqFT"
const FLOW_REACTIVE_POWER_TO_FROM = "FqTF"
const FLOW_ACTIVE_POWER_FROM_TO = "FpFT"
const FLOW_ACTIVE_POWER_TO_FROM = "FpTF"
const FLOW_ACTIVE_POWER = "Fp"
const FLOW_REACTIVE_POWER = "Fq"
const INPUT_POWER_RANGE = "inputpower_range"
const OUTPUT_POWER_RANGE = "outputpower_range"
const RAMP = "ramp"
const RAMP_DOWN = "ramp_dn"
const RAMP_UP = "ramp_up"
const RATE_LIMIT = "RateLimit"
const RATE_LIMIT_FT = "RateLimitFT"
const RATE_LIMIT_TF = "RateLimitTF"
const REACTIVE = "reactive"
const REACTIVE_RANGE = "reactiverange"
const REQUIREMENT = "requirement"
const INFLOW_RANGE = "inflowrange"


# MIT license
# Copyright (c) Microsoft Corporation. All rights reserved.
# See LICENSE in the project root for full license information.

# all other glasses should follow the format below, new glasses must be added to OTHER_GLASSES and OTHER_GLASS_NAMES where the index in the array matches the numeric part of the GlassID

module CARGILLE
using ..GlassCat: Glass, GlassID, OTHER

const OG0608 = Glass(GlassID(OTHER, 1), -2, 1.4451400, 0.0043176, -1.80659e-5, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.32, 1.55, -0.0009083144750540808, 0.0, 0.0, 0.0, 0.0, 0.0, 25.0, 0.008, -1.0, -1.0, 800.0, -1.0, 0, -1.0, [(0.32, 0.03, 10.0), (0.365, 0.16, 100.0), (0.4047, 0.40, 100.0), (0.480, 0.71, 100.0), (0.4861, 0.72, 100.0), (0.5461, 0.80, 100.0), (0.5893, 0.90, 100.0), (0.6328, 0.92, 100.0), (0.6439, 0.95, 100.0), (0.6563, 0.96, 100.0), (0.6943, 0.99, 100.0), (0.840, 0.99, 100.0), (0.10648, 0.74, 100.0), (0.1300, 0.39, 100.0), (0.1550, 0.16, 100.0)], 1.457518, -1.0, -1.0, 0, 57.18978, 0, 0.878, -1)
const OG0607 = Glass(GlassID(OTHER, 2), -2, 1.44503, 0.0044096, -2.85878e-5, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.32, 1.55, -0.0009083144750540808, 0.0, 0.0, 0.0, 0.0, 0.0, 25.0, 0.008, -1.0, -1.0, 700.0, -1.0, 0, -1.0, [(0.32, 0.15, 10.0), (0.365, 0.12, 100.0), (0.4047, 0.42, 100.0), (0.480, 0.78, 100.0), (0.4861, 0.79, 100.0), (0.5461, 0.86, 100.0), (0.5893, 0.90, 100.0), (0.6328, 0.92, 100.0), (0.6439, 0.90, 100.0), (0.6563, 0.92, 100.0), (0.6943, 0.98, 100.0), (0.840, 0.99, 100.0), (0.10648, 0.61, 100.0), (0.1300, 0.39, 100.0), (0.1550, 0.11, 100.0)], 1.457587, -1.0, -1.0, 0, 57.19833, 0, 0.878, -1)
const OG081160 = Glass(GlassID(OTHER, 3), -2, 1.49614, 0.00692199, -8.07052e-5, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.32, 1.55, -0.000885983052189022, 0.0, 0.0, 0.0, 0.0, 0.0, 25.0, 0.014, -1.0, -1.0, 700.0, -1.0, 0, -1.0, [(0.32, 0.04, 100.0), (0.365, 0.13, 100.0), (0.4047, 0.26, 100.0), (0.480, 0.48, 100.0), (0.4861, 0.49, 100.0), (0.5461, 0.60, 100.0), (0.5893, 0.68, 100.0), (0.6328, 0.71, 100.0), (0.6439, 0.73, 100.0), (0.6563, 0.74, 100.0), (0.6943, 0.76, 100.0), (0.840, 0.83, 100.0), (0.10648, 0.86, 100.0), (0.1300, 0.89, 100.0), (0.1550, 0.90, 100.0)], 1.515549, -1.0, -1.0, 0, 36.82493, 0, 1.11, -1)

end

const OTHER_GLASSES = [CARGILLE.OG0607, CARGILLE.OG0608, CARGILLE.OG081160]
const OTHER_GLASS_NAMES = ["CARGILLE.OG0607", "CARGILLE.OG0608", "CARGILLE.OG081160"]


function BatchLSEstimator(arg0::LeastSquaresOptimizer, arg1::Vector{IntegratedPropagatorBuilder})
    return BatchLSEstimator((LeastSquaresOptimizer, Vector{IntegratedPropagatorBuilder}), arg0, arg1)
end

function add_measurement(obj::BatchLSEstimator, arg0::ObservedMeasurement)
    return jcall(obj, "addMeasurement", void, (ObservedMeasurement,), arg0)
end

function equals(obj::Object, arg0::Object)
    return jcall(obj, "equals", jboolean, (Object,), arg0)
end

function estimate(obj::BatchLSEstimator)
    return jcall(obj, "estimate", Vector{AbstractIntegratedPropagator}, ())
end

function get_class(obj::Object)
    return jcall(obj, "getClass", Class, ())
end

function get_evaluations_count(obj::BatchLSEstimator)
    return jcall(obj, "getEvaluationsCount", jint, ())
end

function get_iterations_count(obj::BatchLSEstimator)
    return jcall(obj, "getIterationsCount", jint, ())
end

function get_last_estimations(obj::BatchLSEstimator)
    return jcall(obj, "getLastEstimations", Map, ())
end

function get_measurements_parameters_drivers(obj::BatchLSEstimator, arg0::jboolean)
    return jcall(obj, "getMeasurementsParametersDrivers", ParameterDriversList, (jboolean,), arg0)
end

function get_optimum(obj::BatchLSEstimator)
    return jcall(obj, "getOptimum", LeastSquaresOptimizer_Optimum, ())
end

function get_orbital_parameters_drivers(obj::BatchLSEstimator, arg0::jboolean)
    return jcall(obj, "getOrbitalParametersDrivers", ParameterDriversList, (jboolean,), arg0)
end

function get_physical_covariances(obj::BatchLSEstimator, arg0::jdouble)
    return jcall(obj, "getPhysicalCovariances", RealMatrix, (jdouble,), arg0)
end

function get_propagator_parameters_drivers(obj::BatchLSEstimator, arg0::jboolean)
    return jcall(obj, "getPropagatorParametersDrivers", ParameterDriversList, (jboolean,), arg0)
end

function hash_code(obj::Object)
    return jcall(obj, "hashCode", jint, ())
end

function notify(obj::Object)
    return jcall(obj, "notify", void, ())
end

function notify_all(obj::Object)
    return jcall(obj, "notifyAll", void, ())
end

function set_convergence_checker(obj::BatchLSEstimator, arg0::ConvergenceChecker)
    return jcall(obj, "setConvergenceChecker", void, (ConvergenceChecker,), arg0)
end

function set_max_evaluations(obj::BatchLSEstimator, arg0::jint)
    return jcall(obj, "setMaxEvaluations", void, (jint,), arg0)
end

function set_max_iterations(obj::BatchLSEstimator, arg0::jint)
    return jcall(obj, "setMaxIterations", void, (jint,), arg0)
end

function set_observer(obj::BatchLSEstimator, arg0::BatchLSObserver)
    return jcall(obj, "setObserver", void, (BatchLSObserver,), arg0)
end

function set_parameters_convergence_threshold(obj::BatchLSEstimator, arg0::jdouble)
    return jcall(obj, "setParametersConvergenceThreshold", void, (jdouble,), arg0)
end

function to_string(obj::Object)
    return jcall(obj, "toString", JString, ())
end

function wait(obj::Object)
    return jcall(obj, "wait", void, ())
end

function wait(obj::Object, arg0::jlong)
    return jcall(obj, "wait", void, (jlong,), arg0)
end

function wait(obj::Object, arg0::jlong, arg1::jint)
    return jcall(obj, "wait", void, (jlong, jint), arg0, arg1)
end