function searchsortedfirst_join(_f, v::AbstractVector, x, lo::T, hi::T, o::Ordering, ::Val{T2})::keytype(v) where T<:Integer where T2
    u = T(1)
    lo = lo - u
    hi = hi + u
    @inbounds while lo < hi - u
        m = midpoint(lo, hi)
        fvm = _f(v[m])::T2
        if lt(o, fvm , x)
            lo = m
        else
            hi = m
        end
    end
    return hi
end

# index of the last value of vector a that is less than or equal to x;
# returns 0 if x is less than all values of v.
function searchsortedlast_join(_f, v::AbstractVector, x, lo::T, hi::T, o::Ordering, ::Val{T2})::keytype(v) where T<:Integer where T2
    u = T(1)
    lo = lo - u
    hi = hi + u
    @inbounds while lo < hi - u
        m = midpoint(lo, hi)
        fvm = _f(v[m])::T2
        if lt(o, x, fvm)
            hi = m
        else
            lo = m
        end
    end
    return lo
end

# returns the range of indices of v equal to x
# if v does not contain x, returns a 0-length range
# indicating the insertion point of x
function searchsorted_join(_f, v::AbstractVector, x, ilo::T, ihi::T, o::Ordering, ::Val{T2})::UnitRange{keytype(v)} where T<:Integer where T2
    u = T(1)
    lo = ilo - u
    hi = ihi + u
    @inbounds while lo < hi - u
        m = midpoint(lo, hi)
        fvm = _f(v[m])::T2
        if lt(o, fvm, x)
            lo = m
        elseif lt(o, x, fvm)
            hi = m
        else
            a = searchsortedfirst_join(_f, v, x, max(lo,ilo), m, o, Val(T2))
            b = searchsortedlast_join(_f, v, x, m, min(hi,ihi), o, Val(T2))
            return a : b
        end
    end
    return (lo + 1) : (hi - 1)
end

function _fill_range_for_accelerated_join!(ranges, starts, loc, x, f, sz, chunk; threads = true)
    loc_cumsum = cumsum(loc)
    @_threadsfor threads for i in 1:length(x)
        _index_ = hash(f(x[i])) % chunk + 1
        if loc[_index_]
            actual_start = loc_cumsum[_index_]
            st = starts[actual_start]
            actual_start == length(starts) ? en = sz : en = starts[actual_start+1]-1
            ranges[i] = st:en
        else
            ranges[i] = 1:0
        end
    end
end
# TODO how the hashing behave for Categorical Arrays?
function _find_permute_and_fill_range_for_join!(ranges, dsr, dsl, oncols_right, oncols_left, stable, alg, mapformats, accelerate, chunk = 2^10; nsfpaj = true, threads = true)
    if isempty(dsr)
        idx = []
        fill!(ranges, 1:nrow(dsr))
        last_valid_range = -1
    else
        if accelerate
            if mapformats[2]
                _fr = getformat(dsr, oncols_right[1])
            else
                _fr = identity
            end
            grng = _divide_for_fast_join(_columns(dsr)[oncols_right[1]], _fr, chunk; threads = threads)
            if mapformats[1]
                _fl = getformat(dsl, oncols_left[1])
            else
                _fl = identity
            end
            _fill_range_for_accelerated_join!(ranges, grng.starts, grng.starts_loc, _columns(dsl)[oncols_left[1]], _fl, nrow(dsr), chunk; threads = threads)
            if dsr isa SubDataset
                starts, idx, last_valid_range =  _sortperm_v(dsr, oncols_right, stable = stable, a = alg, mapformats = mapformats[2], notsortpaforjoin = nsfpaj, givenrange = grng, threads = threads)

            else
                starts, idx, last_valid_range =  _sortperm(dsr, oncols_right, stable = stable, a = alg, mapformats = mapformats[2], notsortpaforjoin = nsfpaj, givenrange = grng, threads = threads)
            end
        else
            if dsr isa SubDataset
                starts, idx, last_valid_range =  _sortperm_v(dsr, oncols_right, stable = stable, a = alg, mapformats = mapformats[2], notsortpaforjoin = nsfpaj, threads = threads)
            else
                starts, idx, last_valid_range =  _sortperm(dsr, oncols_right, stable = stable, a = alg, mapformats = mapformats[2], notsortpaforjoin = nsfpaj, threads = threads)
            end
            fill!(ranges, 1:nrow(dsr))
        end
    end
    idx, last_valid_range == length(idx)
end

function _sort_for_join_after_hash(dsr, oncols_right, stable, alg, mapformats, nsfpaj, grng; threads = true)
    if dsr isa SubDataset
        starts, idx, last_valid_range =  _sortperm_v(dsr, oncols_right, stable = stable, a = alg, mapformats = mapformats[2], notsortpaforjoin = nsfpaj, givenrange = grng, threads = threads)
    else
        starts, idx, last_valid_range =  _sortperm(dsr, oncols_right, stable = stable, a = alg, mapformats = mapformats[2], notsortpaforjoin = nsfpaj, givenrange = grng, threads = threads)
    end
end

# find count of each groups id
function _find_counts_for_join(groups, ngroups)
   res = zeros(Int, ngroups)
   for i in 1:length(groups)
       res[groups[i]] += 1
   end
   res
end

function _find_range_for_join!(ranges, ldata, gslots, reps, where, minval, sz; threads = true)
    @_threadsfor threads for i in 1:length(ldata)
        ranges[i] = _query_dictionary_for_join_int(identity, ldata[i], gslots, reps, where, minval, sz)
    end
end

Base.@propagate_inbounds function _fill_val_join!(x, r, val)
    @simd for i in r
        x[i] = val
    end
end

function _fill_val_join!(x, r2, val, inbits, r)
    cnt = 1
    lo = r2.start
    for i in r
        if inbits[i]
            x[cnt+lo-1] = val
            cnt += 1
        end
    end
end
# F1 and F2 are here for type stability when threads = false
function _find_ranges_for_join!(ranges, x, y, _fl::F1, _fr::F2, ::Val{T1}, ::Val{T2}; type = :both, threads = true) where {T1, T2, F1, F2}
    if type == :both
        @_threadsfor threads for i in 1:length(x)
            ranges[i] = searchsorted_join(_fr, y, DataAPI.unwrap(_fl(x[i]))::T1, ranges[i].start, ranges[i].stop, Base.Order.Forward, Val(T2))
        end
    # TODO having another elseif branch is better for performance.
    elseif type == :left || type == :rightstrict
        @_threadsfor threads for i in 1:length(x)
             _flx = _fl(DataAPI.unwrap(x[i]))::T1
            hi = searchsortedlast_join(_fr, y, _flx, ranges[i].start, ranges[i].stop, Base.Order.Forward, Val(T2))
            lo = ranges[i].start
            if type === :rightstrict
                ranges[i] = hi+1:ranges[i].stop
            else
                ranges[i] = lo:hi
            end
        end
    elseif type == :right || type == :leftstrict
        @_threadsfor threads for i in 1:length(x)
            _flx = _fl(DataAPI.unwrap(x[i]))::T1
            lo = searchsortedfirst_join(_fr, y, _flx, ranges[i].start, ranges[i].stop, Base.Order.Forward, Val(T2))
            hi = ranges[i].stop
            if type === :leftstrict
                    ranges[i] = ranges[i].start:lo-1
            else
                ranges[i] = lo:hi
            end
        end
    end
end

function _find_ranges_for_join_pa!(ranges, x, invpool, y, _fl::F1, _fr::F2, ::Val{T1}, ::Val{T2}; type = :both, threads = true) where {T1, T2, F1, F2}
    if type == :both
        @_threadsfor threads for i in 1:length(x)
            revmap_paval_ref = get(invpool, DataAPI.unwrap(_fl(x[i]))::T1, missing)
            if ismissing(revmap_paval_ref)
                ranges[i] = 1:0
            else
                #_fr is identity
                ranges[i] = searchsorted_join(identity, y, revmap_paval_ref, ranges[i].start, ranges[i].stop, Base.Order.Forward, Val(T2))
            end
        end
    elseif type == :left || type == :rightstrict
        @_threadsfor threads for i in 1:length(x)
            revmap_paval_ref = get(invpool, _fl(DataAPI.unwrap(x[i]))::T1, missing)
            if ismissing(revmap_paval_ref)
                ranges[i] = 1:0
            else
                #_fr is identity
                hi = searchsoredlast_join(identity, y, revmap_paval_ref, ranges[i].start, ranges[i].stop, Base.Order.Forward, Val(T2))
                lo = ranges[i].start
                if type === :rightstrict
                    ranges[i] = hi+1:ranges[i].stop
                else
                    ranges[i] = lo:hi
                end
            end
        end
    elseif type == :right || type == :leftstrict
        @_threadsfor threads for i in 1:length(x)
            revmap_paval_ref = get(invpool, _fl(DataAPI.unwrap(x[i]))::T1, missing)
            if ismissing(revmap_paval_ref)
                ranges[i] = 1:0
            else
                #_fr is identity
                lo = searchsortedfirst_join(identity, y, revmap_paval_ref, ranges[i].start, ranges[i].stop, Base.Order.Forward, Val(T2))
                hi = ranges[i].stop
                if type === :leftstrict
                    ranges[i] = ranges[i].start:lo-1
                else
                    range[i] = lo:hi
                end
            end
        end
    end

end

function _fill_oncols_left_table_left!(_res, x, ranges, en, total, fill_val; threads = true)
    @_threadsfor threads for i in 1:length(x)
        i == 1 ? lo = 1 : lo = en[i - 1] + 1
        hi = en[i]
        _fill_val_join!(_res, lo:hi, x[i])
    end
    @_threadsfor threads for i in en[length(x)]+1:total
        _res[i] = fill_val
    end
end

function _fill_oncols_left_table_inner!(_res, x, ranges, en, total; inbits = nothing, en2 = nothing, threads = true)
    if inbits === nothing
        @_threadsfor threads for i in 1:length(x)
            length(ranges[i]) == 0 && continue
            i == 1 ? lo = 1 : lo = en[i - 1] + 1
            hi = en[i]
            _fill_val_join!(_res, lo:hi, x[i])
        end
    else
        @_threadsfor threads for i in 1:length(x)
            length(ranges[i]) == 0 && continue
            if i == 1
                lo = 1
                lo2 = 1
            else
                lo = en[i - 1] + 1
                lo2 = en2[i-1] + 1
            end
            hi = en[i]
            # @show sum(view(inbits, lo:hi))
            # sum(view(inbits, lo:hi)) == 0 && continue
            hi2 = en2[i]
            _fill_val_join!(_res, lo2:hi2, x[i], inbits, lo:hi)
        end
    end
end

function _fill_oncols_left_table_anti!(_res, x, ranges, en, total; threads = true)
    @_threadsfor threads for i in 1:length(x)
        length(ranges[i]) != 0 && continue
        i == 1 ? lo = 1 : lo = en[i - 1] + 1
        hi = en[i]
        _fill_val_join!(_res, lo:hi, x[i])
    end
end

function _fill_right_cols_table_left!(_res, x, ranges, en, total, fill_val; threads = true)
    @_threadsfor threads for i in 1:length(ranges)
        i == 1 ? lo = 1 : lo = en[i - 1] + 1
        hi = en[i]
        length(ranges[i]) == 0 ? _fill_val_join!(_res, lo:hi, fill_val) : copyto!(_res, lo, x, ranges[i].start, length(ranges[i]))
    end
end

function _fill_right_col_range!(_res, r2, x, ranges, inbits, r)
    cnt = 1
    cnt_r = 1
    lo = r2.start
    for i in r
        if inbits[i]
            _res[lo+cnt-1] = x[ranges[cnt_r]]
            cnt += 1
        end
        cnt_r += 1
    end
end

function _fill_right_cols_table_inner!(_res, x, ranges, en, total; inbits = nothing, en2 = nothing, threads = true)
    if inbits === nothing
        @_threadsfor threads for i in 1:length(ranges)
            length(ranges[i]) == 0 && continue
            i == 1 ? lo = 1 : lo = en[i - 1] + 1
            hi = en[i]
            copyto!(_res, lo, x, ranges[i].start, length(ranges[i]))
        end
    else
        @_threadsfor threads for i in 1:length(ranges)
            length(ranges[i]) == 0 && continue
            if i == 1
                lo = 1
                lo2 = 1
            else
                lo = en[i - 1] + 1
                lo2 = en2[i-1] + 1
            end
            hi = en[i]

            hi2 = en2[i]
            _fill_right_col_range!(_res, lo2:hi2, x, ranges[i], inbits, lo:hi)
        end
    end
end

function _create_multiple_match_col_left(ranges, total_length)
    res = allocatecol(Bool, total_length)
    cnt = 0
    for i in 1:length(ranges)
        if length(ranges[i]) == 0
            cnt += 1
            res[cnt] = false
        else
            if length(ranges[i]) == 1
                cnt += 1
                res[cnt] = false
            else
                for j in ranges[i]
                    cnt += 1
                    res[cnt] = true
                end
            end
        end
    end
    res
end
function _create_multiple_match_col_inner(ranges, en, total_length)
    res = allocatecol(Bool, total_length)
    cnt = 0
    if en === nothing
        for i in 1:length(ranges)
            if length(ranges[i]) == 0
                nothing
            else
                if length(ranges[i]) == 1
                    cnt += 1
                    res[cnt] = false
                else
                    for j in ranges[i]
                        cnt += 1
                        res[cnt] = true
                    end
                end
            end
        end
    else
        for i in 1:length(ranges)
            if i == 1
                lo = 1
            else
                lo = en[i - 1] + 1
            end
            hi = en[i]
            if length(lo:hi) == 0
                nothing
            elseif length(lo:hi) == 1
                cnt += 1
                res[cnt] = false
            else
                for j in lo:hi
                    cnt += 1
                    res[cnt] = true
                end
            end
        end
    end

    res
end

ISLE(x, ::Missing) = true
ISLE(x, y) = (x <= y)
ISLE(::Missing, y) = false
ISLE(::Missing, ::Missing) = false

function _mark_lt_part!(inbits, x_l, x_r, _fl::F1, _fr::F2, ranges, r_perms, en, ::Val{T}; strict = false, threads = true) where {T, F1, F2}
    revised_ends = zeros(T, length(en))
    @_threadsfor threads for i in 1:length(ranges)
        if length(ranges[i]) == 0
            if i !== 1
                revised_ends[i] = 0
            end
            continue
        end
        i == 1 ? lo = 1 : lo = en[i - 1] + 1
        hi = en[i]
        total = 0
        cnt = 1
        for j in ranges[i]
            if strict
                inbits[lo + cnt - 1] = isless(_fl(x_l[i]), _fr(x_r[r_perms[j]]))
            else
                inbits[lo + cnt - 1] = ISLE(_fl(x_l[i]), _fr(x_r[r_perms[j]]))
            end
            total += inbits[lo + cnt - 1]
            cnt += 1
        end
        revised_ends[i] = total
        if total == 0
            ranges[i] = 1:0
        end
    end
    cumsum!(revised_ends, revised_ends)
end

function _change_refpool_find_range_for_join!(ranges, dsl, dsr, r_perms, oncols_left, oncols_right, lmf, rmf, j; type = :both, nsfpaj = true, threads = true)
    var_l = _columns(dsl)[oncols_left[j]]
    var_r = _columns(dsr)[oncols_right[j]]
    l_idx = oncols_left[j]
    r_idx = oncols_right[j]
    if lmf
        _fl = getformat(dsl, l_idx)
    else
        _fl = identity
    end
    if rmf
        _fr = getformat(dsr, r_idx)
    else
        _fr = identity
    end

    T1 = Core.Compiler.return_type(DataAPI.unwrap∘_fl, Tuple{eltype(var_l)})

    if DataAPI.refpool(var_r) !== nothing && nsfpaj
        # sort taken care for refs ordering of modified values, but we still need to change refs
        if _fr == identity
            var_r_cpy = var_r
        else
            var_r_cpy = map(_fr, var_r)
        end
        T2 = eltype(DataAPI.refarray(var_r_cpy))
        # now _fr must be identity
        _fr = identity
        # we should use invpool of right column
        _find_ranges_for_join_pa!(ranges, var_l, DataAPI.invrefpool(var_r_cpy), view(DataAPI.refarray(var_r_cpy), r_perms), _fl, _fr, Val(T1), Val(T2); type = type, threads = threads)
    else
        T2 = Core.Compiler.return_type(_fr, Tuple{eltype(var_r)})
        _find_ranges_for_join!(ranges, var_l, view(var_r, r_perms), _fl, _fr, Val(T1), Val(T2); type = type, threads = threads)
    end
end

function _join_left(dsl, dsr, ::Val{T}; onleft, onright, makeunique = false, mapformats = [true, true], stable = false, alg = HeapSort, check = true, accelerate = false, method = :sort, threads = true, multiple_match::Bool = false, multiple_match_name = :multiple, obs_id = [false, false], obs_id_name = :obs_id) where T
    isempty(dsl) && return copy(dsl)
    if method == :hash
        ranges, a, idx, minval, reps, sz, right_cols = _find_ranges_for_join_using_hash(dsl, dsr, onleft, onright, mapformats, makeunique, Val(T); threads = threads)
    elseif method == :sort
        oncols_left = onleft
        oncols_right = onright
        right_cols = setdiff(1:length(index(dsr)), oncols_right)
        if !makeunique && !isempty(intersect(_names(dsl), _names(dsr)[right_cols]))
            throw(ArgumentError("duplicate column names, pass `makeunique = true` to make them unique using a suffix automatically." ))
        end

        ranges = Vector{UnitRange{T}}(undef, nrow(dsl))
        if length(oncols_left) == 1 && nrow(dsr)>1
            success, result = _join_left_dict(dsl, dsr, ranges, oncols_left, oncols_right, right_cols, Val(T); makeunique = makeunique, mapformats = mapformats, check = check, threads = threads, multiple_match = multiple_match, multiple_match_name = multiple_match_name, obs_id = obs_id, obs_id_name = obs_id_name)
            if success
                return result
            end
        end
        idx, uniquemode = _find_permute_and_fill_range_for_join!(ranges, dsr, dsl, oncols_right, oncols_left, stable, alg, mapformats, accelerate; threads = threads)

        for j in 1:length(oncols_left)
            _change_refpool_find_range_for_join!(ranges, dsl, dsr, idx, oncols_left, oncols_right, mapformats[1], mapformats[2], j; threads = threads)
        end
    end
    new_ends = map(x -> max(1, length(x)), ranges)
    cumsum!(new_ends, new_ends)
    total_length = new_ends[end]

    if check
        @assert total_length < 10*nrow(dsl) "the output data set will be very large ($(total_length)×$(ncol(dsl)+length(right_cols))) compared to the left data set size ($(nrow(dsl))×$(ncol(dsl))), make sure that the `on` keyword is selected properly, alternatively, pass `check = false` to ignore this error."
    end

    if multiple_match
        multiple_match_col = _create_multiple_match_col_left(ranges, total_length)
    end

    res = []
    for j in 1:length(index(dsl))
        addmissing = false
        _res = allocatecol(_columns(dsl)[j], total_length, addmissing = false)
        if DataAPI.refpool(_res) !== nothing
            # fill_val = DataAPI.invrefpool(_res)[missing]
            _fill_oncols_left_table_left!(_res.refs, DataAPI.refarray(_columns(dsl)[j]), ranges, new_ends, total_length, missing;  threads = threads)
        else
            _fill_oncols_left_table_left!(_res, _columns(dsl)[j], ranges, new_ends, total_length, missing; threads = threads)
        end
        push!(res, _res)

    end
    if dsl isa SubDataset
        newds = Dataset(res, copy(index(dsl)), copycols = false)
    else
        newds = Dataset(res, Index(copy(index(dsl).lookup), copy(index(dsl).names), copy(index(dsl).format)), copycols = false)
    end

    for j in 1:length(right_cols)
        _res = allocatecol(_columns(dsr)[right_cols[j]], total_length)
        if DataAPI.refpool(_res) !== nothing
            fill_val = DataAPI.invrefpool(_res)[missing]
            _fill_right_cols_table_left!(_res.refs, view(DataAPI.refarray(_columns(dsr)[right_cols[j]]), idx), ranges, new_ends, total_length, fill_val; threads = threads)
        else
            _fill_right_cols_table_left!(_res, view(_columns(dsr)[right_cols[j]], idx), ranges, new_ends, total_length, missing; threads = threads)
        end
        push!(_columns(newds), _res)

        new_var_name = make_unique([_names(dsl); _names(dsr)[right_cols[j]]], makeunique = makeunique)[end]
        push!(index(newds), new_var_name)
        setformat!(newds, index(newds)[new_var_name], getformat(dsr, _names(dsr)[right_cols[j]]))
    end
    if multiple_match
        insertcols!(newds, ncol(newds)+1, multiple_match_name => multiple_match_col, unsupported_copy_cols = false, makeunique = makeunique)
    end
    if obs_id[1]
        obs_id_name1 = Symbol(obs_id_name, "_left")
        obs_id_left = allocatecol(nrow(dsl) < typemax(Int32) ? Int32 : Int64, total_length)
        _fill_oncols_left_table_left!(obs_id_left, 1:nrow(dsl), ranges, new_ends, total_length, missing; threads = threads)
        insertcols!(newds, ncol(newds)+1, obs_id_name1 => obs_id_left, unsupported_copy_cols = false, makeunique = makeunique)
    end
    if obs_id[2]
        obs_id_name2 = Symbol(obs_id_name, "_right")
        obs_id_right = allocatecol(T, total_length)
        _fill_right_cols_table_left!(obs_id_right, idx, ranges, new_ends, total_length, missing, threads = threads)
        insertcols!(newds, ncol(newds)+1, obs_id_name2 => obs_id_right, unsupported_copy_cols = false, makeunique = makeunique)
    end
    newds

end

function _join_left!(dsl::Dataset, dsr::AbstractDataset, ::Val{T}; onleft, onright, makeunique = false, mapformats = [true, true], stable = false, alg = HeapSort, check = true, accelerate = false, method = :sort, threads = true, multiple_match = false, multiple_match_name = :multiple, obs_id = [false, false], obs_id_name = :obs_id) where T
    isempty(dsl) && return dsl
    if method == :hash
        ranges, a, idx, minval, reps, sz, right_cols = _find_ranges_for_join_using_hash(dsl, dsr, onleft, onright, mapformats, makeunique, Val(T); threads = threads)
    elseif method == :sort
        oncols_left = onleft
        oncols_right = onright
        right_cols = setdiff(1:length(index(dsr)), oncols_right)
        if !makeunique && !isempty(intersect(_names(dsl), _names(dsr)[right_cols]))
            throw(ArgumentError("duplicate column names, pass `makeunique = true` to make them unique using a suffix automatically." ))
        end
        ranges = Vector{UnitRange{T}}(undef, nrow(dsl))
        if length(oncols_left) == 1 && nrow(dsr)>1
            success, result = _join_left!_dict(dsl, dsr, ranges, oncols_left, oncols_right, right_cols, Val(T); makeunique = makeunique, mapformats = mapformats, check = check, threads = threads, multiple_match = multiple_match, multiple_match_name = multiple_match_name, obs_id = obs_id, obs_id_name = obs_id_name)
            if success
                return result
            end
        end
        idx, uniquemode = _find_permute_and_fill_range_for_join!(ranges, dsr, dsl, oncols_right, oncols_left, stable, alg, mapformats, accelerate, threads = threads)
        for j in 1:length(oncols_left)
            _change_refpool_find_range_for_join!(ranges, dsl, dsr, idx, oncols_left, oncols_right, mapformats[1], mapformats[2], j, threads = threads)
        end
    end
    if !all(x->length(x) <= 1, ranges)
        throw(ArgumentError("`leftjoin!` can only be used when each observation in left data set matches at most one observation from right data set"))
    end

    new_ends = map(x -> max(1, length(x)), ranges)
    cumsum!(new_ends, new_ends)
    total_length = new_ends[end]

    if check
        @assert total_length < 10*nrow(dsl) "the output data set will be very large ($(total_length)×$(ncol(dsl)+length(right_cols))) compared to the left data set size ($(nrow(dsl))×$(ncol(dsl))), make sure that the `on` keyword is selected properly, alternatively, pass `check = false` to ignore this error."
    end

    if multiple_match
        multiple_match_col = _create_multiple_match_col_left(ranges, total_length)
    end

    for j in 1:length(right_cols)
        _res = allocatecol(_columns(dsr)[right_cols[j]], total_length)
        if DataAPI.refpool(_res) !== nothing
            fill_val = DataAPI.invrefpool(_res)[missing]
            _fill_right_cols_table_left!(_res.refs, view(DataAPI.refarray(_columns(dsr)[right_cols[j]]), idx), ranges, new_ends, total_length, fill_val, threads = threads)
        else
            _fill_right_cols_table_left!(_res, view(_columns(dsr)[right_cols[j]], idx), ranges, new_ends, total_length, missing, threads = threads)
        end
        push!(_columns(dsl), _res)
        new_var_name = make_unique([_names(dsl); _names(dsr)[right_cols[j]]], makeunique = makeunique)[end]
        push!(index(dsl), new_var_name)
        setformat!(dsl, index(dsl)[new_var_name], getformat(dsr, _names(dsr)[right_cols[j]]))
    end
    if multiple_match
        insertcols!(dsl, ncol(dsl)+1, multiple_match_name => multiple_match_col, unsupported_copy_cols = false, makeunique = makeunique)
    end
    if obs_id[1]
        obs_id_name1 = Symbol(obs_id_name, "_left")
        obs_id_left = allocatecol(nrow(dsl) < typemax(Int32) ? Int32 : Int64, total_length)
        _fill_oncols_left_table_left!(obs_id_left, 1:nrow(dsl), ranges, new_ends, total_length, missing; threads = threads)
        insertcols!(dsl, ncol(dsl)+1, obs_id_name1 => obs_id_left, unsupported_copy_cols = false, makeunique = makeunique)
    end
    if obs_id[2]
        obs_id_name2 = Symbol(obs_id_name, "_right")
        obs_id_right = allocatecol(T, total_length)
        _fill_right_cols_table_left!(obs_id_right, idx, ranges, new_ends, total_length, missing, threads = threads)
        insertcols!(dsl, ncol(dsl)+1, obs_id_name2 => obs_id_right, unsupported_copy_cols = false, makeunique = makeunique)
    end

    _modified(_attributes(dsl))
    dsl
end

function _join_inner(dsl, dsr::AbstractDataset, ::Val{T}; onleft, onright, onright_range = nothing , makeunique = false, mapformats = [true, true], stable = false, alg = HeapSort, check = true, accelerate = false, droprangecols = true, strict_inequality = [false, false], method = :sort, threads = true, onlyreturnrange = false, multiple_match = false, multiple_match_name = :multiple, obs_id = [false, false], obs_id_name = :obs_id) where T
    (isempty(dsl) || isempty(dsr)) && throw(ArgumentError("in `innerjoin` both left and right tables must be non-empty"))
    oncols_left = onleft
    oncols_right = onright
    type = :both
    right_range_cols = Int[]
    if onright_range !== nothing
        left_range_col = oncols_left[end]

        right_range_cols = index(dsr)[filter!(!isequal(nothing), collect(onright_range))]
        if droprangecols
            right_cols = setdiff(1:length(index(dsr)), [oncols_right; right_range_cols])
        else
            right_cols = setdiff(1:length(index(dsr)), oncols_right)
        end

        oncols_right = [oncols_right; first(right_range_cols)]
        if onright_range[1] !== nothing
            if strict_inequality[1]
                type = :leftstrict
            else
                type = :left
            end
        else
            if strict_inequality[2]
                type = :rightstrict
            else
                type = :right
            end
        end
    else
        right_cols = setdiff(1:length(index(dsr)), oncols_right)
    end
    if !makeunique && !isempty(intersect(_names(dsl), _names(dsr)[right_cols]))
        throw(ArgumentError("duplicate column names, pass `makeunique = true` to make them unique using a suffix automatically." ))
    end

    nsfpaj = true
    # if the columns for inequality like join are PA we cannot use the fast path
    if type != :both
        if any(i-> DataAPI.refpool(_columns(dsr)[i]) !== nothing, right_range_cols)
            nsfpaj = false
        end
    end
    # if (onright_range === nothing || length(onleft) > 1) is false, then we have inequality kind join with no exact match join
    if method == :hash && (onright_range === nothing || length(onleft) > 1)
        if onright_range !== nothing
            ranges, a, idx, minval, reps, sz, right_cols_2 = _find_ranges_for_join_using_hash(dsl, dsr, onleft[1:end-1], oncols_right[1:end-1], mapformats, true, Val(T); threads = threads)
            filter!(!=(0), reps)
            pushfirst!(reps, 1)
            cumsum!(reps, reps)
            pop!(reps)
            grng = GIVENRANGE(idx, reps, Int[], length(reps))
            starts, idx, last_valid_range = _sort_for_join_after_hash(dsr, right_range_cols[1], stable, alg, mapformats, nsfpaj, grng; threads = threads)
            _change_refpool_find_range_for_join!(ranges, dsl, dsr, idx, oncols_left, oncols_right, mapformats[1], mapformats[2], length(oncols_left); type = type, nsfpaj = nsfpaj, threads = threads)
        else
            ranges, a, idx, minval, reps, sz, right_cols = _find_ranges_for_join_using_hash(dsl, dsr, onleft, onright, mapformats, makeunique, Val(T); threads = threads)
        end
    else
        ranges = Vector{UnitRange{T}}(undef, nrow(dsl))
        if length(oncols_left) == 1 && type == :both && nrow(dsr)>1
            success, result =  _join_inner_dict(dsl, dsr, ranges, oncols_left, oncols_right, right_cols, Val(T); makeunique = makeunique, mapformats = mapformats, check = check, threads = threads, multiple_match = multiple_match, multiple_match_name = multiple_match_name, obs_id = obs_id, obs_id_name = obs_id_name)
            if success
                return result
            end
        end
        idx, uniquemode = _find_permute_and_fill_range_for_join!(ranges, dsr, dsl, oncols_right, oncols_left, stable, alg, mapformats, accelerate && (onright_range == nothing || length(oncols_right)>1); nsfpaj = nsfpaj, threads = threads)

        for j in 1:length(oncols_left)-1
            _change_refpool_find_range_for_join!(ranges, dsl, dsr, idx, oncols_left, oncols_right, mapformats[1], mapformats[2], j; nsfpaj = nsfpaj, threads = threads)
        end
        _change_refpool_find_range_for_join!(ranges, dsl, dsr, idx, oncols_left, oncols_right, mapformats[1], mapformats[2], length(oncols_left); type = type, nsfpaj = nsfpaj, threads = threads)
    end

    new_ends = map(length, ranges)
    cumsum!(new_ends, new_ends)
    total_length = new_ends[end]

    inbits = nothing
    revised_ends = nothing
    if length(right_range_cols) == 2
        inbits = zeros(Bool, total_length)
        # TODO any optimisation is needed for pa?
        _fl = identity
        _fr = identity
        if mapformats[1]
            _fl = getformat(dsl, left_range_col)
        end
        if mapformats[2]
            _fr = getformat(dsr, right_range_cols[2])
        end
        revised_ends = _mark_lt_part!(inbits, _columns(dsl)[left_range_col], _columns(dsr)[right_range_cols[2]], _fl, _fr, ranges, idx, new_ends, total_length < typemax(Int32) ? Val(Int32) : Val(Int64); strict = strict_inequality[2], threads = threads)
    end
    if length(right_range_cols) == 2
        total_length = sum(inbits)
    end

    if onlyreturnrange
        return ranges
    end
    if check
        @assert total_length < 10*nrow(dsl) "the output data set will be very large ($(total_length)×$(ncol(dsl)+length(right_cols))) compared to the left data set size ($(nrow(dsl))×$(ncol(dsl))), make sure that the `on` keyword is selected properly, alternatively, pass `check = false` to ignore this error."
    end
    if multiple_match
        multiple_match_col = _create_multiple_match_col_inner(ranges, revised_ends, total_length)
    end

    res = []
    for j in 1:length(index(dsl))
        _res = allocatecol(_columns(dsl)[j], total_length, addmissing = false)
        if DataAPI.refpool(_res) !== nothing
            _fill_oncols_left_table_inner!(_res.refs, DataAPI.refarray(_columns(dsl)[j]), ranges, new_ends, total_length; inbits = inbits, en2 = revised_ends, threads = threads)
        else
            _fill_oncols_left_table_inner!(_res, _columns(dsl)[j], ranges, new_ends, total_length; inbits = inbits, en2 = revised_ends, threads = threads)
        end
        push!(res, _res)
    end
    if dsl isa SubDataset
        newds = Dataset(res, copy(index(dsl)), copycols = false)
    else
        newds = Dataset(res, Index(copy(index(dsl).lookup), copy(index(dsl).names), copy(index(dsl).format)), copycols = false)
    end

    for j in 1:length(right_cols)
        _res = allocatecol(_columns(dsr)[right_cols[j]], total_length, addmissing = false)
        if DataAPI.refpool(_res) !== nothing
            _fill_right_cols_table_inner!(_res.refs, view(DataAPI.refarray(_columns(dsr)[right_cols[j]]), idx), ranges, new_ends, total_length; inbits = inbits, en2 = revised_ends, threads = threads)
        else
            _fill_right_cols_table_inner!(_res, view(_columns(dsr)[right_cols[j]], idx), ranges, new_ends, total_length; inbits = inbits, en2 = revised_ends, threads = threads)
        end
        push!(_columns(newds), _res)

        new_var_name = make_unique([_names(dsl); _names(dsr)[right_cols[j]]], makeunique = makeunique)[end]
        push!(index(newds), new_var_name)
        setformat!(newds, index(newds)[new_var_name], getformat(dsr, _names(dsr)[right_cols[j]]))
    end
    if multiple_match
        insertcols!(newds, ncol(newds)+1, multiple_match_name => multiple_match_col, unsupported_copy_cols = false, makeunique = makeunique)
    end
    if obs_id[1]
        obs_id_name1 = Symbol(obs_id_name, "_left")
        obs_id_left = allocatecol(nrow(dsl) < typemax(Int32) ? Int32 : Int64, total_length)
        _fill_oncols_left_table_inner!(obs_id_left, 1:nrow(dsl), ranges, new_ends, total_length; inbits = inbits, en2 = revised_ends, threads = threads)
        insertcols!(newds, ncol(newds)+1, obs_id_name1 => obs_id_left, unsupported_copy_cols = false, makeunique = makeunique)
    end
    if obs_id[2]
        obs_id_name2 = Symbol(obs_id_name, "_right")
        obs_id_right = allocatecol(T, total_length)
        _fill_right_cols_table_inner!(obs_id_right, idx, ranges, new_ends, total_length; inbits = inbits, en2 = revised_ends, threads = threads)
        insertcols!(newds, ncol(newds)+1, obs_id_name2 => obs_id_right, unsupported_copy_cols = false, makeunique = makeunique)
    end
    newds

end

function _in(dsl::AbstractDataset, dsr::AbstractDataset, ::Val{T}; onleft, onright, mapformats = [true, true], stable = false, alg = HeapSort, accelerate = false, threads = true) where T
    isempty(dsl) && return Bool[]
    oncols_left = onleft
    oncols_right = onright

    # use Set when there is only one column in `on`
    if length(oncols_right) == 1
        if mapformats[1]
            _fl = getformat(dsl, oncols_left[1])
        else
            _fl = identity
        end
        if mapformats[2]
            _fr = getformat(dsr, oncols_right[1])
        else
            _fr = identity
        end
        return _in_use_Set(_columns(dsl)[oncols_left[1]], _columns(dsr)[oncols_right[1]], _fl, _fr, threads = threads)
    end
    ranges = Vector{UnitRange{T}}(undef, nrow(dsl))
    idx, uniquemode = _find_permute_and_fill_range_for_join!(ranges, dsr, dsl, oncols_right, oncols_left, stable, alg, mapformats, accelerate, threads = threads)
    for j in 1:length(oncols_left)
        _change_refpool_find_range_for_join!(ranges, dsl, dsr, idx, oncols_left, oncols_right, mapformats[1], mapformats[2], j, threads = threads)
    end
    map(x -> length(x) == 0 ? false : true, ranges)
end

function _find_right_not_in_left(ranges, n, idx)
    res = trues(n)
    for i in 1:length(ranges)
        view(res, ranges[i]) .= false
    end
    findall(res)
end

function _fill_oncols_left_table_left_outer!(res, x, notinleft, en, total)
    # TODO when x is a pooled array, we cannot use Threads.@threads (maybe we should create pool first and then use Threads)
    for i in en[end]+1:total
        val = x[notinleft[i - en[end]]]
        res[i] = val
    end
end

function _fill_source_for_outer!(res, ranges, notinleft, lval, rval, en, total)
    cnt = 0
    for i in 1:length(ranges)
        if length(ranges[i]) == 0
            cnt += 1
            res[cnt] = lval
        else
            cnt += length(ranges[i])
        end
    end
    for i in en[end]+1:total
        res[i] = rval
    end
end

function _create_source_for_outer(ranges, notinleft, total_length, en)
    res = allowmissing(PooledArray(["left", "right", "both"]))
    resize!(res.refs, total_length)
    fill!(res.refs, get(res.invpool, "both", missing))
    _fill_source_for_outer!(res.refs, ranges, notinleft, get(res.invpool, "left", missing), get(res.invpool, "right", missing), en, total_length)
    res
end

function _create_multiple_match_col_outer(ranges, notinleft, total_length, new_ends)
    res = _missings(Bool, total_length)
    cnt = 0
    for i in 1:length(ranges)
        if length(ranges[i]) == 0
            cnt += 1
            res[cnt] = false
        else
            if length(ranges[i]) == 1
                cnt += 1
                res[cnt] = false
            else
                for j in ranges[i]
                    cnt += 1
                    res[cnt] = true
                end
            end
        end
    end
    res
end

function _join_outer(dsl, dsr::AbstractDataset, ::Val{T}; onleft, onright, makeunique = false, mapformats = [true, true], stable = false, alg = HeapSort, check = true, accelerate = false, method = :sort, threads = true, source::Bool = false, source_col_name = :source, multiple_match = false, multiple_match_name = :multiple, obs_id = [false, false], obs_id_name = :obs_id) where T
    (isempty(dsl) || isempty(dsr)) && throw(ArgumentError("in `outerjoin` both left and right tables must be non-empty"))
    oncols_left = onleft
    oncols_right = onright
    if method == :hash
        ranges, a, idx, minval, reps, sz, right_cols = _find_ranges_for_join_using_hash(dsl, dsr, onleft, onright, mapformats, makeunique, Val(T); threads = threads)
    elseif method == :sort
        right_cols = setdiff(1:length(index(dsr)), oncols_right)
        if !makeunique && !isempty(intersect(_names(dsl), _names(dsr)[right_cols]))
            throw(ArgumentError("duplicate column names, pass `makeunique = true` to make them unique using a suffix automatically." ))
        end
        ranges = Vector{UnitRange{T}}(undef, nrow(dsl))
        if length(oncols_left) == 1 && nrow(dsr)>1
            success, result = _join_outer_dict(dsl, dsr, ranges, oncols_left, oncols_right, oncols_left, oncols_right, right_cols, Val(T); makeunique = makeunique, mapformats = mapformats, check = check, threads = threads, source = source, source_col_name = source_col_name, multiple_match = multiple_match, multiple_match_name = multiple_match_name, obs_id = obs_id, obs_id_name = obs_id_name)
            if success
                return result
            end
        end
        idx, uniquemode = _find_permute_and_fill_range_for_join!(ranges, dsr, dsl, oncols_right, oncols_left, stable, alg, mapformats, accelerate, threads = threads)
        for j in 1:length(oncols_left)
            _change_refpool_find_range_for_join!(ranges, dsl, dsr, idx, oncols_left, oncols_right, mapformats[1], mapformats[2], j, threads = threads)
        end
    end
    new_ends = map(x -> max(1, length(x)), ranges)
    notinleft = _find_right_not_in_left(ranges, nrow(dsr), idx)
    cumsum!(new_ends, new_ends)
    total_length = new_ends[end] + length(notinleft)

    if check
        @assert total_length < 10*nrow(dsl) "the output data set will be very large ($(total_length)×$(ncol(dsl)+length(right_cols))) compared to the left data set size ($(nrow(dsl))×$(ncol(dsl))), make sure that the `on` keyword is selected properly, alternatively, pass `check = false` to ignore this error."
    end

    if source
        source_col = _create_source_for_outer(ranges, notinleft, total_length, new_ends)
    end
    if multiple_match
        multiple_match_col = _create_multiple_match_col_outer(ranges, notinleft, total_length, new_ends)
    end

    res = []
    for j in 1:length(index(dsl))
        _res = allocatecol(_columns(dsl)[j], total_length)
        if DataAPI.refpool(_res) !== nothing
            fill_val = DataAPI.invrefpool(_res)[missing]
            _fill_oncols_left_table_left!(_res.refs, DataAPI.refarray(_columns(dsl)[j]), ranges, new_ends, total_length, fill_val, threads = threads)
        else
            _fill_oncols_left_table_left!(_res, _columns(dsl)[j], ranges, new_ends, total_length, missing, threads = threads)
        end
        push!(res, _res)
    end
    for j in 1:length(oncols_left)
        _fill_oncols_left_table_left_outer!(res[oncols_left[j]], view(_columns(dsr)[oncols_right[j]], idx), notinleft, new_ends, total_length)
    end
    if dsl isa SubDataset
        newds = Dataset(res, copy(index(dsl)), copycols = false)
    else
        newds = Dataset(res, Index(copy(index(dsl).lookup), copy(index(dsl).names), copy(index(dsl).format)), copycols = false)
    end

    for j in 1:length(right_cols)
        _res = allocatecol(_columns(dsr)[right_cols[j]], total_length)
        if DataAPI.refpool(_res) !== nothing
            fill_val = DataAPI.invrefpool(_res)[missing]
            _fill_right_cols_table_left!(_res.refs, view(DataAPI.refarray(_columns(dsr)[right_cols[j]]), idx), ranges, new_ends, total_length, fill_val, threads = threads)
            _fill_oncols_left_table_left_outer!(_res.refs, view(DataAPI.refarray(_columns(dsr)[right_cols[j]]), idx), notinleft, new_ends, total_length)
        else
            _fill_right_cols_table_left!(_res, view(_columns(dsr)[right_cols[j]], idx), ranges, new_ends, total_length, missing, threads = threads)
            _fill_oncols_left_table_left_outer!(_res, view(_columns(dsr)[right_cols[j]], idx), notinleft, new_ends, total_length)
        end
        push!(_columns(newds), _res)
        new_var_name = make_unique([_names(dsl); _names(dsr)[right_cols[j]]], makeunique = makeunique)[end]
        push!(index(newds), new_var_name)
        setformat!(newds, index(newds)[new_var_name], getformat(dsr, _names(dsr)[right_cols[j]]))
    end
    if source
        insertcols!(newds, ncol(newds)+1, source_col_name => source_col, unsupported_copy_cols = false, makeunique = makeunique)
    end
    if multiple_match
        insertcols!(newds, ncol(newds)+1, multiple_match_name => multiple_match_col, unsupported_copy_cols = false, makeunique = makeunique)
    end
    if obs_id[1]
        # Note that the name convention of obs_id_name1 and name2 are used in other places
        obs_id_name1 = Symbol(obs_id_name, "_left")
        obs_id_left = allocatecol(nrow(dsl) < typemax(Int32) ? Int32 : Int64, total_length)
        _fill_oncols_left_table_left!(obs_id_left, 1:nrow(dsl), ranges, new_ends, total_length, missing, threads = threads)
        insertcols!(newds, ncol(newds)+1, obs_id_name1 => obs_id_left, unsupported_copy_cols = false, makeunique = makeunique)
    end
    if obs_id[2]
        obs_id_name2 = Symbol(obs_id_name, "_right")
        obs_id_right = allocatecol(T, total_length)
        _fill_right_cols_table_left!(obs_id_right, idx, ranges, new_ends, total_length, missing, threads = threads)
        _fill_oncols_left_table_left_outer!(obs_id_right, idx, notinleft, new_ends, total_length)
        insertcols!(newds, ncol(newds)+1, obs_id_name2 => obs_id_right, unsupported_copy_cols = false, makeunique = makeunique)
    end
    newds

end


@with_kw struct CSetup{S,V}
    a::S = 0.5u"cm^2"
    A::S = 71.0u"cm^2"
    fixed_ΔV::V = nothing
end

function specific_capacitance(C, porosity, folder, a, A)
    porosities = sort!(parse.(Int, dirs_in_folder(folder, false)))

    df = CSV.read("$folder/weights.csv")
    mass = (df[!,:after].*u"g" .- df[!,:before].*u"g") .* a ./ A .|> u"μg"
    mass_dict = Dict(porosities.=>mass[1:length(porosities)])

    # @show mass_dict
    C / mass_dict[porosity]
end

function energy(C, ΔV)
    return (C*ΔV^2)/2
end

power(E, Δt) = E / Δt

function add_report!(df, cr)
    add_aux!(df, cr)

    push!(df[!, :C], ustrip(u"μF", cr.C))
    push!(df[!, :C_specific], ustrip(u"F/g", cr.C_specific))
    push!(df[!, :E], ustrip(u"W*hr", cr.E))
    push!(df[!, :E_specific], ustrip(u"W*hr/kg", cr.E_specific))
    push!(df[!, :P], ustrip(u"W", cr.P))
    push!(df[!, :P_specific], ustrip(u"W/kg", cr.P_specific))

    return nothing
end

function add_report!(result_df, datafile, folder, setup)
    if endswith(datafile.filename, "_D")
        df = read_file(datafile)
        cr = CDCapacitanceReport(datafile, df, folder, setup)
        add_report!(result_df, cr)
    end
end

function add_report!(result_df, datafile, quadrant, folder, setup)
    df = read_file(datafile, 3, false)
    cr = CVCapacitanceReport(datafile, df, quadrant, folder, setup)
    add_report!(result_df, cr)
end

function compute_capacitances(folder; cv_setup=[CSetup(),CSetup()], cd_setup=CSetup())
    data = find_files(folder)
    processed_data = find_files(folder, exclude_with=r"!", select_with=".dat", rename=false)

    grouped = groupbyfolder(data["C&D"])
    processed_grouped = groupbyfolder(processed_data["CV"])
    @assert keys(grouped) == keys(processed_grouped)

    cd_units = cd_report_units()
    cv_units = cv_report_units()

    for f in keys(grouped)
        cd_datafiles = grouped[f]
        cv_datafiles = processed_grouped[f]

        cd_report = cd_result()
        cv_report4 = cv_result()
        cv_report2 = cv_result()

        for datafile in cd_datafiles
            add_report!(cd_report, datafile, folder, cd_setup)
        end

        for datafile in cv_datafiles
            add_report!(cv_report4, datafile, 4, folder, cv_setup[1])
            add_report!(cv_report2, datafile, 2, folder, cv_setup[2])
        end

        rename_cols!(df) = rename!(df, Dict(
            "E_specific"=>"Energy density",
            "P_specific"=>"Power density",
            "C"=>"C_abs",
            "C_specific"=>"C",
            "Δt"=>replace_unicode("Δt"),
            "ΔV"=>replace_unicode("ΔV")))
        rename_cols!(cd_report)
        rename_cols!(cv_report4)
        rename_cols!(cv_report2)
        sort!(cd_report, :I)
        sort!(cv_report4, :scan_rate)
        sort!(cv_report2, :scan_rate)

        comment(df) = join(repeat([f*replace_powers(" mA cm^-2")], size(df,2)), ',')

        write_file(
            cd_report,
            (cd_units, comment(cd_report)),
            joinpath(folder, f, "cd_capacitances.csv"),
            ',')
        write_file(
            cv_report4,
            (cv_units, comment(cv_report4)),
            joinpath(folder, f, "cv_capacitances4.csv"),
            ',')
        write_file(
            cv_report2,
            (cv_units, comment(cv_report2)),
            joinpath(folder, f, "cv_capacitances2.csv"),
            ',')
    end

    return nothing
end


RESIDUAL_EXPR = Dict{String, Any}()
residual_expr(model::Model) = RESIDUAL_EXPR[String(name(model)) * "_r"]
jacobian_var_expr(model::Model) = RESIDUAL_EXPR[String(name(model)) * "_rz"]
jacobian_data_expr(model::Model) = RESIDUAL_EXPR[String(name(model)) * "_rθ"]

path_robots = @get_scratch!("robots")

robots = [hopper, hopper1, biped, quadruped, quadruped4, halfquadruped, halfcheetah, halfcheetah4, box, particle, centroidal_quadruped, point_foot_quadruped]

for robot in robots
    path = joinpath(path_robots, String(name(robot)) * ".jld2")

    if !isfile(path)
        # kinematics
        contact_kinematics = eval(Symbol(String(name(robot)) * "_contact_kinematics"))
        contact_kinematics_jacobians = eval(Symbol(String(name(robot)) * "_contact_kinematics_jacobians"))

        # codegen
        r_model, rz_model, rθ_model = codegen_residual(eval(robot), codegen_dynamics(eval(robot))..., contact_kinematics, contact_kinematics_jacobians)
        @save path r_model rz_model rθ_model
    else
        @load path r_model rz_model rθ_model
    end

    RESIDUAL_EXPR[String(name(robot)) * "_r"] = eval(r_model)
    RESIDUAL_EXPR[String(name(robot)) * "_rz"] = eval(rz_model)
    RESIDUAL_EXPR[String(name(robot)) * "_rθ"] = eval(rθ_model)
end


"""
    mutable struct SolidStateDetector{T <: SSDFloat, CS} <: AbstractConfig{T}

CS: Coordinate System: -> :cartesian / :cylindrical
"""
mutable struct SolidStateDetector{T <: SSDFloat, CS} <: AbstractConfig{T}
    name::String  # optional
    inputunits::Dict{String, Unitful.Units}
    world::World{T, 3}

    config_dict::Dict

    medium::NamedTuple # this should become a struct at some point

    semiconductors::Vector{Semiconductor{T}}
    contacts::Vector{Contact{T}}
    passives::Vector{Passive{T}}

    virtual_drift_volumes::Vector{AbstractVirtualVolume{T}}
end

get_precision_type(d::SolidStateDetector{T}) where {T} = T
get_coordinate_system(d::SolidStateDetector{T, CS}) where {T, CS} = CS

function SolidStateDetector{T, S}()::SolidStateDetector{T} where {T <: SSDFloat, S}
    semiconductors::Vector{Semiconductor{T}}, contacts::Vector{Contact{T}}, passives::Vector{Passive{T}} = [], [], []
    virtual_drift_volumes::Vector{AbstractVirtualVolume{T}} = []
    world_limits = get_world_limits_from_objects(Val(S), semiconductors, contacts, passives )
    world = World(Val(S), world_limits)

    return SolidStateDetector{T, S}(
        "EmptyDetector",
        default_unit_dict(),
        world,
        Dict(),
        material_properties[materials["vacuum"]],
        semiconductors,
        contacts,
        passives,
        virtual_drift_volumes
    )
end

function SolidStateDetector{T}()::SolidStateDetector{T} where {T <: SSDFloat}
    S::Symbol = :cartesian
    return SolidStateDetector{T, S}()
end
function SolidStateDetector()::SolidStateDetector{Float32, :cartesian}
    return SolidStateDetector{Float32, :cartesian}()
end

function default_unit_dict()::Dict{String, Unitful.Units}
    return Dict{String, Unitful.Units}(
        "length" => u"m", # change this to u"m" ? SI Units
        "potential" => u"V",
        "angle" => u"°",
        "temperature" => u"K"
    )
end

function construct_units(config_file_dict::Dict)
    dunits::Dict{String, Unitful.Units} = default_unit_dict()
    if haskey(config_file_dict, "units")
        d = config_file_dict["units"]
        if haskey(d, "length") dunits["length"] = unit_conversion[d["length"]] end
        if haskey(d, "angle") dunits["angle"] = unit_conversion[d["angle"]] end
        if haskey(d, "potential") dunits["potential"] = unit_conversion[d["potential"]] end
        if haskey(d, "temperature") dunits["temperature"] = unit_conversion[d["temperature"]] end
    end
    dunits
end

function construct_semiconductor(T, sc::Dict, inputunit_dict::Dict{String, Unitful.Units})
    Semiconductor{T}(sc, inputunit_dict)
end

function construct_passive(T, pass::Dict, inputunit_dict::Dict{String, Unitful.Units})
    Passive{T}(pass, inputunit_dict)
end

function construct_contact(T, contact::Dict, inputunit_dict::Dict{String, Unitful.Units})
    Contact{T}(contact, inputunit_dict)
end

function construct_virtual_volume(T, pass::Dict, inputunit_dict::Dict{String, Unitful.Units})
    construct_virtual_volume(T, pass, inputunit_dict, Val{Symbol(pass["model"])} )
end
function construct_virtual_volume(T, pass::Dict, inputunit_dict::Dict{String, Unitful.Units}, ::Type{Val{:dead}})
    DeadVolume{T}(pass, inputunit_dict)
end
function construct_virtual_volume(T, pass::Dict, inputunit_dict::Dict{String, Unitful.Units}, ::Type{Val{:arbitrary}})
    ArbitraryDriftModificationVolume{T}(pass, inputunit_dict)
end

function construct_objects(T, objects::Vector, semiconductors, contacts, passives, virtual_drift_volumes, inputunit_dict)::Nothing
    for obj in objects
        if obj["type"] == "semiconductor"
            push!(semiconductors, construct_semiconductor(T, obj, inputunit_dict))
        elseif obj["type"] == "contact"
            push!(contacts, construct_contact(T, obj, inputunit_dict))
        elseif obj["type"] == "passive"
            push!(passives, construct_passive(T, obj, inputunit_dict))
        elseif obj["type"] == "virtual_drift_volume"
            push!(virtual_drift_volumes, construct_virtual_volume(T, obj, inputunit_dict))
        else
            @warn "please specify the class to be either a \"semiconductor\", a \"contact\", or \"passive\""
        end
    end
    nothing
end

function get_world_limits_from_objects(S::Val{:cylindrical}, s::Vector{Semiconductor{T}}, c::Vector{Contact{T}}, p::Vector{Passive{T}}) where {T <: SSDFloat}
    ax1l::T, ax1r::T, ax2l::T, ax2r::T, ax3l::T, ax3r::T = 0, 1, 0, 1, 0, 1
    imps_1::Vector{T} = []
    imps_3::Vector{T} = []
    for objects in [s, c, p]
        for object in objects
            for posgeo in object.geometry_positive
                append!(imps_1, get_important_points( posgeo, Val{:r}()))
                append!(imps_3, get_important_points( posgeo, Val{:z}()))
            end
        end
    end
    imps_1 = uniq(sort(imps_1))
    imps_3 = uniq(sort(imps_3))
    if length(imps_1) > 1
        ax1l = minimum(imps_1)
        ax1r = maximum(imps_1)
    elseif length(imps_1) == 1
        ax1l = minimum(imps_1)
        ax1r = maximum(imps_1) + 1
    end
    if length(imps_3) > 1
        ax3l = minimum(imps_3)
        ax3r = maximum(imps_3)
    elseif length(imps_3) == 1
        ax3l = minimum(imps_3)
        ax3r = maximum(imps_3) + 1
    end
    return ax1l, ax1r, ax2l, ax2r, ax3l, ax3r
end
function get_world_limits_from_objects(S::Val{:cartesian}, s::Vector{Semiconductor{T}}, c::Vector{Contact{T}}, p::Vector{Passive{T}}) where {T <: SSDFloat}
    ax1l::T, ax1r::T, ax2l::T, ax2r::T, ax3l::T, ax3r::T = 0, 1, 0, 1, 0, 1
    imps_1::Vector{T} = []
    imps_2::Vector{T} = []
    imps_3::Vector{T} = []
    for objects in [s, c, p]
        for object in objects
            for posgeo in object.geometry_positive
                append!(imps_1, get_important_points( posgeo, Val{:x}()))
                append!(imps_2, get_important_points( posgeo, Val{:y}()))
                append!(imps_3, get_important_points( posgeo, Val{:z}()))
            end
        end
    end
    imps_1 = uniq(sort(imps_1))
    imps_2 = uniq(sort(imps_2))
    imps_3 = uniq(sort(imps_3))
    if length(imps_1) > 1
        ax1l = minimum(imps_1)
        ax1r = maximum(imps_1)
    elseif length(imps_1) == 1
        ax1l = minimum(imps_1)
        ax1r = maximum(imps_1) + 1
    end
    if length(imps_2) > 1
        ax2l = minimum(imps_2)
        ax2r = maximum(imps_2)
    elseif length(imps_2) == 1
        ax2l = minimum(imps_2)
        ax2r = maximum(imps_2) + 1
    end
    if length(imps_3) > 1
        ax3l = minimum(imps_3)
        ax3r = maximum(imps_3)
    elseif length(imps_3) == 1
        ax3l = minimum(imps_3)
        ax3r = maximum(imps_3) + 1
    end
    return ax1l, ax1r, ax2l, ax2r, ax3l, ax3r
end

function SolidStateDetector{T}(config_file::Dict)::SolidStateDetector{T} where{T <: SSDFloat}
    grid_type::Symbol = :cartesian
    semiconductors::Vector{Semiconductor{T}}, contacts::Vector{Contact{T}}, passives::Vector{Passive{T}} = [], [], []
    virtual_drift_volumes::Vector{AbstractVirtualVolume{T}} = []
    medium::NamedTuple = material_properties[materials["vacuum"]]
    inputunits = dunits::Dict{String, Unitful.Units} = Dict{String, Unitful.Units}(
        "length" => u"m", # change this to u"m" ? SI Units
        "potential" => u"V",
        "angle" => u"°",
        "temperature" => u"K"
    )
    inputunits = construct_units(config_file)
    if haskey(config_file, "medium")
        medium = material_properties[materials[config_file["medium"]]]
    end

    if haskey(config_file, "objects")
        construct_objects(T, config_file["objects"], semiconductors, contacts, passives, virtual_drift_volumes, inputunits)
    end

    world = if haskey(config_file, "grid")
        if isa(config_file["grid"], Dict)
            grid_type = Symbol(config_file["grid"]["coordinates"])
            World(T, config_file["grid"], inputunits)
        elseif isa(config_file["grid"], String)
            grid_type = Symbol(config_file["grid"])
            world_limits = get_world_limits_from_objects(Val(grid_type), semiconductors, contacts, passives)
            World(Val(grid_type), world_limits)
        end
    else
        world_limits = get_world_limits_from_objects(Val(grid_type), semiconductors, contacts, passives )
        World(Val(grid_type), world_limits)
    end

    c = SolidStateDetector{T, grid_type}()
    c.name = haskey(config_file, "name") ? config_file["name"] : "NoNameDetector"
    c.config_dict = config_file
    c.semiconductors = semiconductors
    c.contacts = contacts
    c.passives = passives
    c.inputunits = inputunits
    c.medium = medium
    c.world = world
    c.virtual_drift_volumes = virtual_drift_volumes
    return c
end

function SolidStateDetector(parsed_dict::Dict)
    SolidStateDetector{Float32}(parsed_dict)
end

function Base.sort!(v::AbstractVector{<:AbstractGeometry})
    hierarchies::Vector{Int} = map(x->x.hierarchy,v)
    v_result::typeof(v) = []
    for idx in sort!(unique!(hierarchies))
        push!(v_result,filter(x->x.hierarchy == hierarchies[idx],v)...)
    end
    return v_result
end

function SolidStateDetector{T}(parsed_dict::Dict) where T
    SolidStateDetector{T}(parsed_dict)
end

function contains(c::SolidStateDetector, point::AbstractCoordinatePoint{T,3})::Bool where T
    for contact in c.contacts
        if point in contact
            return true
        end
    end
    for sc in c.semiconductors
        if point in sc
            return true
        end
    end
    return false
end

function println(io::IO, d::SolidStateDetector{T, CS}) where {T <: SSDFloat, CS}
    println("________"*d.name*"________\n")
    # println("Class: ",d.class)
    println("---General Properties---")
    println("- Precision type: $(T)")
    println("- Environment Material: $(d.medium.name)")
    println("- Grid Type: $(CS)")
    println()
    println("# Semiconductors: $(length(d.semiconductors))")
    for (isc, sc)  in enumerate(d.semiconductors)
        println("\t_____Semiconductor $(isc)_____\n")
        println(sc)
    end
    println()
    println("# Contacts: $(length(d.contacts))")
    if length(d.contacts)<=5
        for c in d.contacts
            println(c)
        end
    end
    println()
    println("# Passives: $(length(d.passives))")
    if length(d.passives)<=5
        for p in d.passives
            # println(c)
        end
    end
end

function show(io::IO, d::SolidStateDetector{T}) where {T <: SSDFloat} println(io, d) end
function print(io::IO, d::SolidStateDetector{T}) where {T <: SSDFloat} println(io, d) end
function show(io::IO,::MIME"text/plain", d::SolidStateDetector) where {T <: SSDFloat}
    show(io, d)
end

# ToDo: Test it
function generate_random_startpositions(d::SolidStateDetector{T}, n::Int, Volume::NamedTuple=bounding_box(d), rng::AbstractRNG = MersenneTwister(), min_dist_from_boundary = 0.0001) where T
    delta = T(min_dist_from_boundary)
    n_filled::Int = 0
    positions = Vector{CartesianPoint{T}}(undef,n)
    while n_filled < n
        sample=CylindricalPoint{T}(rand(rng,Volume[:r_range].left:0.00001:Volume[:r_range].right),rand(rng,Volume[:φ_range].left:0.00001:Volume[:φ_range].right),rand(rng,Volume[:z_range].left:0.00001:Volume[:z_range].right))
        if !(sample in d.contacts) && contains(d,sample) && contains(d,CylindricalPoint{T}(sample.r+delta,sample.φ,sample.z))&& contains(d,CylindricalPoint{T}(sample.r-delta,sample.φ,sample.z))&& contains(d,CylindricalPoint{T}(sample.r,sample.φ,sample.z+delta))&& contains(d,CylindricalPoint{T}(sample.r,sample.φ,sample.z-delta))
            n_filled += 1
            positions[n_filled]=CartesianPoint(sample)
        end
    end
    positions
end

function paint_object(det::SolidStateDetector{T}, object::AbstractObject, grid::Grid{T, 3, S}) where {T <: SSDFloat, S}
    samples = []
    axes_syms = S == :cylindrical ? [:r, :φ, :z] : [:x, :y, :z]
    all_imps = [get_important_points(det, s) for s in axes_syms]
    for g in object.geometry_positive
        stepsizes = T[]
        for (iax, sax) in enumerate(axes_syms)
            imps_ax = all_imps[iax]
            ax::Vector{T} = grid.axes[iax].ticks
            imin::Int = searchsortednearest(ax, minimum(imps_ax))
            imax::Int = searchsortednearest(ax, maximum(imps_ax))

            ax = ax[imin:imax]
            delete_inds::Vector{Int} = Int[]
            for imp in imps_ax
                push!(delete_inds, searchsortednearest(ax, imp))
            end
            unique!(sort!(delete_inds))
            deleteat!(ax, delete_inds)
            stepsize::T = length(ax) <= 1 ? T(1) : geom_round((minimum(diff(ax)) / 4))
            imps_g = get_important_points(g, Val(sax))
            unique!(sort!(imps_g))
            if length(imps_g) > 1
                min_imps_g::T = minimum(diff(imps_g)) / 4
                if min_imps_g < stepsize
                    stepsize = geom_round(min_imps_g)
                end
            end
            imps_g_min::T, imps_g_max::T = if length(imps_g) > 0
                minimum(imps_g), maximum(imps_g)
            else
                0, 0
            end
            Δg::T = (imps_g_max - imps_g_min)
            if Δg > 0
                g_imin::Int = searchsortednearest(ax, imps_g_min)
                g_imax::Int = searchsortednearest(ax, imps_g_max)
                n_grid_points::Int = g_imax - g_imin + 1
                n::Int = Int(round(Δg / stepsize))
                if n > 2 * n_grid_points
                    stepsize = Δg / (4 * n_grid_points)
                end
            else
                stepsize = 1
            end
            if iszero(stepsize) stepsize = 1 end
            push!(stepsizes, stepsize)
        end
        append!(samples, filter( x-> x in object.geometry, sample(g, stepsizes)) )
    end
    object_gridpoints = unique!([find_closest_gridpoint(sample_point, grid) for sample_point in samples])
    return object_gridpoints
end

function paint_object(det::SolidStateDetector{T}, object::AbstractObject{T}, grid::CylindricalGrid{T}, ::Val{:φ}, φ::T )  where {T <: SSDFloat}
    closest_φ_idx=searchsortednearest(grid.axes[2].ticks, φ)
    stepsize::Vector{T}= [minimum(diff(grid.axes[1].ticks)), IntervalSets.width(grid.axes[2].interval) == 0.0 ? 0.05236 : minimum(diff(grid.axes[2].ticks)), minimum(diff(grid.axes[3].ticks))]
    stepsize /= 2
    samples = filter(x-> x in object.geometry, vcat([sample(g, stepsize) for g in object.geometry_positive]...))
    object_gridpoints = unique!([find_closest_gridpoint(sample_point,grid) for sample_point in samples])
    return filter(x -> x[2]==closest_φ_idx, object_gridpoints)
end
function paint_object(det::SolidStateDetector{T}, object::AbstractObject{T}, grid::CylindricalGrid{T}, ::Val{:r}, r::T )  where {T <: SSDFloat}
    return CartesianPoint{T}[]
end
function paint_object(det::SolidStateDetector{T}, object::AbstractObject{T}, grid::Grid{T}, ::Val{:z}, z::T )  where {T <: SSDFloat}
    return CartesianPoint{T}[]
end
function paint_object(det::SolidStateDetector{T}, object::AbstractObject{T}, grid::CartesianGrid{T}, ::Val{:x}, x::T )  where {T <: SSDFloat}
    return CartesianPoint{T}[]
end
function paint_object(det::SolidStateDetector{T}, object::AbstractObject{T}, grid::CartesianGrid{T}, ::Val{:y}, y::T )  where {T <: SSDFloat}
    return CartesianPoint{T}[]
end


# TODO: replace with Mjolnir version.

abstract type Diff end
struct UndefinedChange <: Diff end
struct NoChange <: Diff end
struct SetDiff{V} <: Diff
    added::Set{V}
    deleted::Set{V}
end
struct DictDiff{K, V} <: Diff
    added::AbstractDict{K, V}
    deleted::AbstractSet{K}
    updated::AbstractDict{K, Diff}
end
struct VectorDiff <: Diff
    new_length::Int
    prev_length::Int
    updated::Dict{Int,Diff}
    VectorDiff(nl::Int, pl::Int, updated::Dict{Int, Diff}) = new(nl, pl, updated)
end
struct IntDiff <: Diff
    difference::Int # new - old
end

struct Diffed{V, DV}
    val::V
    diff::DV
end

# Register diff propagation with Mjolnir
#struct DiffPrimitives end
#
#@abstract DiffPrimitives function *(d1::Partial{<:Diffed{V, NoChange}}, d2::Partial{<:Diffed{V, NoChange}}) where V
#    Diffed(d1.value.val * d2.value.val, NoChange())
#end
#@abstract DiffPrimitives function +(d1::Partial{<:Diffed{V, NoChange}}, d2::Partial{<:Diffed{V, NoChange}}) where V
#    Diffed(d1.value.val + d2.value.val, NoChange())
#end
#@abstract DiffPrimitives function -(d1::Partial{<:Diffed{V, NoChange}}, d2::Partial{<:Diffed{V, NoChange}}) where V
#    Diffed(d1.value.val - d2.value.val, NoChange())
#end
#@abstract DiffPrimitives function /(d1::Partial{<:Diffed{V, NoChange}}, d2::Partial{<:Diffed{V, NoChange}}) where V
#    Diffed(d1.value.val / d2.value.val, NoChange())
#end


module Ext
using ..Mosek

immutable VarByIndex index :: Int32 end
immutable VarByName name :: String end
immutable BarvarByIndex index :: Int64 end
immutable BarvarByName name :: String end
immutable ConByIndex index :: Int32 end
immutable ConByName  name  :: String end
immutable ConeByIndex index :: Int32 end
immutable ConeByName  name  :: String end

immutable Variable
    t::Mosek.Task
    index :: Int32
end

immutable SemidefiniteVariable
    t::Mosek.Task
    index :: Int32
end

immutable Constraint
    t::Mosek.Task
    index :: Int32
end

immutable ConeConstraint
    t::Mosek.Task
    index :: Int32
end

immutable Obj
end

immutable Objective
    t::Mosek.Task
end

immutable Symmat
    index :: Int
end

Base.getindex(t::Mosek.Task, index :: VarByIndex) = Variable(t,index.index)
function Base.getindex(t::Mosek.Task, name :: VarByName)
    ok,index = getvarnameindex(t,name.name)
    if ok > 0
        getindex(t,VarByIndex(index))
    else
        throw(KeyError("'$name' not found"))
    end
end

Base.getindex(t::Mosek.Task, index :: BarvarByIndex) = SemidefiniteVariable(t,index.index)
function Base.getindex(t::Mosek.Task, name :: BarvarByName)
    ok,index = getbarvarnameindex(t,name.name)
    if ok > 0
        getindex(t,BarvarByIndex(index))
    else
        throw(KeyError("'$name' not found"))
    end
end

Base.getindex(t::Mosek.Task, index :: ConByIndex) = Constraint(t,index.index)
function Base.getindex(t::Mosek.Task, name :: ConByName)
    ok,index = getconnameindex(t,name.name)
    if ok > 0
        getindex(t,ConByIndex(index))
    else
        throw(KeyError("'$name' not found"))
    end
end

Base.getindex(t::Mosek.Task, index :: ConeByIndex) = ConeConstraint(t,index.index)
function Base.getindex(t::Mosek.Task, name :: ConeByName)
    ok,index = getconenameindex(t,name.name)
    if ok > 0
        getindex(t,ConeByIndex(index))
    else
        throw(KeyError("'$name' not found"))
    end
end

Base.getindex(t::Mosek.Task, index :: Obj) = Objective(t)
function Base.getindex(t::Mosek.Task, index :: Symmat)
    dim,nz,tp = getsymmatinfo(t,index.index)
    subi,subj,valij = getsparsesymmat(t,index.index)
    for i in 1:nz
        if subi[i] != subj[i]
            push!(subi,subj[i])
            push!(subj,subi[i])
            push!(valij,valij[i])
        end
    end
    sparse(subi,subj,valij,dim,dim)
end

function Base.show(f::IO, var :: Variable)
    bk,bl,bu = getvarbound(var.t,var.index)
    name = getvarname(var.t,var.index)

    if bk == MSK_BK_FX
        print(f,"Variable('$name' = $bl)")
    else
        blstr =
            if     bk == MSK_BK_LO || bk == MSK_BK_RA "[$bl"
            else   "]-Inf"
            end
        bustr =
            if     bk == MSK_BK_UP || bk == MSK_BK_RA "$bu]"
            else   "+Inf["
            end

        print(f,"Variable('$name' ∈ $blstr;$bustr)")
    end
end

function Base.show(f::IO, var :: SemidefiniteVariable)
    if var.index > 0 && var.index < getnumbarvar(var.t)
        dim,ns,tp = getsymmatinfo(var.t,var.index)
        name = mkbarvarname(var.t,var.index)

        print(f,"SemidefiniteVariable('$name' ∈ S($dim))")
    else
        error("Invalid reference")
    end
end

function Base.show(f::IO, cone :: ConeConstraint)
    name = getconename(cone.t,cone.index)
    ct,cp,nummem,submem = getcone(cone.t,cone.index)

    dom =
        if ct == MSK_CT_QUAD "Q"
        elseif ct == MSK_CT_RQUAD "Q_r"
        else "?"
        end

    varnames = join(map(
        j -> let varname = getvarname(cone.t,j)
               if length(varname) == 0
                 "#x$j"
               else
                 Mosek.escapename(varname)
               end
             end,
        submem),",")

    print(f,"ConeByIndex('$name': ($varnames) ∈ $dom)")
end

mkvarname(t,j) =
    let n = getbarvarname(t,j)
        if length(n) == 0
            "#x$j"
        else
            Mosek.escapename(n)
        end
    end

mkbarvarname(t,j) =
    let n = getbarvarname(t,j)
        if length(n) == 0
            "#X̂$j"
        else
            Mosek.escapename(n)
        end
    end

function Base.show(f::IO, con :: Objective)
    t = con.t
    name = getobjname(t)

    print(f,"Objective('$name': ")

    for j in 1:getnumvar(t)
        cj = getcj(t,j)
        if cj < 0 || cj > 0
            varname = Mosek.escapename(getvarname(t,j))
            if length(varname) == 0
                varname = "#x$j"
            end
            print(f,"$(Mosek.fmtcof(cj)) $varname ")
        end
    end

    if getnumbarvar(t) > 0
        _,barcidx = getbarcsparsity(t)

        for idx in barcidx
            (barcj,num,sub,w) = getbarcidx(t,idx)

            barvarname = mkbarvarname(t,barcj)

            if num == 1
                print(f,"$(Mosek.fmtcof(w[1])) #MX$(sub[1]) ⋅ $barvarname) ")
            elseif num > 1
                print(f,"(")
                printf(f,"$(Mosek.fmtcof(w[1])) #MX$(sub[1])")
                print(f,") ⋅ $barvarname ")
            end
        end
    end

    numqobjnz = getnumqobjnz(t)
    if numqobjnz > 0
        (nnz,qcsubi,qcsubj,qcval) = getqobj(t)
        for k in 1:nnz
            if qcsubi[k] == qcsubj[k]
                ni = mkvarname(t,qcsubi[k])
                print(f,"$(Mosek.fmtcof(qcval[k])) $ni ^ 2")
            else
                ni = mkvarname(t,qcsubi[k])
                nj = mkvarname(t,qcsubj[k])
                print(f,"$(Mosek.fmtcof(qcval[k])) $ni ⋅ $nj")
            end
        end
    end

    print(f,")")
end

function Base.show(f::IO, con :: Constraint)
    t = con.t
    i = con.index
    bk,bl,bu = getconbound(t,i)
    name = getconname(t,i)

    print(f,"Constraint('$name': ")
    if bk == MSK_BK_FX
        print(f,"$bl = ")
    else
        blstr =
            if     bk == MSK_BK_LO || bk == MSK_BK_RA "[$bl"
            else   "]-Inf"
            end
        bustr =
            if     bk == MSK_BK_UP || bk == MSK_BK_RA "$bu]"
            else   "+Inf["
            end

        print(f,"$blstr;$bustr ∋ ")
    end

    (nzi,subi,vali) = getarow(t,Int32(i))

    for k in 1:nzi
        varname = Mosek.escapename(getvarname(t,subi[k]))
        if length(varname) == 0
            varname = "#x$(subi[k])"
        end
        print(f,"$(Mosek.fmtcof(vali[k])) $varname ")
    end

    numqconnz = getnumqconknz(t,i)
    if numqconnz > 0
        (nnz,qcsubi,qcsubj,qcval) = getqconk(t,i)
        for k in 1:nnz
            if qcsubi[k] == qcsubj[k]
                ni = mkvarname(t,qcsubi[k])
                print(f,"$(Mosek.fmtcof(qcval[k])) $ni ^ 2")
            else
                ni = mkvarname(t,qcsubi[k])
                nj = mkvarname(t,qcsubj[k])
                print(f,"$(Mosek.fmtcof(qcval[k])) $ni ⋅ $nj")
            end
        end
    end

    _,baraidx = getbarasparsity(t)
    if length(baraidx) > 0
        # count
        numbarnzi = count(idx -> getbaraidxij(t,idx)[1] == i, baraidx)
        idxs = Vector{Int}(numbarnzi)
        k = 0
        for idx in baraidx
            (barai,baraj) = getbaraidxij(t,idx)
            if barai == i
                k += 1
                idxs[k] = idx
            end
        end

        for idx in idxs
            (barai,baraj,num,sub,w) = getbaraidx(t,idx)
            barvarname = mkbarvarname(t,baraj)

            if num == 1
                print(f,"$(Mosek.fmtcof(w[1])) #MX$(sub[1]) ⋅ $barvarname ")
            elseif num > 1
                print(f,"(")
                printf(f,"$(Mosek.fmtcof(w[1])) #MX$(sub[1])")
                print(f,") ⋅ $barvarname ")
            end
        end
    end

    print(f,")")
end

function findvars(t::Mosek.Task, name::String)
    res = VarByIndex[]
    for j in 1:getnumvar(t)
        if getvarname(t,j) == name
            push!(res,VarByIndex(j))
        end
    end
    res
end

function findvars(t::Mosek.Task, regex::Regex)
    res = VarByIndex[]
    for j in 1:getnumvar(t)
        if ismatch(regex,getvarname(t,j))
            push!(res,VarByIndex(j))
        end
    end
    res
end

function findcons(t::Mosek.Task, name::String)
    res = ConByIndex[]
    for j in 1:getnumcon(t)
        if getconname(t,j) == name
            push!(res,ConByIndex(j))
        end
    end
    res
end

function findcons(t::Mosek.Task, regex::Regex)
    res = ConByIndex[]
    for j in 1:getnumcon(t)
        if ismatch(regex,getconname(t,j))
            push!(res,ConByIndex(j))
        end
    end
    res
end

function findcones(t::Mosek.Task, name::String)
    res = ConeByIndex[]
    for j in 1:getnumcone(t)
        if getconename(t,j) == name
            push!(res,ConeByIndex(j))
        end
    end
    res
end

function findcones(t::Mosek.Task, regex::Regex)
    res = ConeByIndex[]
    for j in 1:getnumcone(t)
        if ismatch(regex,getconename(t,j))
            push!(res,ConeByIndex(j))
        end
    end
    res
end

Var{I <: Integer}(index::I) = VarByIndex(convert(Int32,index))
Var(name::String) = VarByName(name)

Barvar{I <: Integer}(index::I) = BarvarByIndex(convert(Int32,index))
Barvar(name::String) = BarvarByName(name)

Con{I <: Integer}(index::I) = ConByIndex(convert(Int32,index))
Con(name::String) = ConByName(name)

Cone{I <: Integer}(index::I) = ConeByIndex(convert(Int32,index))
Cone(name::String) = ConeByName(name)

export Var,Barvar,Con,Cone,Obj,Symmat,Barvar,findvars,findcons,findcones

end


"""
`module  JAC.PairAnnihilation1Photon`  ... a submodel of JAC that contains all methods for computing positron-bound-electron pair annihilation
                                           (PEPA) with single-photon emission cross sections and rates; e^+ + |i(N)> --> |f(N-1)> + photon;
                                           it is using JAC, JAC.ManyElectron, JAC.Radial.
"""
module PairAnnihilation1Photon

    using JAC, JAC.ManyElectron, JAC.Radial
    global JAC_counter = 0

    """
    `struct  PairAnnihilation1Photon.Settings`  ... defines a type for the details and parameters in computing positron-bound-electron pair
             annihilation (PEPA) with single-photon emission cross sections and rates; e^+ + |i(N)> --> |f(N-1)> + photon.

        + multipoles              ::Array{EmMultipole}           ... Specifies the multipoles of the radiation field that are to be included.
        + gauges                  ::Array{UseGauge}              ... Specifies the gauges to be included into the computations.
        + positronEnergies        ::Array{Float64,1}             ... List of positron energies.
        + printBeforeComputation  ::Bool                         ... True, if all energies and lines are printed before their evaluation.
        + selectLines             ::Bool                         ... True, if lines are selected individually for the computations.
        + selectedLines           ::Array{Tuple{Int64,Int64},1}  ... List of lines, given by tupels (inital-level, final-level).
    """
    struct Settings
        multipoles                ::Array{EmMultipole}
        gauges                    ::Array{UseGauge}
        positronEnergies          ::Array{Float64,1}
        printBeforeComputation    ::Bool
        selectLines               ::Bool
        selectedLines             ::Array{Tuple{Int64,Int64},1}
    end

    """
    `JAC.PairAnnihilation1Photon.Settings()`  ... constructor for the default values of pair-annihilation photon line computations
    """
    function Settings()
        Settings(EmMultipole[], UseGauge[], Float64[], false, false, Tuple{Int64,Int64}[])
    end

     # `Base.show(io::IO, settings::PairAnnihilation1Photon.Settings)`
     #		... prepares a proper printout of the variable settings::PairAnnihilation1Photon.Settings.
    function Base.show(io::IO, settings::PairAnnihilation1Photon.Settings)
        println(io, "multipoles:               $(settings.multipoles)  ")
        println(io, "gauges:                   $(settings.gauges)  ")
        println(io, "positronEnergies:         $(settings.positronEnergies)  ")
        println(io, "printBeforeComputation:   $(settings.printBeforeComputation)  ")
        println(io, "selectLines:              $(settings.selectLines)  ")
        println(io, "selectedLines:            $(settings.selectedLines)  ")
    end

    """
    `struct  PairAnnihilation1Photon.Channel`  ... defines a type for a positron-bound-electron pair annihilation (PEPA) with single-photon
             emission channel that specifies all quantum numbers, phases and amplitudes.

        + multipole      ::EmMultipole          ... Multipole of the photon absorption.
        + gauge          ::EmGauge              ... Gauge for dealing with the (coupled) radiation field.
        + kappa          ::Int64                ... partial-wave of the incoming free positron
        + symmetry       ::LevelSymmetry        ... total angular momentum and parity of the scattering state
        + phase          ::Float64              ... phase of the partial wave
        + amplitude      ::Complex{Float64}     ... PairAnnihilation1PhotonChannel amplitude associated with the given channel.
   """
    struct  Channel
        multipole        ::EmMultipole
        gauge            ::EmGauge
        kappa            ::Int64
        symmetry         ::LevelSymmetry
        phase            ::Float64
        amplitude        ::Complex{Float64}
    end

    """
    `struct  PairAnnihilation1Photon.Line`  ... defines a type for a positron-bound-electron pair-annihilation (photon) line that may
                                                include the definition of channels.

        + initialLevel   ::Level                 ... initial-(state) level
        + finalLevel     ::Level                 ... final-(state) level
        + positronEnergy ::Float64                ... Energy of the (incoming free) positron.
        + photonEnergy   ::Float64                ... Energy of the emitted photon.
        + crossSection   ::EmProperty             ... Cross section for this pair-annihilation (photon) line.
        + hasChannels    ::Bool                   ... Determines whether the individual (sub-) channels are defined in terms of their
                                                      free-positron energy, kappa, multipole, etc., or not.
        + channels       ::Array{PairAnnihilation1Photon.Channel,1}  ... List of PairAnnihilation1Photon.Channels of this line.
    """
    struct  Line
        initialLevel     ::Level
        finalLevel       ::Level
        positronEnergy   ::Float64
        photonEnergy     ::Float64
        crossSection     ::EmProperty
        hasChannels      ::Bool
        channels         ::Array{PairAnnihilation1Photon.Channel,1}
    end

    """
    `JAC.PairAnnihilation1Photon.Line()`  ... 'empty' constructor for a pair-annihilation (photon) line between a specified initial and
                                              final level.
    """
    function Line()
        Line(Level(), Level(), 0., 0., EmProperty(0., 0.), false, PairAnnihilation1Photon[] )
    end

    # `Base.show(io::IO, line::PairAnnihilation1Photon.Line)`  ... prepares a proper printout of the variable line::PairAnnihilation1Photon.Line.
    function Base.show(io::IO, line::PairAnnihilation1Photon.Line)
        println(io, "initialLevel:      $(line.initialLevel)  ")
        println(io, "finalLevel:        $(line.finalLevel)  ")
        println(io, "positronEnergy:    $(line.positronEnergy)  ")
        println(io, "photonEnergy:      $(line.photonEnergy)  ")
        println(io, "crossSection:      $(line.crossSection)  ")
        println(io, "hasChannels:       $(line.hasChannels)  ")
        println(io, "channels:          $(line.channels)  ")
    end

    """
    `JAC.PairAnnihilation1Photon.computeAmplitudesProperties(line::PairAnnihilation1Photon.Line, grid::Radial.Grid,
                                                             settings::PairAnnihilation1Photon.Settings)`  ... to compute all amplitudes and
         properties of the given line; a line::PairAnnihilation1Photon.Line is returned for which the amplitudes and properties are now evaluated.
    """
    function  computeAmplitudesProperties(line::PairAnnihilation1Photon.Line, grid::Radial.Grid, settings::PairAnnihilation1Photon.Settings)
        global JAC_counter
        newChannels = PairAnnihilation1Photon.Channel[]
        for channel in line.channels
            # Generate a continuum orbital
            JAC_counter = JAC_counter + 1
            if   JAC_counter < 20   println("PairAnnihilation1Photon.computeAmplitudes..-aa: warning ... no cont. orbital is generated.")  end
            phase  = 0.
            # Define a proper continuum basis from the initialLevel.basis and the continuum orbital
            JAC_counter = JAC_counter + 1
            if   JAC_counter < 20   println("PairAnnihilation1Photon.computeAmplitudes..-ab: warning ... no coninuum basis is yet generated.") end
            # Compute the transition matrix for the continuum and the initial-state basis
            JAC_counter = JAC_counter + 1
            if   JAC_counter < 20   println("PairAnnihilation1Photon.computeAmplitudes..-ac: warning ... no transition matrix is computed.") end
            # matrix    = JAC.PhotoIonization.computeMatrix(channel.multipole, channel.gauge, line.omega, line.finalLevel.basis,
            #                                               line.initialLevel.basis, grid, settings)
            # amplitude = line.finalLevel.mc * matrix * line.initialLevel.mc
            amplitude = 1.0
            push!( newChannels, PairAnnihilation1Photon.Channel( channel.multipole, channel.gauge, channel.kappa, channel.symmetry,
                                                                 phase, amplitude) )
        end
        # Calculate the photonrate and angular beta if requested
        JAC_counter = JAC_counter + 1
        if   JAC_counter < 20   println("PairAnnihilation1Photon.computeAmplitudesProperties-ba: warning ... cs set to -1.") end
        crossSection = EmProperty(-1., -1.)
        line = PairAnnihilation1Photon.Line( line.initialLevel, line.finalLevel, line.positronEnergy, line.photonEnergy,
                                             crossSection, true, newChannels)
        return( line )
    end

    """
    `JAC.PairAnnihilation1Photon.computeLines(finalMultiplet::Multiplet, initialMultiplet::Multiplet, grid::Radial.Grid,
                                              settings::PairAnnihilation1Photon.Settings; output::Bool=true)`  ... to compute the
         pair-annihilation single-photon emission amplitudes and all properties as requested by the given settings. A list of
         lines::Array{PairAnnihilation1Photon.Lines} is returned.
    """
    function  computeLines(finalMultiplet::Multiplet, initialMultiplet::Multiplet, grid::Radial.Grid, settings::PairAnnihilation1Photon.Settings;
                           output::Bool=true)
        lines = JAC.PairAnnihilation1Photon.determineLines(finalMultiplet, initialMultiplet, settings)
        # Display all selected lines before the computations start
        if  settings.printBeforeComputation    JAC.PairAnnihilation1Photon.displayLines(lines)    end
        # Calculate all amplitudes and requested properties
        newLines = PairAnnihilation1Photon.Line[]
        for  line in lines
            newLine = JAC.PairAnnihilation1Photon.computeAmplitudesProperties(line, grid, settings)
            push!( newLines, newLine)
        end
        # Print all results to screen
        JAC.PairAnnihilation1Photon.displayResults(lines)
        #
        if    output    return( lines )
        else            return( nothing )
        end
    end

    """
    `JAC.PairAnnihilation1Photon.determineChannels(finalLevel::Level, initialLevel::Level, settings::PairAnnihilation1Photon.Settings)`
         ... to determine a list of pair-annihilation single-photon emission Channel for a transitions from the initial to final level and by
         taking into account the particular settings of for this computation; an Array{PairAnnihilation1Photon.Channel,1} is returned.
    """
    function determineChannels(finalLevel::Level, initialLevel::Level, settings::PairAnnihilation1Photon.Settings)
        channels = PairAnnihilation1Photon.Channel[];
        symi = LevelSymmetry(initialLevel.J, initialLevel.parity);    symf = LevelSymmetry(finalLevel.J, finalLevel.parity)
        for  mp in settings.multipoles
            for  gauge in settings.gauges
                symList = JAC.AngularMomentum.allowedMultipoleSymmetries(symi, mp)
                ##x println("mp = $mp   symi = $symi   symList = $symList")
                for  symt in symList
                    kappaList = JAC.AngularMomentum.allowedKappaSymmetries(symt, symf)
                    for  kappa in kappaList
                        # Include further restrictions if appropriate
                        if     string(mp)[1] == 'E'  &&   gauge == Basics.UseCoulomb
                            push!(channels, PairAnnihilation1Photon.Channel(mp, JAC.Coulomb,   kappa, symt, 0., Complex(0.)) )
                        elseif string(mp)[1] == 'E'  &&   gauge == Basics.UseBabushkin
                            push!(channels, PairAnnihilation1Photon.Channel(mp, JAC.Babushkin, kappa, symt, 0., Complex(0.)) )
                        elseif string(mp)[1] == 'M'
                            push!(channels, PairAnnihilation1Photon.Channel(mp, JAC.Magnetic,  kappa, symt, 0., Complex(0.)) )
                        end
                    end
                end
            end
        end
        return( channels )
    end

    """
    `JAC.PairAnnihilation1Photon.determineLines(finalMultiplet::Multiplet, initialMultiplet::Multiplet,
                                                settings::PairAnnihilation1Photon.Settings)` ... to determine a list of
         PairAnnihilation1Photon.Line's for transitions between levels from the initial- and final-state multiplets, and  by taking into account
         the particular selections and settings for this computation; an Array{PairAnnihilation1Photon.Line,1} is returned. Apart from the
         level specification, all physical properties are set to zero during the initialization process.
    """
    function  determineLines(finalMultiplet::Multiplet, initialMultiplet::Multiplet, settings::PairAnnihilation1Photon.Settings)
        if    settings.selectLines    selectLines   = true;   selectedLines = Basics.determine("selected lines", settings.selectedLines)
        else                          selectLines   = false
        end

        lines = PairAnnihilation1Photon.Line[]
        for  i = 1:length(initialMultiplet.levels)
            for  f = 1:length(finalMultiplet.levels)
                if  selectLines  &&  !(haskey(selectedLines, (i,f)) )    continue   end
                for  ep in settings.positronEnergies
                    omega = ep - (finalMultiplet.levels[f].energy - initialMultiplet.levels[i].energy) + 2* Defaults.getDefaults("mc^2")
                    if  omega < 0    continue   end

                    channels = JAC.PairAnnihilation1Photon.determineChannels(finalMultiplet.levels[f], initialMultiplet.levels[i], settings)
                    push!( lines, PairAnnihilation1Photon.Line(initialMultiplet.levels[i], finalMultiplet.levels[f], ep, omega,
                                                               EmProperty(0., 0.), true, channels) )
                end
            end
        end
        return( lines )
    end

    """
    `JAC.PairAnnihilation1Photon.displayLines(lines::Array{PairAnnihilation1Photon.Line,1})`  ... to display a list of lines and channels that
         have been selected due to the prior settings. A neat table of all selected transitions and energies is printed but nothing is returned
         otherwise.
    """
    function  displayLines(lines::Array{PairAnnihilation1Photon.Line,1})
        println(" ")
        println("  Selected pair-annihilation single-photon emission lines:")
        println(" ")
        println("  ", JAC.TableStrings.hLine(175))
        sa = "  ";   sb = "  "
        sa = sa * JAC.TableStrings.center(18, "i-level-f"; na=2);                                sb = sb * JAC.TableStrings.hBlank(20)
        sa = sa * JAC.TableStrings.center(18, "i--J^P--f"; na=4);                                sb = sb * JAC.TableStrings.hBlank(22)
        sa = sa * JAC.TableStrings.center(38, "Energies  " * JAC.TableStrings.inUnits("energy"); na=4);
        sb = sb * JAC.TableStrings.center(38, " i--f       positron      omega"; na=4)
        sa = sa * JAC.TableStrings.flushleft(57, "List of multipoles, gauges, kappas and total symmetries"; na=4)
        sb = sb * JAC.TableStrings.flushleft(57, "partial (multipole, gauge, total J^P)                  "; na=4)
        println(sa);    println(sb);    println("  ", JAC.TableStrings.hLine(175))
        #
        for  line in lines
            sa  = "  ";    isym = LevelSymmetry( line.initialLevel.J, line.initialLevel.parity)
                           fsym = LevelSymmetry( line.finalLevel.J,   line.finalLevel.parity)
            sa = sa * JAC.TableStrings.center(18, JAC.TableStrings.levels_if(line.initialLevel.index, line.finalLevel.index); na=2)
            sa = sa * JAC.TableStrings.center(18, JAC.TableStrings.symmetries_if(isym, fsym); na=4)
            energy = line.initialLevel.energy - line.finalLevel.energy
            sa = sa * @sprintf("%.5e", Defaults.convertUnits("energy: from atomic", energy))              * "  "
            sa = sa * @sprintf("%.5e", Defaults.convertUnits("energy: from atomic", line.positronEnergy)) * "  "
            sa = sa * @sprintf("%.5e", Defaults.convertUnits("energy: from atomic", line.photonEnergy))   * "    "
            kappaMultipoleSymmetryList = Tuple{Int64,EmMultipole,EmGauge,LevelSymmetry}[]
            for  i in 1:length(line.channels)
                push!( kappaMultipoleSymmetryList, (line.channels[i].kappa, line.channels[i].multipole, line.channels[i].gauge,
                                                    line.channels[i].symmetry) )
            end
            wa = JAC.TableStrings.kappaMultipoleSymmetryTupels(85, kappaMultipoleSymmetryList)
            sb = sa * wa[1];    println( sb )
            for  i = 2:length(wa)
                sb = JAC.TableStrings.hBlank( length(sa) ) * wa[i];    println( sb )
            end
        end
        println("  ", JAC.TableStrings.hLine(175))
        #
        return( nothing )
    end

    """
    `JAC.PairAnnihilation1Photon.displayResults(lines::Array{PairAnnihilation1Photon.Line,1})`  ... to list all results, energies, cross sections,
         etc. of the selected lines. A neat table is printed but nothing is returned otherwise.
    """
    function  displayResults(lines::Array{PairAnnihilation1Photon.Line,1})
        println(" ")
        println("  Pair-annihilation single-photon emission cross sections:")
        println(" ")
        println("  ", JAC.TableStrings.hLine(128))
        sa = "  ";   sb = "  "
        sa = sa * JAC.TableStrings.center(18, "i-level-f"; na=2);                                sb = sb * JAC.TableStrings.hBlank(20)
        sa = sa * JAC.TableStrings.center(18, "i--J^P--f"; na=4);                                sb = sb * JAC.TableStrings.hBlank(22)
        sa = sa * JAC.TableStrings.center(38, "Energies  " * JAC.TableStrings.inUnits("energy"); na=4);
        sb = sb * JAC.TableStrings.center(38, " i--f       positron      omega"; na=4)
        sa = sa * JAC.TableStrings.center(10, "Multipoles"; na=1);                              sb = sb * JAC.TableStrings.hBlank(13)
        sa = sa * JAC.TableStrings.center(30, "Cou -- Cross section -- Bab"; na=3)
        sb = sb * JAC.TableStrings.center(30, JAC.TableStrings.inUnits("cross section") * "          " *
                                              JAC.TableStrings.inUnits("cross section"); na=3)
        println(sa);    println(sb);    println("  ", JAC.TableStrings.hLine(128))
        #
        for  line in lines
            sa  = "  ";    isym = LevelSymmetry( line.initialLevel.J, line.initialLevel.parity)
                           fsym = LevelSymmetry( line.finalLevel.J,   line.finalLevel.parity)
            sa = sa * JAC.TableStrings.center(18, JAC.TableStrings.levels_if(line.initialLevel.index, line.finalLevel.index); na=2)
            sa = sa * JAC.TableStrings.center(18, JAC.TableStrings.symmetries_if(isym, fsym); na=4)
            energy = line.initialLevel.energy - line.finalLevel.energy
            sa = sa * @sprintf("%.5e", Defaults.convertUnits("energy: from atomic", energy))              * "  "
            sa = sa * @sprintf("%.5e", Defaults.convertUnits("energy: from atomic", line.positronEnergy)) * "  "
            sa = sa * @sprintf("%.5e", Defaults.convertUnits("energy: from atomic", line.photonEnergy))   * "     "
            multipoles = EmMultipole[]
            for  ch in line.channels
                multipoles = push!( multipoles, ch.multipole)
            end
            multipoles = unique(multipoles);   mpString = JAC.TableStrings.multipoleList(multipoles) * "          "
            sa = sa * JAC.TableStrings.flushleft(11, mpString[1:10];  na=2)
            sa = sa * @sprintf("%.6e", Defaults.convertUnits("cross section: from atomic", line.crossSection.Coulomb))     * "    "
            sa = sa * @sprintf("%.6e", Defaults.convertUnits("cross section: from atomic", line.crossSection.Babushkin))   * "    "
            println(sa)
        end
        println("  ", JAC.TableStrings.hLine(128))
        #
        return( nothing )
    end

end # module