__precompile__()
module Gym

using PyCall
using IntervalSets

export GymEnv, GymSpec, reset!, step!, action_space, observation_space, render, spec

mutable struct GymEnv
    pyenv
    done
    spec # specifications
end

function GymEnv(name::String)
    specs = [:id, :nondeterministic, :reward_threshold, :tags, :timestep_limit, :trials]
    pyenv = gym.make(name)
    pyspec = pyenv[:spec]
    return GymEnv(pyenv, true,  Dict(s => pyspec[s] for s in specs))
end

spec(env::GymEnv) = env.spec

Base.srand(env::GymEnv, seed) = env.pyenv[:seed](seed)

Base.done(env::GymEnv) = env.done

reset!(env::GymEnv) = env.pyenv[:reset]()

function step!(env::GymEnv, action)
    return obs, r, env.done, info = env.pyenv[:step](action)
end

action_space(env::GymEnv) = pyset_to_julia(env.pyenv[:action_space])
observation_space(env::GymEnv) = pyset_to_julia(env.pyenv[:observation_space])

# use keyword argument close=true to close
render(env::GymEnv, args...; kws...) = env.pyenv[:render](args...; kws...)

function pyset_to_julia(A::PyObject)
    if haskey(A, :n)
        # choose from n actions
        return 0:A[:n]-1
    elseif haskey(A, :spaces)
        # array of action sets
        return ([pyset_to_julia(a) for a in A[:spaces]]...)
    elseif haskey(A, :high)
        # continuous interval
        return A[:low]..A[:high] # interval from IntervalSets
    # elseif haskey(A, :buttonmasks)
    #     # assumed VNC actions... keys to press, buttons to mask, and screen position
    #     # keyboard = DiscreteSet(A[:keys])
    #     keyboard = KeyboardActionSet(A[:keys])
    #     buttons = DiscreteSet(Int[bm for bm in A[:buttonmasks]])
    #     width,height = A[:screen_shape]
    #     mouse = MouseActionSet(width, height, buttons)
    #     TupleSet(keyboard, mouse)
    # elseif haskey(A, :actions)
    #     # Hardcoded
    #     TupleSet(DiscreteSet(A[:actions]))
    else
        @show A
        @show keys(A)
        error("Unknown set type: $A")
    end
end

function __init__()
    global const gym = PyCall.pywrap(PyCall.pyimport("gym"))
end

end # module


println("Dv) Test of the PairAnnihilation1Photon module with ASF from an internally generated initial- and final-state multiplet.")

@warn("\n\n !!! This example does not work properly at present !!! \n\n")
#
wa = Atomic.Computation(Atomic.Computation(), name="xx", grid=JAC.Radial.Grid(true), nuclearModel=Nuclear.Model(36.) )

wb = perform(wa)


module Model

using LightGraphs
using GraphIO

using DataStructures
using SparseArrays
using Optim
using LineSearches
using StatsBase
using Statistics
using JSON
using Dates
using Serialization
import Optim: converged

include("./structures.jl")
include("./nodemap.jl")

setprecision(BigFloat, 64)

mutable struct GraphModel
    dataset::String # constant
    structure_alphabet_size::Int64 # constant
    G::LightGraphs.SimpleGraph{Int64} # constant
    A::SparseArrays.SparseMatrixCSC{Int64,Int64} # constant
    n::Int64 # constant
    m::Int64 # constant
    maximum_m_simple::Int64 # constant
    maximum_m_loopy::Int64 # constant
    lambdas::Array{BigFloat,1} # growing at end (and values changing by optimization)
    constraint_values::Array{BigFloat,1} # growing at end
    micro_structures::Array{Union{Bool, Set{Tuple{Int64,Int64}}},1} # growing at end
    edge_lambda_indices::SortedDict{Tuple{Int64,Int64},Array{Int64,1}} # new edges inserted, growing at selected value ends
    edge_equivalence_classes::Dict{Set{Int64},Array{Tuple{Int64,Int64},1}} # currently reconstructed but could be smarter
    backup_optimization_result::Optim.MultivariateOptimizationResults # reconstructed
    last_optimization_result::Optim.MultivariateOptimizationResults # reconstructed
    entropy::BigFloat # changing
    equivalence_class_edge_probabilities::Dict{Set{Int64},BigFloat} # reconstructed
    surprise_matrix::SparseArrays.SparseMatrixCSC{BigFloat,Int64} # reconstructed
    surprise_matrix_transpose::SparseArrays.SparseMatrixCSC{BigFloat,Int64} # reconstructed
    macro_structures::Array{Structures.Structure} # growing at end
    macro_structure_description_lengths::Array{BigFloat} # growing at end
    edge_probability_description_length_maxent::BigFloat # changing
    structure_type_description_length::BigFloat # changing
    node_surprises::Array{Tuple{Int64,BigFloat},1} # reconstructed
    surprising_edges::Array{Tuple{Int64,Int64},1} # reconstructed
    start_description_length_maxent::BigFloat # constant
    total_description_length_maxent::BigFloat # changing
    description_lengths_over_time::Array{BigFloat,1} # growing at end
    parameters::Dict{String,Float64} # constant resp. set only upon saving
    io::IO # constant
    hub_spoke_edges::Set{Tuple{Int64,Int64}} # changing
    star_spoke_edges::Set{Tuple{Int64,Int64}} # changing
    zero_edges::Set{Tuple{Int64,Int64}} # changing
    last_hub_spoke_edges::Set{Tuple{Int64,Int64}} # changing
    last_star_spoke_edges::Set{Tuple{Int64,Int64}} # changing
    last_zero_edges::Set{Tuple{Int64,Int64}} # changing - note that we'll be a bit imprecise here: if a zero edge is discovered after already having been covered, it will remain also in its previous equivalence class, if any
    covered_nodes::Set{Int64} # growing
    function GraphModel(dataset::String, io=stdout)
        setup_graph!(new(), dataset, io)
    end
    function GraphModel(kwargs::Dict)
        M = new()
        for (k,v) in kwargs
            setproperty!(M, k, v)
        end
        return M
    end
end

Base.show(io::IO, M::GraphModel) = print(io, "Model for $(M.dataset):\n$(typeof(M.G)) with n = $(M.n), m=$(M.m)")
JSON.lower(M::GraphModel) = Dict(key=> getfield(M, key) for key in fieldnames(GraphModel) if key in
            (:dataset, :n, :m, :macro_structures, :macro_structure_description_lengths,
                :edge_probability_description_length_maxent,
                :start_description_length_maxent,
                :total_description_length_maxent,
                :description_lengths_over_time,
                :lambdas,
                :parameters,
                :structure_alphabet_size,
                :structure_type_description_length
            )
)

"""
helper providing basic logging
"""
function log_progress!(M::GraphModel, msg::String)
    println(M.io, now(), " - ", msg)
    flush(M.io)
end

"""
save model as jldb (serialization format) or json
"""
function save_model(M::GraphModel, directory::String; final=true, overwrite=true)
    path, file = splitdir(M.dataset)
    last_folder = splitdir(path)[end]
    file_base, ext = splitext(file)
    size_threshold = M.parameters["size_threshold"]
    stop_after_n_structures = M.parameters["stop_after_n_structures"]
    filepath = "$(directory)/$(last_folder)/$(file_base)_size-$(size_threshold)_max-$(stop_after_n_structures)"
    mkpath(splitdir(filepath)[1])
    if !final
        open(filepath*".jldb", "w") do io
            serialize(io, M)
        end
    else
        open(filepath*".json", "w") do io
            write(io, json(M, 4))
        end
        log_progress!(M, "Saved model to $(filepath).json")
    end
    mv("$(path)/$(file_base)-nodemap.csv", "$(directory)/$(last_folder)/$(file_base)-nodemap.csv", force=overwrite)
end

"""
restore model from json
"""
function restore_model(path_to_file::String)
    model_dict = JSON.parsefile(path_to_file)
    M = Model.GraphModel(model_dict["dataset"])
    structures = Array{Structures.Structure,1}()
    for sd in model_dict["macro_structures"]
        if sd["structure_type"] == "clique"
            structure = Structures.Clique(sd["n_edges_total"], Array{Int64,1}(sd["nodes"]))
        elseif sd["structure_type"] == "star"
            structure = Structures.Star(sd["n_edges_total"], sd["hub"], Array{Int64,1}(sd["spokes"]))
        elseif sd["structure_type"] == "biclique"
            structure = Structures.Biclique(sd["n_edges_total"], Array{Int64,1}(sd["left_nodes"]), Array{Int64,1}(sd["right_nodes"]), sd["n_edges_in_left"], sd["n_edges_in_right"])
        elseif sd["structure_type"] == "starclique"
            structure = Structures.Starclique(sd["n_edges_total"], Array{Int64,1}(sd["left_nodes"]), Array{Int64,1}(sd["right_nodes"]), sd["n_edges_in_left"], sd["n_edges_in_right"])
        # generic structures are already present
        else
            continue
        end
        push!(structures, structure)
    end
    add_macro_structures!(M, structures; lambdas=get!(model_dict, "lambdas", []))
    if model_dict["lambdas"] == []
        println("WARNING: Recomputing lambdas when restoring model - true edge probability description length for computed lambdas might differ from restored edge probability description length due to different optimization process...")
    end
    M.parameters["surprise_threshold"] = model_dict["parameters"]["surprise_threshold"]  === nothing ? Inf : model_dict["parameters"]["surprise_threshold"]
    M.parameters["size_threshold"] = model_dict["parameters"]["size_threshold"]  === nothing ? Inf : model_dict["parameters"]["size_threshold"]
    M.description_lengths_over_time = model_dict["description_lengths_over_time"]
    M.macro_structure_description_lengths = model_dict["macro_structure_description_lengths"]
    M.edge_probability_description_length_maxent = model_dict["edge_probability_description_length_maxent"]
    M.total_description_length_maxent = M.edge_probability_description_length_maxent + M.structure_type_description_length + sum(M.macro_structure_description_lengths)
    return M
end

"""
initial graph setup with first optimization
"""
function setup_graph!(M::GraphModel, dataset::String, io::IO)
    set_constant_terms!(M, dataset, io)
    M.lambdas = zeros(2)
    M.constraint_values = [M.m, 0]
    M.micro_structures = [true, Set([(i,i) for i=1:M.n])]
    M.edge_lambda_indices = SortedDict((i,i)=>[1,2] for i=1:M.n)
    M.edge_equivalence_classes = Dict(Set([1,2])=>[(i,i) for i=1:M.n])
    M.macro_structures = [Structures.GenericStructure("Overall number of edges", M.m), Structures.GenericStructure("Loop-freeness", 0)]
    # below, we add the cost of transmitting n to the cost of transmitting m (since it is not a proper structure but necessary to specify m)
    M.macro_structure_description_lengths = [Structures.MDL.universal_integer(M.n) + Structures.MDL.log2(M.maximum_m_loopy), 1.]
    M.structure_type_description_length = 0
    M.description_lengths_over_time = []
    M.star_spoke_edges = Set()
    M.hub_spoke_edges = Set()
    M.zero_edges = Set()
    M.last_star_spoke_edges = Set()
    M.last_hub_spoke_edges = Set()
    M.last_zero_edges = Set()
    M.covered_nodes = Set()
    update_variable_terms!(M)
    M.backup_optimization_result = deepcopy(M.last_optimization_result)
    M.start_description_length_maxent = M.total_description_length_maxent
    # the below is to account for the fact that we have two initial constraints but perform only one optimization
    append!(M.description_lengths_over_time, [M.start_description_length_maxent])
    M.parameters = Dict("surprise_threshold" => Inf, "size_threshold" => Inf)
    return M
end

"""
initial graph setup: set constant terms - NB: dataset now needs to be full path with extension
"""
function set_constant_terms!(M::GraphModel, dataset::String, io::IO)
    M.dataset = dataset
    M.structure_alphabet_size = 4 # cliques, stars, bicliques, starcliques
    M.io = io
    M.G = SimpleGraph(NodeMap.loadgraph(dataset, GraphIO.EdgeList.EdgeListFormat())[1])
    NodeMap.create_node_map(dataset)
    remove_loops!(M.G)
    M.A = LightGraphs.LinAlg.adjacency_matrix(M.G)
    M.n = nv(M.G)
    M.m = ne(M.G)
    M.maximum_m_simple = Structures.MDL.choose(M.n, 2)
    M.maximum_m_loopy = M.maximum_m_simple + M.n
end

"""
make graph loop-free
"""
function remove_loops!(G::SimpleGraph)
    for e in [e for e in edges(G) if src(e) == dst(e)]
        rem_edge!(G, e)
    end
end

"""
each step: update model (re-optimize, update description length, update surprises)
"""
function update_variable_terms!(M::GraphModel)
    log_progress!(M, "starting optimization")
    optimize_maxent!(M)
    log_progress!(M, "finished optimization")
    log_progress!(M, "updating description length")
    update_description_length!(M)
end

"""
optimization wrapper function
"""
function optimize_maxent!(M::Model.GraphModel)
    n_iter = 50
    td = TwiceDifferentiable(x -> lagrange_dual_equiv(x, M), M.lambdas; autodiff = :forward)
        result = optimize(
        td, M.lambdas,
        Newton(linesearch=LineSearches.BackTracking()), # the solver - using BackTracking instead of the default HagerZhang because that sometimes throws a weird assertion error
        Optim.Options(allow_f_increases=true,
            show_trace=false, show_every=100, iterations=n_iter, x_tol=1e-8, f_tol=1e-8, g_tol=1e-6)
    )
    M.last_optimization_result = result
    M.lambdas = M.last_optimization_result.minimizer
    M.entropy = M.last_optimization_result.minimum
    if !converged(result)
        log_progress!(M, "NO CONVERGENCE AFTER $(n_iter) ITERATIONS")
    end
end

"""
optimization computation: wrapper
"""
function lagrange_dual_equiv(lambdas, M::Model.GraphModel)
    return (get_first_term(lambdas, M) - get_second_term(lambdas, M))
end

"""
optimization computation: first term
"""
function get_first_term(lambdas, M::Model.GraphModel)
    #log_progress!(M, "computing first term in optimization")
    rest_lambda = lambdas[1]
    if isempty(M.edge_equivalence_classes)
        return M.maximum_m_loopy * log(get_normalizer(rest_lambda))
    else
        lambda_total_contribution = BigFloat(0)
        lambda_total_sizes = 0
        for equivalence_class_key in keys(M.edge_equivalence_classes)
            size = get_equiv_size(equivalence_class_key, M.edge_equivalence_classes)
            lambda_sum = get_lambda_sum_from_equiv(equivalence_class_key, lambdas)
            normalized_sum = get_normalizer(lambda_sum)
            contribution = log(normalized_sum)
            lambda_total_contribution += size * contribution
            lambda_total_sizes += size
        end
        rest_class_n = M.maximum_m_loopy - sum(lambda_total_sizes)
        rest_contribution = rest_class_n * log(get_normalizer(rest_lambda))
        return lambda_total_contribution + rest_contribution
    end
end

"""
optimization helper (normalization)
"""
function get_normalizer(lambda_sum)
    return (1 + exp(lambda_sum))
end

"""
optimization helper (size of equivalence class)
"""
function get_equiv_size(equivalence_class_key::Set{Int64}, equivalence_classes::Dict{Set{Int64},Array{Tuple{Int64,Int64},1}})
    return length(equivalence_classes[equivalence_class_key])
end

"""
optimization helper (lambda sum for equivalence class)
"""
function get_lambda_sum_from_equiv(equivalence_class_key::Set{Int64}, lambdas)
    lambda_sum = BigFloat(0)
    for idx in equivalence_class_key
        lambda_sum += lambdas[idx]
    end
    return lambda_sum
end

"""
optimization computation: second term
"""
function get_second_term(lambdas, M::Model.GraphModel)
    #log_progress!(M, "computing second term in optimization")
    second_term = BigFloat(0)
    for (lambda, constraint_value) in zip(lambdas, M.constraint_values)
        second_term += lambda * constraint_value
    end
    return second_term
end

"""
description length computation wrapper
"""
function update_description_length!(M::Model.GraphModel)
    update_equivalence_class_edge_probabilities!(M)
    update_edge_probability_description_length!(M)
    update_structure_type_description_length!(M)
    update_total_description_length!(M)
    # NB: the below gives misleading results when multiple structures are added at a time, e.g., during replay -> we have taken care of that in restore_model
    append!(M.description_lengths_over_time, M.total_description_length_maxent)
end

"""
description length helper: structure type encoding costs
"""
function update_structure_type_description_length!(M::Model.GraphModel)
    n_different_structures = length(M.macro_structures) - 2 # we don't count the generic macro structures as their type is equivalence_class_edge_probabilities
    M.structure_type_description_length = (Structures.MDL.universal_integer(n_different_structures + 1) + Structures.MDL.log2_choose(M.structure_alphabet_size + n_different_structures - 1, n_different_structures - 1)
    )
end

"""
description length helper: edge probabilities for equivalence classes
"""
function update_equivalence_class_edge_probabilities!(M::Model.GraphModel)
    probabilities = Dict{Array{Int64,1},BigFloat}(
        equivalence_class=>edge_probability_for_equiv(equivalence_class, M.lambdas) for equivalence_class in keys(M.edge_equivalence_classes))
    probabilities[[1,]] = edge_probability_for_equiv(Set([1,]), M.lambdas)
    M.equivalence_class_edge_probabilities = probabilities
end

"""
description length helper: edge probability for one equivalence class
"""
function edge_probability_for_equiv(equivalence_class, lambdas)
    if equivalence_class == Set([1,])
        lambda_sum = lambdas[1]
    else
        lambda_sum = get_lambda_sum_from_equiv(equivalence_class, lambdas)
    end
    edge_prop = exp(lambda_sum) / get_normalizer(lambda_sum)
    return edge_prop
end

"""
description length helper: surprise matrices update
"""
function update_surprise_matrices!(M::Model.GraphModel)
    # NB: we currently keep track only of surprising EDGES (else we get a non-sparse matrix :( )
    surprise_matrix = SparseMatrixCSC{BigFloat,Int64}(M.A)
    for idx in filter!(x->(x[1] < x[2]), findall(isone, M.A))
        edge = (idx[1],idx[2])
        # due to symmetry
        eq_class = Set(get(M.edge_lambda_indices, edge, [1,]))
        rest_class_probability = M.equivalence_class_edge_probabilities[Set([1,])]
        if eq_class == Set([1,])
            surprise_matrix[edge...] = 1 - rest_class_probability
        else
            surprise_matrix[edge...] = 0
        end
        surprise_matrix[reverse(edge)...] = 0
    end
    M.surprise_matrix = surprise_matrix
    M.surprise_matrix_transpose = sparse(transpose(M.surprise_matrix))
end

"""
description length helper: total description length of edge probabilities (error in MDL, L(D|M))
"""
function update_edge_probability_description_length!(M::Model.GraphModel)
    dl = 0.
    rest_size = M.maximum_m_loopy # because we have loop-freeness as a separate constraint
    n_guaranteed_edges = length(M.hub_spoke_edges) # guaranteed to be there and not appearing in any equivalence class
    n_zero_edges = length(M.zero_edges) # guaranteed to be not there and not appearing in any equivalence class
    rest_size -=  (n_guaranteed_edges + n_zero_edges)
    rest_edges = M.m - n_guaranteed_edges
    for (eq_class, edges) in M.edge_equivalence_classes
        size = length(edges)
        size_ones = length(filter(edge -> has_edge(M.G, edge...), edges))
        size_zeros = size - size_ones
        eq_class_probability = M.equivalence_class_edge_probabilities[eq_class]
        contribution = size_ones * -Structures.MDL.log2_zero(eq_class_probability) + size_zeros * -Structures.MDL.log2_zero(1 - eq_class_probability)
        dl += contribution
        rest_size -= size
        rest_edges -= size_ones
    end
    rest_probability = M.equivalence_class_edge_probabilities[Set([1,])]
    rest_contribution = rest_edges * -Structures.MDL.log2_zero(rest_probability) + (rest_size - rest_edges) * -Structures.MDL.log2_zero(1 - rest_probability)
    dl += rest_contribution
    M.edge_probability_description_length_maxent = dl
end

"""
description length helper: total description length (L(M) + L(D|M))
"""
function update_total_description_length!(M::Model.GraphModel)
    M.total_description_length_maxent = sum(M.macro_structure_description_lengths) + M.edge_probability_description_length_maxent + M.structure_type_description_length
end

"""
sanity check whether probability mass approximately sums to 1
"""
function sanity_check(M::Model.GraphModel)
    if !isempty(M.edge_equivalence_classes)
        equiv = sum([edge_probability_for_equiv(equivalence_class, M.lambdas) * get_equiv_size(equivalence_class, M.edge_equivalence_classes)
            for equivalence_class in keys(M.edge_equivalence_classes)])
        rest_size = M.maximum_m_loopy - sum([get_equiv_size(equivalence_class, M.edge_equivalence_classes)
            for equivalence_class in keys(M.edge_equivalence_classes)])
    else
        equiv = 0
        rest_size = M.maximum_m_loopy
    end
    rest = edge_probability_for_equiv([1,], M.lambdas)
    probability_mass_sum = equiv + rest * rest_size
    print("Sanity Check: probability mass sums to $(probability_mass_sum), expected $(M.m)\n")
    return abs(probability_mass_sum-M.m) <= 1.0e-2
end

"""
generate micro structures and constraints from clique candidate
"""
function generate_micro_structures(structure::Structures.Clique, M::Model.GraphModel)
    micro_structures = [Set([(n1,n2) for n1 in structure.nodes for n2 in structure.nodes if n1 < n2])]
    clique_edges_in_hs_edges = intersect(micro_structures[1], M.hub_spoke_edges)
    setdiff!(micro_structures[1], clique_edges_in_hs_edges)
    constraint_values = [structure.n_edges_total - length(clique_edges_in_hs_edges)]
    return micro_structures, constraint_values
end

"""
generate micro structures and constraints from star candidate
"""
function generate_micro_structures(structure::Structures.Star, M::Model.GraphModel)
    hub = structure.hub
    micro_structures = [Set([(hub < n ? (hub,n) : (n, hub)) for n in structure.spokes]),
        Set([(n1,n2) for n1 in structure.spokes for n2 in structure.spokes if (n1 < n2) && (n1,n2) ∉ M.star_spoke_edges])]
    hs_edges_in_hs_edges = intersect(micro_structures[1], M.hub_spoke_edges)
    setdiff!(micro_structures[1],hs_edges_in_hs_edges)
    ss_edges_in_hs_edges = intersect(micro_structures[end], M.hub_spoke_edges)
    setdiff!(micro_structures[end],ss_edges_in_hs_edges)
    constraint_values = [structure.n_nodes_in_spokes-length(hs_edges_in_hs_edges), structure.n_edges_in_spokes-length(ss_edges_in_hs_edges)] # note that this is theoretically wrong - we do it for performance reasons
    return micro_structures, constraint_values
end

"""
generate micro structures and constraints from biclique candidate
"""
function generate_micro_structures(structure::Union{Structures.Biclique,Structures.Starclique}, M::Model.GraphModel)
    if structure isa Structures.Biclique
        left = Set([(n1, n2) for n1 in structure.left_nodes for n2 in structure.left_nodes if n1 < n2 && (n1,n2) ∉ M.star_spoke_edges])
    else # assuming Starclique
        left = Set([(n1, n2) for n1 in structure.left_nodes for n2 in structure.left_nodes if n1 < n2])
    end
    left_edges_in_hs_edges = intersect(left, M.hub_spoke_edges)
    setdiff!(left,left_edges_in_hs_edges)
    right = Set([(n1, n2) for n1 in structure.right_nodes for n2 in structure.right_nodes if n1 < n2 && (n1,n2) ∉ M.star_spoke_edges])
    right_edges_in_hs_edges = intersect(right, M.hub_spoke_edges)
    setdiff!(right,right_edges_in_hs_edges)
    across = Set([(n1 < n2) ? (n1, n2) : (n2, n1) for n1 in structure.left_nodes for n2 in structure.right_nodes])
    across_edges_in_hs_edges = intersect(across, M.hub_spoke_edges)
    setdiff!(across, across_edges_in_hs_edges)
    micro_structures = [left, right, across]
    constraint_values = [   structure.n_edges_in_left - length(left_edges_in_hs_edges),
                            structure.n_edges_in_right - length(right_edges_in_hs_edges),
                            structure.n_edges_across - length(across_edges_in_hs_edges)
                        ]
    return micro_structures, constraint_values
end

"""
try adding structure to model, keeping the result only if it reduces the description length
"""
function test_adding_structure!(M::Model.GraphModel, structure::Structures.Structure, size_threshold::Float64=1.)
    log_progress!(M, "saving previous state...")
    M.backup_optimization_result = deepcopy(M.last_optimization_result)
    log_progress!(M, "testing structure $(structure)...")
    if structure.n_nodes_total < size_threshold
        log_progress!(M, "failure: structure too small")
        return false
    end
    add_macro_structures!(M, Array{Structures.Structure,1}([structure]))
    if converged(M.last_optimization_result) && M.description_lengths_over_time[end] <= M.description_lengths_over_time[end-1]
        log_progress!(M, "success: new description length $(M.total_description_length_maxent)")
        return true
    else
        roll_back_model!(M)
        log_progress!(M, "failure: no decrease in description length")
        return false
    end
end

"""
add macro structures to model (assuming no connector knowledge)
we allow for adding multiple structures at once to enable faster replay of macro_structure_description_lengths
(in the model building phase, we test adding one structure at a time only)
"""
function add_macro_structures!(M::Model.GraphModel, structures::Array{Structures.Structure,1}; lambdas=[])
    log_progress!(M, "updating micro structures et al")
    for structure in structures
        micro_structures, constraint_values = generate_micro_structures(structure, M)
        if structure isa Structures.Star
            M.last_hub_spoke_edges = deepcopy(M.hub_spoke_edges)
            M.last_star_spoke_edges = deepcopy(M.star_spoke_edges)
            n_edges_in_spokes = structure.n_edges_in_spokes
            union!(M.star_spoke_edges, micro_structures[end])
            union!(M.hub_spoke_edges, micro_structures[1])
            if n_edges_in_spokes > 0 # if the star is perfect, we know exactly what its associated probabilities are
                append!(M.micro_structures, [micro_structures[end]])
                append!(M.constraint_values, [constraint_values[end]])
                append!(M.lambdas, zeros(1))
                update_edge_lambda_indices!(M, [micro_structures[end]], [constraint_values[end]])
                update_equivalence_classes_from_edge_lambda_indices!(M)
            else
                M.last_zero_edges = deepcopy(M.last_zero_edges)
                union!(M.zero_edges, micro_structures[1])
            end
        elseif structure isa Structures.Starclique || structure isa Structures.Biclique
            M.last_star_spoke_edges = deepcopy(M.star_spoke_edges)
            union!(M.star_spoke_edges, micro_structures[2])
            if structure isa Structures.Biclique
                union!(M.star_spoke_edges, micro_structures[1])
            end
            append!(M.micro_structures, micro_structures)
            append!(M.constraint_values, constraint_values)
            append!(M.lambdas, zeros(length(micro_structures)))
            update_edge_lambda_indices!(M, micro_structures, constraint_values)
            update_equivalence_classes_from_edge_lambda_indices!(M)
        else
            append!(M.micro_structures, micro_structures)
            append!(M.constraint_values, constraint_values)
            append!(M.lambdas, zeros(length(micro_structures)))
            update_edge_lambda_indices!(M, micro_structures, constraint_values)
            update_equivalence_classes_from_edge_lambda_indices!(M)
        end
    end
    if !isempty(lambdas)
        M.lambdas = lambdas
    end
    update_variable_terms!(M)
    for structure in structures
        append!(M.macro_structures, [structure])
        append!(M.macro_structure_description_lengths, [Structures.compute_description_length(structure, M.n, false)])
    end
end

function update_edge_lambda_indices!(M::Model.GraphModel, new_micro_structures, new_constraint_values)
    log_progress!(M, "updating edge lambda indices")
    start_lambda_index = length(M.lambdas) - length(new_constraint_values)
    for (idx, structure) in enumerate(new_micro_structures)
        new_idx = start_lambda_index+idx
        present_edges = intersect(keys(M.edge_lambda_indices), structure)
        for edge in present_edges
            append!(M.edge_lambda_indices[edge], new_idx)
        end
        absent_edge_dict = Dict(e=>[1,new_idx] for e in setdiff(structure,present_edges))
        merge!(M.edge_lambda_indices,absent_edge_dict)
    end
end

"""
roll back the model if the added structure is no good
eliminates last macro structure added and all its consequences
"""
function roll_back_model!(M::Model.GraphModel)
    structure_to_remove = M.macro_structures[end]
    micros_to_remove = determine_micros_to_remove(structure_to_remove)
    if structure_to_remove isa Structures.Star
        M.hub_spoke_edges = deepcopy(M.last_hub_spoke_edges)
        M.star_spoke_edges = deepcopy(M.last_star_spoke_edges)
        if structure_to_remove.n_edges_in_spokes == 0
            micros_to_remove = 0
            M.last_zero_edges = deepcopy(M.last_zero_edges)
        end
    elseif structure_to_remove isa Structures.Starclique || structure_to_remove isa Structures.Biclique
        M.star_spoke_edges = deepcopy(M.last_star_spoke_edges)
    end
    if micros_to_remove > 0
        roll_back_edge_lambda_indices!(M, micros_to_remove)
        roll_back_lambda_related!(M, micros_to_remove)
        update_equivalence_classes_from_edge_lambda_indices!(M)
    end
    update_equivalence_class_edge_probabilities!(M)
    update_edge_probability_description_length!(M)
    pop!(M.macro_structures)
    pop!(M.macro_structure_description_lengths)
    pop!(M.description_lengths_over_time)
    roll_back_total_description_length!(M)
    roll_back_optimization_result!(M)
end

function determine_micros_to_remove(structure_to_remove::Structures.Structure)
    if structure_to_remove isa Structures.Clique || structure_to_remove isa Structures.Star
        micros_to_remove = 1
    elseif structure_to_remove isa Structures.Biclique || structure_to_remove isa Structures.Starclique
        micros_to_remove = 3
    else
        throw(ErrorException("Cannot roll back for structure of type $(typeof(structure_to_remove))!"))
    end
    return micros_to_remove
end

function roll_back_edge_lambda_indices!(M::Model.GraphModel, micros_to_remove::Int64)
    for micro_structure in M.micro_structures[end-micros_to_remove+1:end]
        for edge in micro_structure
            pop!(M.edge_lambda_indices[edge])
            if M.edge_lambda_indices[edge] == [1]
                delete!(M.edge_lambda_indices, edge)
            end
        end
    end
end

function roll_back_lambda_related!(M::Model.GraphModel, micros_to_remove::Int64)
    first_to_delete = length(M.micro_structures)-micros_to_remove+1
    last_to_delete = length(M.micro_structures)
    deleteat!(M.lambdas, first_to_delete:last_to_delete)
    deleteat!(M.constraint_values, first_to_delete:last_to_delete)
    deleteat!(M.micro_structures, first_to_delete:last_to_delete)
end

function update_equivalence_classes_from_edge_lambda_indices!(M::Model.GraphModel)
    equivalence_classes = Dict{Set{Int64},Array{Tuple{Int64,Int64},1}}()
    for (k,v) in M.edge_lambda_indices
        append!(get!(equivalence_classes, Set(v), Array{Int64,1}()), [k])
    end
    M.edge_equivalence_classes = equivalence_classes
end

function update_equivalence_class_edge_probabilities!(M::Model.GraphModel)
    probabilities = Dict{Set{Int64},BigFloat}(
        equivalence_class=>edge_probability_for_equiv(equivalence_class, M.lambdas) for equivalence_class in keys(M.edge_equivalence_classes))
    probabilities[Set([1,])] = edge_probability_for_equiv(Set([1,]), M.lambdas)
    M.equivalence_class_edge_probabilities = probabilities
end

function roll_back_total_description_length!(M::Model.GraphModel)
    M.total_description_length_maxent = M.description_lengths_over_time[end]
end

function roll_back_optimization_result!(M::Model.GraphModel)
    M.last_optimization_result = deepcopy(M.backup_optimization_result)
    M.entropy = M.last_optimization_result.minimum
end

end


# All functions needed for the invasion experiments.
# Create the global species pool, carry out species introductions, and disturbance.

let speciespool = Individual[]
    #XXX should `speciespool` be part of the `world` object holding all grid cells?
    """
        initglobalspeciespool!()

    Initialise the foreign species pool (i.e. an array of random individuals that
    are used as the propagule source for invasion events.
    """
    function initglobalspeciespool!()
        for i in 1:setting("global-species-pool")
            push!(speciespool, createind())
        end
    end

    """
        invade!(patch, pressure)

    Select a given amount of individuals from the global species pool and add
    them to the patch.
    """
    function invade!(patch::Patch, pressure::Int)
        invaders = deepcopy(rand(speciespool, pressure))
        append!(patch.community, invaders)
    end

    """
        invade!(world)

    Introduce non-native species from the species pool to all patches marked
    as invasible.
    """
    global function invade!(world::Array{Patch,1})
        # This function has to be global to escape the let-block of the species pool
        (setting("propagule-pressure") == 0 || setting("global-species-pool") == 0) && return
        (length(speciespool) == 0) && initglobalspeciespool!()
        for patch in world
            patch.invasible && invade!(patch, setting("propagule-pressure"))
        end
    end
end

"""
    disturb!(patch, intensity)

Species-independent mortality due to disturbance on the given patch. The intensity
gives the mortality percentage.
"""
function disturb!(patch::Patch, intensity::Int)
    length(patch.community) <= 0 && return
    deaths = Integer(round(length(patch.community) * (intensity/100)))
    dead = unique(rand(1:length(patch.community), deaths))
    while length(dead) < deaths
        newdead = rand(1:length(patch.community), deaths-length(dead))
        dead = unique(vcat(dead, newdead))
    end
    sort!(dead)
    deleteat!(patch.community, dead)
end

"""
    disturb!(world)

Disturb all patches in the world.
"""
function disturb!(world::Array{Patch,1})
    (setting("disturbance") == 0) && return
    if setting("disturbance") > 100
        simlog("disturbance must be no more than 100%", 'w')
        updatesetting("disturbance", 100)
    end
    for patch in world
        (patch.isisland || !setting("static")) && disturb!(patch, setting("disturbance"))
    end
end


include("trackingcell.jl");
include("lineage.jl");
include("tiffxml.jl");
include("segmentation3d.jl")

data_dir = "/datahub/rawdata/tandeng/mRNA_imaging/mRNA_confocal_hamamatsu-60X-TIRF";
ret_dir = "/datahub/rawdata/tandeng/mRNA_imaging/mRNA_confocal_hamamatsu-60X-TIRF/20200316_result";

paths_all = []
for s in 1
	img = load(File(format"TIFF","$data_dir/20200316/HE7-11-1-80uw-PWM_1_s$s.ome.tiff"));
	img_edge, mask_edge, mask_clear = split_cell_LoG(img, size(img)[3]÷20);
	save("$ret_dir/img_edge_s$s.tiff", img_edge);

	nucleus_pos = [ component_centroids(label_components(mask_clear[:,:,i])) for i in 1:size(mask_clear)[3] ];

	paths = generate_path(nucleus_pos);
	push!(paths_all, paths)

	for i in 1:length(paths)
		save("$ret_dir/s$s-c$i.tiff", N0f16.(export_cell(paths[i], img, mask_edge)));
		embedxml(700, 700, 20, length(paths[i]), "$ret_dir/s$s-c$i.tiff");
	end
end


export rtdm_write, rtdm_read, rtdm_read!, isrtdmok, RTDMInterface

type CRC
  crc  :: UInt16 # running crc
  data :: UInt8  # input byte for bit shifting
end

const polynomial = 0xa001
function compute_crc(crc::CRC)
  for i in 1:8
    if ((crc.crc $ crc.data)&0x0001) == 0x0001
      crc.crc = (crc.crc>>>1) $ polynomial
    else
      crc.crc = crc.crc>>>1
    end
    crc.data >>>=1
  end
  return crc.crc
end

immutable RTDMInterface
  address :: Dict{Symbol,UInt32} # connects symbols to addresses
  iouart  :: IO # serial port microcontroller is connected to
  iopipe  :: IOBuffer # pipe to serialize data for crc and uart
  iopipe2 :: IOBuffer
  buffer1 :: Array{UInt8,1}
  buffer4 :: Array{UInt8,1}
  buffer6 :: Array{UInt8,1}
  buffercrc :: Array{UInt16,1}
  crc :: CRC
  function RTDMInterface(addressdict::Dict{Symbol,UInt32}, iouart::IO)
    new(addressdict, iouart, PipeBuffer(), PipeBuffer(),
      Array(UInt8,1), Array(UInt8,4), Array(UInt8,6),
      Array(UInt16,1), CRC(0xffff,0x00))
  end
end

function Base.show(io::IO, x::RTDMInterface)
  println(io, "Dict{Symbol,UInt32} with ",length(x.address)," entries")
  show(io, x.iouart)
  return nothing
end

function Base.close(i::RTDMInterface)
  close(i.iouart)
  close(i.iopipe)
end

function pipecrc!(i::RTDMInterface, data)
  write(i.iopipe,data)
  while ~eof(i.iopipe)
    i.crc.data = read(i.iopipe, UInt8)
    compute_crc(i.crc)
  end
  return nothing
end

function sendpipewithcrc(i::RTDMInterface)  # always attach and send crc
  while ~eof(i.iopipe)
    byte = read(i.iopipe, UInt8)
    write(i.iopipe2, byte)
    i.crc.data = byte
    compute_crc(i.crc)
  end
  write(i.iopipe2, i.crc.crc)
  write(i.iouart, takebuf_array(i.iopipe2))
  resetcrc!(i)
  return nothing
end

function loadpipe(i::RTDMInterface, bytestoload::Integer, last2crc = bytestoload>1)
  # set last2crc = true to not include last two bytes read in crc calculation
  data = read(i.iouart, UInt8, bytestoload)
  for index in 1:length(data)
    if (index < bytestoload-1) | ~last2crc
      i.crc.data = data[index]
      compute_crc(i.crc)
    end
  end
  write(i.iopipe,data)
  return nothing
end

function resetcrc!(i::RTDMInterface)
  i.crc.crc = 0xffff
  return nothing
end

getcrc(i::RTDMInterface) = i.crc.crc

function checkforerrorcode(i::RTDMInterface)
  # errorcode = 0   no error
  # errorcode < 0   somthing wrong with received message. from this module.
  # errorcode > 0   error code returned by target.  from microcontroller.
  # returns errorcode and crc up to this point
  resetcrc!(i)
  errorcode = 0
  loadpipe(i,1)
  replycode = read(i.iopipe, UInt8)
  if replycode != 0x2b  # '+'
    loadpipe(i,6)
    read!(i.iopipe, i.buffer4)
    read!(i.iopipe, i.buffercrc)
    if i.buffer4[1] != 0x24 # '$'
      errorcode = -1
    elseif i.buffer4[2] != 0x45 # 'E'
      errorcode = -1
    elseif i.buffer4[4] != 0x23 # '#'
      errorcode = -1
    elseif i.buffercrc[1] != getcrc(i)
      errorcode = -2
    else
      errorcode = convert(Int,i.buffer4[3])
    end
  end
  return errorcode
end

const communication_link_sanity_check = b"$s#"
const linkok = b"$RTDM#"
const clsc_writereplybuffer = Array(UInt8,length(linkok))

function isrtdmok(i::RTDMInterface; retry = 1)
  errorcode = -99
  for attempt in 1:retry
    resetcrc!(i)
    write(i.iopipe, communication_link_sanity_check)
    sendpipewithcrc(i)
    errorcode = checkforerrorcode(i)
    if errorcode == 0
      loadpipe(i, 8)
      read!(i.iopipe, i.buffer6)
      read!(i.iopipe, i.buffercrc)
      if getcrc(i) != i.buffercrc[1] # did crc match?
        errorcode = -3
      end
    end
    if errorcode == 0
      break
    end
  end
  return errorcode == 0
end

const startreadrequest =  b"$m"
const endofmessage = b"#"
const startreadreply =b"$"
function rtdm_read{T}(i::RTDMInterface, ::Type{T}, address::Integer; retry = 1)
  address32 = convert(UInt32,address)
  buffersize16 = UInt16(sizeof(T))
  errorcode = -99
  data = T(0)
  for attempt in 1:retry
    resetcrc!(i)
    # request read
    write(i.iopipe, startreadrequest)
    write(i.iopipe, address32)
    write(i.iopipe, buffersize16)
    write(i.iopipe, endofmessage)
    sendpipewithcrc(i)
    # process reply
    errorcode = checkforerrorcode(i)
    if errorcode == 0
      loadpipe(i, sizeof(T) + 4)
      read!(i.iopipe, i.buffer1)
      data = read(i.iopipe, T)
      read!(i.iopipe, i.buffer1)
      read!(i.iopipe, i.buffercrc)
      if getcrc(i) != i.buffercrc[1]
        errorcode = -4
      end
    end
    if errorcode == 0
      break
    end
  end
  checkerrorcode(errorcode,retry)
  return data
end

function rtdm_read!{T}(i::RTDMInterface, buffer::Array{T}, address::Integer; retry = 1)
  address32 = UInt32(address)
  buffersize16 = UInt16(sizeof(buffer))
  errorcode = -99
  for attempt in 1:retry
    resetcrc!(i)
    # request read
    write(i.iopipe, startreadrequest)
    write(i.iopipe, address32)
    write(i.iopipe, buffersize16)
    write(i.iopipe, endofmessage)
    sendpipewithcrc(i)
    # process reply
    errorcode = checkforerrorcode(i)
    if errorcode == 0
      loadpipe(i, buffersize16 + 4)
      read!(i.iopipe, i.buffer1) # start code
      read!(i.iopipe, buffer) # data
      read!(i.iopipe, i.buffer1) # end code
      read!(i.iopipe, i.buffercrc) # crc
      if getcrc(i) != i.buffercrc[1]
        errorcode = -4
      end
    end
    if errorcode == 0
      break
    end
  end
  checkerrorcode(errorcode,retry)
  return nothing
end

const startwrite = b"$M"
const endofwrite = b"#"

function rtdm_write{T}(i::RTDMInterface, buffer::Array{T}, address::Integer; retry = 1)
  address32 = UInt32(address)
  buffersize16 = UInt16(sizeof(buffer))
  errorcode = -99
  for attempt in 1:retry
    resetcrc!(i)
    # write memory
    write(i.iopipe, startwrite)
    write(i.iopipe, address32)
    write(i.iopipe, buffersize16)
    write(i.iopipe, buffer)
    write(i.iopipe, endofwrite)
    sendpipewithcrc(i)
    # process reply
    errorcode = checkforerrorcode(i)
    if errorcode == 0
      loadpipe(i,6)
      read!(i.iopipe, i.buffer4)
      read!(i.iopipe, i.buffercrc)
      if getcrc(i) != i.buffercrc[1]
        errorcode = -5
      end
    end
    if errorcode != 0
      break
    end
  end
  checkerrorcode(errorcode,retry)
  return nothing
end

function rtdm_write(i::RTDMInterface, data, address::Integer; retry = 1)
  address32 = UInt32(address)
  buffersize16 = UInt16(sizeof(data))
  errorcode = -99
  for attempt in 1:retry
    resetcrc!(i)
    # write memory
    write(i.iopipe, startwrite)
    write(i.iopipe, address32)
    write(i.iopipe, buffersize16)
    write(i.iopipe, data)
    write(i.iopipe, endofwrite)
    sendpipewithcrc(i)
    # process reply
    errorcode = checkforerrorcode(i)
    if errorcode == 0
      loadpipe(i,6)
      read!(i.iopipe, i.buffer4)
      read!(i.iopipe, i.buffercrc)
      if getcrc(i) != i.buffercrc[1]
        errorcode = -5
      end
    end
    if errorcode != 0
      break
    end
  end
  checkerrorcode(errorcode,retry)
  return nothing
end

function rtdm_write(i::RTDMInterface, data, address::Symbol; retry = 1)
  rtdm_write(i, data, i.address[address]; retry = retry)
end
function rtdm_write{T}(i::RTDMInterface, buffer::Array{T}, address::Symbol; retry = 1)
  rtdm_write(i, buffer, i.address[address]; retry = retry)
end
function rtdm_read!{T}(i::RTDMInterface, buffer::Array{T}, address::Symbol; retry = 1)
  rtdm_read!(i, buffer, i.address[address]; retry = retry)
end
function rtdm_read{T}(i::RTDMInterface, ::Type{T}, address::Symbol; retry = 1)
  rtdm_read(i, T, i.address[address]; retry = retry)
end


## Features.jl
## -----------
## Feature Types and basic functionality
## related directly to IO of Feature Types
using DataFrames
using IntervalTrees

abstract Feature
abstract SequenceFeature <: Feature

type SequencePos
     start::Int32
     stop::Int32

     function SequencePos( start::String,stop::String)
        start_int = parseint(start)
        stop_int  = parseint(stop)
        start     = int32(start_int)
        stop      = int32(stop_int)
        new(start, stop)
    end
    function SequencePos(start::Int,stop::Int)
       new(int32(start),int32(stop))
    end
end

#DEPRECATE
type GenomePos
     chr::Int16
     start::Int32
     stop::Int32

     function GenomePos( chr::String,start::String,stop::String, organism="none")
        start_int = parseint(start)
        stop_int  = parseint(stop)
        start     = int32(start_int)
        stop      = int32(stop_int)
        if organism == "none"
            chr       = chr_convert_int16(chr)
        else
            chr  = chr_convert_int16(chr, organism)
        end
        new(chr, start, stop)
    end
    function GenomePos(chr::Int,start::Int,stop::Int)
       new(int16(chr),int32(start),int32(stop))
    end
    function GenomePos(chr::String, start::Int,stop::Int)
       new( chr_convert_int16(chr),int32(start),int32(stop))
    end
end

type CytosineCount <: SequenceFeature
    pos::SequencePos
    strand::Bool
    C_count::Int16
    T_count::Int16
end

type CytosineMethylation <: SequenceFeature
     pos::SequencePos
     strand::Bool
     mc::Float32
     hmc::Float32
     function CytosineMethylation(pos::SequencePos,strand::Bool, mc,hmc)
       new(pos,strand,float32(mc),float32(hmc))
    end
end

type CytosineCount2
    pos::GenomePos
    strand::Bool
    C_count::Int16
    T_count::Int16
end

type CytosineMethylation2
    pos::GenomePos
    strand::Bool
    mc::Float32
    hmc::Float32
    #constructor to ensure float32s
    function CytosineMethylation(pos::GenomePos,strand::Bool, mc,hmc)
       new(pos,strand,float32(mc),float32(hmc))
    end
end

# This holds features in a dataframe
type FeatureTable
  description::Dict{ASCIIString,Any}
  dataframe::DataFrame
end

# This holds Features organised by sequence and ordered by co-ordinate
# Idea being this might be efficient for holding many features
type SequenceFeatures
    description::Dict{ASCIIString,Any}
    features::Dict{Any,Any}
end

function add_sequence_feature(sequence_features::SequenceFeatures,  sequence_name::String, feature::SequenceFeature )

     if haskey( sequence_features.features, sequence_name )
         sequence_feature_vector = sequence_features.features[sequence_name]
         push!( sequence_feature_vector, feature)
     else
         vec = Array(SequenceFeature,0)
         push!(vec, feature)
         sequence_features.features[sequence_name] = vec
     end
end

# Similar to SequenceFeatures
# Based on an interval tree
type FeatureRegions
  description::Dict{ASCIIString,Any}
  regions::Dict{Any,Any}

  function FeatureRegions(FT::FeatureTable)
     sequence_regions = Dict()
     DF = FT.dataframe
     nrows = size(DF,1)
     for row = 1:nrows
         gene = DF[row, :]
         #println(gene)
         gene_name = gene[:name][1]
         start     = gene[:start][1]
         stop      = gene[:stop][1]
         chr       = gene[:chr][1]
         insertintervals!(sequence_regions, chr, start, stop, gene )
     end
     new( ["organism"=>""], sequence_regions )
  end
end

# I think the reason we can't just stick anything into value
# and we have multiple functions for with different value types is because
# IntervalTree didn't let us do that. Check that.
function insertintervals!( sequence_regions_dict, chr, start, stop, value::DataFrame)
    if haskey(sequence_regions_dict, chr)
        interval_tree = sequence_regions_dict[chr]
        interval_tree[(start,stop)] = value
    else
        try
           interval_tree = IntervalTree{Int32,DataFrame}()
           interval_tree[(start,stop)] = value
           sequence_regions_dict[chr]  = interval_tree
        catch e
           println("Exception chr: $chr, start: $start, stop: $stop, value $value ")
           println( join( [  "Exception, ", typeof(chr), " ", typeof(start), " ", typeof(stop) ], "\t" ) )
           error(e)
        end
    end
end

function insertintervals!( sequence_regions_dict, chr, start, stop, value::ASCIIString)
    if haskey(sequence_regions_dict, chr)
        interval_tree = sequence_regions_dict[chr]
        interval_tree[(start,stop)] = value
    else
        interval_tree = IntervalTree{Int32,ASCIIString}()
        interval_tree[(start,stop)] = value
        sequence_regions_dict[chr] = interval_tree
    end
end

function getoverlaps(feature_regions,chr,start,stop)
     # if (haskey(genome_intervals, chr) == true )
          interval_tree = feature_regions.regions[chr]
          iter = intersect(interval_tree, (start,stop))
          return iter
      #else
      #   println("WARN: No key found for $key")
      #   return []
      #end
end

function chr_convert_int16( chr)
    rgx1 = match(r"chr(.)",chr)
    if rgx1 != nothing
      # if it's just numbers
        chr_name = rgx1.captures[1]
        if chr_name == "X"
            return int16(200)
         end
        if chr_name == "Y"
            return int16(201)
        end
        if chr_name == "M"
            return int16(202)
        end
        rgx2 = match(r"^[0-9]+$", string( chr_name ))
        if rgx2 != nothing
           #chr_num = rgx2.captures[1]
            return convert(Int16, parseint(chr_name))
        else
           error("no valid chr number for $chr")
        end
    else
         error("Chromosome field not in valid format e.g. chr1")
    end
end

function feature_table_to_bed( features::FeatureTable, path, delim="\t"; overwrite=false)
        if isfile(path) && !overwrite
             println("feature_table_to_bed: $path exists")
             return 0;
         end

         df = features.dataframe
         nrows = size(df,1)
         iostream = open(path,"w")
         count = 0
         for row =  1:nrows
         	  row    = df[row,:]
            name   = row[:name][1]
            start  = row[:start][1]
            stop   = row[:stop][1]
            strand = row[:strand][1]
            strand = strand ? "+" : "-"
         	  #chr    = CURRENT_ANNOTATION_CHR_NAMES["mm10"][ row[:chr][1] ]
            chr    = row[:chr][1]
            line   = join([chr,start,stop,name,0,strand,"\n"],delim)
            print(iostream, line)
            count += 1
         end
         close(iostream)
         return count
end


# defines variables found in INDEXB.INC for use in Julia

const JBTOT  =  102
const JBFUSE =  35

const VARS = OffsetArrays.OffsetArray(Array{String,1}(undef, JBTOT+1), 0:JBTOT)
VARS[ 0]  = "      s    "; const JSA    =  0;
VARS[ 1]  = "      x    "; const JXA    =  1;
VARS[ 2]  = "      y    "; const JYA    =  2;
VARS[ 3]  = "      z    "; const JZA    =  3;
VARS[ 4]  = "    twist  "; const JTW    =  4;
VARS[ 5]  = "     EIcc  "; const JECC   =  5;
VARS[ 6]  = "     EInn  "; const JENN   =  6;
VARS[ 7]  = "     EIcn  "; const JECN   =  7;
VARS[ 8]  = "     EIcs  "; const JECS   =  8;
VARS[ 9]  = "     EIsn  "; const JESN   =  9;
VARS[10]  = "      GJ   "; const JGJ    = 10;
VARS[11]  = "      EA   "; const JEA    = 11;
VARS[12]  = "      GKc  "; const JGKC   = 12;
VARS[13]  = "      GKn  "; const JGKN   = 13;
VARS[14]  = "     mgcc  "; const JMCC1  = 14;
VARS[15]  = "     mgnn  "; const JMNN1  = 15;
VARS[16]  = "      mg   "; const JMG1   = 16;
VARS[17]  = "     Ccg   "; const JCCG1  = 17;
VARS[18]  = "     Ncg   "; const JNCG1  = 18;
VARS[19]  = "    Dmgcc  "; const JMCC2  = 19;
VARS[20]  = "    Dmgnn  "; const JMNN2  = 20;
VARS[21]  = "     Dmg   "; const JMG2   = 21;
VARS[22]  = "    DCcg   "; const JCCG2  = 22;
VARS[23]  = "    DNcg   "; const JNCG2  = 23;
VARS[24]  = "     Cea   "; const JCEA   = 24;
VARS[25]  = "     Nea   "; const JNEA   = 25;
VARS[26]  = "     Cta   "; const JCTA   = 26;
VARS[27]  = "     Nta   "; const JNTA   = 27;
VARS[28]  = "    tdeps  "; const JTDE   = 28;
VARS[29]  = "    tdgam  "; const JTDG   = 29;
VARS[30]  = "   Cshell  "; const JCSH   = 30;
VARS[31]  = "   Nshell  "; const JNSH   = 31;
VARS[32]  = "  Atshell  "; const JASH   = 32;
VARS[33]  = "   radius  "; const JRAD   = 33;
VARS[34]  = "     Cdf   "; const JCDF   = 34;
VARS[35]  = "     Cdp   "; const JCDP   = 35;
VARS[36]  = "    chord  "; const JCH    = 36;
VARS[37]  = "     Xax   "; const JXAX   = 37;
VARS[38]  = "    alpha  "; const JAL    = 38;
VARS[39]  = "      Cm   "; const JCM    = 39;
VARS[40]  = "    CLmax  "; const JCMX   = 40;
VARS[41]  = "    CLmin  "; const JCMN   = 41;
VARS[42]  = "    dCLda  "; const JDCL   = 42;
VARS[43]  = "   dCLdF1  "; const JCLF1  = 43;
VARS[44]  = "   dCLdF2  "; const JCLF2  = 44;
VARS[45]  = "   dCLdF3  "; const JCLF3  = 45;
VARS[46]  = "   dCLdF4  "; const JCLF4  = 46;
VARS[47]  = "   dCLdF5  "; const JCLF5  = 47;
VARS[48]  = "   dCLdF6  "; const JCLF6  = 48;
VARS[49]  = "   dCLdF7  "; const JCLF7  = 49;
VARS[50]  = "   dCLdF8  "; const JCLF8  = 50;
VARS[51]  = "   dCLdF9  "; const JCLF9  = 51;
VARS[52]  = "   dCLdF10 "; const JCLF10 = 52;
VARS[53]  = "   dCLdF11 "; const JCLF11 = 53;
VARS[54]  = "   dCLdF12 "; const JCLF12 = 54;
VARS[55]  = "   dCLdF13 "; const JCLF13 = 55;
VARS[56]  = "   dCLdF14 "; const JCLF14 = 56;
VARS[57]  = "   dCLdF15 "; const JCLF15 = 57;
VARS[58]  = "   dCLdF16 "; const JCLF16 = 58;
VARS[59]  = "   dCLdF17 "; const JCLF17 = 59;
VARS[60]  = "   dCLdF18 "; const JCLF18 = 60;
VARS[61]  = "   dCLdF19 "; const JCLF19 = 61;
VARS[62]  = "   dCLdF20 "; const JCLF20 = 62;
VARS[63]  = "   dCMdF1  "; const JCMF1  = 63;
VARS[64]  = "   dCMdF2  "; const JCMF2  = 64;
VARS[65]  = "   dCMdF3  "; const JCMF3  = 65;
VARS[66]  = "   dCMdF4  "; const JCMF4  = 66;
VARS[67]  = "   dCMdF5  "; const JCMF5  = 67;
VARS[68]  = "   dCMdF6  "; const JCMF6  = 68;
VARS[69]  = "   dCMdF7  "; const JCMF7  = 69;
VARS[70]  = "   dCMdF8  "; const JCMF8  = 70;
VARS[71]  = "   dCMdF9  "; const JCMF9  = 71;
VARS[72]  = "   dCMdF10 "; const JCMF10 = 72;
VARS[73]  = "   dCMdF11 "; const JCMF11 = 73;
VARS[74]  = "   dCMdF12 "; const JCMF12 = 74;
VARS[75]  = "   dCMdF13 "; const JCMF13 = 75;
VARS[76]  = "   dCMdF14 "; const JCMF14 = 76;
VARS[77]  = "   dCMdF15 "; const JCMF15 = 77;
VARS[78]  = "   dCMdF16 "; const JCMF16 = 78;
VARS[79]  = "   dCMdF17 "; const JCMF17 = 79;
VARS[80]  = "   dCMdF18 "; const JCMF18 = 80;
VARS[81]  = "   dCMdF19 "; const JCMF19 = 81;
VARS[82]  = "   dCMdF20 "; const JCMF20 = 82;
VARS[83]  = "   dCDdF1  "; const JCDF1  = 83;
VARS[84]  = "   dCDdF2  "; const JCDF2  = 84;
VARS[85]  = "   dCDdF3  "; const JCDF3  = 85;
VARS[86]  = "   dCDdF4  "; const JCDF4  = 86;
VARS[87]  = "   dCDdF5  "; const JCDF5  = 87;
VARS[88]  = "   dCDdF6  "; const JCDF6  = 88;
VARS[89]  = "   dCDdF7  "; const JCDF7  = 89;
VARS[90]  = "   dCDdF8  "; const JCDF8  = 90;
VARS[91]  = "   dCDdF9  "; const JCDF9  = 91;
VARS[92]  = "   dCDdF10 "; const JCDF10 = 92;
VARS[93]  = "   dCDdF11 "; const JCDF11 = 93;
VARS[94]  = "   dCDdF12 "; const JCDF12 = 94;
VARS[95]  = "   dCDdF13 "; const JCDF13 = 95;
VARS[96]  = "   dCDdF14 "; const JCDF14 = 96;
VARS[97]  = "   dCDdF15 "; const JCDF15 = 97;
VARS[98]  = "   dCDdF16 "; const JCDF16 = 98;
VARS[99]  = "   dCDdF17 "; const JCDF17 = 99;
VARS[100] = "   dCDdF18 "; const JCDF18 = 100;
VARS[101] = "   dCDdF19 "; const JCDF19 = 101;
VARS[102] = "   dCDdF20 "; const JCDF20 = 102;


using Unitful
using YAML
using PyCall
using StaticArrays

module Calc
    include("Calculators/RateUncertainty.jl")
    include("Calculators/ThermoUncertainty.jl")
    include("Calculators/Thermo.jl")
    include("Calculators/Diffusion.jl")
    include("Calculators/Rate.jl")
    include("Calculators/Viscosity.jl")
    include("Calculators/Thermovec.jl")
    include("Calculators/Ratevec.jl")
end
module Spc
    include("Calculators/ThermoUncertainty.jl")
    include("Calculators/Thermo.jl")
    include("Calculators/Diffusion.jl")
    include("Species.jl")
end
module Rxn
    include("Calculators/RateUncertainty.jl")
    include("Calculators/ThermoUncertainty.jl")
    include("Calculators/Thermo.jl")
    include("Calculators/Diffusion.jl")
    include("Calculators/Rate.jl")
    include("Species.jl")
    include("Reaction.jl")
end
module Solv
    include("Calculators/Viscosity.jl")
    include("Solvent.jl")
end

const unitsdict = Dict()
const elementdict = Dict([1=>"H",6=>"C",8=>"O",7=>"N",17=>"Cl",16=>"S",18=>"Ar",10=>"Ne",2=>"He",
        15=>"P",9=>"F",35=>"Br",53=>"I",289=>"Fl"])

const allowedfcnlist = vcat(names(Calc),names(Spc),names(Rxn),names(Solv))

"""
Zhao et al 2003
"""
const mcgowanvolumes = Dict(["H"=> upreferred(8.71u"mL/mol").val, "He"=> upreferred(6.75u"mL/mol").val,
        "C"=> upreferred(16.35u"mL/mol").val, "N"=> upreferred(14.39u"mL/mol").val,
         "O"=> upreferred(12.43u"mL/mol").val, "F"=> upreferred(10.47u"mL/mol").val,
         "Ne"=> upreferred(8.51u"mL/mol").val,"Si"=> upreferred(26.83u"mL/mol").val, "P"=> upreferred(24.87u"mL/mol").val,
        "S"=> upreferred(22.91u"mL/mol").val, "Cl"=> upreferred(20.95u"mL/mol").val,
         "Ar"=> upreferred(18.99u"mL/mol").val,"Br"=> upreferred(26.21u"mL/mol").val,])

function parseentry(q,units=nothing)
    """
    parses single YAML lines into strings, numbers and arrays of Numbers in SI units
    """
    if isa(q,String)
        q = parsestring(q)
    end

    if isa(q,String)
        return q
    elseif isa(q,Quantity)
        return upreferred(q).val
    elseif isa(q,Number)
        if units == nothing
            return q
        else
            return upreferred(Quantity(q,units)).val
        end
    elseif isa(q,AbstractArray)
        if units == nothing
            if typeof(q[1]) <: AbstractArray
                return convert(Array,hcat(q...)')
            else
                return q
            end
        else
            if typeof(q[1]) <: AbstractArray
                return convert(Array,hcat(upreferred(Quantity(q,units)).val...)')
            else
                return upreferred(Quantity(q,units)).val
            end
        end
    elseif isa(q,Dict)
        return q
    else
        tq = typeof(q)
        throw(error("unable to parse $q of type $tq"))
    end

end
export parseentry

function parsestring(s)
    """
    Detects if a given string is actually a quantity input using metaprogramming
    the string is parsed by julia's parser and then analyzed to check if it is in
    the format of a quantity input
    """
    try
        ex = Meta.parse(s)
        if !isa(ex,Symbol) && !isa(ex,String) && length(ex.args) == 3 && ex.args[1] == :* &&
            (isa(ex.args[2],Number) || (ex.args[2].head == :vec && all([isa(x,Number) for x in ex.args[2].args]))) &&
            ex.args[3].args[1] == Symbol("@u_str") &&
            isa(ex.args[3].args[2],LineNumberNode) && ex.args[3].args[2].line == 1 &&
            ex.args[3].args[2].file == :none && isa(ex.args[3].args[3],String)
            return eval(ex)
        else
            return s
        end
    catch
        return s
    end
end
export parsestring

function getatomdictfromrdkit(mol)
    """
    retrives the number of each type of atom and the number of bonds of an rdkit molecule
    """
    atmD = Dict{String,Int64}()
    for atm in mol.GetAtoms()
        v = elementdict[atm.GetAtomicNum()]
        if v in keys(atmD)
            atmD[v] += 1
        else
            atmD[v] = 1
        end
    end
    nbonds = length(mol.GetBonds())
    return atmD,nbonds
end
export getatomdictfromrdkit

getatomdictsmiles(smiles) = getatomdictfromrdkit(Chem.AddHs(Chem.MolFromSmiles(smiles)))
export getatomdictsmiles
getatomdictinchi(inchi) = getatomdictfromrdkit(Chem.AddHs(Chem.MolFromInchi(inchi)))
export getatomdictinchi

function getspeciesradius(atomdict::Dict{String,Int64},nbonds::Int64)
    """
    estimates the McGowan radius by calculating the McGowan Volume
    from the number of each type of atom and the number of bonds
    as described in Zhao et al. 2003
    """
    Vtot = 0.0
    for (atm,n) in atomdict
        Vtot += n*mcgowanvolumes[atm]
    end
    Vtot -= nbonds*upreferred(6.56u"mL/mol").val
    V = Vtot/Na
    r = (0.75/Base.pi*V)^(1.0/3.0)
    return r
end
export getspeciesradius

function fcndict2obj(d::T,ymlunitsdict::Q) where {T,Q<:Any}
    """
    constructs an object from a dictionary by recursively constructing
    the objects in its fields
    the dictionary entry for "type" denotes the object being constructed
    which must be in the allowedfcnlist
    """
    strfcn = d["type"]
    fcn = Symbol(strfcn)
    @assert fcn in allowedfcnlist fcn
    kwexprs = Array{Expr,1}()
    for (key,val) in d
        if key == "type"
            continue
        end
        if isa(val,AbstractDict) && "type" in keys(val)
            val = fcndict2obj(val,ymlunitsdict)
        elseif isa(val,AbstractArray) && typeof(val).parameters[1] <: AbstractDict
            val = [fcndict2obj(x,ymlunitsdict) for x in val]
        else
            if (strfcn,key) in keys(unitsdict)
                if unitsdict[(strfcn,key)] in keys(ymlunitsdict)
                    val = parseentry(val,ymlunitsdict[unitsdict[(strfcn,key)]])
                else
                    val = parseentry(val)
                end
            else
                #@warn("assuming values for $strfcn,$key are in SI units")
                val = parseentry(val)
            end
        end
        kwexpr = Expr(:kw,Symbol(key),val)
        push!(kwexprs,kwexpr)
    end
    ex = Expr(:call,fcn,Expr(:parameters,kwexprs...))
    return eval(ex)
end
export fcndict2obj

"""
examines the input file and parses as appropriate
for chemkin (.inp files) with or without species dictionaries
calls rmgpy to convert it to yml and then parses the yml file
for rms (.rms, .yml, .rmg) it parses it directly
"""
function readinput(fname::String;spcdict::String="",output::String="chem.rms")
    extension = split(fname,".")[end]
    if extension in ["rms","yml","rmg"]
        return readinputyml(fname)
    elseif extension in ["inp"]
        if spcdict == ""
            convertchemkin2yml(fname,output=output)
        else
            convertchemkin2yml(fname,spcdictpath=spcdict,output=output)
        end
        return readinputyml(output)
    else
        throw(error("file extension $extension not understood, use .inp for chemkin files or .rms for rms yaml files"))
    end
end
export readinput

"""
parses a YAML input file into a dictionary containing
partitions of Species and Reaction objects
"""
function readinputyml(fname::String)
    D = YAML.load(open(fname))
    outdict = Dict()

    #Units
    ymlunitsdict=D["Units"] #for now don't use

    #Solvents
    if "Solvents" in keys(D)
        s = D["Solvents"]
        for sol in s
            sol["type"] = "Solvent"
        end
        solvents = [fcndict2obj(q,ymlunitsdict) for q in s]
        outdict["Solvents"] = solvents
    end

    #phases
    spcindex = 1
    spcdict = Dict()
    for p in D["Phases"]
        name = p["name"]
        spclist = Array{Species,1}()
        for d in p["Species"]
            d["index"] = spcindex
            spcindex += 1
            spcname = d["name"]
            #attempt to generate molecular information from rdkit if possible
            if !("atomnums" in keys(d)) || !("bondnum" in keys(d))
                if "smiles" in keys(d)
                    try
                        d["atomnums"],d["bondnum"] = getatomdictsmiles(d["smiles"])
                    catch
                        @warn("failed to generate molecular information from smiles for species $spcname")
                    end
                elseif "inchi" in keys(d)
                    try
                        d["atomnums"],d["bondnum"] = getatomdictinchi(d["inchi"])
                    catch
                        @warn("failed to generate molecular information from inchi for species $spcname")
                    end
                end
            end
            #if diffusion model is not present and we can calculate molecular radius and stokesdiffusivity
            if !("diffusion" in keys(d)) && "atomnums" in keys(d) && "bondnum" in keys(d)
                try
                    diffD = Dict()
                    diffD["type"] = "StokesDiffusivity"
                    diffD["r"] = getspeciesradius(d["atomnums"],d["bondnum"])
                    d["diffusion"] = diffD
                    if !("radius" in keys(d))
                        d["radius"] = diffD["r"]
                    end
                catch
                    @warn("failed to generate StokesDiffusivity model for species $spcname")
                end
            end

            spc = fcndict2obj(d,ymlunitsdict)
            push!(spclist,spc)
            spcdict[spc.name] = (spc,p["name"])
        end
        outdict[name] = Dict()
        outdict[name]["Species"] = spclist
        outdict[name]["Reactions"] = Array{ElementaryReaction,1}()
    end

    #reactions
    rxnindex = 1
    for rxn in D["Reactions"]
        phs = Array{String,1}()
        rxn["reactants"] = convert(Array{Any},rxn["reactants"])
        rxn["index"] = rxnindex
        rxnindex += 1
        for (i,item) in enumerate(rxn["reactants"])
            spc,ph = spcdict[item]
            push!(phs,ph)
            rxn["reactants"][i] = spc
        end
        rxn["products"] = convert(Array{Any},rxn["products"])
        for (i,item) in enumerate(rxn["products"])
            spc,ph = spcdict[item]
            push!(phs,ph)
            rxn["products"][i] = spc
        end
        rxn["reactants"] = [r for r in rxn["reactants"]]
        rxn["reactants"] = SVector(rxn["reactants"]...)
        rxn["products"] = [r for r in rxn["products"]]
        rxn["products"] = SVector(rxn["products"]...)
        rxn["reactantinds"] =  [r.index for r in rxn["reactants"]]
        rxn["reactantinds"] = SVector(rxn["reactantinds"]...)
        rxn["productinds"] =  [r.index for r in rxn["products"]]
        rxn["productinds"] = SVector(rxn["productinds"]...)
        if "efficiencies" in keys(rxn["kinetics"])
            rxn["kinetics"]["efficiencies"] = Dict{Int64,Float64}([spcdict[name][1].index=>val-1.0 for (name,val) in rxn["kinetics"]["efficiencies"]]) #in RMS we correct [M] rather than calculate it so we subtract 1
        end
        r = fcndict2obj(rxn,ymlunitsdict)
        unique!(phs)
        if length(phs) == 1
            push!(outdict[phs[1]]["Reactions"],r)
        else
            if Set(phs) in keys(outDict)
                push!(outdict[Set(phs)],r)
            else
                outdict[Set(phs)]=[r]
            end
        end
    end

    return outdict
end

export readinputyml


############################################################################################
"""
$(TYPEDEF)

SMC kernel containing particles, dynamics and buffers for sampling.

# Fields
$(TYPEDFIELDS)
"""
struct SMCParticles{B<:AbstractAlgorithm,A<:ModelWrapper,D<:SMCBuffer}
    "SMC particles used for sampling."
    model::Vector{A}
    "Individual kernels associated to an SMC particle."
    kernel::Vector{B}
    "Weights struct for all SMC particles."
    weights::BaytesCore.ParameterWeights
    "Buffer values for allocation free evaluations."
    buffer::D
    function SMCParticles(
        model::Vector{A},
        algorithm::Vector{B},
        weights::BaytesCore.ParameterWeights,
        buffer::D,
    ) where {B<:AbstractAlgorithm,A<:ModelWrapper,D<:SMCBuffer}
        return new{B,A,D}(model, algorithm, weights, buffer)
    end
end

############################################################################################
"""
$(SIGNATURES)
Initialize SMC particles.

# Examples
```julia
```

"""
function SMCParticles(
    _rng::Random.AbstractRNG,
    JitterKernel::AbstractConstructor,
    objective::Objective,
    info::BaytesCore.SampleDefault,
    tune::SMCTune
)
    ## Initialize individual MCMC algorithm
    @unpack data, temperature = objective
    @unpack tagged, Ntuning = tune
    Nchains = tune.chains.Nchains
    ## Adjust observed data to current knowledge
    modelᵗᵉᵐᵖ = deepcopy(objective.model)
    ## Initialize individual Models ~ Assign copies for  Model and Algorithms ~ necessary in initiation so no pointer error
    modelᵛ = [
        ModelWrapper(deepcopy(modelᵗᵉᵐᵖ.val), modelᵗᵉᵐᵖ.info, modelᵗᵉᵐᵖ.id) for
        _ in Base.OneTo(Nchains)
    ]
    algorithmᵛ = [JitterKernel(_rng, modelᵛ[iter], data, temperature, info) for iter in Base.OneTo(Nchains)]
    ## Assign weights
    weights = BaytesCore.ParameterWeights(Nchains)
    ## Assign buffer for inplace resampling
    buffer = SMCBuffer(
        _rng, algorithmᵛ[1], modelᵗᵉᵐᵖ, data, Nchains
    )
    ## Loop through all models
    #Polyester.@batch per=thread minbatch=tune.batchsize
    Base.Threads.@threads for iter in eachindex(algorithmᵛ)
        ## Tune kernel
        propose!(_rng, algorithmᵛ[iter], modelᵛ[iter], data, temperature, BaytesCore.UpdateTrue())
        for _ in 2:Ntuning
            propose!(_rng, algorithmᵛ[iter], modelᵛ[iter], data, temperature, tune.capture)
        end
        ## Assign weight
        #!NOTE: Objective itself only stores logposterior function, no problem to call it as long as θ comes from modelᵛ.
        _objective = Objective(modelᵛ[iter], objective.data, tune.tagged, objective.temperature)
        buffer.cumweights[iter], weights.buffer[iter] = SMCweight(
            _rng,
            _objective,
            algorithmᵛ[iter],
            _objective.temperature * _objective(objective.model.val)
        )
    end
    ## Normalize weights
    weights(weights.buffer)
    ## Return container
    return SMCParticles(modelᵛ, algorithmᵛ, weights, buffer)
end

############################################################################################
function SMCDiagnostics(
    smc::SMCParticles, temperature::T, ESS::Float64, accepted::Bool, jittersteps::Int64, iter::Int64, generated::G
) where {T<:AbstractFloat, G}
    return SMCDiagnostics(
        BaytesCore.BaseDiagnostics(
            Statistics.mean(smc.buffer.cumweights),
            temperature,
            copy(smc.buffer.predictions),
            iter-1
        ),
        BaytesCore.weightedincrement(smc.weights),
        #!NOTE: There is not really any way around making a copy from buffer if no pointer issue for diagnostics
        copy(smc.buffer.cumweights),
        copy(smc.weights.ℓweightsₙ),
        deepcopy(smc.buffer.jitterdiagnostics),
        jittersteps,
        copy(smc.buffer.correlation.ρ),
        ESS,
        accepted,
        generated
    )
end

############################################################################################
#export
export SMCParticles, init, resample!, propagate!, weight!


#
#=
Examples for developing the JAC tools
=#
println("A)  Tests & development of the electronic structure part.")
println("---------------------------------------------------------")
println("Aa) Test of the radial integration in CI computations (Randolf).")
println("Ab) Read in a full multiplet representation from files compatible to Ratip2012, Grasp2018, ....")
println("Ac) Test of several radial e-e potentials.")
println("Ad) Tests towards a SCF field: B-spline primitives and one-particle spectra in a local potential.")
println("Ae) Test of the CI part for an internally generated neon multiplet without Breit interaction.")
println("Af) Test of the frequency-independent Breit interaction for an internally generated neon multiplet.")
println("Ag) Test of the QED model corrections to the level structure of atoms and ions.")
println("Ah) Generate and normalize continuum orbitals in a local potential.")
println("Ai) Test of the jj-LS transformation of levels from a given multiplet.")
println("An) Test of a mean-field basis and configuration-interaction (CI) expansion.")
println("Ap) Test of restricted-active-space (RAS) expansion.")
println("Aq) Test of the Green(function) expansion.")
println("As) Test of spin-angular coefficients.")
println("At) Test of parallel computating with Julia.")
println("")
println("B)  Tests & development of atomic amplitudes")
println("--------------------------------------------")
println("Ba) Tests of the dipole, em multipole and momentum-transfer amplitudes.")
println("Bb) Tests of the parity non-conservation, Schiff moment and anapole moment amplitudes.")
println("")
println("C)   Tests & development of atomic properties")
println("---------------------------------------------")
println("Ca)  Test of the Einstein module with ASF from an internally generated multiplet.")
println("Cb)  Test of the Hfs module for HFS A,B parameters and hyperfine representation with ASF from an internally generated multiplet.")
println("Cc)  Test of the IsotopeShift module with ASF from an internally generated multiplet.")
println("Cd)  Test of the FormFactor module with ASF from an internally generated multiplet.")
println("Ce)  Test of the PlasmaShift module with ASF from an internally generated multiplet.")
println("Cfg) Test of the LandeZeeman module with ASF from an internally generated multiplet.")
println("Ch)  Test of the AlphaVariation module with ASF from an internally generated multiplet.")
println("Ci)  Test of the DecayYields module with ASF from an internally generated multiplet.")
println("Ck)  Test of the MultipolePolarizibility module with ASF from an internally generated multiplet.")
println("")
println("D)   Tests & development of atomic processes")
println("--------------------------------------------")
println("Da)  Test of the PhotoEmission module with ASF from an internally generated initial- and final-state multiplet.")
println("Db)  Test of the PhotoExcitation module with ASF from an internally generated initial- and final-state multiplet.")
println("Dc)  Test of the Photoionization module with ASF from an internally generated initial- and final-state multiplet.")
println("Dcp) Test of the Photoionization and PlasmaShift module with ASF from an internally generated initial- and final-state multiplet.")
println("Dd)  Test of the PhotoRecombination module with ASF from an internally generated initial- and final-state multiplet.")
println("De)  Test of the Auger module with ASF from an internally generated initial- and final-state multiplet.")
println("Dep) Test of the Auger and PlasmaShift modules with ASF from an internally generated initial- and final-state multiplet.")
println("Df)  Test of the Dielectronic module with ASF from an internally generated initial-, intermediate and final-state multiplets.")
println("Dg)  Test of the PhotoExcitationFluores module with ASF from an internally generated initial-, intermediate and final-state multiplets.")
println("Dh)  Test of the PhotoExcitationAutoIon module with ASF from an internally generated initial-, intermediate and final-state multiplets.")
println("Di)  Test of the RayleighCompton module with ASF from an internally generated initial- and final-state multiplets.")
println("Dj)  Test of the MultiPhotonDeExcitation module with ASF from an internally generated initial- and final-state multiplet.")
println("Dk)  Test of the CoulombExcitation module with ASF from an internally generated initial- and final-state multiplet.")
println("Dl)  Test of the PhotoIonizationFluores module with ASF from an internally generated initial-, intermediate and final-state multiplets.")
println("Dm)  Test of the PhotoIonizationAutoion module with ASF from an internally generated initial-, intermediate and final-state multiplets.")
println("Dn)  Test of the ImpactExcitation module with ASF from an internally generated initial- and final-state multiplet.")
println("Dp)  Test of the ImpactExcitationAutoion module with ASF from an internally generated initial-, intermediate and final-state multiplet.")
println("Dq)  Test of the RadiativeAuger module with ASF from an internally generated initial- and final-state multiplet.")
println("Dr)  Test of the MultiPhotonIonization module with ASF from an internally generated initial- and final-state multiplet.")
println("Ds)  Test of the MultiPhotonDoubleIon module with ASF from an internally generated initial- and final-state multiplet.")
println("Dt)  Test of the InternalConversion module with ASF from an internally generated initial- and final-state multiplet.")
println("Du) Test of the CoulombIonization module with ASF from an internally generated initial- and final-state multiplet.")
println("Dv) Test of the PairAnnihilation1Photon.jl module with ASF from an internally generated initial- and final-state multiplet.")
println("")
println("E)  Tests & development of atomic cascades")
println("------------------------------------------")
println("Ea) Three-step cascade after 1s-3p photo-excitation of Si^+: Configuration model.")
println("Eb) Simulation of the four-step cascade after 1s-3p photo-excitation of Si^+: Configuration model.")
println("Ec) Three-step cascade computations and simulations for the decay of the neon 1s^-1 3p hole states: AverageSCA model.")
println("Ed) Photoabsorption of Ne^+: AverageSCA model.")
println("")
println("F)  Tests & development of symbolic evaluations of Racah expressions")
println("--------------------------------------------------------------------")
println("Fa) Symbolic evaluation by means of special values & recursion relations of the Wigner n-j symbols.")
println("Fb) Symbolic simplification of Racah expressions by means of sum rules.")
println("Fc) Symbolic simplification of recoupling coefficients by means of sum rules.")
println("Fd) Symbolic simplification of Racah expressions including spherical harmonics and Wigner rotation matrices.")
println("Fe) Symbolic simplification of spherical tensor operators, matrix elements and spherical amplitudes.")
println("")
println("G)  Tests & development of semiempirical computations")
println("-----------------------------------------------------")
println("Ga) Tests of the semi-empirical computation of ADK rates.")
println("")
println("H)  Tests & development of atomic response computations")
println("-----------------------------------------------------")
println("Ha) Tests of the HighHarmonic module to calculate a high-harmonic spectrum in single-electron approximation.")
println("Hb) Tests of the StrongField module to calculate energy and momentum distributions of photoelectron in SFA.")