# [
#  [ MPIStateArray Name, Field Name, Maximum, Minimum, Mean, Standard Deviation ],
#  [         :                :          :        :      :          :           ],
# ]
parr = [
    ["Q", "u[1]", 12, 12, 0, 12],
    ["Q", "u[2]", 0, 0, 0, 0],
    ["Q", :η, 12, 12, 0, 12],
    ["Q", :θ, 15, 15, 15, 15],
    ["s_aux", :y, 15, 15, 15, 15],
    ["s_aux", :w, 12, 12, 0, 12],
    ["s_aux", :pkin, 15, 15, 15, 15],
    ["s_aux", :wz0, 12, 12, 0, 12],
    ["aux", "uᵈ[1]", 15, 15, 15, 15],
    ["aux", "uᵈ[2]", 15, 15, 15, 15],
    ["aux", "ΔGᵘ[1]", 15, 15, 15, 15],
    ["aux", "ΔGᵘ[2]", 15, 15, 15, 15],
]

### fully explicit
explicit = [
    [
        "state",
        "u[1]",
        -9.58544066049463849843e-01,
        9.58544066049465071089e-01,
        -6.13908923696726568442e-17,
        4.45400263687296238402e-01,
    ],
    [
        "state",
        "u[2]",
        -4.33260855197780914020e-14,
        1.99397359064900580713e-14,
        -1.30343101521973575132e-16,
        2.95525689323845820606e-15,
    ],
    [
        "state",
        :η,
        -8.52732886154656810618e-01,
        8.52845586939211197652e-01,
        2.20052243093959998331e-14,
        6.02992088522925295813e-01,
    ],
    [
        "state",
        :θ,
        0.00000000000000000000e+00,
        0.00000000000000000000e+00,
        0.00000000000000000000e+00,
        0.00000000000000000000e+00,
    ],
    [
        "aux",
        :y,
        0.00000000000000000000e+00,
        1.00000000000000011642e+06,
        5.00000000000000000000e+05,
        2.92775877460665535182e+05,
    ],
    [
        "aux",
        :w,
        -4.04553460063758398447e-04,
        4.04714358463272711169e-04,
        4.75730566051879549438e-19,
        1.63958655681888576441e-04,
    ],
    [
        "aux",
        :pkin,
        0.00000000000000000000e+00,
        0.00000000000000000000e+00,
        0.00000000000000000000e+00,
        0.00000000000000000000e+00,
    ],
    [
        "aux",
        :wz0,
        -2.01164684799271339293e-04,
        2.01041968159484089494e-04,
        -2.10942374678779754294e-20,
        1.42228420244455277133e-04,
    ],
    [
        "aux",
        "uᵈ[1]",
        0.00000000000000000000e+00,
        0.00000000000000000000e+00,
        0.00000000000000000000e+00,
        0.00000000000000000000e+00,
    ],
    [
        "aux",
        "uᵈ[2]",
        0.00000000000000000000e+00,
        0.00000000000000000000e+00,
        0.00000000000000000000e+00,
        0.00000000000000000000e+00,
    ],
    [
        "aux",
        "ΔGᵘ[1]",
        0.00000000000000000000e+00,
        0.00000000000000000000e+00,
        0.00000000000000000000e+00,
        0.00000000000000000000e+00,
    ],
    [
        "aux",
        "ΔGᵘ[2]",
        0.00000000000000000000e+00,
        0.00000000000000000000e+00,
        0.00000000000000000000e+00,
        0.00000000000000000000e+00,
    ],
]

imex = [
    [
        "state",
        "u[1]",
        -9.57705359685807500192e-01,
        9.57705359685806500991e-01,
        4.54747350886464140750e-17,
        4.45323727947873004851e-01,
    ],
    [
        "state",
        "u[2]",
        -4.07596731389788922865e-14,
        2.53992382675900717845e-14,
        9.88087105439151860723e-18,
        2.63742927063789745855e-15,
    ],
    [
        "state",
        :η,
        -8.55941716778505390373e-01,
        8.56014908798837681481e-01,
        2.16732587432488803528e-14,
        6.05260814296588400829e-01,
    ],
    [
        "state",
        :θ,
        0.00000000000000000000e+00,
        0.00000000000000000000e+00,
        0.00000000000000000000e+00,
        0.00000000000000000000e+00,
    ],
    [
        "aux",
        :y,
        0.00000000000000000000e+00,
        1.00000000000000011642e+06,
        5.00000000000000000000e+05,
        2.92775877460665535182e+05,
    ],
    [
        "aux",
        :w,
        -4.03419301824637895407e-04,
        4.03626959596159180614e-04,
        -3.33066907387546970565e-21,
        1.63816237485524198066e-04,
    ],
    [
        "aux",
        :pkin,
        0.00000000000000000000e+00,
        0.00000000000000000000e+00,
        0.00000000000000000000e+00,
        0.00000000000000000000e+00,
    ],
    [
        "aux",
        :wz0,
        -1.97016630922629948884e-04,
        1.97066086934914155761e-04,
        -7.49400541621980656312e-19,
        1.39393318574472925390e-04,
    ],
    [
        "aux",
        "uᵈ[1]",
        0.00000000000000000000e+00,
        0.00000000000000000000e+00,
        0.00000000000000000000e+00,
        0.00000000000000000000e+00,
    ],
    [
        "aux",
        "uᵈ[2]",
        0.00000000000000000000e+00,
        0.00000000000000000000e+00,
        0.00000000000000000000e+00,
        0.00000000000000000000e+00,
    ],
    [
        "aux",
        "ΔGᵘ[1]",
        0.00000000000000000000e+00,
        0.00000000000000000000e+00,
        0.00000000000000000000e+00,
        0.00000000000000000000e+00,
    ],
    [
        "aux",
        "ΔGᵘ[2]",
        0.00000000000000000000e+00,
        0.00000000000000000000e+00,
        0.00000000000000000000e+00,
        0.00000000000000000000e+00,
    ],
]

refVals = (explicit = (explicit, parr), imex = (imex, parr))


module Pi

using DataFrames
using CSV

export readPiFile

struct ModelANDParents
   relation::Any
   parents::Any
end

struct ModelORParents
   relation::Any
   posParents::Any
   negParents::Any
end

struct PIparents
   andPosParents::Array
   andNegParents::Array
   orPosParents::Array
   orNegParents::Array
end

# Assumptions:
## If there is only one parent it will be an AND relation

function readFile(fname)
    df = CSV.read(fname,
                  delim="\t",
                  header=["parentName", "childName", "posneg", "andor"],
                  types=[String, String, Int, Int])

    PIs = Dict{AbstractString,Any}()

    for row in eachrow(df)
        posnegbool = row[:posneg] == 1 ? true : false
        andBool = row[:andor] == 0 ? true : false
        if haskey(PIs, row[:childName])
            if posnegbool
                parents = andBool ?
                             PIs[row[:childName]].andPosParents :
                             PIs[row[:childName]].orPosParents
            else
                parents = andBool ?
                             PIs[row[:childName]].andNegParents :
                             PIs[row[:childName]].orNegParents
            end
            push!(parents, row[:parentName])
        else
            posParents = AbstractString[]
            negParents = AbstractString[]
            if posnegbool
                push!(posParents, row[:parentName])
            else
                push!(negParents, row[:parentName])
            end

            node = PIparents(andBool ? posParents : AbstractString[],
                             andBool ? negParents : AbstractString[],
                             andBool ? AbstractString[] : posParents,
                             andBool ? AbstractString[] : negParents)
            PIs[row[:childName]] = node
        end
    end

    for row in eachrow(df)
        if !haskey(PIs, row[:parentName])
            PIs[row[:parentName]] = PIparents(AbstractString[],AbstractString[],AbstractString[],AbstractString[])
        end
    end

    nodes = Dict{AbstractString,Any}()
    for nodeName in keys(PIs)
        parents = PIs[nodeName].andPosParents
        if (length(PIs[nodeName].andPosParents) + length(PIs[nodeName].andNegParents)) == 0
            if (length(PIs[nodeName].orPosParents) + length(PIs[nodeName].orNegParents)) > 0
                nodes[nodeName] = ModelORParents("OR", PIs[nodeName].orPosParents, PIs[nodeName].orNegParents)
            else
                nodes[nodeName] = ModelANDParents("ROOT", parents)
            end
        else
            if (length(PIs[nodeName].orPosParents) + length(PIs[nodeName].orNegParents)) > 0
                orNodeName = string(nodeName, "_OR")
                nodes[orNodeName] = ModelORParents("OR", PIs[nodeName].orPosParents, PIs[nodeName].orNegParents)
                push!(parents, orNodeName)
            end

            if ((length(PIs[nodeName].andNegParents) > 1) && (length(parents) == 0))
                pseudoParent = "PSEUDONODE_PARENT_$nodeName"
                push!(PIs[nodeName].andPosParents, pseudoParent)
                nodes[pseudoParent] = ModelANDParents("ROOT", AbstractString[])
                parents = PIs[nodeName].andPosParents
            end

            negParents = PIs[nodeName].andNegParents
            if length(parents) == 1 && length(negParents) == 1
                nodes[nodeName] = ModelANDParents("ANDNEG", union(parents, negParents))
            elseif length(parents) == 0 && length(negParents) == 1
                nodes[nodeName] = ModelANDParents("NEG", negParents)
            elseif length(parents) == 1 && length(negParents) == 0
                nodes[nodeName] = ModelANDParents("AND", parents)
            else
               pseudonodeIndex = 1
               childNodeName = parents[1]
               for i = 2:length(parents)
                   parenti = parents[i]
                   if length(negParents) == 0 && i == length(parents)
                       nodes[nodeName]= ModelANDParents("AND", AbstractString[childNodeName, parenti])
                       continue
                   else
                       newChildNodeName = string(childNodeName, "_PSEUDONODE_", parenti)
                       nodes[newChildNodeName]= ModelANDParents("AND", AbstractString[childNodeName, parenti])
                       childNodeName = newChildNodeName
                   end
               end

               for i = 1:length(negParents)
                   parenti = negParents[i]
                   if i == length(negParents)
                       nodes[nodeName]= ModelANDParents("ANDNEG", AbstractString[childNodeName, parenti])
                   else
                       newChildNodeName = string(childNodeName, "_PSEUDONODE_", parenti)
                       nodes[newChildNodeName]= ModelANDParents("ANDNEG", AbstractString[childNodeName, parenti])
                       childNodeName = newChildNodeName
                   end
               end
            end
        end

    end

    return nodes
end

end


#=

####
#### Reads profiles needed for RRTMGP and related
#### calculations assuming a certain netCDF file layout
####

This file reads an example file containing atmospheric conditions (temperature, pressure, gas concentrations)
 and surface properties (emissivity, temperature), defined on `nlay` layers across a set of `ncol` columns subject to
 `nexp` perturbations, and returns them in data structures suitable for use in rte and rrtmpg. The input data
 are partitioned into a user-specified number of blocks.
For the moment only quantities relevant to longwave calculations are provided.

The example files comes from the Radiative Forcing MIP (https://www.earthsystemcog.org/projects/rfmip/)
 The protocol for this experiment allows for different specifications of which gases to consider:
all gases, (CO2, CH4, N2O) + {CFC11eq; CFC12eq + HFC-134eq}. Ozone is always included
The protocol does not specify the treatment of gases like CO

# Convention

Note that `ds` is used to denote an NC dataset.
=#

using RRTMGP.GasConcentrations
using RRTMGP.Utilities
using RRTMGP.Gases
using NCDatasets

"""
    read_atmos(ds, FT, I, gases_prescribed)

Read profiles for all columns
 - `p_lay` pressure (layers)
 - `t_lay` temperature (layers)
 - `p_lev` pressure (levels)
 - `t_lev` temperature (levels)
 - `col_dry` gas concentrations
"""
function read_atmos(ds, FT, I, gases_prescribed)

    ncol = ds.dim["col"]
    nlay = ds.dim["lay"]
    nlev = ds.dim["lev"]
    @assert nlev == nlay + 1

    p_lay = Array{FT}(ds["p_lay"][:])
    t_lay = Array{FT}(ds["t_lay"][:])
    p_lev = Array{FT}(ds["p_lev"][:])
    t_lev = Array{FT}(ds["t_lev"][:])

    gases_to_look_for = [
        h2o(),
        co2(),
        o3(),
        n2o(),
        co(),
        ch4(),
        o2(),
        n2(),
        ccl4(),
        cfc11(),
        cfc12(),
        cfc22(),
        hfc143a(),
        hfc125(),
        hfc23(),
        hfc32(),
        hfc134a(),
        cf4(),
        no2(),
    ]

    gases_in_database = filter(ug -> haskey(ds, "vmr_" * chem_name(ug)), gases_to_look_for)

    gas_concs = GasConcs(FT, I, gases_prescribed, ncol, nlay, length(gases_in_database))

    for eg in gases_in_database
        set_vmr!(gas_concs, eg, Array{FT}(ds["vmr_" * chem_name(eg)][:]))
    end

    # col_dry has unchanged allocation status on return if the variable isn't present in the netCDF file
    col_dry = haskey(ds, "col_dry") ? Array{FT}(ds["col_dry"][:]) : nothing

    return p_lay, t_lay, p_lev, t_lev, gas_concs, col_dry
end

"""
    is_sw(ds)

Does this file contain variables needed to do SW calculations?
"""
is_sw(ds) = haskey(ds, "solar_zenith_angle")

"""
    is_lw(ds)

Does this file contain variables needed to do LW calculations?
"""
is_lw(ds) = !is_sw(ds)

"""
    read_lw_bc(ds)

Read LW boundary conditions for all columns
"""
read_lw_bc(ds) = ds["t_sfc"][:], ds["emis_sfc"][:]

"""
    read_lw_rt(ds)

Read LW radiative transfer parameters
"""
read_lw_rt(ds) = length(size(ds["angle"]))

"""
    read_sw_bc(ds)

Read SW boundary conditions for all columns
"""
read_sw_bc(ds) = (
    ds["solar_zenith_angle"][:],
    ds["total_solar_irradiance"][:],
    ds["sfc_alb_direct"],
    ds["sfc_alb_diffuse"],
    haskey(ds, "tsi_scaling") ? ds["tsi_scaling"][:] : nothing,
)

"""
    read_spectral_disc!(ds, spectral_disc::AbstractOpticalProps)

Read spectral discretization
"""
function read_spectral_disc!(ds, spectral_disc::AbstractOpticalProps)
    @assert ds.dim["pair"] == 2
    band_lims_wvn = ds["band_lims_wvn"][:]
    band_lims_gpt = ds["band_lims_gpt"][:]
    spectral_disc.base = OpticalPropsBase("read_spectral_disc!", band_lims_wvn, band_lims_gpt)
end

"""
    read_sfc_test_file(ds)

Read surface SW albedo and LW emissivity spectra from the surface test file
"""
read_sfc_test_file(ds) = (ds["SW_albedo"][:], ds["LW_emissivity"][:])

"""
    read_direction(ds)

Which direction is up? Stored as a global attribute.
"""
read_direction(ds) = (ds.attrib["top_at_1"] == 1)

"""
    read_sources(ds)

Sources of upward and downward diffuse radiation, for each layer and at the surface
"""
read_sources(ds) = ds["source_up"][:], ds["source_dn"][:], ds["source_sfc"][:]

"""
    read_lw_Planck_sources!(ds, sources::SourceFuncLongWave{FT}) where FT

Longwave sources at layer centers; edges in two directions; surface
   Also directionality since this will be needed for solution
"""
function read_lw_Planck_sources!(ds, sources::SourceFuncLongWave{FT}) where {FT}
    ncol = ds.dim["col"]
    nlay = ds.dim["lay"]
    ngpt = ds.dim["gpt"]
    nband = ds.dim["band"]
    # Error checking
    @assert ds.dim["pair"] == 2
    @assert haskey(ds, "lay_src")

    # Spectral discretization
    band_lims_wvn = ds["band_lims_wvn"][:]
    band_lims_gpt = ds["band_lims_gpt"][:]

    sources.optical_props = OpticalPropsBase("SourceFuncLongWave", band_lims_wvn, band_lims_gpt)
    sources.τ .= Array{FT}(undef, ncol, nlay)

    sources.lay_source .= ds["lay_src"][:]
    sources.lev_source_inc .= ds["lev_src_inc"][:]
    sources.lev_source_dec .= ds["lev_src_dec"][:]
    sources.sfc_source .= ds["sfc_src"][:]
    return nothing

end

"""
    read_sw_solar_sources(ds)

Shortwave source at TOA
   Also directionality since this will be needed for solution
"""
read_sw_solar_sources(ds) = ds["toa_src"][:]

"""
    read_two_stream(ds)

Two-stream results: reflection and transmission for diffuse and direct radiation; also extinction
"""
function read_two_stream(ds)
    Rdif = ds["Rdif"][:]
    Tdif = ds["Tdif"][:]
    Rdir = haskey(ds, "Rdir") ? ds["Rdir"][:] : nothing
    Tdir = haskey(ds, "Tdir") ? ds["Tdir"][:] : nothing
    Tnoscat = haskey(ds, "Tnoscat") ? ds["Tnoscat"][:] : nothing
    return Rdif, Tdif, Rdir, Tdir, Tnoscat
end

"""
    read_gpt_fluxes(ds)

g-point fluxes
"""
read_gpt_fluxes(ds) =
    (ds["gpt_flux_up"][:], ds["gpt_flux_dn"][:], haskey(ds, "gpt_flux_dn_dir") ? ds["gpt_flux_dn_dir"][:] : nothing)

"""
    read_size(ds)

Find the size of the problem: columns, layers, perturbations (experiments)
"""
function read_size(ds)
    ncol = Int(ds.dim["site"])
    nlay = Int(ds.dim["layer"])
    nexp = Int(ds.dim["expt"])
    @assert ds.dim["level"] == nlay + 1
    return ncol, nlay, nexp
end

"""
    read_and_block_pt(ds)

Return layer and level

 - `p_lay` layer pressure dimensions (ncol, nlay, nblocks)
 - `p_lev` level pressure dimensions (ncol, nlay+1, nblocks)
 - `t_lay` layer temperature dimensions (ncol, nlay, nblocks)
 - `t_lev` level temperature dimensions (ncol, nlay+1, nblocks)
"""
function read_and_block_pt(ds)
    FT = Float64
    ncol_l, nlay_l, nexp_l = read_size(ds)
    @assert !any([ncol_l, nlay_l, nexp_l] .== 0)

    # Read p, T data; reshape to suit RRTMGP dimensions
    p_lay = Array(transpose(kron(ones(1, nexp_l), Array{FT}(ds["pres_layer"][:]))))
    p_lev = Array(transpose(kron(ones(1, nexp_l), Array{FT}(ds["pres_level"][:]))))
    t_lay = Array(transpose(reshape(Array{FT}(ds["temp_layer"][:]), nlay_l, ncol_l * nexp_l)))
    t_lev = Array(transpose(reshape(Array{FT}(ds["temp_level"][:]), nlay_l + 1, ncol_l * nexp_l)))

    return p_lay, p_lev, t_lay, t_lev
end

"""
    read_and_block_sw_bc(ds, blocksize)

Read and reshape shortwave boundary conditions

 - `surface_albedo` surface albedo
 - `total_solar_irradiance` total solar irradiance
 - `solar_zenith_angle` solar zenith angle
"""
function read_and_block_sw_bc(ds)
    FT = Float64
    ncol_l, nlay_l, nexp_l = read_size(ds)
    blocksize = ncol_l * nexp_l
    @assert !any([ncol_l, nlay_l, nexp_l] .== 0)

    # Check that output arrays are sized correctly : blocksize, nlay, (ncol * nexp)/blocksize
    surface_albedo = repeat(Array{FT}(ds["surface_albedo"][:]), nexp_l)
    total_solar_irradiance = repeat(Array{FT}(ds["total_solar_irradiance"][:]), nexp_l)
    solar_zenith_angle = repeat(Array{FT}(ds["solar_zenith_angle"][:]), nexp_l)
    return surface_albedo, total_solar_irradiance, solar_zenith_angle
end

"""
    read_and_block_lw_bc(ds, blocksize)

Read and reshape longwave boundary conditions

 - `surface_emissivity` surface emissivity
 - `surface_temperature` surface temperature
"""
function read_and_block_lw_bc(ds)
    FT = Float64
    ncol_l, nlay_l, nexp_l = read_size(ds)
    ncols = ncol_l * nexp_l
    @assert !any([ncol_l, nlay_l, nexp_l] .== 0)

    surface_emissivity = Array{FT}(reshape(repeat(ds["surface_emissivity"][:], 1, nexp_l), ncols))
    surface_temperature = Array{FT}(reshape(ds["surface_temperature"][:], ncols)) # alternate version

    return surface_emissivity, surface_temperature
end

"""
    determine_gas_names(ds, forcing_index)

Create a pair of string arrays - one containing the chemical name of each gas, used by the k-distribution, and
one containing the name as contained in the RFMIP input files - depending on the forcing scenario:

 - `forcing_index` (1 = all available greenhouse gases;
                    2 = CO2, CH4, N2O, CFC11eq
                    3 = CO2, CH4, N2O, CFC12eq, HFC-134eq
                    All scenarios use 3D values of ozone, water vapor so those aren't listed here
"""
function determine_gas_names(ds, forcing_index)

    @assert any(forcing_index .== [1, 2, 3])
    if forcing_index == 1
        names_in_kdist = convert.(AbstractGas, read_kdist_gas_names(ds))

        # elseif forcing_index == 2

        #   # Not part of the RFMIP specification, but oxygen is included because it's a major
        #   #    gas in some bands in the SW
        #   names_in_kdist = [co2(), ch4(), n2o(), o2(), cfc12(), cfc11()]
        # elseif forcing_index == 3

        #   # Not part of the RFMIP specification, but oxygen is included because it's a major
        #   #    gas in some bands in the SW
        #   names_in_kdist = [co2(), ch4(), n2o(), o2(), cfc12(), hfc134a()]
    end
    return names_in_kdist

end

"""
    read_kdist_gas_names(ds)

Read the names of the gases known to the k-distribution
"""
read_kdist_gas_names(ds) = lowercase.(strip.(String[join(ds["gas_names"][:][:, i]) for i in 1:ds.dim["absorber"]]))

"""
    read_and_block_gases_ty(ds, gas_names)

Read and reshape gas concentrations. RRTMGP requires gas concentrations to be supplied via a class
(GasConcs). Gas concentrations are set via a call to set_vmr!(gas_concs, name, values)
where `name` is nominally the chemical formula for the gas in question and `values` may be
a scalar, a 1-d profile assumed to apply to all columns, or an array of dimension (`ncol`, `nlay`).

This routine outputs a vector `nblocks` long of these types so each element of the array can be passed to
the rrtmgp gas optics calculation in turn.

This routine exploits RFMIP conventions: only water vapor and ozone vary by column within
each experiment.

Fields in the RFMIP file have a trailing _GM (global mean); some fields use a chemical formula and other
a descriptive name, so a map is provided between these.

 - `gas_names` Names used by the k-distribution/gas concentration type
 - `gas_conc_array` vector of [`GasConcs`](@ref)
"""
function read_and_block_gases_ty(ds, gas_names::Vector{AbstractGas})
    ncol_l, nlay_l, nexp_l = read_size(ds)
    ncols = ncol_l * nexp_l
    @assert !any([ncol_l, nlay_l, nexp_l] .== 0)
    FT = Float64
    I = Int
    gas_conc_array = GasConcs(FT, I, gas_names, ncols, nlay_l)

    # Experiment index for each column
    exp_num = repeat(collect(1:nexp_l)', ncol_l, 1)[:]
    # Water vapor and ozone depend on col, lay, exp: look just like other fields
    for gas in (h2o(), o3())
        gas_conc = Array{FT}(ds[rfmip_name(gas)][:])
        gas_conc_scaling = read_scaling(ds, FT, rfmip_name(gas))
        gas_conc_temp_3d = reshape(gas_conc, nlay_l, ncols) .* gas_conc_scaling
        gas_conc_temp_3d_t = convert(Array, transpose(gas_conc_temp_3d))
        set_vmr!(gas_conc_array, gas, gas_conc_temp_3d_t)
    end

    # All other gases are a function of experiment only
    for gas in gas_names

        # Skip 3D fields above, also NO2 since RFMIP doesn't have this
        if gas in [h2o(), o3(), no2()]
            continue
        end

        # Read the values as a function of experiment
        gas_conc_scaling = read_scaling(ds, FT, rfmip_name(gas) * "_GM")
        gas_conc_temp_1d = ds[rfmip_name(gas) * "_GM"][:] * gas_conc_scaling

        # Does every value in this block belong to the same experiment?
        if all(exp_num[2:end] .- exp_num[1] .== 0)
            # Provide a scalar value
            set_vmr!(gas_conc_array, gas, gas_conc_temp_1d[exp_num[1]])
        else
            # Create 2D field, ncols x nlay, with scalar values from each experiment
            set_vmr!(gas_conc_array, gas, repeat(gas_conc_temp_1d[exp_num[:]], 1, nlay_l))
        end
    end
    return gas_conc_array
end

read_scaling(ds, FT, varName) = parse(FT, ds[varName].attrib["units"])


__precompile__(true)
module Alazar

# Type aliases go here
export U32, U16, U8, S32, S16
export dsp_module_handle
export AlazarBits, Alazar8Bit, Alazar12Bit, Alazar16Bit
export AlazarFFTBits, U16Log, U16Amp2, U8Log, U8Amp2
export S32Real, S32Imag, FloatLog, FloatAmp2

export alazaropen

const U32 = Culong
const U16 = Cushort
const U8  = Cuchar
const S32 = Clong
const S16 = Cshort
const dsp_module_handle = Ptr{Void}

abstract type AlazarBits end
abstract type AlazarFFTBits <: AlazarBits end

struct Alazar8Bit <: AlazarBits
    b::U8
end
struct Alazar12Bit <: AlazarBits
    b::U16
end
struct Alazar16Bit <: AlazarBits
    b::U16
end
struct U16Log <: AlazarFFTBits
    b::U16
end
struct U16Amp2 <: AlazarFFTBits
    b::U16
end
struct U8Log <: AlazarFFTBits
    b::U8
end
struct U8Amp2 <: AlazarFFTBits
    b::U8
end
struct S32Real <: AlazarFFTBits
    b::S32
end
struct S32Imag <: AlazarFFTBits
    b::S32
end
struct FloatLog <: AlazarFFTBits
    b::Cfloat
end
struct FloatAmp2 <: AlazarFFTBits
    b::Cfloat
end

Base.convert(::Type{UInt8}, x::Alazar8Bit) = ltoh(convert(UInt8, x.b))
Base.convert(::Type{T}, x::Alazar8Bit) where {T <: Integer} = convert(T, convert(UInt8, x))
Base.convert(::Type{T}, x::Alazar8Bit) where {T <: AbstractFloat} = T(convert(UInt8,x)/0xFF*2-1)

Base.convert(::Type{UInt16}, x::Alazar12Bit) = ltoh(convert(UInt16, x.b)) >> 4
Base.convert(::Type{T}, x::Alazar12Bit) where {T <: Integer} = convert(T, convert(UInt16, x))
Base.convert(::Type{T}, x::Alazar12Bit) where {T <: AbstractFloat} = T(convert(UInt16,x)/0xFFF*2-1)

Base.convert(::Type{UInt16}, x::Alazar16Bit) = ltoh(convert(UInt16, x.b))
Base.convert(::Type{T}, x::Alazar16Bit) where {T <: Integer} = convert(T, convert(UInt16, x))
Base.convert(::Type{T}, x::Alazar16Bit) where {T <: AbstractFloat} = T(convert(UInt16,x)/0xFFFF*2-1)

Base.convert(::Type{S}, x::T) where {S <: Real,T <: AlazarBits} = convert(S, x.b)
Base.convert(::Type{S}, x::T) where {S <: Complex,T <: S32Real} = S(x.b,0)
Base.convert(::Type{S}, x::T) where {S <: Complex,T <: S32Imag} = S(0,x.b)
Base.convert(::Type{T}, x::T) where {T <: AlazarBits} = x
Base.convert(::Type{S}, x::T) where {S <: AlazarBits,T <: AlazarBits} = S(x.b)

# Constants and exceptions go here
include("constants.jl")

# Load libraries
const ats = is_windows() ? "ATSApi.dll" : "libATSApi.so"
const linux_libc = "libc.so.6"
# The library should only be loaded once. We don't load it automatically because
# if another worker process loads this module, there could be problems.
#
# Instead the library should be given the chance to load (if it hasn't already)
# whenever Julia objects representing instruments are created.
alazaropen() = begin
    @static is_windows() ? begin
        atsHandle = Libdl.dlopen(ats)
        atexit(()->Libdl.dlclose(atsHandle))
    end : (@static is_linux() ? begin
        atsHandle = Libdl.dlopen(ats)
        libcHandle = Libdl.dlopen(linux_libc)
        atexit(()->begin
            Libdl.dlclose(atsHandle)
            Libdl.dlclose(libcHandle)
        end)
    end : error("OS not supported"))
end

include("buffer.jl")
include("errors.jl")
include("api.jl")
include("dsp.jl")

end


@testset "fit" begin
    θ = [1.0,1.0]
    θ2 = [1.0,1.0,1.0,1.0]
    @testset "Error handling" begin
        @test_throws ArgumentError fit(ones(10,2),1,model=OU,x₀=θ)
        @test_throws ArgumentError fit(ones(10,3),1,model=CorrelatedOU,x₀=[1.0,1.0,0.5])
        @test_throws ArgumentError fit(ones(10),1,model=OU,x₀=[θ;1.1])
        @test_throws ArgumentError fit(ones(10),1,model=OU,x₀=θ,lowerΩcutoff = 2, upperΩcutoff = 1)
        @test_throws ArgumentError fit(ones(10),1,model=OU,x₀=θ,x_lowerbounds = ones(3))
        @test_throws ArgumentError fit(ones(10),1,model=OU,x₀=θ,x_upperbounds = ones(3))
        @test_throws ArgumentError fit(ones(10),1,model=OU,x₀=θ,taper="Dpss_4")
    end
    @testset "fitting" begin
        ts = simulate_gp(OU(θ),1000,1.0,1)[1]
        @test fit(ts,model=OU,x₀=θ) isa WhittleLikelihoodInference.Optim.MultivariateOptimizationResults
        @test fit(ts,model=OU,x₀=θ,taper=:dpss_4) isa WhittleLikelihoodInference.Optim.MultivariateOptimizationResults
        @test fit(ts,model=OU,x₀=θ,taper="dpss_4.2",options=WhittleLikelihoodInference.Optim.Options(iterations=4000)) isa WhittleLikelihoodInference.Optim.MultivariateOptimizationResults
        @test fit(ts.ts,ts.Δ,model=OU,x₀=θ,taper="dpss_4.2") isa WhittleLikelihoodInference.Optim.MultivariateOptimizationResults
        @test lowerbounds(TwoOU) == [lowerbounds(OU);lowerbounds(OU)]
        @test upperbounds(TwoOU) == [upperbounds(OU);upperbounds(OU)]
        ts2 = simulate_gp(TwoOU(θ2),1000,1.0,1)[1]
        @test fit(ts2.ts,ts2.Δ,model=TwoOU,x₀=θ2) isa WhittleLikelihoodInference.Optim.MultivariateOptimizationResults
    end
end


module Auth

# use both cookie and header to validate user incase some server stripped out
# the cookie e.g. Google server then use the header to stored the verified user
# with the validate token

using StructTypes, Dates, JWTs, HTTP
using ..Model

const JWT_AUTH_KEYS = Ref{JWKSet}()

function init(authkeysfile)
    JWT_AUTH_KEYS[] = JWKSet(authkeysfile)
    refresh!(JWT_AUTH_KEYS[])
    return
end

# generating a new token for a user
const DATE_FORMAT = DateFormat("e, dd u yyyy HH:MM:SS G\\MT") # Wed, 21 Oct 2015 07:28:00 GMT
const JWT_TOKEN_COOKIE_NAME = "X-MusicAlbums-Jwt-Token"

function addtoken!(resp::HTTP.Response, user::User)
    exp = Dates.now(Dates.UTC) + Dates.Hour(12)
    payload = Dict("iss"=>"MusicAlbums.jl", "exp"=>Dates.datetime2unix(exp), "sub"=>"managing albums", "aud"=>user.username, "uid"=>user.id)
    jwt = JWT(; payload=payload)
    keyid = first(first(JWT_AUTH_KEYS[].keys))
    sign!(jwt, JWT_AUTH_KEYS[], keyid)
    # handle the case where Google cloud stripped out all cookies to their server automatically
    # check not just the cookie but header to see if it valid
    HTTP.setheader(resp, "Set-Cookie" => "$JWT_TOKEN_COOKIE_NAME=$(join([jwt.header, jwt.payload, jwt.signature], '.')); Expires=$(Dates.format(exp, DATE_FORMAT))")
    HTTP.setheader(resp, JWT_TOKEN_COOKIE_NAME => join([jwt.header, jwt.payload, jwt.signature], '.'))
    return resp
end

struct Unauthenticated <: Exception end

# parsing token from HTTP request
# and validate the user
# then add the validate token to the cookie and header
function User(req::HTTP.Request)
    if HTTP.hasheader(req, "Cookie")
        cookies = filter(x->x.name == JWT_TOKEN_COOKIE_NAME, HTTP.cookies(req))
        if !isempty(cookies) && !isempty(cookies[1].value)
            jwt = JWT(; jwt=cookies[1].value)
            verified = false                  # start to validate the user cookie
            for kid in JWT_AUTH_KEYS[].keys
                validate!(jwt, JWT_AUTH_KEYS[], kid[1])
                verified |= isverified(jwt)
            end
            if verified
                parts = claims(jwt)  # user is good add token
                return User(parts["uid"], parts["aud"])
            end
        end
    elseif HTTP.hasheader(req, JWT_TOKEN_COOKIE_NAME)
        jwt = JWT(; jwt=String(HTTP.header(req, JWT_TOKEN_COOKIE_NAME)))
        verified = false                # in case cookie is stripped out use the header
        for kid in JWT_AUTH_KEYS[].keys
            validate!(jwt, JWT_AUTH_KEYS[], kid[1])
            verified |= isverified(jwt)
        end
        if verified
            parts = claims(jwt)   # ssame thing with the cookie
            return User(parts["uid"], parts["aud"])
        end
    end
    throw(Unauthenticated())
end

end # module


function testdump()
    println("\n in testdump")
    dim = 10
    hnsw = createHnswApi(Float32, 8, 16, "DistL1")
    # block // insertion
    nbinsert = 500
    parallel = true
    if parallel
        datas = rand(Float32, (dim, nbinsert));
        data_insert = map(i -> ( rand(Float32, dim) , UInt(i) ), 1:size(datas)[2])
        insert(hnsw, data_insert)
    else
        for i in 1:nbinsert
            data = rand(Float32, dim);
            insert(hnsw, data, UInt64(i))
        end
    end
    #
    v1 = [Float32(0.5) for i in 1:dim]
    neighbours = search(hnsw, v1, 8, 16)
    @info " 0.5 vector neighbours list : " neighbours
    #
    res = fileDump(hnsw, "testdumpfromjulia")
    if res > 0
        true
    else
        @warn "file dump failed : " "testdumpfromjulia"
        false
    end
end

function testreload()
    println("\n in testreload")
    # we know type and distance we reload from so we do not need to call
    # description = getDescription("testdumpfromjulia")
    res = loadHnsw("testdumpfromjulia", Float32, "DistL1")
    if res != nothing
        println("loadHnsw from file succeeded")
        description , hnswapi = res
    else
        return false
    end
    dim = description.data_dimension
    v1 = [Float32(0.5) for i in 1:dim]
    neighbours = search(hnswapi, v1, 8, 16)
    @info " 0.5 vector neighbours list : " neighbours
    true
end


"""
    nshots(x)
    nshots!(x, val)

The number of RF excitations needed to reconstruct a slice or volume.
Please mind that this is not the same as Echo Train Length which denotes the
number of lines of k-space collected after an excitation.
"""
@defprop NumberShots{:nshots}::Int

"""
    effective_echo_spacing(x)
    effective_echo_spacing!(x, val)

The "effective" sampling interval, specified in seconds, between lines in the
phase-encoding direction, defined based on the size of the reconstructed image
in the phase direction. It is frequently, but incorrectly, referred to as
"dwell time" (see `dwell_time` parameter below for actual dwell time). It is
required for unwarping distortions using field maps. Note that beyond just
in-plane acceleration, a variety of other manipulations to the phase encoding
need to be accounted for properly, including partial fourier, phase
oversampling, phase resolution, phase field-of-view and interpolation.2
This parameter is REQUIRED if corresponding fieldmap data is present.

2Conveniently, for Siemens’ data, this value is easily obtained as
1/[`BWPPPE` * `ReconMatrixPE`], where BWPPPE is the
"`BandwidthPerPixelPhaseEncode` in DICOM tag (0019,1028) and ReconMatrixPE is
the size of the actual reconstructed data in the phase direction (which is NOT
reflected in a single DICOM tag for all possible aforementioned scan
manipulations). See [here](https://lcni.uoregon.edu/kb-articles/kb-0003) and
[here](https://github.com/neurolabusc/dcm_qa/tree/master/In/total_readout_time)
"""
@defprop EffectiveEchoSpacing{:effective_echo_spacing}::F64Second

"""
    parallel_acquisition_technique(x)
    parallel_acquisition_technique!(x, val)

The type of parallel imaging used (e.g. GRAPPA, SENSE).
"""
@defprop ParallelAcquisitionTechnique{:parallel_acquisition_technique}::String

"""
    parallel_reduction_factor_in_plane(x)
    parallel_reduction_factor_in_plane!(x, val)

The parallel imaging (e.g, GRAPPA) factor. Use the denominator of the fraction
of k-space encoded for each slice. For example, 2 means half of k-space is encoded.
"""
@defprop ParallelReductionFactor{:parallel_reduction_factor_in_plane}::Int

"""
    partial_fourier(x)
    partial_fourier!(x, val)

The fraction of partial Fourier information collected.
"""
@defprop PartialFourier{:partial_fourier}::Float64

"""
    partial_fourier_direction(x)
    partial_fourier_direction!(x, val)

The direction where only partial Fourier information was collected.
"""
@defprop PartialFourierDirection{:partial_fourier_direction}::String

"""
    total_readout_time(x)
    total_readout_time!(x, val)

The total readout time. This is actually the "effective" total readout time ,
defined as the readout duration, specified in seconds, that would have generated
data with the given level of distortion. It is NOT the actual, physical duration
of the readout train. If `effective_echo_spacing` has been properly computed, it
is just `effective_echo_spacing` * (ReconMatrixPE - 1).3 .

* This parameter is
REQUIRED if corresponding "field/distortion" maps acquired with opposing phase
encoding directions are present.

3We use the "FSL definition", i.e, the time between the center of the
first "effective" echo and the center of the last "effective" echo.
"""
@defprop TotalReadoutTime{:total_readout_time}::F64Second


"""
    MCGP(args...; kwargs...)

Monte-Carlo Gaussian Process

## Arguments
- `X::AbstractArray` : Input features, if `X` is a matrix the choice of colwise/rowwise is given by the `obsdim` keyword
- `y::AbstractVector` : Output labels
- `kernel::Kernel` : Covariance function, can be any kernel from KernelFunctions.jl
- `likelihood` : Likelihood of the model. For compatibilities, see [`Likelihood Types`](@ref likelihood_user)
- `inference` : Inference for the model, at the moment only [`GibbsSampling`](@ref) is available (see the [`Compatibility Table`](@ref compat_table))

## Keyword arguments
- `verbose::Int` : How much does the model print (0:nothing, 1:very basic, 2:medium, 3:everything)
- `optimiser` : Optimiser used for the kernel parameters. Should be an Optimiser object from the [Flux.jl](https://github.com/FluxML/Flux.jl) library, see list here [Optimisers](https://fluxml.ai/Flux.jl/stable/training/optimisers/) and on [this list](https://github.com/theogf/AugmentedGaussianProcesses.jl/tree/master/src/inference/optimisers.jl). Default is `ADAM(0.001)`
- `atfrequency::Int=1` : Choose how many variational parameters iterations are between hyperparameters optimization
- `mean=ZeroMean()` : PriorMean object, check the documentation on it [`MeanPrior`](@ref meanprior)
- `obsdim::Int=1` : Dimension of the data. 1 : X ∈ DxN, 2: X ∈ NxD
"""
mutable struct MCGP{
    T<:Real,
    TLikelihood<:AbstractLikelihood{T},
    TInference<:AbstractInference{T},
    TData<:AbstractDataContainer,
    N,
} <: AbstractGP{T,TLikelihood,TInference,N}
    data::TData
    f::NTuple{N,SampledLatent{T}} # Vector of latent GPs
    likelihood::TLikelihood
    inference::TInference
    verbose::Int64 #Level of printing information
    atfrequency::Int64
    trained::Bool
end

function MCGP(
    X::AbstractArray,
    y::AbstractVector,
    kernel::Kernel,
    likelihood::Union{AbstractLikelihood,Distribution},
    inference::AbstractInference;
    verbose::Int=0,
    optimiser=ADAM(0.01),
    atfrequency::Integer=1,
    mean::Union{<:Real,AbstractVector{<:Real},PriorMean}=ZeroMean(),
    obsdim::Int=1,
)
    X, T = wrap_X(X, obsdim)
    y, nLatent, likelihood = check_data!(y, likelihood)

    inference isa SamplingInference || error(
        "The inference object should be of type `SamplingInference` : either `GibbsSampling` or `HMCSampling`",
    )
    !isa(likelihood, GaussianLikelihood) || error(
        "For a Gaussian Likelihood you should directly use the `GP` model or the `SVGP` model for large datasets",
    )
    implemented(likelihood, inference) ||
        error("The $likelihood is not compatible or implemented with the $inference")
    !isa(likelihood, Distribution) ||
        error("Using Distributions.jl distributions is unfortunately not yet implemented")
    data = wrap_data(X, y)
    nFeatures = nSamples(data)

    if isa(optimiser, Bool)
        optimiser = optimiser ? ADAM(0.01) : nothing
    end

    if typeof(mean) <: Real
        mean = ConstantMean(mean)
    elseif typeof(mean) <: AbstractVector{<:Real}
        mean = EmpiricalMean(mean)
    end

    latentf = ntuple(_ -> SampledLatent(T, nFeatures, kernel, mean), nLatent)

    likelihood = init_likelihood(likelihood, inference, nLatent, nSamples(data))
    xview = view_x(data, 1:nSamples(data))
    yview = view_y(likelihood, data, 1:nSamples(data))
    inference = tuple_inference(
        inference, nLatent, nSamples(data), nSamples(data), nSamples(data), xview, yview
    )
    return MCGP{T,typeof(likelihood),typeof(inference),typeof(data),nLatent}(
        data, latentf, likelihood, inference, verbose, atfrequency, false
    )
end

function Base.show(io::IO, model::MCGP)
    return print(
        io,
        "Monte Carlo Gaussian Process with a $(likelihood(model)) sampled via $(inference(model)) ",
    )
end

Zviews(model::MCGP) = [input(model)]
objective(::MCGP{T}) where {T} = NaN

@traitimpl IsFull{MCGP}


module GroundDataV
"""
Module for the download and processing of atmospheric data gathered by measuring stations located in Veneto, Italy.
"""

using CombinedParsers
using CombinedParsers.Regexp
using CSV
using DataFrames
using Dates
using HTTP
using JSON
using JSONTables
using Revise

export getData,
       getRegionAttributes, getRegionIds, getRegionStationInfo

@syntax informations = Sequence(
    :info => Sequence(
        Sequence( "<", "PERIODO", ">" ),
        "<![CDATA[",
        re"[^\[\]]+",
        "]]>",
        "</PERIODO>"
    ),
    :begin => Sequence(
        Sequence( "<", "INIZIO", ">" ),
        Numeric(Int64),
        "</INIZIO>"
    ),
    :end => Sequence(
        Sequence( "<", "FINE", ">" ),
        Numeric(Int64),
        "</FINE>"
    ),
    :proj => Sequence(
        Sequence( "<", "PROJECTION", ">" ),
        re"[^<:>]+",
        ":",
        Numeric(Int64),
        "</PROJECTION>"
    )
)

@syntax stations = Repeat(
    Sequence(
        re"<[^<>]+>",
        Either(
            Numeric(Int64),
            Numeric(Float64),
            Sequence( "<![CDATA[", re"[^\[\]]+", "]]>" ),
            re"[^<>]+"
        ),
        re"</[^<>]+>"
    )
)

@syntax sensor = Sequence(
    Repeat(
        re"<(?!D)[^>]+>",
        Either(
            Numeric(Int64),
            Sequence( "<![CDATA[", re"[^\[\]]+", "]]>" ),
            re"[^>]+"
        ),
        re"</[^>]+>"
    ),
    Repeat(
        Sequence( "<DATI ISTANTE=\"", Numeric(Int64), "\">" ),
        Sequence( "<VM>", Numeric(Float64), "</VM>" ),
        "</DATI>"
    )
)

"""
    getRegionAttributes( [ type::Symbol=:METEO ] )

Obtain the names of the columns of the region's dataframe required by `GroundData.createMap`'s `attributes` parameter to create `GroundData.standardize`'s `map` parameter.
"""
function getRegionAttributes( type::Symbol=:METEO )
    return type == :METEO ? [ :paramnm, :unitnm, :value, nothing, :instant, :x, :y, :quota, nothing, nothing, :rmh ] :
        type == :AIRQUALITY ? [ :param, nothing, :value, nothing, :date, :lon, :lat, nothing, nothing, :tipozona ] :
            throw(DomainError(type, "`type` must be either `:METEO` OR `:AIRQUALITY`."))
end

"""
    getRegionAttributes( [ type::Symbol=:METEO ] )

Obtain the names of the columns of the dataframe required for `GroundData.standardize`'s `bridge` parameter.
"""
function getRegionIds( type::Symbol=:METEO )
    return type == :METEO ? :idstaz :
        type == :AIRQUALITY ? :codseqst : throw(DomainError(type, "`type` must be either `:METEO` OR `:AIRQUALITY`."))
end

"""
    getRegionStationInfo( [ type::Symbol=:METEO  ] )

Obtain the names of the columns of the region's stations dataframe required by `GroundData.createMap`'s `attributes` parameter to be used in `GroundData.generateUuidsTable`.
"""
function getRegionStationsInfo( type::Symbol=:METEO )
    return type == :METEO ? [ :idstaz, :nome, :x, :y ] :
        type == :AIRQUALITY ? [ :codseqst, :nome, :lon, :lat ] :
            throw(DomainError(type, "`type` must be either `:METEO` OR `:AIRQUALITY`."))
end

"""
    getStationsInfo()

Obtain the dataframe containing informations on all available ARPAV stations.
"""
function getMeteoStationsData()
 #Get the information on the available stations
    # Get the page containing informations on all the available stations
    page = String( HTTP.get( "https://www.arpa.veneto.it/bollettini/meteo/h24/img07/stazioni.xml" ).body )
    # Keep only the useful portion of the page
    useful = split( page, "</PERIODO>" )[2][1:end-15]
    # Split the downloaded string in substrings one for each station
    arr = split( useful, r"</?STAZIONE>", keepempty=false )
    #info = informations(arr[1])
    stats = stations.(arr[2:end])
    #Create the dictionary of the stations, checking the type of `attribute[2]` (contains the value of the attribute)
    df = DataFrame([
        Dict(
            Symbol( lowercase( String( attribute[1][2] ) ) )
            =>
            isa(attribute[2], Number) ? attribute[2] : # If it's a number leave it as is
                isa(attribute[2], Array) ? lowercase( String( attribute[2] ) ) : #If it is an array (of strings) convert it to String
                    titlecase( String( attribute[2][2] ) ) # If it is a Tuple ( Es. `("<![CDATA[", "Arabba", "]]>")` ) take the actual value ( `"Arabba"` )
            for attribute in station
        ) for station in stats
    ])
    insertcols!(df, :rmh => "0m")
    return df
end

# res = getMeteoStationData()

"""
    getStationsData( stats::AbstractVector{String} )

Obtain the dataframe containing the data of all the stations described by the elements of `stats`.
"""
function getMeteoData( ids::AbstractVector{Int64} )
    pages_vect = [ String( HTTP.get("https://www.arpa.veneto.it/bollettini/meteo/h24/img07/$(lpad(id, 4, "0")).xml").body ) for id in ids ]
    # For each of the station remove the first part of the string and the closing tags
    stat_strings_vect = [ split(page, "</ATTIVAZIONE>")[2][1:end-26] for page in pages_vect ]
    # From each station generate the corresponding array of sensors
    sensor_vect = split.( stat_strings_vect, r"</?SENSORE>", keepempty=false )
    # vect[i][j] i-th station, j-th sensor
    # vect[i][j][1] j-th sensor's info (Vector)
    # vect[i][j][2] j-th sensor's data (Vector)
    # vect[i][j][2][l] j-th sensor'data, l-th measurement (Vector)
    # vect[i][j][2][l][4] l-th measurement's values (Vector 1 => mean, 2 => min, 3 => max)
    vect = [ sensor.(sensor_group) for sensor_group in sensor_vect ]
    df = DataFrame([
        push!(
            Dict(
                Symbol( lowercase( String( attribute[3] ) ) )
                =>
                isa( attribute[5], Number ) ? attribute[5] :
                    isa( attribute[5], Array ) ? String( attribute[5] ) :
                        titlecase( String( attribute[5][2] ) )
              for attribute in sensor[1]
            ),
            :idstaz => id,
            :instant => entry[2],
            :value => entry[5]
        )
        for (id, station) in zip(ids, vect)
            for sensor in station
                for entry in sensor[2]
    ])
    transform!(
        df,
        [:unitnm] => ByRow( x -> x = replace( replace( x, "\xb0" => "°" ), "2" => "²" ) ) => :unitnm,
        [:instant] => ByRow( x -> DateTime( string(x), "yyyymmddHHMM" ) ) => :instant
    )
    return df
end

#   df = getMeteoStationsData()
#   data = getMeteoSensorsInfo( df[1:3, :idstaz] )

"""
    getAqStationsData()

Obtain a dataframe containing informations on the measuring stations in the area.
"""
function getAqStationsData()
    page = String(HTTP.get("http://213.217.132.81/aria-json/exported/aria/stats.json").body)
    df = DataFrame(jsontable(page[14:end]))
    return df
end

#   res = getAqStationsData()

"""
    getAqData()

Obtain the data gathered by the stations in the area.
"""
function getAqData()
    page = String(HTTP.get("http://213.217.132.81/aria-json/exported/aria/data.json").body)
    # js["stazioni"][i]                                            i-th station
    # js["stazioni"][i]["codseqst"]                                i-th station's id
    # js["stazioni"][i]["misurazioni"][j]                          i-th station's j-th measurement
    # js["stazioni"][i]["misurazioni"][j]["pm10"/"ozono"][l]       j-th measurement's l-th entry
    js = JSON.parse(page)["stazioni"]
    arr = Dict[]
    for station in js
        if !isempty(station["misurazioni"])
            for measurement in station["misurazioni"]
                for key in collect(keys(measurement))
                    for entry in measurement[key]
                        push!(
                            arr,
                            Dict(
                                :codseqst => station["codseqst"],
                                :param => key,
                                :date => entry["data"],
                                :value => entry["mis"]
                            )
                        )
                    end
                end
            end
        else
            push!( arr, Dict( :codseqst => station["codseqst"], :param => missing, :date => missing, :value => missing ) )
        end
    end
    return DataFrame(arr)
end

#   df = getAqSensorsData()

"""
    getData(; <keyword arguments> )

Obtain data of category `type` and source of category `kind`.

# Arguments
- `type::Symbol=:METEO`: defines the type of data to be downloaded, it must either be `:METEO` or `:AIRQUALITY`.
- `kind::Symbol=:STATIONS`: defines if the data to be downloaded has to regard the stations or their actual measurements, it must either be `:STATIONS` or `:SENSORS`.
"""
function getData(; type::Symbol=:METEO, kind::Symbol=:STATIONS )
    if type == :METEO
        stations = getMeteoStationsData()
        if kind == :STATIONS
            return stations
        elseif kind == :SENSORS
            return getMeteoData( stations[:, :idstaz] )
        else
            throw(DomainError(kind, "`kind` must either be `:STATIONS` or `:SENSORS`."))
        end
    elseif type == :AIRQUALITY
        if kind == :STATIONS
            return getAqStationsData()
        elseif kind == :SENSORS
            return getAqData()
        else
            throw(DomainError(kind, "`kind` must either be `:STATIONS` or `:SENSORS`."))
        end
    else
        throw(DomainError(type, "`type` must either be `:METEO` or `:AIRQUALITY`."))
    end
end

#   ressta = getData()
#   ressen = getData( source=:SENSORS )
#   ressta = getData( type=:AIRQUALITY, source=:STATIONS )
#   ressen = getData( type=:AIRQUALITY, source=:SENSORS )

end # moduled


module MarketDatas
using DataFrames
using TimeSeries
using SQLite

export MarketData, storeMarketData, loadMarketData, loadHistoryData
export storeHistoryData
export DataSource, Logs, CREST, EveCentral, Unknown
@enum DataSource Logs CREST EveCentral Unknown
type MarketData
  itemID :: Int64
  source :: DataSource
  timeFetched :: DateTime
  sells :: AbstractDataFrame
  buys :: AbstractDataFrame
end

# MARKET DATA SAVING
const schema = ["""
CREATE TABLE metadata (
id INTEGER PRIMARY KEY ASC,
itemID INTEGER,
timeFetched TEXT
source INTEGER
);""",
"""CREATE TABLE sells (
id INTEGER PRIMARY KEY ASC,
mID INTEGER,
OrderID INTEGER,
Price INTEGER,
Location INTEGER,
Quantity INTEGER,
MinQuantity INTEGER,
Duration INTEGER,
Issued TEXT
);""",
"""CREATE TABLE buys (
id INTEGER PRIMARY KEY ASC,
mID INTEGER,
OrderID INTEGER,
Price INTEGER,
Location INTEGER,
Quantity INTEGER,
MinQuantity INTEGER,
Duration INTEGER,
Issued TEXT
);
""",
"""
CREATE TABLE history_data (
  id INTEGER PRIMARY KEY ASC,
  ItemID INTEGER,
  Date TEXT,
  LowPrice Real,
  HighPrice Real,
  AvgPrice Real,
  Volume INTEGER,
  OrderCount INTEGER,
  CONSTRAINT one_record UNIQUE (date, itemID)
);
"""]

# Code to save and load market data #
function initializeDB(filename :: ASCIIString)
  db = SQLite.DB(filename)
  for e in schema
    query(db, e)
  end
  db
end

function storeHistoryData(db, itemID :: Int, data :: TimeArray)
  q = "INSERT OR REPLACE INTO history_data " *
      "(ItemID, Date, LowPrice, HighPrice, AvgPrice, Volume, OrderCount) " *
      "VALUES (?, ?, ?, ?, ?, ?, ?)"
  s = SQLite.Stmt(db, q)
  for row = 1:length(data)
    SQLite.bind!(s, 1, itemID)
    SQLite.bind!(s, 2, string(data[row].timestamp[1]))
    SQLite.bind!(s, 3, data[row]["LowPrice"].values[1])
    SQLite.bind!(s, 4, data[row]["HighPrice"].values[1])
    SQLite.bind!(s, 5, data[row]["AvgPrice"].values[1])
    SQLite.bind!(s, 6, round(data[row]["Volume"].values[1]))
    SQLite.bind!(s, 7, round(data[row]["OrderCount"].values[1]))
    SQLite.execute!(s)
  end
end

function storeHistoryData(db :: SQLite.DB, data :: Dict{Int64, TimeArray})
  query(db, "BEGIN TRANSACTION")
  for (idx, i) in data
    storeHistoryData(db, idx, i)
  end
  query(db, "END TRANSACTION")
end

function storePriceData(db, table, mID, data :: DataFrame)
  q = "INSERT INTO $table "*
      "(mID, OrderID, Price, Location, Quantity, MinQuantity, Duration, Issued)"*
      "VALUES (?, ?, ?, ?, ?, ?, ?, ?)"
  s = SQLite.Stmt(db, q)
  for row = 1:length(data[1])
    SQLite.bind!(s, 1, mID)
    SQLite.bind!(s, 2, data[:OrderID][row])
    SQLite.bind!(s, 3, data[:Price][row])
    SQLite.bind!(s, 4, data[:Location][row])
    SQLite.bind!(s, 5, data[:Quantity][row])
    SQLite.bind!(s, 6, data[:MinQuantity][row])
    SQLite.bind!(s, 7, Int(data[:Duration][row]))
    SQLite.bind!(s, 8, string(data[:Issued][row]))
    SQLite.execute!(s)
  end
end

function storeMarketData(db :: SQLite.DB, data :: MarketData)
  query(db, "INSERT INTO metadata (itemID, timeFetched, source) VALUES (?, ?, ?)",
        [data.itemID, string(data.timeFetched), Int(data.source)])
  r = query(db, "SELECT last_insert_rowid()")
  mID :: Int = get(r.data[1][1])
  storePriceData(db, "sells", mID, data.sells)
  storePriceData(db,"buys", mID, data.buys)
end
function storeMarketData(db :: SQLite.DB, data)
  SQLite.query(db, "BEGIN TRANSACTION")
  for (idx,i :: MarketData) in data
    storeMarketData(db, i)
  end
  SQLite.query(db, "END TRANSACTION")
end
function storeMarketData(dbname :: ASCIIString, data)
  local db
  if isfile(dbname)
    db = SQLite.DB(dbname)
  else
    db = initializeDB(dbname)
  end
  storeMarketData(db, data)
end

function loadHistoryData(db, itemID :: Int)
  df = DataFrame(query(db, "SELECT * FROM history_data WHERE ItemID = ?", [itemID]))
  #convertMarketHistory(df)
end

function loadHistoryData(db, itemID :: Int, since :: DateTime)
  local timestamps :: Vector{DateTime}
  df = DataFrame(query(db, "SELECT * FROM history_data WHERE ItemID = ? AND Date > datetime(?)", [itemID, string(since)]))
  #convertMarketHistory(df)
end

function loadMarketData(db :: SQLite.DB, itemID :: Int, fresh :: DateTime)
  metadata = DataFrame(query(db, "SELECT * FROM metadata WHERE ItemID = ? AND timeFetched > datetime(?) ORDER BY timeFetched DESC", [itemID, string(since)]))
  (rows, cols) = size(metadata)
  if rows == 0
    return MarketData(0, DateTime(2003, 5, 5), Unknown,
                      DataFrame(OrderID = [], Price = [], Location = [], Quantity = [], MinQuantity = [], Duration = [], Issued = []),
                      DataFrame(OrderID = [], Price = [], Location = [], Quantity = [], MinQuantity = [], Duration = [], Issued = []))
  else
    df = Dates.DateFormat("yyyy-mm-ddTHH:MM:SS")
    sells = DataFrame(query(db, "SELECT * FROM sells WHERE mID = ?", [Int(metadata[:id][1])]))
    buys = DataFrame(query(db, "SELECT * FROM buys WHERE mID = ?", [Int(metadata[:id][1])]))
    return MarketData(Int(metadata[:itemID][1]),
                      DataSource(metadata[:source][1]),
                      DateTime(metadata[:timeFetched][1]).
                      sells,
                      buys)
  end
end
end


function NumericalPropagator(arg0::ODEIntegrator)
    return NumericalPropagator((ODEIntegrator,), arg0)
end

function NumericalPropagator(arg0::ODEIntegrator, arg1::AttitudeProvider)
    return NumericalPropagator((ODEIntegrator, AttitudeProvider), arg0, arg1)
end

function add_force_model(obj::NumericalPropagator, arg0::ForceModel)
    return jcall(obj, "addForceModel", void, (ForceModel,), arg0)
end

function get_all_force_models(obj::NumericalPropagator)
    return jcall(obj, "getAllForceModels", List, ())
end

function get_orbit_type(obj::NumericalPropagator)
    return jcall(obj, "getOrbitType", OrbitType, ())
end

function get_position_angle_type(obj::NumericalPropagator)
    return jcall(obj, "getPositionAngleType", PositionAngle, ())
end

function get_pv_coordinates(obj::NumericalPropagator, arg0::AbsoluteDate, arg1::Frame)
    return jcall(obj, "getPVCoordinates", TimeStampedPVCoordinates, (AbsoluteDate, Frame), arg0, arg1)
end

function propagate(obj::AbstractIntegratedPropagator, arg0::AbsoluteDate)
    return jcall(obj, "propagate", SpacecraftState, (AbsoluteDate,), arg0)
end

function remove_force_models(obj::NumericalPropagator)
    return jcall(obj, "removeForceModels", void, ())
end

function reset_initial_state(obj::NumericalPropagator, arg0::SpacecraftState)
    return jcall(obj, "resetInitialState", void, (SpacecraftState,), arg0)
end

function set_ignore_central_attraction(obj::NumericalPropagator, arg0::jboolean)
    return jcall(obj, "setIgnoreCentralAttraction", void, (jboolean,), arg0)
end

function set_initial_state(obj::NumericalPropagator, arg0::SpacecraftState)
    return jcall(obj, "setInitialState", void, (SpacecraftState,), arg0)
end

function set_mu(obj::NumericalPropagator, arg0::jdouble)
    return jcall(obj, "setMu", void, (jdouble,), arg0)
end

function set_orbit_type(obj::NumericalPropagator, arg0::OrbitType)
    return jcall(obj, "setOrbitType", void, (OrbitType,), arg0)
end

function set_position_angle_type(obj::NumericalPropagator, arg0::PositionAngle)
    return jcall(obj, "setPositionAngleType", void, (PositionAngle,), arg0)
end

function tolerances(::Type{NumericalPropagator}, arg0::jdouble, arg1::AbsolutePVCoordinates)
    return jcall(NumericalPropagator, "tolerances", Vector{Vector{jdouble}}, (jdouble, AbsolutePVCoordinates), arg0, arg1)
end

function tolerances(::Type{NumericalPropagator}, arg0::jdouble, arg1::Orbit, arg2::OrbitType)
    return jcall(NumericalPropagator, "tolerances", Vector{Vector{jdouble}}, (jdouble, Orbit, OrbitType), arg0, arg1, arg2)
end


#not currently used

#=

function get_data_info_oldversion(at_set, dim)

    # very  minor changes, see differs. This is soley for format conversion from old coefs to new format, which has extra 3body terms.

    n_2body = 5
    n_2body_onsite = 4
    n_2body_S = 7
    n_3body = 5 #differs. is 7 in the main version
    n_3body_same = 5
    n_3body_onsite = 4
    n_3body_onsite_same = 2

    data_info = Dict{Tuple, Array{Int64,1}}()
    orbs = []
    if dim == 2 #2body

        at_list = [i for i in at_set]
#        println(at_list)
        if length(at_list) == 1
            at_list = [at_list[1], at_list[1]]
        end
        sort!(at_list)
#        println(at_list)

        orbs1 = atoms[at_list[1]].orbitals
        orbs2 = atoms[at_list[2]].orbitals

        at1 = at_list[1]
        at2 = at_list[2]

        if at1 == at2
            same_at = true
        else
            same_at = false
        end

#        orbs = []

        #2body part
        function get2bdy(n, symb)
            tot=0
            for o1 in orbs1
                for o2 in orbs2
                    if same_at && ((o2 == :s && o1 == :p) || (o2 == :s && o1 == :d) || (o2 == :p && o1 == :d))
                        continue
                    end
#                    push!(orbs, (o1, o2, symb))
                    if o1 == :s && o2 == :s
                        data_info[(at1, o1, at2, o2, symb)] = tot+1:tot+n
                        data_info[(at2, o2, at1, o1, symb)] = tot+1:tot+n
                        tot += n

                    elseif (o1 == :s && o2 == :p ) || (o1 == :p && o2 == :s )
                        data_info[(at1, o1, at2, o2, symb)] = tot+1:tot+n
                        data_info[(at2, o2, at1, o1, symb)] = tot+1:tot+n
                        tot += n
#                        if same_at
#                            data_info[(o2, o1, symb)] = data_info[(o1, o2, symb)]
#                        end

                    elseif (o1 == :p && o2 == :p )
                        data_info[(at1, o1, at2, o2, symb)] = tot+1:tot+n*2
                        data_info[(at2, o2, at1, o1, symb)] = tot+1:tot+n*2
                        tot += n*2

#                    elseif (o1 == :p && o2 == :p )
#                        data_info[(at1, o1, at2, o2, symb)] = tot+1:tot+n*2
#                        data_info[(at2, o2, at1, o1, symb)] = tot+1:tot+n*2
#                        tot += n*2

                    elseif (o1 == :s && o2 == :d ) || (o1 == :d && o2 == :s )
                        data_info[(at1, o1, at2, o2, symb)] = tot+1:tot+n
                        data_info[(at2, o2, at1, o1, symb)] = tot+1:tot+n
                        tot += n

                    elseif (o1 == :p && o2 == :d ) || (o1 == :d && o2 == :p )
                        data_info[(at1, o1, at2, o2, symb)] = tot+1:tot+n*2
                        data_info[(at2, o2, at1, o1, symb)] = tot+1:tot+n*2
                        tot += n*2

                    elseif (o1 == :d && o2 == :d )
                        data_info[(at1, o1, at2, o2, symb)] = tot+1:tot+n*3
                        data_info[(at2, o2, at1, o1, symb)] = tot+1:tot+n*3
                        tot += n*3

                    end
                end
            end
            return tot
        end

        totH = get2bdy(n_2body, :H)
        totS = get2bdy(n_2body_S, :S)
#        println("totH $totH totS $totS")
        #onsite part
        function getonsite(atX,orbsX, tot, n)
            for o1 in orbsX
                for o2 in orbsX
                    if (o2 == :s && o1 == :p) || (o2 == :s && o1 == :d) || (o2 == :p && o1 == :d)
                        continue
                    end

#                    push!(orbs, (o1, o2, :O))
                    if o1 == :s && o2 == :s
                        data_info[(atX, o1, o2, :O)] = tot+1:tot+n
#                        println("data_info" , (atX, o1, o2, :O), tot+1:tot+n)
                        tot += n
                    elseif (o1 == :s && o2 == :p )
                        data_info[(atX, o1, o2, :O)] = tot+1:tot+n
                        data_info[(atX, o2, o1, :O)] = data_info[(atX, o1, o2, :O)]
                        tot += n
                    elseif (o1 == :p && o2 == :p )
                        data_info[(atX, o1, o2, :O)] = tot+1:tot+n*2
                        tot += n*2

                    elseif o1 == :s && o2 == :d
                        data_info[(atX, o1, o2, :O)] = tot+1:tot+n
                        data_info[(atX, o2, o1, :O)] = data_info[(atX, o1, o2, :O)]
                        tot += n

                    elseif o1 == :p && o2 == :d
                        data_info[(atX, o1, o2, :O)] = tot+1:tot+n
                        data_info[(atX, o2, o1, :O)] = data_info[(atX, o1, o2, :O)]
                        tot += n

                    elseif o1 == :d && o2 == :d
                        data_info[(atX, o1, o2, :O)] = tot+1:tot+n*2
                        data_info[(atX, o2, o1, :O)] = data_info[(atX, o1, o2, :O)]
                        tot += n*2

                    end
                end
            end
            return tot
        end

        if same_at #true onsite terms
           for o in orbs1
#                println("true onsite ", o)
                data_info[(at1, o, :A)] = [totH+1]
                totH += 1
            end
        end

        totHO = getonsite(at1, orbs1, totH, n_2body_onsite)

        if !(same_at) #need reverse if not same atom
            totHO = getonsite(at2, orbs2, totHO, n_2body_onsite)
        end

    elseif dim == 3 #3body

        totS = 0 #no 3body overlap terms

        at_list = [i for i in at_set]
        sort!(at_list)

        if length(at_list) == 1

            #permutations are trivial
            perm_ij = [[at_list[1], at_list[1], at_list[1]]]
            perm_on = [[at_list[1], at_list[1], at_list[1]]]
        elseif length(at_list) == 2

            #unique permutations
            perm_ij = [[at_list[1], at_list[1], at_list[2]] ,
                       [at_list[2], at_list[2], at_list[1]] ,
                       [at_list[1], at_list[2], at_list[1]] ,
                       [at_list[1], at_list[2], at_list[2]] ]

            perm_on = [[at_list[1], at_list[2], at_list[2]] ,
                        [at_list[1], at_list[1], at_list[2]] ,
                        [at_list[2], at_list[1], at_list[1]] ,
                        [at_list[2], at_list[1], at_list[2]] ]

        elseif length(at_list) == 3

            #all permutations exist hij
            perm_ij = [[at_list[1], at_list[2], at_list[3]] ,
                       [at_list[1], at_list[3], at_list[2]] ,
                       [at_list[2], at_list[1], at_list[3]] ,
                       [at_list[2], at_list[3], at_list[1]] ,
                       [at_list[3], at_list[1], at_list[2]] ,
                       [at_list[3], at_list[2], at_list[1]] ]

            #onsite can flip last 2 atoms
            perm_on = [[at_list[1], at_list[2], at_list[3]] ,
                       [at_list[2], at_list[1], at_list[3]] ,
                       [at_list[3], at_list[1], at_list[2]]]

        else
            println("ERROR  get_data_info $dim $at_set $at_list")
        end

        function get3bdy(n, symb, start, at1, at2, at3)
            tot=start

            orbs1 = atoms[at1].orbitals
            orbs2 = atoms[at2].orbitals

            if at1 == at2
                same_at = true
            else
                same_at = false
            end

            for o1 in orbs1
                for o2 in orbs2
                    if same_at && ((o2 == :s && o1 == :p) || (o2 == :s && o1 == :d) || (o2 == :p && o1 == :d))
                        continue
                    end

#                    push!(orbs, (o1, o2, symb))

                    if same_at
                        data_info[(at1, o1, at2, o2, at3,  symb)] = collect(tot+1:tot+n)
                        data_info[(at2, o2, at1, o1, at3,  symb)] = collect(tot+1:tot+n)
                    else                                                  #[1 2 3 4 5 6 7 8  9  10 11 12 13 14 15 16 17 18]
                        data_info[(at1, o1, at2, o2, at3,  symb)] = collect(tot+1:tot+n)
#                        data_info[(at2, o2, at1, o1, at3,  symb)] = tot .+ [1 4 6 2 5 3 7 10 12 8  11 9  13 16 18 14 17 15]' #switch 2 4 and 3 6
#                        data_info[(at2, o2, at1, o1, at3,  symb)] = tot .+ [1 4 6 2 5 3 7 10 12 8  11 9  ]' #switch 2 4 and 3 6
#                        data_info[(at2, o2, at1, o1, at3,  symb)] = tot .+ [1 3 2 4 5 7 6 8 9]' #switch 2 4 and 3 6
#                        data_info[(at2, o2, at1, o1, at3,  symb)] = tot .+ [1 3 2 4 5 7 6]' #switch 2 4 and 3 6
#                        data_info[(at2, o2, at1, o1, at3,  symb)] = tot .+ [1 3 2  4 6 5]' #switch 2 4 and 3 6
#                        data_info[(at2, o2, at1, o1, at3,  symb)] = tot .+ [1 3 2 4 6 5  7 9 8 ]' #switch 2 4 and 3 6

#                        data_info[(at2, o2, at1, o1, at3,  symb)] = tot .+ [1, 3, 2, 4, 5, 7, 6  ] #switch 2 4 and 3 6

                        data_info[(at2, o2, at1, o1, at3,  symb)] = tot .+ [1, 3, 2, 4, 5  ] #differs
                    end

                    tot += n

#                    if same_at
#                        data_info[(o2, o1, symb)] = data_info[(o1, o2, symb)]
#                    end

                end
            end
            return tot
        end

#        if at_list[2] == at_list[3]
#            same_at_on = true
#        else
#            same_at_on = false
 #       end

        function get3bdy_onsite(n, same_at,symb, start, at1, at2, at3)
#            if at2 == at3  #|| at1 == at2 || at1 == at3
#                same_at = true
#            else
#                same_at = false
#            end

            orbs1 = atoms[at1].orbitals

#            println("get3bdy_onsite $at1 $at2 $at3 $n")

            tot=start
            for o1 in orbs1
#                data_info[(at1, o1,at2, at3,  symb)] = collect(tot+1:tot+n)
#                data_info[(at1, o1,at3, at2,  symb)] = collect(tot+1:tot+n)

#                push!(orbs, (at1, o1,at2, at3,  symb))
#                push!(orbs, (at1, o1,at3, at2,  symb))

                if same_at
                    data_info[(at1, o1,at2, at3,  symb)] = collect(tot+1:tot+n)
                    data_info[(at1, o1,at3, at2,  symb)] = collect(tot+1:tot+n)
                else
                    data_info[(at1, o1,at2, at3,  symb)] = collect(tot+1:tot+n)
#                    data_info[(at1, o1,at3, at2,  symb)] = collect(tot+1:tot+n)
                    data_info[(at1, o1,at3, at2,  symb)] = tot .+ [1, 3, 2, 4]

#                    data_info[(at1, o1,at2, at3,  symb)] = collect(tot+1:tot+n)
#                    data_info[(at1, o1,at3, at2,  symb)] = tot .+ [1 3 2 4]'
                end
                tot += n                               #       1 2 3 4 5 6 7 8

            end
            return tot
        end

        tot_size = 0
        for p in perm_ij
            if p[1] == p[2]
                tot_size = get3bdy(n_3body_same, :H, tot_size, p[1], p[2], p[3])
            else
                tot_size = get3bdy(n_3body, :H, tot_size, p[1], p[2], p[3])
            end
        end

        for p in perm_on
            if  (p[1] == p[2] ||  p[2] == p[3] || p[1] == p[3])
                tot_size = get3bdy_onsite(n_3body_onsite_same,true, :O, tot_size, p[1], p[2], p[3]) #all diff
            else
                tot_size = get3bdy_onsite(n_3body_onsite,false, :O, tot_size, p[1], p[2], p[3]) #
            end
        end

        totHO = tot_size

    else
        println("error, only 2 or 3 body terms, you gave me : ", at_list)
    end
    return totHO ,totS, data_info, orbs

end

function fix_format_change(datH, totHnew, dim, at_list, data_info)

    #this converts from old data_info to new data_info, setting the extra terms to zero.

    totH,totS, data_info_old, orbs = get_data_info_oldversion(at_list, dim)

    datHnew = zeros(totHnew)

    for k in keys(data_info)
        dnew = data_info[k]
        dold = data_info_old[k]

        n=length(dold)
        datHnew[dnew[1:n] ] = datH[dold]  #the extra terms are left as zero
    end

    return datHnew

end
=#


@testset "simple_varinfo.jl" begin
    @testset "constructor & indexing" begin
        @testset "NamedTuple" begin
            svi = SimpleVarInfo(; m=1.0)
            @test getlogp(svi) == 0.0
            @test haskey(svi, @varname(m))
            @test !haskey(svi, @varname(m[1]))

            svi = SimpleVarInfo(; m=[1.0])
            @test getlogp(svi) == 0.0
            @test haskey(svi, @varname(m))
            @test haskey(svi, @varname(m[1]))
            @test !haskey(svi, @varname(m[2]))
            @test svi[@varname(m)][1] == svi[@varname(m[1])]

            svi = SimpleVarInfo(; m=(a=[1.0],))
            @test haskey(svi, @varname(m))
            @test haskey(svi, @varname(m.a))
            @test haskey(svi, @varname(m.a[1]))
            @test !haskey(svi, @varname(m.a[2]))
            @test !haskey(svi, @varname(m.a.b))

            svi = SimpleVarInfo{Float32}(; m=1.0)
            @test getlogp(svi) isa Float32

            svi = SimpleVarInfo((m=1.0,), 1.0)
            @test getlogp(svi) == 1.0
        end

        @testset "Dict" begin
            svi = SimpleVarInfo(Dict(@varname(m) => 1.0))
            @test getlogp(svi) == 0.0
            @test haskey(svi, @varname(m))
            @test !haskey(svi, @varname(m[1]))

            svi = SimpleVarInfo(Dict(@varname(m) => [1.0]))
            @test getlogp(svi) == 0.0
            @test haskey(svi, @varname(m))
            @test haskey(svi, @varname(m[1]))
            @test !haskey(svi, @varname(m[2]))
            @test svi[@varname(m)][1] == svi[@varname(m[1])]

            svi = SimpleVarInfo(Dict(@varname(m) => (a=[1.0],)))
            @test haskey(svi, @varname(m))
            @test haskey(svi, @varname(m.a))
            @test haskey(svi, @varname(m.a[1]))
            @test !haskey(svi, @varname(m.a[2]))
            @test !haskey(svi, @varname(m.a.b))

            svi = SimpleVarInfo(Dict(@varname(m.a) => [1.0]))
            # Now we only have a variable `m.a` which is subsumed by `m`,
            # but we can't guarantee that we have the "entire" `m`.
            @test !haskey(svi, @varname(m))
            @test haskey(svi, @varname(m.a))
            @test haskey(svi, @varname(m.a[1]))
            @test !haskey(svi, @varname(m.a[2]))
            @test !haskey(svi, @varname(m.a.b))
        end
    end

    @testset "SimpleVarInfo on $(nameof(model))" for model in
                                                     DynamicPPL.TestUtils.DEMO_MODELS
        # We might need to pre-allocate for the variable `m`, so we need
        # to see whether this is the case.
        m = model().m
        svi_nt = if m isa AbstractArray
            SimpleVarInfo((m=similar(m),))
        else
            SimpleVarInfo()
        end
        svi_dict = SimpleVarInfo(VarInfo(model), Dict)

        @testset "$(nameof(typeof(svi.values)))" for svi in (svi_nt, svi_dict)
            # Random seed is set in each `@testset`, so we need to sample
            # a new realization for `m` here.
            m = model().m

            ### Sampling ###
            # Sample a new varinfo!
            _, svi_new = DynamicPPL.evaluate!!(model, svi, SamplingContext())

            # If the `m[1]` varname doesn't exist, this is a univariate model.
            # TODO: Find a better way of dealing with this that is not dependent
            # on knowledge of internals of `model`.
            isunivariate = !haskey(svi_new, @varname(m[1]))

            # Realization for `m` should be different wp. 1.
            if isunivariate
                @test svi_new[@varname(m)] != m
            else
                @test svi_new[@varname(m[1])] != m[1]
                @test svi_new[@varname(m[2])] != m[2]
            end
            # Logjoint should be non-zero wp. 1.
            @test getlogp(svi_new) != 0

            ### Evaluation ###
            # Sample some random testing values.
            m_eval = if m isa AbstractArray
                randn!(similar(m))
            else
                randn(eltype(m))
            end

            # Update the realizations in `svi_new`.
            svi_eval = if isunivariate
                DynamicPPL.setindex!!(svi_new, m_eval, @varname(m))
            else
                DynamicPPL.setindex!!(svi_new, m_eval, [@varname(m[1]), @varname(m[2])])
            end
            # Reset the logp field.
            svi_eval = DynamicPPL.resetlogp!!(svi_eval)

            # Compute `logjoint` using the varinfo.
            logπ = logjoint(model, svi_eval)
            # Extract the parameters from `svi_eval`.
            m_vi = if isunivariate
                svi_eval[@varname(m)]
            else
                svi_eval[[@varname(m[1]), @varname(m[2])]]
            end
            # These should not have changed.
            @test m_vi == m_eval
            # Compute the true `logjoint` and compare.
            logπ_true = DynamicPPL.TestUtils.logjoint_true(model, m_vi)
            @test logπ ≈ logπ_true
        end
    end
end


const ascii_signs = [0b100_000_000 0b000_100_000 0b000_000_100;
                     0b010_000_000 0b000_010_000 0b000_000_010;
                     0b001_000_000 0b000_001_000 0b000_000_001]

const ascii_lookup = Dict{UInt16,Char}()
const ascii_decode = Array(Char, 512)
ascii_lookup[0b101_000_000] = '"'
ascii_lookup[0b111_111_111] = '@'
ascii_lookup[0b011_110_011] = '$'
ascii_lookup[0b010_000_000] = '\''
ascii_lookup[0b010_100_010] = '('
ascii_lookup[0b010_001_010] = ')'
ascii_lookup[0b000_010_000] = '*'
ascii_lookup[0b010_111_010] = '+'
ascii_lookup[0b000_010_010] = ','
ascii_lookup[0b000_100_100] = ','
ascii_lookup[0b000_001_001] = ','
ascii_lookup[0b000_111_000] = '-'
ascii_lookup[0b000_000_010] = '.'
ascii_lookup[0b000_000_100] = '.'
ascii_lookup[0b000_000_001] = '.'
ascii_lookup[0b001_010_100] = '/'
ascii_lookup[0b010_100_000] = '/'
ascii_lookup[0b001_010_110] = '/'
ascii_lookup[0b011_010_010] = '/'
ascii_lookup[0b001_010_010] = '/'
ascii_lookup[0b110_010_111] = '1'
ascii_lookup[0b111_010_100] = '7'
ascii_lookup[0b010_000_010] = ':'
ascii_lookup[0b111_000_111] = '='
ascii_lookup[0b010_111_101] = 'A'
ascii_lookup[0b011_100_011] = 'C'
ascii_lookup[0b110_101_110] = 'D'
ascii_lookup[0b111_110_100] = 'F'
ascii_lookup[0b011_101_011] = 'G'
ascii_lookup[0b101_111_101] = 'H'
ascii_lookup[0b111_010_111] = 'I'
ascii_lookup[0b011_001_111] = 'J'
ascii_lookup[0b101_110_101] = 'K'
ascii_lookup[0b100_100_111] = 'L'
ascii_lookup[0b111_111_101] = 'M'
ascii_lookup[0b101_101_101] = 'N'
ascii_lookup[0b111_101_111] = 'O'
ascii_lookup[0b111_111_100] = 'P'
ascii_lookup[0b111_010_010] = 'T'
ascii_lookup[0b101_101_111] = 'U'
ascii_lookup[0b101_101_010] = 'V'
ascii_lookup[0b101_111_111] = 'W'
ascii_lookup[0b101_010_101] = 'X'
ascii_lookup[0b101_010_010] = 'Y'
ascii_lookup[0b110_100_110] = '['
ascii_lookup[0b010_001_000] = '\\'
ascii_lookup[0b100_010_001] = '\\'
ascii_lookup[0b110_010_010] = '\\'
ascii_lookup[0b100_010_011] = '\\'
ascii_lookup[0b100_010_010] = '\\'
ascii_lookup[0b011_001_011] = ']'
ascii_lookup[0b010_101_000] = '^'
ascii_lookup[0b000_000_111] = '_'
ascii_lookup[0b100_000_000] = '`'
ascii_lookup[0b000_111_111] = 'a'
ascii_lookup[0b100_111_111] = 'b'
ascii_lookup[0b001_111_111] = 'd'
ascii_lookup[0b001_111_010] = 'f'
ascii_lookup[0b100_111_101] = 'h'
ascii_lookup[0b100_101_101] = 'k'
ascii_lookup[0b110_010_011] = 'l'
ascii_lookup[0b000_111_101] = 'n'
ascii_lookup[0b000_111_100] = 'r'
ascii_lookup[0b000_101_111] = 'u'
ascii_lookup[0b000_101_010] = 'v'
ascii_lookup[0b011_110_011] = '{'
ascii_lookup[0b010_010_010] = '|'
ascii_lookup[0b100_100_100] = '|'
ascii_lookup[0b001_001_001] = '|'
ascii_lookup[0b110_011_110] = '}'

ascii_decode[0b1] = ' '
for i in 1:511
    min_dist = typemax(Int)
    min_char = ' '
    for (k, v) in ascii_lookup
        cur_dist = count_ones(UInt16(i) $ k)
        if cur_dist < min_dist
            min_dist = cur_dist
            min_char = v
        end
    end
    min_char
    ascii_decode[i + 1] = min_char
end

"""
As the name suggests the `AsciiCanvas` only uses
ASCII characters to draw it's content. Naturally,
it doesn't look quite as nice as the Unicode-based
ones. However, in some situations it might yield
better results. Printing plots to a file is one
of those situations.

The AsciiCanvas is best utilized in combination
with `lineplot`.
For `scatterplot` we suggest to use the `DotCanvas`
instead.
"""
type AsciiCanvas <: LookupCanvas
    grid::Array{UInt16,2}
    colors::Array{UInt8,2}
    pixel_width::Int
    pixel_height::Int
    origin_x::Float64
    origin_y::Float64
    width::Float64
    height::Float64
end

@inline x_pixel_per_char(::Type{AsciiCanvas}) = 3
@inline y_pixel_per_char(::Type{AsciiCanvas}) = 3

@inline lookup_encode(c::AsciiCanvas) = ascii_signs
@inline lookup_decode(c::AsciiCanvas) = ascii_decode

function AsciiCanvas(args...; nargs...)
    CreateLookupCanvas(AsciiCanvas, args...; nargs...)
end


cfg = NeuroEvolution.get_config("../cfg/test.yaml")
cfg["n_out"] = 1
cfg["n_in"] = 2
cfg["log_gen"] = 0
cfg["save_gen"] = 0

@testset "Evolution creation" begin
    # test_fitness::Function = x::NEATIndiv -> fitness_xor(x, cfg["n_in"])
    e::Evolution = NEAT(cfg, fitness_xor, cfg["n_in"])
    # @test length(e.population) == cfg["n_population"]
    @test all(typeof.(e.population) .== NEATIndividual)
end

@testset "Evaluate" begin
    # test_fitness::Function = x::NEATIndiv -> fitness_xor(x, cfg["n_in"])
    e::Evolution = NEAT(cfg, fitness_xor, cfg["n_in"])
    e.evaluate(e)
    @test all(getfield.(e.population, :fitness) .!= [-Inf])
end

@testset "Populate" begin
    cfg = NeuroEvolution.get_config("../cfg/test.yaml")
    # test_fitness::Function = x::NEATIndiv -> fitness_xor(x, cfg["n_in"])
    e::Evolution = NEAT(cfg, fitness_xor, cfg["n_in"])
    e.evaluate(e)
    e.populate(e)
    @test length(e.population) > 0
end

@testset "NEAT on XOR" begin
    cfg = NeuroEvolution.get_config("../cfg/test.yaml")
    cfg["verbose"]=true
    # Run evolution
    # test_fitness::Function = x::NEATIndiv -> fitness_xor(x, cfg["n_in"])
    e = NEAT(cfg, fitness_xor, cfg["n_in"])
    Cambrian.run!(e)

    @test length(e.population) <= 1.1 * cfg["n_population"]

    X, y = xor_dataset(cfg["n_in"], 100)
    max_f = sum(log_fitness.(y, y)) / length(X)
    y_false = []
    for i in y
        push!(y_false, 1 .- i)
    end
    min_f = sum(log_fitness.(y, y_false)) / length(X)
    if cfg["verbose"]
        println("\n-- NEAT on XOR --")
        println("Worst possible fitness: ", min_f)
        println("Best  possible fitness: ", max_f)
    end
    # Analyse results
    best = sort(e.population, rev=true)
    @test best[1].fitness[1] <= max_f
    @test best[1].fitness[1] > min_f

    y = process(best[1], [1., 1.])
    @test typeof(y[1])==Float64

    if cfg["verbose"]
        println("Final fitness: ", best[1].fitness[1])
        println("Population size: ", length(e.population))
        println("Genome size: ", length(best[1].genes))
        println("Max innovation: ", e.cfg["innovation_max"])
        println("Species: ", length(e.cfg["Species"]))
        println("\nBest individuals: ", getfield.(best[1:10], :fitness), "\n")
        println("\n\n", best[1])
    end
end

@testset "Cambrian.GA x NEAT" begin
    cfg = NeuroEvolution.get_config("../cfg/ga.yaml")

    e = GA_NEAT(NEATIndividual, cfg, fitness_xor; id="test")
    step!(e)
    @test length(e.population) == cfg["n_population"]
    run!(e)
    best = sort(e.population, rev=true)

    X, y = xor_dataset(cfg["n_in"], 100)
    max_f = sum(log_fitness.(y, y)) / length(X)
    y_false = []
    for i in y
        push!(y_false, 1 .- i)
    end
    min_f = sum(log_fitness.(y, y_false)) / length(X)
    @test best[1].fitness[1] <= max_f
    @test best[1].fitness[1] > min_f

    y = process(best[1], [1., 1.])
    @test typeof(y[1])==Float64
end


"""
	traversal(potree::String, params::ParametersOrthophoto)

Trie traversal.
If entire point cloud falls in volume process all files of Potree project
else travers trie, depth search first, and process nodes falling in region of interest.

Input:
 - params: initial parameters
 - potree: potree hierarchy
"""
function traversal(potree::String, params::ParametersOrthophoto)
    flushprintln("= ")
    flushprintln("= PROJECT: $potree")
    flushprintln("= ")

    metadata = CloudMetadata(potree) # metadata of current potree project
    trie = potree2trie(potree)
    params.numNodes = length(keys(trie))

    # if model contains the whole point cloud ( == 2)
    #	process all files
    # else
    # 	navigate potree

    intersection =
        Common.modelsdetection(params.model, metadata.tightBoundingBox)

    if intersection == 2
        flushprintln("FULL model")
        for k in keys(trie)
            params.numFilesProcessed = params.numFilesProcessed + 1
            if params.numFilesProcessed % 100 == 0
                flushprintln(
                    params.numFilesProcessed,
                    " files processed of ",
                    params.numNodes,
                )
            end

            file = trie[k]
            updateWithoutControl!(params, file)

        end
    elseif intersection == 1
        flushprintln("DFS")
        dfs(trie, params)

        if params.numNodes - params.numFilesProcessed > 0
            flushprintln("$(params.numNodes-params.numFilesProcessed) file of $(params.numNodes) not processed - out of region of interest")
        end
    elseif intersection == 0
        flushprintln("OUT OF REGION OF INTEREST")
    end

end

"""
	dfs(t::DataStructures.Trie{String},
	params::Union{ParametersOrthophoto,ParametersExtraction},
	s::Union{Nothing,IOStream},n::Int64,nfiles::Int64,l::Int64)

Depth search first.
"""
function dfs(trie::DataStructures.Trie{String}, params::ParametersOrthophoto)# due callback: 1 con controllo e 1 senza controllo

    file = trie.value # path to node file
    nodebb = FileManager.las2aabb(file) # aabb of current octree
    inter = Common.modelsdetection(params.model, nodebb)

    if inter == 1
        # intersecato ma non contenuto
        # alcuni punti ricadono nel modello altri no
        params.numFilesProcessed = params.numFilesProcessed + 1
        if params.numFilesProcessed % 100 == 0
            flushprintln(
                params.numFilesProcessed,
                " files processed of ",
                params.numNodes,
            )
        end

        updateWithControl!(params, file) # update with check
        for key in collect(keys(trie.children)) # for all children
            dfs(trie.children[key], params)
        end
    elseif inter == 2
        # contenuto: tutti i punti del albero sono nel modello
        for k in keys(trie)
            params.numFilesProcessed = params.numFilesProcessed + 1
            if params.numFilesProcessed % 100 == 0
                flushprintln(
                    params.numFilesProcessed,
                    " files processed of ",
                    params.numNodes,
                )
            end
            file = trie[k]
            updateWithoutControl!(params, file) # update without check
        end
    end

end


/**
 * This class models Crypto operations.
 */
class CryptoLibrary {
    //static int cl1TESTOUTPUT = 7;

    // All the interesting constraints are coded up in CEDataFlow
    public static int encrypt(int publicKey, int plaintext) {
 	// here is where the magic encrypting happens.
	// the result has no relation to the plaintext.
	return  ({L})plaintext+1;
    }

    public static int decrypt(int privateKey, int ciphertext) {
	return ciphertext-1;
    }
}


export execute, initialise_sampler, timer_output

using FileIO
using TimerOutputs
using CUDA

using QXContexts.Param
using QXContexts.ComputeGraphs
using QXContexts.Sampling

# const timer_output = TimerOutput()
# if haskey(ENV, "QXRUN_TIMER")
#     timeit_debug_enabled() = return true # Function fails to be defined by enable_debug_timings
#     TimerOutputs.enable_debug_timings(Execution)
# end

"""
    write_results(results, output_file)

Save results from calculations for the given
"""
function write_results(results, output_file)
    @assert splitext(output_file)[end] == ".jld2" "Output file should have jld2 suffix"
    if results !== nothing
        save(output_file, "results", results)
    end
end

"""
    initialise_sampler(dsl_file::String,
                       input_file::String,
                       param_file::String;
                       use_mpi::Bool=false,
                       sub_comm_size::Int=1,
                       use_gpu::Bool=false,
                       elt::Type=ComplexF32)

Initialise the sampler
"""
function initialise_sampler(dsl_file::String,
                            input_file::String,
                            param_file::String;
                            use_mpi::Bool=false,
                            sub_comm_size::Int=1,
                            use_gpu::Bool=false,
                            elt::Type=ComplexF32)
    # read dsl file of commands and data file with initial tensors
    @timeit "Parse input files" cg, _ = parse_dsl_files(dsl_file, input_file)

    # Create a context to execute the commands in
    T = if use_gpu
        @assert CUDA.functional() "CUDA installation is not functional, ensure you have a GPU and appropriate drivers"
        CuArray{elt}
    else
        Array{elt}
    end
    @timeit "Create Context" begin
        ctx = QXContext{T}(cg)
        if use_mpi
            ctx = QXMPIContext(ctx, sub_comm_size=sub_comm_size)
        end
    end

    # read sampler parameters from parameter file
    sampler_args = parse_parameters(param_file)

    # Create a sampler to produce bitstrings to get amplitudes for and a variable to store
    # the results.
    @timeit "Create sampler" create_sampler(ctx, sampler_args)
end

"""
    execute(dsl_file::String,
            input_file::Union{String, Nothing}=nothing,
            param_file::Union{String, Nothing}=nothing,
            output_file::String="";
            max_amplitudes::Union{Int, Nothing}=nothing,
            max_slices::Union{Int, Nothing}=nothing,
            timings::Bool=false,
            kwargs...)

Main entry point for running calculations. Loads input data, runs computations and
writes results to output files.
"""
function execute(dsl_file::String,
                 input_file::Union{String, Nothing}=nothing,
                 param_file::Union{String, Nothing}=nothing,
                 output_file::String="";
                 max_amplitudes::Union{Int, Nothing}=nothing,
                 max_slices::Union{Int, Nothing}=nothing,
                 timings::Bool=false,
                 kwargs...)

    if input_file === nothing
        input_file = splitext(dsl_file)[1] * ".jld2"
    end

    if param_file === nothing
        param_file = splitext(dsl_file)[1] * ".yml"
    end
    @timeit "Init sampler" sampler = initialise_sampler(dsl_file, input_file, param_file;
                                 kwargs...)

    @timeit "Simulation" results = sampler(max_amplitudes=max_amplitudes,
                                           max_slices=max_slices)

    if output_file != "" && results !== nothing
        @timeit "Write results" write_results(results, output_file)
        if timings
            print_timer()
        end
    end
    results
end


###### This file is part of the MendelImpute.jl package.
###### It contains relevant code to search breakpoints.
###### Currently code for double breakpoint search is broken and commented out.

"""
    continue_haplotype(X, compressed_Hunique, window, happair_prev, happair_next)

Searches the breakpoint between `happair_prev` and `happair_next`, if there is
one.

# Arguments:
- `X`: Genotype vector spanning 2 windows.
- `compressed_Hunique`: A `CompressedHaplotypes` keeping track of unique
    haplotypes for each window and some other information
- `window`: The current window being considered. It is the 2nd window of `X`.
- `happair_prev`: Haplotype pair for first window
- `happair_next`: Haplotype pair for second window

# Optional arguments:
- `phased`: Boolean indicating whether `happair_prev` and `happair_next` have
    been phased. If `false`, will try 2 different orientations.
- `search_double_bkpts`: If `true`, will search double breakpoints. If false
    the output `bkpt = (-2, -2)`

# Output
- `happair_next`: If `phase = false`, this is equal to `happair_next` for input.
    Otherwise it may be flipped
- `bkpt`: Tuple of integer indicating optimal breakpoint starting from the
    previous window. `-1` indicates no breakpoints and `-2` indicates double
    breakpoint.
"""
function continue_haplotype(
    X::AbstractVector,
    compressed_Hunique::CompressedHaplotypes,
    window::Int,
    happair_prev::Tuple,
    happair_next::Tuple;
    phased::Bool = false,
    search_double_bkpts::Bool = false
    )

    # indices for complete reference panel
    i, j = happair_prev
    k, l = happair_next

    # both strands match
    if i == k && j == l
        return (k, l), (-1, -1)
    end
    if i == l && j == k && !phased
        return (l, k), (-1, -1)
    end

    # get unique haplotypes with only typed snps
    overlap = compressed_Hunique.overlap
    if overlap
        # create views on non-overlapping regions
        rprev = nonoverlap_range(compressed_Hunique, window - 1)
        rcurr = nonoverlap_range(compressed_Hunique, window)
        Hprev = view(compressed_Hunique.CW_typed[window - 1].uniqueH, rprev, :)
        Hcurr = view(compressed_Hunique.CW_typed[window].uniqueH, rcurr, :)
    else
        Hprev = compressed_Hunique.CW_typed[window - 1].uniqueH
        Hcurr = compressed_Hunique.CW_typed[window].uniqueH
    end

    # only one strand matches
    if i == k && j ≠ l
        iu = complete_idx_to_unique_typed_idx(i, window - 1, compressed_Hunique)
        ku = complete_idx_to_unique_typed_idx(k, window, compressed_Hunique)
        ju1 = complete_idx_to_unique_typed_idx(j, window - 1, compressed_Hunique)
        ju2 = complete_idx_to_unique_typed_idx(j, window, compressed_Hunique)
        lu1 = complete_idx_to_unique_typed_idx(l, window - 1, compressed_Hunique)
        lu2 = complete_idx_to_unique_typed_idx(l, window, compressed_Hunique)

        # lazy concatenation
        s1  = ApplyArray(vcat, view(Hprev, :, iu),  view(Hcurr, :, ku))
        s21 = ApplyArray(vcat, view(Hprev, :, ju1), view(Hcurr, :, ju2))
        s22 = ApplyArray(vcat, view(Hprev, :, lu1), view(Hcurr, :, lu2))

        breakpt, errors = search_breakpoint(X, s1, s21, s22)
        return (k, l), (-1, breakpt)
    elseif i == l && j ≠ k && !phased
        iu = complete_idx_to_unique_typed_idx(i, window - 1, compressed_Hunique)
        lu = complete_idx_to_unique_typed_idx(l, window, compressed_Hunique)
        ju1 = complete_idx_to_unique_typed_idx(j, window - 1, compressed_Hunique)
        ju2 = complete_idx_to_unique_typed_idx(j, window, compressed_Hunique)
        ku1 = complete_idx_to_unique_typed_idx(k, window - 1, compressed_Hunique)
        ku2 = complete_idx_to_unique_typed_idx(k, window, compressed_Hunique)

        # lazy concatenation
        s1  = ApplyArray(vcat, view(Hprev, :, iu),  view(Hcurr, :, lu))
        s21 = ApplyArray(vcat, view(Hprev, :, ju1), view(Hcurr, :, ju2))
        s22 = ApplyArray(vcat, view(Hprev, :, ku1), view(Hcurr, :, ku2))

        breakpt, errors = search_breakpoint(X, s1, s21, s22)
        return (l, k), (-1, breakpt)
    elseif j == k && i ≠ l && !phased
        ju = complete_idx_to_unique_typed_idx(j, window - 1, compressed_Hunique)
        ku = complete_idx_to_unique_typed_idx(k, window, compressed_Hunique)
        iu1 = complete_idx_to_unique_typed_idx(i, window - 1, compressed_Hunique)
        iu2 = complete_idx_to_unique_typed_idx(i, window, compressed_Hunique)
        lu1 = complete_idx_to_unique_typed_idx(l, window - 1, compressed_Hunique)
        lu2 = complete_idx_to_unique_typed_idx(l, window, compressed_Hunique)

        # lazy concatenation
        s1  = ApplyArray(vcat, view(Hprev, :, ju),  view(Hcurr, :, ku))
        s21 = ApplyArray(vcat, view(Hprev, :, iu1), view(Hcurr, :, iu2))
        s22 = ApplyArray(vcat, view(Hprev, :, lu1), view(Hcurr, :, lu2))

        breakpt, errors = search_breakpoint(X, s1, s21, s22)
        return (l, k), (breakpt, -1)
    elseif j == l && i ≠ k
        ju = complete_idx_to_unique_typed_idx(j, window - 1, compressed_Hunique)
        lu = complete_idx_to_unique_typed_idx(l, window, compressed_Hunique)
        iu1 = complete_idx_to_unique_typed_idx(i, window - 1, compressed_Hunique)
        iu2 = complete_idx_to_unique_typed_idx(i, window, compressed_Hunique)
        ku1 = complete_idx_to_unique_typed_idx(k, window - 1, compressed_Hunique)
        ku2 = complete_idx_to_unique_typed_idx(k, window, compressed_Hunique)

        # lazy concatenation
        s1  = ApplyArray(vcat, view(Hprev, :, ju),  view(Hcurr, :, lu))
        s21 = ApplyArray(vcat, view(Hprev, :, iu1), view(Hcurr, :, iu2))
        s22 = ApplyArray(vcat, view(Hprev, :, ku1), view(Hcurr, :, ku2))

        breakpt, errors = search_breakpoint(X, s1, s21, s22)
        return (k, l), (breakpt, -1)
    end

    # both strand mismatch
    if search_double_bkpts
        iu1 = complete_idx_to_unique_typed_idx(i, window - 1, compressed_Hunique)
        iu2 = complete_idx_to_unique_typed_idx(i, window, compressed_Hunique)
        ku1 = complete_idx_to_unique_typed_idx(k, window - 1, compressed_Hunique)
        ku2 = complete_idx_to_unique_typed_idx(k, window, compressed_Hunique)
        ju1 = complete_idx_to_unique_typed_idx(j, window - 1, compressed_Hunique)
        ju2 = complete_idx_to_unique_typed_idx(j, window, compressed_Hunique)
        lu1 = complete_idx_to_unique_typed_idx(l, window - 1, compressed_Hunique)
        lu2 = complete_idx_to_unique_typed_idx(l, window, compressed_Hunique)

        # lazy concatenation
        s11 = ApplyArray(vcat, view(Hprev, :, iu1), view(Hcurr, :, iu2))
        s12 = ApplyArray(vcat, view(Hprev, :, ku1), view(Hcurr, :, ku2))
        s21 = ApplyArray(vcat, view(Hprev, :, ju1), view(Hcurr, :, ju2))
        s22 = ApplyArray(vcat, view(Hprev, :, lu1), view(Hcurr, :, lu2))

        breakpt, errors = search_breakpoint(X, s11, s12, s21, s22)
        return (k, l), breakpt

        # Always assume double bkpt searches are phased.
        # breakpt1, errors1 = search_breakpoint(X, H, (i, k), (j, l))
        # breakpt2, errors2 = search_breakpoint(X, H, (i, l), (j, k))
        # if errors1 < errors2
        #     return (k, l), breakpt1
        # else
        #     return (l, k), breakpt2
        # end
    else
        return (k, l), (-2, -2)
    end
end

"""
    search_single_breakpoint(X, s1, s21, s22)

Find the optimal break point between s21 and s22 in configuration
s1 | s21
s1 | s22
"""
function search_breakpoint(
    X::AbstractVector,
    s1::AbstractVector,
    s21::AbstractVector,
    s22::AbstractVector,
    )

    n = length(X)
    length(s1) == length(s21) == length(s22) == n ||error("search_breakpoint:" *
        " all vectors should have same length but length X = $n, s1 =" *
        " $(length(s1)), s21 = $(length(s21)), s22 = $(length(s22))")

    # count number of errors if second haplotype is all from s22
    errors = 0
    @inbounds for pos in 1:n
        if !ismissing(X[pos])
            errors += X[pos] ≠ s1[pos] + s22[pos]
        end
    end
    bkpt_optim, err_optim = 0, errors

    # quick return if perfect match
    err_optim == 0 && return 0, 0

    # extend haplotype s21 position by position
    @inbounds for bkpt in 1:n
        if !ismissing(X[bkpt]) && s21[bkpt] ≠ s22[bkpt]
            errors -= X[bkpt] ≠ s1[bkpt] + s22[bkpt]
            errors += X[bkpt] ≠ s1[bkpt] + s21[bkpt]
            if errors :: Int < err_optim
                bkpt_optim, err_optim = bkpt, errors
                # quick return if perfect match
                err_optim == 0 && return bkpt_optim, err_optim :: Int
            end
        end
    end

    return bkpt_optim, err_optim :: Int
end

"""
    search_breakpoint(X, s11, s12, s21, s22)

Find the optimal break point between s11 and s12 as well as between s21 and s22
in configuration
s11 | s21
s12 | s22
"""
function search_breakpoint(
    X::AbstractVector,
    s11::AbstractVector,
    s12::AbstractVector,
    s21::AbstractVector,
    s22::AbstractVector,
    )
    n = length(X)
    length(s11) == length(s12) == length(s21) == length(s22) == n ||
        error("search_breakpoint: all vectors should have same length but " *
        "length X = $n, s11 = $(length(s11)), s12 = $(length(s12)), s21 = " *
        "$(length(s21)), s22 = $(length(s22))")

    err_optim   = typemax(Int)
    bkpts_optim = (0, 0)

    # search over all combintations of break points in two strands
    @inbounds for bkpt1 in 0:n

        # count number of errors if second haplotype is all from s22
        errors = 0
        for pos in 1:bkpt1
            if !ismissing(X[pos])
                errors += X[pos] ≠ s11[pos] + s22[pos]
            end
        end
        for pos in (bkpt1 + 1):length(X)
            if !ismissing(X[pos])
                errors += X[pos] ≠ s12[pos] +s22[pos]
            end
        end
        if errors :: Int < err_optim
            err_optim = errors
            bkpts_optim = (bkpt1, 0)

            # quick return if perfect match
            err_optim == 0 && return bkpts_optim, err_optim :: Int
        end

        # extend haplotype s21 position by position
        for bkpt2 in 1:bkpt1
            if !ismissing(X[bkpt2]) && s21[bkpt2] ≠ s22[bkpt2]
                errors -= X[bkpt2] ≠ s11[bkpt2] + s22[bkpt2]
                errors += X[bkpt2] ≠ s11[bkpt2] + s21[bkpt2]
                if errors :: Int < err_optim
                    err_optim = errors
                    bkpts_optim = (bkpt1, bkpt2)
                end
            end
        end
        for bkpt2 in (bkpt1 + 1):length(X)
            if !ismissing(X[bkpt2]) && s21[bkpt2] ≠ s22[bkpt2]
                errors -= X[bkpt2] ≠ s12[bkpt2] + s22[bkpt2]
                errors += X[bkpt2] ≠ s12[bkpt2] + s21[bkpt2]
                if errors :: Int < err_optim
                    err_optim = errors
                    bkpts_optim = (bkpt1, bkpt2)
                    # quick return if perfect match
                    err_optim == 0 && return bkpts_optim, err_optim :: Int
                end
            end
        end
    end

    return bkpts_optim, err_optim :: Int
end


using GLVisualize, GeometryTypes, GLAbstraction, Colors, FileIO

window = glscreen()

mesh = GLNormalMesh(loadasset("./resources/quadrotor.obj")

view(visualize(mesh), window)

renderloop(window)