"""
    update!(
        KS::X,
        ctr::Y,
        face::Z,
        dt,
        residual; # 1D / 2D
        coll = :bgk::Symbol,
        bc = :fix::Symbol,
    ) where {
        X<:AbstractSolverSet,
        Y<:AbstractArray{ControlVolume1D1F,1},
        Z<:AbstractArray{Interface1D1F,1},
    }

    update!(
        KS::X,
        ctr::Y,
        face::Z,
        dt,
        residual; # 1D / 2D
        coll = :bgk::Symbol,
        bc = :extra::Symbol,
    ) where {
        X<:AbstractSolverSet,
        Y<:AbstractArray{ControlVolume1D2F,1},
        Z<:AbstractArray{Interface1D2F,1},
    }

    update!(
        KS::X,
        ctr::Y,
        face::Z,
        dt,
        residual; # 1D / 2D
        coll = :bgk::Symbol,
        bc = :extra::Symbol,
        isMHD = true::Bool,
    ) where {
        X<:AbstractSolverSet,
        Y<:AbstractArray{ControlVolume1D3F,1},
        Z<:AbstractArray{Interface1D3F,1},
    }

    update!(
        KS::X,
        ctr::Y,
        face::Z,
        dt,
        residual; # 1D / 2D
        coll = :bgk::Symbol,
        bc = :extra::Symbol,
        isMHD = true::Bool,
    ) where {
        X<:AbstractSolverSet,
        Y<:AbstractArray{ControlVolume1D4F,1},
        Z<:AbstractArray{Interface1D4F,1},
    }

    update!(
        KS::X,
        ctr::Y,
        a1face::Z,
        a2face::Z,
        dt,
        residual; # 1D / 2D
        coll = :bgk::Symbol,
        bc = :fix::Symbol,
    ) where {
        X<:AbstractSolverSet,
        Y<:AbstractArray{ControlVolume2D2F,2},
        Z<:AbstractArray{Interface2D2F,2},
    }

Update flow variables over control volumes

"""
function update!(
    KS::X,
    ctr::Y,
    face::Z,
    dt,
    residual::T; # 1D / 2D
    coll = symbolize(KS.set.collision),
    bc = symbolize(KS.set.boundary),
) where {
    X<:AbstractSolverSet,
    Y<:AbstractArray{ControlVolume1D,1},
    Z<:AbstractArray{Interface1D,1},
    T<:AbstractFloat,
}

    sumRes = zero(ctr[1].w)
    sumAvg = zero(ctr[1].w)

    @inbounds Threads.@threads for i = 1:KS.pSpace.nx
        ctr[i].w, sumRes, sumAvg = step!(
            face[i].fw,
            ctr[i].w,
            ctr[i].prim,
            face[i+1].fw,
            KS.gas.a,
            KS.ps.dx[i],
            sumRes,
            sumAvg,
        )
    end

    residual = sqrt(sumRes * KS.pSpace.nx) / (sumAvg + 1.e-7)

    if bc[1] == :period
        ctr[0].w = ctr[KS.pSpace.nx].w
        ctr[0].sw = ctr[KS.pSpace.nx].sw
        ctr[KS.pSpace.nx+1].w = ctr[1].w
        ctr[KS.pSpace.nx+1].sw = ctr[1].sw
    elseif bc[1] == :extra
        ctr[0].w = ctr[1].w
    end

    if bc[2] == :extra
        ctr[KS.pSpace.nx+1].w = ctr[KS.pSpace.nx].w
    end

    return residual

end

function update!(
    KS::X,
    ctr::Y,
    face::Z,
    dt,
    residual::AbstractArray{T}; # 1D / 2D
    coll = symbolize(KS.set.collision),
    bc = symbolize(KS.set.boundary),
) where {
    X<:AbstractSolverSet,
    Y<:AbstractArray{ControlVolume1D,1},
    Z<:AbstractArray{Interface1D,1},
    T<:AbstractFloat,
}

    sumRes = zero(ctr[1].w)
    sumAvg = zero(ctr[1].w)

    if ndims(sumRes) == 1
        @inbounds Threads.@threads for i = 2:KS.pSpace.nx-1
            step!(
                face[i].fw,
                ctr[i].w,
                ctr[i].prim,
                face[i+1].fw,
                KS.gas.γ,
                KS.ps.dx[i],
                sumRes,
                sumAvg,
            )
        end
    elseif ndims(sumRes) == 2
        @inbounds Threads.@threads for i = 2:KS.pSpace.nx-1
            step!(
                face[i].fw,
                ctr[i].w,
                ctr[i].prim,
                face[i+1].fw,
                KS.gas.γ,
                KS.gas.mi,
                KS.gas.ni,
                KS.gas.me,
                KS.gas.ne,
                KS.gas.Kn[1],
                KS.ps.dx[i],
                dt,
                sumRes,
                sumAvg,
            )
        end
    end

    for i in eachindex(residual)
        residual[i] = sqrt(sumRes[i] * KS.pSpace.nx) / (sumAvg[i] + 1.e-7)
    end

    if bc isa Symbol
        update_boundary!(KS, ctr, face, dt, residual; bc = [bc, bc])
    else
        update_boundary!(KS, ctr, face, dt, residual; bc = bc)
    end

    return nothing

end

function update!(
    KS::X,
    ctr::Y,
    face::Z,
    dt,
    residual; # 1D / 2D
    coll = symbolize(KS.set.collision),
    bc = symbolize(KS.set.boundary),
) where {
    X<:AbstractSolverSet,
    Y<:AbstractArray{ControlVolume1D1F,1},
    Z<:AbstractArray{Interface1D1F,1},
}

    sumRes = zero(ctr[1].w)
    sumAvg = zero(ctr[1].w)

    phase = KS.set.space[3:end]
    if phase == "1f1v"
        @inbounds Threads.@threads for i = 2:KS.pSpace.nx-1
            step!(
                face[i].fw,
                face[i].ff,
                ctr[i].w,
                ctr[i].prim,
                ctr[i].f,
                face[i+1].fw,
                face[i+1].ff,
                KS.vSpace.u,
                KS.vSpace.weights,
                KS.gas.γ,
                KS.gas.μᵣ,
                KS.gas.ω,
                KS.gas.Pr,
                KS.ps.dx[i],
                dt,
                sumRes,
                sumAvg,
                coll,
            )
        end
    elseif phase == "1f3v"
        @inbounds Threads.@threads for i = 2:KS.pSpace.nx-1
            step!(
                face[i].fw,
                face[i].ff,
                ctr[i].w,
                ctr[i].prim,
                ctr[i].f,
                face[i+1].fw,
                face[i+1].ff,
                KS.vSpace.u,
                KS.vSpace.v,
                KS.vSpace.w,
                KS.vSpace.weights,
                KS.gas.γ,
                KS.gas.μᵣ,
                KS.gas.ω,
                KS.gas.Pr,
                KS.ps.dx[i],
                dt,
                sumRes,
                sumAvg,
                coll,
            )
        end
    end

    for i in eachindex(residual)
        residual[i] = sqrt(sumRes[i] * KS.pSpace.nx) / (sumAvg[i] + 1.e-7)
    end

    if bc isa Symbol
        update_boundary!(KS, ctr, face, dt, residual; coll = coll, bc = [bc, bc])
    else
        update_boundary!(KS, ctr, face, dt, residual; coll = coll, bc = bc)
    end

    return nothing

end

function update!(
    KS::X,
    ctr::Y,
    face::Z,
    dt,
    residual; # 1D / 2D
    coll = symbolize(KS.set.collision),
    bc = symbolize(KS.set.boundary),
) where {
    X<:AbstractSolverSet,
    Y<:AbstractArray{ControlVolume1D2F,1},
    Z<:AbstractArray{Interface1D2F,1},
}

    sumRes = zero(ctr[1].w)
    sumAvg = zero(ctr[1].w)

    if ndims(sumRes) == 1
        @inbounds Threads.@threads for i = 2:KS.pSpace.nx-1
            step!(
                face[i].fw,
                face[i].fh,
                face[i].fb,
                ctr[i].w,
                ctr[i].prim,
                ctr[i].h,
                ctr[i].b,
                face[i+1].fw,
                face[i+1].fh,
                face[i+1].fb,
                KS.vSpace.u,
                KS.vSpace.weights,
                KS.gas.K,
                KS.gas.γ,
                KS.gas.μᵣ,
                KS.gas.ω,
                KS.gas.Pr,
                KS.ps.dx[i],
                dt,
                sumRes,
                sumAvg,
                coll,
            )
        end
    elseif ndims(sumRes) == 2
        @inbounds Threads.@threads for i = 2:KS.pSpace.nx-1
            step!(
                face[i].fw,
                face[i].fh,
                face[i].fb,
                ctr[i].w,
                ctr[i].prim,
                ctr[i].h,
                ctr[i].b,
                face[i+1].fw,
                face[i+1].fh,
                face[i+1].fb,
                KS.vSpace.u,
                KS.vSpace.weights,
                KS.gas.K,
                KS.gas.γ,
                KS.gas.mi,
                KS.gas.ni,
                KS.gas.me,
                KS.gas.ne,
                KS.gas.Kn[1],
                KS.gas.Pr,
                KS.ps.dx[i],
                dt,
                sumRes,
                sumAvg,
                coll,
            )
        end
    end

    for i in eachindex(residual)
        residual[i] = sqrt(sumRes[i] * KS.pSpace.nx) / (sumAvg[i] + 1.e-7)
    end

    if bc isa Symbol
        update_boundary!(KS, ctr, face, dt, residual; coll = coll, bc = [bc, bc])
    else
        update_boundary!(KS, ctr, face, dt, residual; coll = coll, bc = bc)
    end

    return nothing

end

function update!(
    KS::X,
    ctr::Y,
    face::Z,
    dt,
    residual; # 1D / 2D
    coll = symbolize(KS.set.collision),
    bc = symbolize(KS.set.boundary),
    isMHD = true::Bool,
) where {
    X<:AbstractSolverSet,
    Y<:AbstractArray{ControlVolume1D3F,1},
    Z<:AbstractArray{Interface1D3F,1},
}

    sumRes = zero(ctr[1].w)
    sumAvg = zero(ctr[1].w)

    @inbounds Threads.@threads for i = 2:KS.pSpace.nx-1
        step!(KS, face[i], ctr[i], face[i+1], KS.ps.dx[i], dt, sumRes, sumAvg, coll, isMHD)
    end

    for i in eachindex(residual)
        residual[i] = sqrt(sumRes[i] * KS.pSpace.nx) / (sumAvg[i] + 1.e-7)
    end

    if bc isa Symbol
        update_boundary!(
            KS,
            ctr,
            face,
            dt,
            residual;
            coll = coll,
            bc = [bc, bc],
            isMHD = isMHD,
        )
    else
        update_boundary!(KS, ctr, face, dt, residual; coll = coll, bc = bc, isMHD = isMHD)
    end

    #=
    ng = 1 - first(eachindex(KS.pSpace.x))
    if bc == :extra
        for i in 1:ng
            ctr[1-i].w .= ctr[1].w
            ctr[1-i].prim .= ctr[1].prim
            ctr[1-i].h0 .= ctr[1].h0
            ctr[1-i].h1 .= ctr[1].h1
            ctr[1-i].h2 .= ctr[1].h2
            ctr[1-i].E .= ctr[1].E
            ctr[1-i].B .= ctr[1].B
            ctr[1-i].ϕ = deepcopy(ctr[1].ϕ)
            ctr[1-i].ψ = deepcopy(ctr[1].ψ)
            ctr[1-i].lorenz .= ctr[1].lorenz

            ctr[KS.pSpace.nx+i].w .= ctr[KS.pSpace.nx].w
            ctr[KS.pSpace.nx+i].prim .= ctr[KS.pSpace.nx].prim
            ctr[KS.pSpace.nx+i].h0 .= ctr[KS.pSpace.nx].h0
            ctr[KS.pSpace.nx+i].h1 .= ctr[KS.pSpace.nx].h1
            ctr[KS.pSpace.nx+i].h2 .= ctr[KS.pSpace.nx].h2
            ctr[KS.pSpace.nx+i].E .= ctr[KS.pSpace.nx].E
            ctr[KS.pSpace.nx+i].B .= ctr[KS.pSpace.nx].B
            ctr[KS.pSpace.nx+i].ϕ = deepcopy(ctr[KS.pSpace.nx].ϕ)
            ctr[KS.pSpace.nx+i].ψ = deepcopy(ctr[KS.pSpace.nx].ψ)
            ctr[KS.pSpace.nx+i].lorenz .= ctr[KS.pSpace.nx].lorenz
        end
    elseif bc == :period
        for i in 1:ng
            ctr[1-i].w .= ctr[KS.pSpace.nx+1-i].w
            ctr[1-i].prim .= ctr[KS.pSpace.nx+1-i].prim
            ctr[1-i].h0 .= ctr[KS.pSpace.nx+1-i].h0
            ctr[1-i].h1 .= ctr[KS.pSpace.nx+1-i].h1
            ctr[1-i].h2 .= ctr[KS.pSpace.nx+1-i].h2
            ctr[1-i].E .= ctr[KS.pSpace.nx+1-i].E
            ctr[1-i].B .= ctr[KS.pSpace.nx+1-i].B
            ctr[1-i].ϕ = deepcopy(ctr[KS.pSpace.nx+1-i].ϕ)
            ctr[1-i].ψ = deepcopy(ctr[KS.pSpace.nx+1-i].ψ)
            ctr[1-i].lorenz .= ctr[KS.pSpace.nx+1-i].lorenz

            ctr[KS.pSpace.nx+i].w .= ctr[i].w
            ctr[KS.pSpace.nx+i].prim .= ctr[i].prim
            ctr[KS.pSpace.nx+i].h0 .= ctr[i].h0
            ctr[KS.pSpace.nx+i].h1 .= ctr[i].h1
            ctr[KS.pSpace.nx+i].h2 .= ctr[i].h2
            ctr[KS.pSpace.nx+i].E .= ctr[i].E
            ctr[KS.pSpace.nx+i].B .= ctr[i].B
            ctr[KS.pSpace.nx+i].ϕ = deepcopy(ctr[i].ϕ)
            ctr[KS.pSpace.nx+i].ψ = deepcopy(ctr[i].ψ)
            ctr[KS.pSpace.nx+i].lorenz .= ctr[i].lorenz
        end
    elseif bc == :balance
        @. ctr[0].w = 0.5 * (ctr[-1].w + ctr[1].w)
        @. ctr[0].prim = 0.5 * (ctr[-1].prim + ctr[1].prim)
        @. ctr[0].h0 = 0.5 * (ctr[-1].h0 + ctr[1].h0)
        @. ctr[0].h1 = 0.5 * (ctr[-1].h1 + ctr[1].h1)
        @. ctr[0].h2 = 0.5 * (ctr[-1].h2 + ctr[1].h2)
        @. ctr[0].E = 0.5 * (ctr[-1].E + ctr[1].E)
        @. ctr[0].B = 0.5 * (ctr[-1].B + ctr[1].B)
        ctr[0].ϕ = 0.5 * (ctr[-1].ϕ + ctr[1].ϕ)
        ctr[0].ψ = 0.5 * (ctr[-1].ψ + ctr[1].ψ)
        @. ctr[0].lorenz = 0.5 * (ctr[-1].lorenz + ctr[1].lorenz)

        @. ctr[KS.pSpace.nx+1].w = 0.5 * (ctr[KS.pSpace.nx].w + ctr[KS.pSpace.nx+2].w)
        @. ctr[KS.pSpace.nx+1].prim = 0.5 * (ctr[KS.pSpace.nx].prim + ctr[KS.pSpace.nx+2].prim)
        @. ctr[KS.pSpace.nx+1].h0 = 0.5 * (ctr[KS.pSpace.nx].h0 + ctr[KS.pSpace.nx+2].h0)
        @. ctr[KS.pSpace.nx+1].h1 = 0.5 * (ctr[KS.pSpace.nx].h1 + ctr[KS.pSpace.nx+2].h1)
        @. ctr[KS.pSpace.nx+1].h2 = 0.5 * (ctr[KS.pSpace.nx].h2 + ctr[KS.pSpace.nx+2].h2)
        @. ctr[KS.pSpace.nx+1].E = 0.5 * (ctr[KS.pSpace.nx].E + ctr[KS.pSpace.nx+2].E)
        @. ctr[KS.pSpace.nx+1].B = 0.5 * (ctr[KS.pSpace.nx].B + ctr[KS.pSpace.nx+2].B)
        ctr[KS.pSpace.nx+1].ϕ = 0.5 * (ctr[KS.pSpace.nx].ϕ + ctr[KS.pSpace.nx+2].ϕ)
        ctr[KS.pSpace.nx+1].ψ = 0.5 * (ctr[KS.pSpace.nx].ψ + ctr[KS.pSpace.nx+2].ψ)
        @. ctr[KS.pSpace.nx+1].lorenz = 0.5 * (ctr[KS.pSpace.nx].lorenz + ctr[KS.pSpace.nx+2].lorenz)
    else
    end
    =#
    return nothing

end

function update!(
    KS::X,
    ctr::Y,
    face::Z,
    dt,
    residual; # 1D / 2D
    coll = symbolize(KS.set.collision),
    bc = symbolize(KS.set.boundary),
    isMHD = true::Bool,
) where {
    X<:AbstractSolverSet,
    Y<:AbstractArray{ControlVolume1D4F,1},
    Z<:AbstractArray{Interface1D4F,1},
}

    sumRes = zero(ctr[1].w)
    sumAvg = zero(ctr[1].w)

    @inbounds Threads.@threads for i = 2:KS.pSpace.nx-1
        step!(KS, face[i], ctr[i], face[i+1], KS.ps.dx[i], dt, sumRes, sumAvg, coll, isMHD)
    end

    for i in eachindex(residual)
        residual[i] = sqrt(sumRes[i] * KS.pSpace.nx) / (sumAvg[i] + 1.e-7)
    end

    if bc isa Symbol
        update_boundary!(
            KS,
            ctr,
            face,
            dt,
            residual;
            coll = coll,
            bc = [bc, bc],
            isMHD = isMHD,
        )
    else
        update_boundary!(KS, ctr, face, dt, residual; coll = coll, bc = bc, isMHD = isMHD)
    end

    return nothing

end

function update!(
    KS::X,
    ctr::Y,
    a1face::Z,
    a2face::Z,
    dt,
    residual;
    coll = symbolize(KS.set.collision),
    bc = symbolize(KS.set.boundary),
) where {
    X<:AbstractSolverSet,
    Y<:AbstractArray{ControlVolume2D,2},
    Z<:AbstractArray{Interface2D,2},
}

    nx, ny, dx, dy = begin
        if KS.ps isa CSpace2D
            KS.ps.nr, KS.ps.nθ, KS.ps.dr, KS.ps.darc
        else
            KS.ps.nx, KS.ps.ny, KS.ps.dx, KS.ps.dy
        end
    end

    sumRes = zero(ctr[1].w)
    sumAvg = zero(ctr[1].w)

    if ndims(sumRes) == 1

        @inbounds for j = 2:ny-1
            for i = 2:nx-1
                step!(
                    ctr[i, j].w,
                    ctr[i, j].prim,
                    a1face[i, j].fw,
                    a1face[i+1, j].fw,
                    a2face[i, j].fw,
                    a2face[i, j+1].fw,
                    KS.gas.γ,
                    dx[i, j] * dy[i, j],
                    sumRes,
                    sumAvg,
                    coll,
                )
            end
        end

    end

    for i in eachindex(residual)
        residual[i] = sqrt(sumRes[i] * nx * ny) / (sumAvg[i] + 1.e-7)
    end

    if bc isa Symbol
        update_boundary!(
            KS,
            ctr,
            a1face,
            a2face,
            dt,
            residual;
            coll = coll,
            bc = [bc, bc, bc, bc],
        )
    else
        update_boundary!(KS, ctr, a1face, a2face, dt, residual; coll = coll, bc = bc)
    end

    return nothing

end

function update!(
    KS::X,
    ctr::Y,
    a1face::Z,
    a2face::Z,
    dt,
    residual;
    coll = symbolize(KS.set.collision),
    bc = symbolize(KS.set.boundary),
) where {
    X<:AbstractSolverSet,
    Y<:AbstractArray{ControlVolume2D1F,2},
    Z<:AbstractArray{Interface2D1F,2},
}

    nx, ny, dx, dy = begin
        if KS.ps isa CSpace2D
            KS.ps.nr, KS.ps.nθ, KS.ps.dr, KS.ps.darc
        else
            KS.ps.nx, KS.ps.ny, KS.ps.dx, KS.ps.dy
        end
    end

    sumRes = zero(ctr[1].w)
    sumAvg = zero(ctr[1].w)

    if ndims(sumRes) == 1

        @inbounds for j = 2:ny-1
            for i = 2:nx-1
                step!(
                    ctr[i, j].w,
                    ctr[i, j].prim,
                    ctr[i, j].f,
                    a1face[i, j].fw,
                    a1face[i, j].ff,
                    a1face[i+1, j].fw,
                    a1face[i+1, j].ff,
                    a2face[i, j].fw,
                    a2face[i, j].ff,
                    a2face[i, j+1].fw,
                    a2face[i, j+1].ff,
                    KS.vSpace.u,
                    KS.vSpace.v,
                    KS.vSpace.weights,
                    KS.gas.γ,
                    KS.gas.μᵣ,
                    KS.gas.ω,
                    KS.gas.Pr,
                    dx[i, j] * dy[i, j],
                    dt,
                    sumRes,
                    sumAvg,
                    coll,
                )
            end
        end

    end

    for i in eachindex(residual)
        residual[i] = sqrt(sumRes[i] * nx * ny) / (sumAvg[i] + 1.e-7)
    end

    if bc isa Symbol
        update_boundary!(
            KS,
            ctr,
            a1face,
            a2face,
            dt,
            residual;
            coll = coll,
            bc = [bc, bc, bc, bc],
        )
    else
        update_boundary!(KS, ctr, a1face, a2face, dt, residual; coll = coll, bc = bc)
    end

    return nothing

end

#--- 2d2f ---#
function update!(
    KS::X,
    ctr::Y,
    a1face::Z,
    a2face::Z,
    dt,
    residual;
    coll = symbolize(KS.set.collision),
    bc = symbolize(KS.set.boundary),
) where {
    X<:AbstractSolverSet,
    Y<:AbstractArray{ControlVolume2D2F,2},
    Z<:AbstractArray{Interface2D2F,2},
}

    nx, ny, dx, dy = begin
        if KS.ps isa CSpace2D
            KS.ps.nr, KS.ps.nθ, KS.ps.dr, KS.ps.darc
        else
            KS.ps.nx, KS.ps.ny, KS.ps.dx, KS.ps.dy
        end
    end

    sumRes = zero(ctr[1].w)
    sumAvg = zero(ctr[1].w)

    if ndims(sumRes) == 1

        @inbounds for j = 2:ny-1
            for i = 2:nx-1
                step!(
                    ctr[i, j].w,
                    ctr[i, j].prim,
                    ctr[i, j].h,
                    ctr[i, j].b,
                    a1face[i, j].fw,
                    a1face[i, j].fh,
                    a1face[i, j].fb,
                    a1face[i+1, j].fw,
                    a1face[i+1, j].fh,
                    a1face[i+1, j].fb,
                    a2face[i, j].fw,
                    a2face[i, j].fh,
                    a2face[i, j].fb,
                    a2face[i, j+1].fw,
                    a2face[i, j+1].fh,
                    a2face[i, j+1].fb,
                    KS.vSpace.u,
                    KS.vSpace.v,
                    KS.vSpace.weights,
                    KS.gas.K,
                    KS.gas.γ,
                    KS.gas.μᵣ,
                    KS.gas.ω,
                    KS.gas.Pr,
                    dx[i, j] * dy[i, j],
                    dt,
                    sumRes,
                    sumAvg,
                    coll,
                )
            end
        end

    end

    for i in eachindex(residual)
        residual[i] = sqrt(sumRes[i] * nx * ny) / (sumAvg[i] + 1.e-7)
    end

    if bc isa Symbol
        update_boundary!(
            KS,
            ctr,
            a1face,
            a2face,
            dt,
            residual;
            coll = coll,
            bc = [bc, bc, bc, bc],
        )
    else
        update_boundary!(KS, ctr, a1face, a2face, dt, residual; coll = coll, bc = bc)
    end

    return nothing

end

function update!(
    KS::X,
    ctr::Y,
    face::Z,
    dt,
    residual; # 1D / 2D
    coll = symbolize(KS.set.collision),
    bc = symbolize(KS.set.boundary),
) where {
    X<:AbstractSolverSet,
    Y<:AbstractArray{ControlVolumeUS,1},
    Z<:AbstractArray{Interface2D,1},
}

    sumRes = zero(ctr[1].w)
    sumAvg = zero(ctr[1].w)

    @inbounds Threads.@threads for i in eachindex(ctr)
        if KS.ps.cellType[i] in (0, 2)
            dirc = [sign(dot(ctr[i].n[j], face[KS.ps.cellFaces[i, j]].n)) for j = 1:3]

            step!(
                ctr[i].w,
                ctr[i].prim,
                face[KS.ps.cellFaces[i, 1]].fw,
                face[KS.ps.cellFaces[i, 2]].fw,
                face[KS.ps.cellFaces[i, 3]].fw,
                KS.gas.γ,
                KS.pSpace.cellArea[i],
                dirc,
                sumRes,
                sumAvg,
            )
        end
    end

    for i in eachindex(residual)
        residual[i] = sqrt(sumRes[i] * size(KS.ps.cellid, 1)) / (sumAvg[i] + 1.e-7)
    end

    update_boundary!(KS, ctr, face, dt, residual; coll = coll, bc = bc)

    return nothing

end

function update!(
    KS::X,
    ctr::Y,
    face::Z,
    dt,
    residual; # 1D / 2D
    coll = symbolize(KS.set.collision),
    bc = symbolize(KS.set.boundary),
) where {
    X<:AbstractSolverSet,
    Y<:AbstractArray{ControlVolumeUS1F,1},
    Z<:AbstractArray{Interface2D1F,1},
}

    sumRes = zero(ctr[1].w)
    sumAvg = zero(ctr[1].w)

    @inbounds Threads.@threads for i in eachindex(ctr)
        if KS.ps.cellType[i] in (0, 2)
            dirc = [sign(dot(ctr[i].n[j], face[KS.ps.cellFaces[i, j]].n)) for j = 1:3]

            step!(
                ctr[i].w,
                ctr[i].prim,
                ctr[i].f,
                face[KS.ps.cellFaces[i, 1]].fw,
                face[KS.ps.cellFaces[i, 1]].ff,
                face[KS.ps.cellFaces[i, 2]].fw,
                face[KS.ps.cellFaces[i, 2]].ff,
                face[KS.ps.cellFaces[i, 3]].fw,
                face[KS.ps.cellFaces[i, 3]].ff,
                KS.vSpace.u,
                KS.vSpace.v,
                KS.vSpace.weights,
                KS.gas.K,
                KS.gas.γ,
                KS.gas.μᵣ,
                KS.gas.ω,
                KS.gas.Pr,
                KS.pSpace.cellArea[i],
                dirc,
                dt,
                sumRes,
                sumAvg,
                coll,
            )
        end
    end

    for i in eachindex(residual)
        residual[i] = sqrt(sumRes[i] * size(KS.ps.cellid, 1)) / (sumAvg[i] + 1.e-7)
    end

    update_boundary!(KS, ctr, face, dt, residual; coll = coll, bc = bc)

    return nothing

end

function update!(
    KS::X,
    ctr::Y,
    face::Z,
    dt,
    residual; # 1D / 2D
    coll = symbolize(KS.set.collision),
    bc = symbolize(KS.set.boundary),
) where {
    X<:AbstractSolverSet,
    Y<:AbstractArray{ControlVolumeUS2F,1},
    Z<:AbstractArray{Interface2D2F,1},
}

    sumRes = zero(ctr[1].w)
    sumAvg = zero(ctr[1].w)

    @inbounds Threads.@threads for i in eachindex(ctr)
        if KS.ps.cellType[i] in (0, 2)
            dirc = [sign(dot(ctr[i].n[j], face[KS.ps.cellFaces[i, j]].n)) for j = 1:3]

            step!(
                ctr[i].w,
                ctr[i].prim,
                ctr[i].h,
                ctr[i].b,
                face[KS.ps.cellFaces[i, 1]].fw,
                face[KS.ps.cellFaces[i, 1]].fh,
                face[KS.ps.cellFaces[i, 1]].fb,
                face[KS.ps.cellFaces[i, 2]].fw,
                face[KS.ps.cellFaces[i, 2]].fh,
                face[KS.ps.cellFaces[i, 2]].fb,
                face[KS.ps.cellFaces[i, 3]].fw,
                face[KS.ps.cellFaces[i, 3]].fh,
                face[KS.ps.cellFaces[i, 3]].fb,
                KS.vSpace.u,
                KS.vSpace.v,
                KS.vSpace.weights,
                KS.gas.K,
                KS.gas.γ,
                KS.gas.μᵣ,
                KS.gas.ω,
                KS.gas.Pr,
                KS.pSpace.cellArea[i],
                dirc,
                dt,
                sumRes,
                sumAvg,
                coll,
            )
        end
    end

    for i in eachindex(residual)
        residual[i] = sqrt(sumRes[i] * size(KS.ps.cellid, 1)) / (sumAvg[i] + 1.e-7)
    end

    update_boundary!(KS, ctr, face, dt, residual; coll = coll, bc = bc)

    return nothing

end


#  MIPLearn: Extensible Framework for Learning-Enhanced Mixed-Integer Optimization
#  Copyright (C) 2020-2021, UChicago Argonne, LLC. All rights reserved.
#  Released under the modified BSD license. See COPYING.md for more details.

using Cbc
using JuMP
using MIPLearn
using PyCall
using Test

if is_gurobi_available
    using Gurobi
end

miplearn_tests = pyimport("miplearn.solvers.tests")
traceback = pyimport("traceback")

function _test_solver(optimizer_factory)
    MIPLearn.@python_call miplearn_tests.run_internal_solver_tests(
        JuMPSolver(optimizer_factory),
    )
end

@testset "JuMPSolver" begin
    @testset "Cbc" begin
        _test_solver(Cbc.Optimizer)
    end
    if is_gurobi_available
        @testset "Gurobi" begin
            _test_solver(Gurobi.Optimizer)
        end
    end
end


# GlaThiDa ice thickness loader
#
# This is a bit of a mess as it is hard to find the right glacier.
# Strategy is to just use Matthias' compiled data.

"""
Returns thickness data if there is any.  Uses Matthias's compiled data instead of directly GlaThiDa.
"""
function load!(ds::DataSet{ThicknessData, GlaThiDaLoader}, dem,
               glaciermask, target_proj, outline::Outline{RGIGlacier}, cropbox,
               pp::Phys)
    if isfile(ds.files[1])
        # get radar lines
        out, head = readdlm(ds.files[1], F, header=true)
        x = out[:,2]
        y = out[:,3]
        z = out[:,4]
        z[z.==-9999] = FILL
        bed = out[:,5]
        bed[bed.==-9999] = FILL
        thick = out[:,6]
        thick[thick.==-9999] = FILL
        if all(z.==FILL) && all(thick.!=FILL) && all(bed.!=FILL)
            z .= thick.+bed
        elseif all(thick.==FILL)
            thick .= z.-bed
        end
        keep = Int[]
        for (i,t) in enumerate(thick)
            t!=FILL && push!(keep,i)
        end

        # coordinate transform
        for (i,(xx,yy,zz)) in enumerate(zip(x,y,z))
            x[i],y[i],z[i] = transform_proj([xx,yy,zz], "+proj=longlat +datum=WGS84" , target_proj)
        end
        tr = Traj(x=x[keep],y=y[keep],v=thick[keep],err=z[keep])

        # some radar data has all zero, if so discard
        if all(abs.(tr.v).<1)
            println("All thickness data less than 1m, not using GlaThiDa measurements.")
            return SomeData(nothing)
        end

        # Sort it.  Do this a few times, makes things better
        if length(tr)>3
            for i = 1:3
                if !VAWTools.issorted_traj(tr, 1:min(length(tr), 15)) && length(tr)<2*10^4
                    tr = VAWTools.sort_traj(tr)
                end

                # purge points closer than some distance apart
                len = sqrt(outline.gid.attrs.area)
                window = len/100
                tr = VAWTools.downsample(tr, window)
            end
        end

        SomeData(vals=tr, sigma=ds.opts[:sigma])
    elseif isfile(ds.files[2])
        return SomeData(nothing)

        # # Get a mean thickness in AVG_*.dat
        # out, head = readdlm(ds.files[2], F, header=true)
        # rginr = out[:,3]
        # @assert length(unique(rginr))==length(rginr)
        # i = findfirst(rginr.==getrgi_nr(outline.gid))
        # if i==0
        #     return SomeData(nothing)
        # else
        #     meanh = out[i,end]
        #     return SomeData(vals=meanh, sigma=ds.opts[:sigma_mean_h_rel]*meanh)
        # end
    else
        return SomeData(nothing)
    end
end

# ##
# ttt = CSV.read(GZip.open("/home/mauro/julia/mypkgs/BITEModel/scripts/data/GlaThiDa_2016/TTT_2016_corr.csv.gz"), datarow=5,  rows_for_type_detect=10^6, header=3)
# pu = by(ttt, :GlaThiDa_ID, d->DataFrame(c=first(d[:POLITICAL_UNIT]),
#                                    lat=mean(d[:POINT_LAT]),
#                                    lon=mean(d[:POINT_LON])
#                                    ))
# pu[pu[:c].=="NO",:] # two in Svalbard

# t = CSV.read("/home/mauro/julia/mypkgs/BITEModel/scripts/data/GlaThiDa_2016/T_2016_corr.csv", datarow=3,  rows_for_type_detect=10^6, header=1)
# no = t[t[:POLITICAL_UNIT].=="NO",:]
# no = no[no[:LAT].>75,:]


mutable struct ParamType{Tdim, Tsol, Tres, Tmsh} <: AbstractParamType{Tdim}
  f::IOStream
  t::Float64  # current time value
  order::Int  # accuracy of elements (p=1,2,3...)

  q_vals::Array{Tsol, 1}  # resuable temporary storage for q variables at a node
  qg::Array{Tsol, 1}  # reusable temporary storage for boundary condition
  v_vals::Array{Tsol, 1}  # reusable storage for convert back to entropy vars.

  res_vals1::Array{Tres, 1}  # reusable residual type storage
  res_vals2::Array{Tres, 1}  # reusable residual type storage

  flux_vals1::Array{Tres, 1}  # reusable storage for flux values
  flux_vals2::Array{Tres, 1}  # reusable storage for flux values

  sat_vals::Array{Tres, 1}  # reusable storage for SAT term

  A0::Array{Tsol, 2}  # reusable storage for the A0 matrix
  A0inv::Array{Tsol, 2}  # reusable storage for inv(A0)
  A1::Array{Tsol, 2}  # reusable storage for a flux jacobian
  A2::Array{Tsol, 2}  # reusable storage for a flux jacobian

  A_mats::Array{Tsol, 3}  # reusable storage for flux jacobians

  Rmat1::Array{Tres, 2}  # reusable storage for a matrix of type Tres
  Rmat2::Array{Tres, 2}

  nrm::Array{Tmsh, 1}  # a normal vectora

  edgestab_gamma::Float64  # edge stabilization parameter
  # debugging options
  writeflux::Bool  # write Euler flux
  writeboundary::Bool  # write boundary data
  writeq::Bool # write solution variables
  use_edgestab::Bool  # use edge stabilization
  use_filter::Bool  # use filtering
  use_res_filter::Bool # use residual filtering

  filter_mat::Array{Float64, 2}  # matrix that performs filtering operation
  # includes transformations to/from modal representation

  use_dissipation::Bool  # use artificial dissipation
  dissipation_const::Float64  # constant used for dissipation filter matrix

  tau_type::Int  # type of tau to use for GLS stabilization

  krylov_itr::Int  # Krylov iteration number for iterative solve
  krylov_type::Int # 1 = explicit jacobian, 2 = jac-vec prod
  time::Timings

  function ParamType{Tdim, Tsol, Tres, Tmsh}(mesh, sbp, opts, order::Integer) where {Tdim, Tsol, Tres, Tmsh}
    # create values, apply defaults

    t = 0.0
    myrank = mesh.myrank
    f = open("log_$myrank.dat", "w")
    q_vals = Array{Tsol}(4)
    qg = Array{Tsol}(4)
    v_vals = Array{Tsol}(4)

    res_vals1 = Array{Tres}(4)
    res_vals2 = Array{Tres}(4)

    flux_vals1 = Array{Tres}(4)
    flux_vals2 = Array{Tres}(4)

    sat_vals = Array{Tres}(4)

    A0 = zeros(Tsol, 4, 4)
    A0inv = zeros(Tsol, 4, 4)
    A1 = zeros(Tsol, 4, 4)
    A2 = zeros(Tsol, 4, 4)
    A_mats = zeros(Tsol, 4, 4, Tdim)

    Rmat1 = zeros(Tres, 4, 4)
    Rmat2 = zeros(Tres, 4, 4)

    nrm = zeros(Tmsh, 2)

    edgestab_gamma = opts["edgestab_gamma"]

    # debugging options
    writeflux = opts[ "writeflux"]
    writeboundary = opts[ "writeboundary"]
    writeq = opts["writeq"]
    use_edgestab = opts["use_edgestab"]
    if use_edgestab println("edge stabilization enabled") end

    use_filter = opts["use_filter"]
    if use_filter println("solution variables filter enabled") end

    use_res_filter = opts["use_res_filter"]
    if use_res_filter println("residual filter enabled") end

    if use_filter || use_res_filter || opts["use_filter_prec"]
      filter_fname = opts["filter_name"]
      filter_mat = calcFilter(sbp, filter_fname, opts)
    else
      filter_mat = Array{Float64}(0,0)
    end

    use_dissipation = opts["use_dissipation"]
    if use_dissipation println("artificial dissipation enabled") end

    dissipation_const = opts["dissipation_const"]

    tau_type = opts["tau_type"]

    krylov_itr = 0
    krylov_type = 1 # 1 = explicit jacobian, 2 = jac-vec prod

    time = Timings()
    return new(f, t, order, q_vals, qg, v_vals, res_vals1, res_vals2, sat_vals, flux_vals1,
               flux_vals2, A0, A0inv, A1, A2, A_mats, Rmat1, Rmat2, nrm,
               edgestab_gamma, writeflux, writeboundary,
               writeq, use_edgestab, use_filter, use_res_filter, filter_mat,
               use_dissipation, dissipation_const, tau_type,
               krylov_itr, krylov_type,
               time)

  end   # end of ParamType function

end  # end type declaration

mutable struct EllipticData_{Tsol, Tres, Tdim, Tmsh} <: EllipticData{Tsol, Tres, Tdim}
  params::ParamType{Tdim, Tsol, Tres, Tmsh}
  comm::MPI.Comm
  commsize::Int
  myrank::Int
  #
  # Debug Use Only
  #
  xy_face::Array{Tsol, 4} # (x/y, L/R, numnodes, iface)
  #
  # End Debug
  #
  area_sum::Array{Tmsh, 1}
  w::Array{Tmsh, 2} # H/|J|, (numNodesPerElement, numEl)
  q::Array{Tsol, 3}
  q_irk::Array{Tsol, 4}
  q1::Array{Float64, 3}
  q2::Array{Float64, 3}
  q_face::Array{Tsol, 4} # (numDofs, left/right, numNodesPerFace, numFaces)
  # flux_parametric::Array{Tsol, 2}

  xflux_face::Array{Tres, 3}    # stores (u+ - u-)nx*, (numDofs, numNodes, numFaces)
  yflux_face::Array{Tres, 3}    # stores (u+ - u-)ny*
  flux_face ::Array{Tres, 3}    # stores σ⋅norm

  xflux_bndry::Array{Tres, 3}    # stores (u+ - u-)nx*, (numDofs, numNodes, numFaces)
  yflux_bndry::Array{Tres, 3}    # stores (u+ - u-)ny*
  flux_bndry ::Array{Tres, 3}    # stores σ⋅norm

  res::Array{Tres, 3}            # result of computation (numDofs, numNodesPerElem, numElems)
  res_irk::Array{Tres, 4}            # result of computation (numDofs, numNodesPerElem, numElems)
  res1::Array{Float64, 3}            # result of computation (numDofs, numNodesPerElem, numElems)
  res_vec::Array{Tres, 1}         # result of computation in vector form
  q_vec::Array{Tres,1}            # initial condition in vector form
  q_bndry::Array{Tsol, 3}         # store solution variables interpolated to

  shared_data::Array{SharedFaceData{Tsol}, 1}  # MPI send and receive buffers
  #q_face_send::Array{Array{Tsol, 3}, 1}    # send buffers for sending q values
  # to other processes
  #q_face_recv::Array{Array{Tsol, 3}, 1}    # recieve buffers for q values
  #flux_sharedface::Array{Array{Tres, 3}, 1}    # hold shared face flux
  # derivative
  src_func::SRCType    # functor for source term
  src::Array{Tsol, 3} # (numDofPerNode, numNodesPerElement, numEl)
  #
  # Declaration of variables exclusively used in elliptic problems
  #
  # flux_func::FluxType    # functor for the face flux
  diffusion_func::AbstractDiffn
  functional::AbstractFunctional
  calc_energy::AbstractFunctional
  energy::Array{Tsol, 1}
  # diffusion coefficients
  lambda::Array{Tsol, 5} # (d1, d2, ndof, numnodes, n_elemes)
  lambda_face::Array{Tsol, 6} # (d2, d1, ndof, L/R, numNodesPerFace, n_faces)
  lambda_bndry::Array{Tsol, 5} # (d1, d2, ndof, numNodesPerface, n_bndfaces)
  q_grad::Array{Tsol, 4}    # gradient of q, (dim, numDofPerNode, numNodesPerElem, numElems)
  #
  # TBD: Not sure if we store gradient of q at face cubature points
  #
  q_grad_face::Array{Tsol, 5} #(dim, numDofPerNode, left/right, numNodesPerFace, numFaces)
  q_grad_bndry::Array{Tsol, 4} #(dim, numDofPerNode, numNodesPerFace, numBndries)
  #
  # end declaration of variables exclusively used in elliptic problems
  #
  Minv::Array{Float64, 1}
  M::Array{Float64, 1}
  res_edge::Array{Tres, 4}       # edge based residual used for stabilization
  # (numdof, numNodesPerElement, numEl, numEdgesPerEl)
  flux_func::FluxType
  majorIterationCallback::Function
  # assembleSolution::Function
  disassembleSolution::Function
  multiplyA0inv::Function

  q_face_send::Array{Array{Tsol, 3}, 1}  # send buffers for sending q values to other processes
  q_face_recv::Array{Array{Tsol, 3}, 1}  # recieve buffers for q values

  nstages::UInt8
  istage::UInt8

  function EllipticData_{Tsol, Tres, Tdim, Tmsh}(mesh::AbstractMesh, sbp::AbstractOperator, opts) where {Tsol, Tres, Tdim, Tmsh}
    println("\nConstructing EllipticData object")
    println("  Tsol = ", Tsol)
    println("  Tres = ", Tres)
    println("  Tdim = ", Tdim)
    println("  Tmsh = ", Tmsh)

    eqn = new()

    eqn.comm = mesh.comm
    eqn.commsize = mesh.commsize
    eqn.myrank = mesh.myrank

    eqn.params = ParamType{Tdim, Tsol, Tres, Tmsh}(mesh, sbp, opts, mesh.order)

    numfacenodes = mesh.numNodesPerFace
    numfaces = mesh.numInterfaces
    numBndFaces = mesh.numBoundaryFaces
    numvars  = mesh.numDofPerNode
    #
    # volume variables
    #
    # TODO: switch the last two dimensions in all gradient variable
    # in order to keep consistency with lambda
    #
    eqn.w      = zeros(Tmsh, mesh.numNodesPerElement, mesh.numEl)
    eqn.q      = zeros(Tsol, numvars, sbp.numnodes, mesh.numEl)
    if haskey(opts, "TimeAdvance") && opts["TimeAdvance"] == "BDF2"
      eqn.q1      = zeros(Float64, numvars, sbp.numnodes, mesh.numEl)
      eqn.q2      = zeros(Float64, numvars, sbp.numnodes, mesh.numEl)
    end
    if haskey(opts, "TimeAdvance") && opts["TimeAdvance"] == "CN"
      eqn.q1      = zeros(Float64, numvars, sbp.numnodes, mesh.numEl)
    end
    eqn.res    = zeros(Tsol, numvars, sbp.numnodes, mesh.numEl)
    if haskey(opts, "TimeAdvance") && opts["TimeAdvance"] == "SDIRK4"
      # eqn.res_irk = zeros(Tsol, numvars, sbp.numnodes, mesh.numEl, 6)
      eqn.q_irk   = zeros(Tsol, numvars, sbp.numnodes, mesh.numEl, 6)
    end
    eqn.res1    = zeros(Float64, numvars, sbp.numnodes, mesh.numEl)
    eqn.src    = zeros(Tsol, numvars, sbp.numnodes, mesh.numEl)
    eqn.q_grad = zeros(Tsol, numvars, sbp.numnodes, mesh.numEl, Tdim)
    eqn.lambda = zeros(Tsol, Tdim, Tdim, numvars, sbp.numnodes, mesh.numEl)
    #
    # interface variables
    #
    eqn.q_face    = zeros(Tsol, numvars, 2, numfacenodes, numfaces)
    eqn.q_grad_face = zeros(Tsol, numvars, 2, numfacenodes, numfaces, Tdim)
    eqn.flux_face = zeros(Tsol, numvars, numfacenodes, numfaces)
    eqn.xflux_face = zeros(Tsol, numvars, numfacenodes, numfaces)
    eqn.yflux_face = zeros(Tsol, numvars, numfacenodes, numfaces)
    eqn.lambda_face = zeros(Tsol, Tdim, Tdim, numvars, 2, numfacenodes, numfaces)
    #
    # boundary variables
    #
    eqn.q_bndry    = zeros(Tsol, numvars, numfacenodes, numBndFaces)
    eqn.q_grad_bndry = zeros(Tsol, numvars, numfacenodes, numBndFaces, Tdim)
    eqn.flux_bndry = zeros(Tsol, numvars, numfacenodes, numBndFaces)
    eqn.xflux_bndry = zeros(Tsol, numvars, numfacenodes, numBndFaces)
    eqn.yflux_bndry = zeros(Tsol, numvars, numfacenodes, numBndFaces)
    eqn.lambda_bndry = zeros(Tsol, Tdim, Tdim, numvars, numfacenodes, numBndFaces)
    if mesh.isDG
      for i=1:mesh.npeers
        if opts["precompute_face_flux"]
          eqn.flux_sharedface[i] = zeros(Tres, mesh.numDofPerNode, numfacenodes,
                                         mesh.peer_face_counts[i])
        end
        # eqn.aux_vars_sharedface[i] = zeros(Tres, mesh.numDofPerNode,
                                        # numfacenodes, mesh.peer_face_counts[i])
      end
      eqn.shared_data = getSharedFaceData(Tsol, mesh, sbp, opts,
                                          opts["parallel_data"])
    else
      eqn.shared_data = Array{SharedFaceData}(0)
    end
#
    # reshape of q and res
    #
    eqn.q_vec = reshape(eqn.q, mesh.numDof)
    eqn.res_vec = reshape(eqn.res, mesh.numDof)

    # flux_func = FluxDict["SIPG"]
    eqn.majorIterationCallback = majorIterationCallback
    # eqn.assembleSolution = assembleSolution
    # eqn.disassembleSolution = disassembleSolution
    eqn.multiplyA0inv = multiplyA0inv

    eqn.Minv = calcMassMatrixInverse(mesh, sbp, eqn)
    eqn.M = calcMassMatrix(mesh, sbp, eqn)

    if opts["use_edge_res"]
      eqn.res_edge = zeros(Tres, mesh.numDofPerNode, sbp.numnodes, mesh.numEl,
                           mesh.numTypePerElement[2])
    else
      eqn.res_edge = zeros(Tres, 0, 0, 0, 0)
    end

    eqn.xy_face = zeros(Tsol, 2, 2, numfacenodes, numfaces)

    eqn.area_sum = zeros(Tmsh, mesh.numEl)
    calcWetArea(mesh, sbp, eqn)

    for el = 1 : mesh.numEl
      for n = 1 : mesh.numNodesPerElement
        eqn.w[n, el] = sbp.w[n]/mesh.jac[n, el]
      end
    end
    eqn.energy = zeros(Tsol, mesh.numDofPerNode)
    eqn.calc_energy = FunctionalDict["energy"]

    eqn.istage = 1
    eqn.nstages = 1
    if haskey(opts, "TimeAdvance") && opts["TimeAdvance"] == "SDIRK4"
      eqn.nstages = 5
    end

    # TODO: parallel related variables
    #
    eqn.q_face_send = Array{Array{Tsol, 3}}(1)
    eqn.q_face_recv = Array{Array{Tsol, 3}}(1)
    return eqn
  end # end function
end # end type

function calcWetArea(mesh::AbstractMesh{Tmsh},
                     sbp::AbstractOperator,
                     eqn::EllipticData{Tsol, Tres, Tdim}) where {Tmsh, Tsol, Tres, Tdim}
  nfaces = length(mesh.interfaces)
  nrm = zeros(Tmsh, mesh.numNodesPerFace, Tdim)
  area = zeros(Tmsh, mesh.numNodesPerFace)
  face_area = zero(Tmsh)
  sbpface = mesh.sbpface

  #
  # Compute the wet area of each element
  #
  for f = 1:nfaces
    face = mesh.interfaces[f]
    eL = face.elementL
    eR = face.elementR
    fL = face.faceL
    fR = face.faceR
    #
    # Compute the size of face
    face_area = 0.0
    for n=1:mesh.numNodesPerFace

      dxidx = sview(mesh.dxidx_face, :, :, n, f)
      # norm vector in reference element
      # nrm_xi = sview(sbp.facenormal, :, fL)
      # nrm_xi = sview(mesh.sbpface.normal, :, fL)
      # nrm[n,1] = dxidx[1, 1]*nrm_xi[1] + dxidx[2, 1]*nrm_xi[2]
      # nrm[n,2] = dxidx[1, 2]*nrm_xi[1] + dxidx[2, 2]*nrm_xi[2]
      # area[n] = sqrt(nrm_xy[n,1]*nrm_xy[n,1] + nrm_xy[n,2]*nrm_xy[n,2])

      nrm_xy = ro_sview(mesh.nrm_face, :, n, f)
      area[n] = norm(nrm_xy)
      face_area += sbpface.wface[n]*area[n]
    end
    eqn.area_sum[eL] += face_area
    eqn.area_sum[eR] += face_area
  end

  for bc = 1:mesh.numBC
    indx0 = mesh.bndry_offsets[bc]
    indx1 = mesh.bndry_offsets[bc+1] - 1

    for f = indx0:indx1
      face = mesh.bndryfaces[f].face
      elem = mesh.bndryfaces[f].element
      #
      # Compute the size of face
      face_area = 0.0
      for n=1:mesh.numNodesPerFace
        # dxidx = sview(mesh.dxidx_bndry, :, :, n, f)
        # # norm vector in reference element
        # nrm_xi = sview(mesh.sbpface.normal, :, face)
        # nrm[n,1] = dxidx[1, 1]*nrm_xi[1] + dxidx[2, 1]*nrm_xi[2]
        # nrm[n,2] = dxidx[1, 2]*nrm_xi[1] + dxidx[2, 2]*nrm_xi[2]
        # area[n] = sqrt(nrm[n,1]*nrm[n,1] + nrm[n,2]*nrm[n,2])
        nrm_xy = ro_sview(mesh.nrm_bndry, :, :, f)
        area[n] = norm(nrm_xy)
        face_area += sbpface.wface[n]*area[n]
      end
      eqn.area_sum[elem] += 2.0*face_area
    end
  end
  return nothing
end

function majorIterationCallback(itr::Integer,
                                mesh::AbstractMesh,
                                sbp::AbstractOperator,
                                eqn::AbstractEllipticData,
                                opts,
                                f::IO)
  if opts["write_vis"] && ((itr % opts["output_freq"]) == 0 || itr == 1)
    vals = real(eqn.q_vec)
    saveSolutionToMesh(mesh, vals)
    fname = string("solution_", itr)
    writeVisFiles(mesh, fname)
  end

  return nothing
end

function multiplyA0inv(mesh::AbstractMesh{Tmsh},
                       sbp::AbstractOperator,
                       eqn::EllipticData{Tsol, Tres, Tdim},
                       opts,
                       res::AbstractArray{Tsol, 3}) where {Tmsh, Tsol, Tdim, Tres}
  return nothing
end


# generators.jl
#
# Core package functionality that implements algorithmic counterfactual.
module Generators

using ..Models
using ..Losses
using Flux
using LinearAlgebra

export AbstractGenerator, AbstractGradientBasedGenerator, GenericGenerator, GreedyGenerator, CounterfactualState,
    generate_perturbations, conditions_satisified, mutability_constraints

include("base.jl")

end


###
### LongSequence
###
###
### A general purpose biological sequence representation.
###
### It is cheap to create a subsequence from a sequence because sequences can
### share underlying data: creating a subsequence is just allocating a new
### sequence object defining a part of the original sequence without copy.
### Destructive operations create a copy of the underlying data if and only if it
### is shared between (sub)sequences. This is often called as copy-on-write
### strategy in computer science and should be transparent to the user.
###
### This file is a part of BioJulia.
### License is MIT: https://github.com/BioJulia/BioSequences.jl/blob/master/LICENSE.md

# About Internals
# ---------------
#
# The `data` field of a `LongSequence{A}` object contains binary representation
# of a biological character sequence. Each character is encoded with an encoder
# corresponding to the alphabet `A` and compactly packed into `data`. To extract
# a character from a sequence, you should decode this binary sequence with a
# decoder that is a pair of the encoder. The length of encoded binary bits is
# fixed, and hence a character at arbitrary position can be extracted in a
# constant time. To know the exact location of a character at a position, you
# can use the `bitindex(seq, i)` function, which returns a pair of element's index
# containing binary bits and bits' offset. As a whole, character extraction
# `seq[i]` can be written as:
#
#     j = bitindex(seq, i)
#     decode(A, (seq.data[index(j)] >> offset(j)) & mask(A))
#
#  index :        index(j) - 1       index(j)       index(j) + 1
#   data : ....|xxxxx...........|xxXxxxxxxxxxxxxx|............xxxx|....
# offset :                          |<-offset(j)-|
#  width :      |<---- 64 ---->| |<---- 64 ---->| |<---- 64 ---->|
#
#  * '.' : unused (4 bits/char)
#  * 'x' : used
#  * 'X' : used and pointed by index `i`

"""
Biological sequence data structure indexed by an alphabet type `A`.
"""
mutable struct LongSequence{A <: Alphabet} <: BioSequence{A}
    data::Vector{UInt64}  # encoded character sequence data
    part::UnitRange{Int}  # interval within `data` defining the (sub)sequence
    shared::Bool          # true if and only if `data` is shared between sequences

    function LongSequence{A}(data::Vector{UInt64},
                            part::UnitRange{Int},
                            shared::Bool) where A
        return new(data, part, shared)
    end
end

const LongNucleotideSequence = LongSequence{<:NucleicAcidAlphabet}
const LongDNASeq       = LongSequence{DNAAlphabet{4}}
const LongRNASeq       = LongSequence{RNAAlphabet{4}}
const LongAminoAcidSeq = LongSequence{AminoAcidAlphabet}
const LongCharSeq      = LongSequence{CharAlphabet}

###
### Required type traits and methods
###

"Gets the alphabet encoding of a given BioSequence."
BioSymbols.alphabet(::Type{LongSequence{A}}) where {A} = alphabet(A)
Alphabet(::Type{LongSequence{A}}) where {A <: Alphabet} = A()
Base.length(seq::LongSequence) = last(seq.part) - first(seq.part) + 1
bindata(seq::LongSequence) = seq.data
Base.eltype(::Type{LongSequence{A}}) where {A} = eltype(A)

@inline function seq_data_len(::Type{A}, len::Integer) where A <: Alphabet
    return cld(len, div(64, bits_per_symbol(A())))
end

@inline function encoded_data(seq::LongSequence)
    return seq.data
end

# Replace a LongSequence's data with a copy, copying only what's needed.
# The user should never need to call this, as it has no outward effect on the
# sequence.
function orphan!(seq::LongSequence,
		 size::Integer = length(seq),
		 force::Bool = false)
	if !seq.shared & !force
	    return seq
	end
	return _orphan!(seq, size)
end

function _orphan!(seq::LongSequence{A},
		 size::Integer = length(seq)) where {A}

    j, r = bitindex(seq, 1)
    data = Vector{UInt64}(undef, seq_data_len(A, size))

    @inbounds if !isempty(seq) & !isempty(data)
        x = seq.data[j] >> r
        m = index(bitindex(seq, lastindex(seq))) - j + 1
        l = min(lastindex(data), m)
        @simd for i in 1:l-1
            y = seq.data[j + i]
            data[i] = x | y << (64 - r)
            x = y >> (r & 63)
        end
        if m <= l
            data[l] = x
        else
            y = seq.data[j + l]
            data[l] = x | y << (64 - r)
        end
    end

    seq.data = data
    seq.part = 1:size
    seq.shared = false
    return seq
end

include("indexing.jl")
include("constructors.jl")
include("printing.jl")
include("copying.jl")
include("conversion.jl")
include("stringliterals.jl")
include("transformations.jl")
include("operators.jl")
include("counting.jl")


#=
This file is auto-generated. Do not edit.
=#
"""
    mutable struct HydroDispatch <: HydroGen
        name::String
        available::Bool
        bus::Bus
        active_power::Float64
        reactive_power::Float64
        rating::Float64
        prime_mover::PrimeMovers
        active_power_limits::NamedTuple{(:min, :max), Tuple{Float64, Float64}}
        reactive_power_limits::Union{Nothing, NamedTuple{(:min, :max), Tuple{Float64, Float64}}}
        ramp_limits::Union{Nothing, NamedTuple{(:up, :down), Tuple{Float64, Float64}}}
        time_limits::Union{Nothing, NamedTuple{(:up, :down), Tuple{Float64, Float64}}}
        base_power::Float64
        operation_cost::OperationalCost
        services::Vector{Service}
        dynamic_injector::Union{Nothing, DynamicInjection}
        ext::Dict{String, Any}
        time_series_container::InfrastructureSystems.TimeSeriesContainer
        internal::InfrastructureSystemsInternal
    end

# Arguments
- `name::String`
- `available::Bool`
- `bus::Bus`
- `active_power::Float64`
- `reactive_power::Float64`, validation range: `reactive_power_limits`, action if invalid: `warn`
- `rating::Float64`: Thermal limited MVA Power Output of the unit. <= Capacity, validation range: `(0, nothing)`, action if invalid: `error`
- `prime_mover::PrimeMovers`: Prime mover technology according to EIA 923
- `active_power_limits::NamedTuple{(:min, :max), Tuple{Float64, Float64}}`, validation range: `(0, nothing)`, action if invalid: `warn`
- `reactive_power_limits::Union{Nothing, NamedTuple{(:min, :max), Tuple{Float64, Float64}}}`, action if invalid: `warn`
- `ramp_limits::Union{Nothing, NamedTuple{(:up, :down), Tuple{Float64, Float64}}}`: ramp up and ramp down limits in MW (in component base per unit) per minute, validation range: `(0, nothing)`, action if invalid: `error`
- `time_limits::Union{Nothing, NamedTuple{(:up, :down), Tuple{Float64, Float64}}}`: Minimum up and Minimum down time limits in hours, validation range: `(0, nothing)`, action if invalid: `error`
- `base_power::Float64`: Base power of the unit in MVA, validation range: `(0, nothing)`, action if invalid: `warn`
- `operation_cost::OperationalCost`: Operation Cost of Generation [`OperationalCost`](@ref)
- `services::Vector{Service}`: Services that this device contributes to
- `dynamic_injector::Union{Nothing, DynamicInjection}`: corresponding dynamic injection device
- `ext::Dict{String, Any}`
- `time_series_container::InfrastructureSystems.TimeSeriesContainer`: internal time_series storage
- `internal::InfrastructureSystemsInternal`: power system internal reference, do not modify
"""
mutable struct HydroDispatch <: HydroGen
    name::String
    available::Bool
    bus::Bus
    active_power::Float64
    reactive_power::Float64
    "Thermal limited MVA Power Output of the unit. <= Capacity"
    rating::Float64
    "Prime mover technology according to EIA 923"
    prime_mover::PrimeMovers
    active_power_limits::NamedTuple{(:min, :max), Tuple{Float64, Float64}}
    reactive_power_limits::Union{Nothing, NamedTuple{(:min, :max), Tuple{Float64, Float64}}}
    "ramp up and ramp down limits in MW (in component base per unit) per minute"
    ramp_limits::Union{Nothing, NamedTuple{(:up, :down), Tuple{Float64, Float64}}}
    "Minimum up and Minimum down time limits in hours"
    time_limits::Union{Nothing, NamedTuple{(:up, :down), Tuple{Float64, Float64}}}
    "Base power of the unit in MVA"
    base_power::Float64
    "Operation Cost of Generation [`OperationalCost`](@ref)"
    operation_cost::OperationalCost
    "Services that this device contributes to"
    services::Vector{Service}
    "corresponding dynamic injection device"
    dynamic_injector::Union{Nothing, DynamicInjection}
    ext::Dict{String, Any}
    "internal time_series storage"
    time_series_container::InfrastructureSystems.TimeSeriesContainer
    "power system internal reference, do not modify"
    internal::InfrastructureSystemsInternal
end

function HydroDispatch(name, available, bus, active_power, reactive_power, rating, prime_mover, active_power_limits, reactive_power_limits, ramp_limits, time_limits, base_power, operation_cost=TwoPartCost(0.0, 0.0), services=Device[], dynamic_injector=nothing, ext=Dict{String, Any}(), time_series_container=InfrastructureSystems.TimeSeriesContainer(), )
    HydroDispatch(name, available, bus, active_power, reactive_power, rating, prime_mover, active_power_limits, reactive_power_limits, ramp_limits, time_limits, base_power, operation_cost, services, dynamic_injector, ext, time_series_container, InfrastructureSystemsInternal(), )
end

function HydroDispatch(; name, available, bus, active_power, reactive_power, rating, prime_mover, active_power_limits, reactive_power_limits, ramp_limits, time_limits, base_power, operation_cost=TwoPartCost(0.0, 0.0), services=Device[], dynamic_injector=nothing, ext=Dict{String, Any}(), time_series_container=InfrastructureSystems.TimeSeriesContainer(), internal=InfrastructureSystemsInternal(), )
    HydroDispatch(name, available, bus, active_power, reactive_power, rating, prime_mover, active_power_limits, reactive_power_limits, ramp_limits, time_limits, base_power, operation_cost, services, dynamic_injector, ext, time_series_container, internal, )
end

# Constructor for demo purposes; non-functional.
function HydroDispatch(::Nothing)
    HydroDispatch(;
        name="init",
        available=false,
        bus=Bus(nothing),
        active_power=0.0,
        reactive_power=0.0,
        rating=0.0,
        prime_mover=PrimeMovers.HY,
        active_power_limits=(min=0.0, max=0.0),
        reactive_power_limits=nothing,
        ramp_limits=nothing,
        time_limits=nothing,
        base_power=0.0,
        operation_cost=TwoPartCost(nothing),
        services=Device[],
        dynamic_injector=nothing,
        ext=Dict{String, Any}(),
        time_series_container=InfrastructureSystems.TimeSeriesContainer(),
    )
end

"""Get [`HydroDispatch`](@ref) `name`."""
get_name(value::HydroDispatch) = value.name
"""Get [`HydroDispatch`](@ref) `available`."""
get_available(value::HydroDispatch) = value.available
"""Get [`HydroDispatch`](@ref) `bus`."""
get_bus(value::HydroDispatch) = value.bus
"""Get [`HydroDispatch`](@ref) `active_power`."""
get_active_power(value::HydroDispatch) = get_value(value, value.active_power)
"""Get [`HydroDispatch`](@ref) `reactive_power`."""
get_reactive_power(value::HydroDispatch) = get_value(value, value.reactive_power)
"""Get [`HydroDispatch`](@ref) `rating`."""
get_rating(value::HydroDispatch) = get_value(value, value.rating)
"""Get [`HydroDispatch`](@ref) `prime_mover`."""
get_prime_mover(value::HydroDispatch) = value.prime_mover
"""Get [`HydroDispatch`](@ref) `active_power_limits`."""
get_active_power_limits(value::HydroDispatch) = get_value(value, value.active_power_limits)
"""Get [`HydroDispatch`](@ref) `reactive_power_limits`."""
get_reactive_power_limits(value::HydroDispatch) = get_value(value, value.reactive_power_limits)
"""Get [`HydroDispatch`](@ref) `ramp_limits`."""
get_ramp_limits(value::HydroDispatch) = get_value(value, value.ramp_limits)
"""Get [`HydroDispatch`](@ref) `time_limits`."""
get_time_limits(value::HydroDispatch) = value.time_limits
"""Get [`HydroDispatch`](@ref) `base_power`."""
get_base_power(value::HydroDispatch) = value.base_power
"""Get [`HydroDispatch`](@ref) `operation_cost`."""
get_operation_cost(value::HydroDispatch) = value.operation_cost
"""Get [`HydroDispatch`](@ref) `services`."""
get_services(value::HydroDispatch) = value.services
"""Get [`HydroDispatch`](@ref) `dynamic_injector`."""
get_dynamic_injector(value::HydroDispatch) = value.dynamic_injector
"""Get [`HydroDispatch`](@ref) `ext`."""
get_ext(value::HydroDispatch) = value.ext
"""Get [`HydroDispatch`](@ref) `time_series_container`."""
get_time_series_container(value::HydroDispatch) = value.time_series_container
"""Get [`HydroDispatch`](@ref) `internal`."""
get_internal(value::HydroDispatch) = value.internal

"""Set [`HydroDispatch`](@ref) `name`."""
set_name!(value::HydroDispatch, val) = value.name = val
"""Set [`HydroDispatch`](@ref) `available`."""
set_available!(value::HydroDispatch, val) = value.available = val
"""Set [`HydroDispatch`](@ref) `bus`."""
set_bus!(value::HydroDispatch, val) = value.bus = val
"""Set [`HydroDispatch`](@ref) `active_power`."""
set_active_power!(value::HydroDispatch, val) = value.active_power = set_value(value, val)
"""Set [`HydroDispatch`](@ref) `reactive_power`."""
set_reactive_power!(value::HydroDispatch, val) = value.reactive_power = set_value(value, val)
"""Set [`HydroDispatch`](@ref) `rating`."""
set_rating!(value::HydroDispatch, val) = value.rating = set_value(value, val)
"""Set [`HydroDispatch`](@ref) `prime_mover`."""
set_prime_mover!(value::HydroDispatch, val) = value.prime_mover = val
"""Set [`HydroDispatch`](@ref) `active_power_limits`."""
set_active_power_limits!(value::HydroDispatch, val) = value.active_power_limits = set_value(value, val)
"""Set [`HydroDispatch`](@ref) `reactive_power_limits`."""
set_reactive_power_limits!(value::HydroDispatch, val) = value.reactive_power_limits = set_value(value, val)
"""Set [`HydroDispatch`](@ref) `ramp_limits`."""
set_ramp_limits!(value::HydroDispatch, val) = value.ramp_limits = set_value(value, val)
"""Set [`HydroDispatch`](@ref) `time_limits`."""
set_time_limits!(value::HydroDispatch, val) = value.time_limits = val
"""Set [`HydroDispatch`](@ref) `base_power`."""
set_base_power!(value::HydroDispatch, val) = value.base_power = val
"""Set [`HydroDispatch`](@ref) `operation_cost`."""
set_operation_cost!(value::HydroDispatch, val) = value.operation_cost = val
"""Set [`HydroDispatch`](@ref) `services`."""
set_services!(value::HydroDispatch, val) = value.services = val
"""Set [`HydroDispatch`](@ref) `ext`."""
set_ext!(value::HydroDispatch, val) = value.ext = val
"""Set [`HydroDispatch`](@ref) `time_series_container`."""
set_time_series_container!(value::HydroDispatch, val) = value.time_series_container = val


(1 ; 2)y


using DSGE, ClusterManagers, HDF5

##########################################################################################
## SETUP
##########################################################################################

# What do you want to do?
run_estimation     = false
run_modal_forecast = false
run_full_forecast  = true

# Initialize model object
# Note that the default for m1010 uses 6 anticipated shocks
m = Model1010("ss20")

# Settings for data, paths, etc.
dataroot = joinpath(dirname(@__FILE__()), "input_data")
saveroot = dirname(@__FILE__())
m <= Setting(:dataroot, dataroot, "Input data directory path")
m <= Setting(:saveroot, saveroot, "Output data directory path")
m <= Setting(:data_vintage, "161223")
m <= Setting(:use_population_forecast, false)

# Settings for estimation
# set to false => will load pre-computed mode and hessian before MCMC
m <= Setting(:reoptimize, true)
m <= Setting(:calculate_hessian, true)

# Settings for forecast dates
m <= Setting(:date_forecast_start,  quartertodate("2016-Q4"))
m <= Setting(:date_conditional_end, quartertodate("2016-Q4"))
m <= Setting(:shockdec_startdate,   Nullable(date_mainsample_start(m)))

# Parallelization
m <= Setting(:forecast_block_size,  500)
nworkers = 1
addprocsfcn = addprocs_sge # choose to work with your scheduler; see ClusterManagers.jl

##########################################################################################
## RUN
##########################################################################################

# Run estimation
if run_estimation

    if reoptimize(m)
        # Start from ss18 mode
        mode_file = rawpath(m, "estimate", "paramsmode.h5")
        mode_file = replace(mode_file, "ss20", "ss18")
        update!(m, h5read(mode_file, "params"))
    else
        # Use calculated ss18 mode
        mode_file = joinpath(dataroot, "user", "paramsmode_vint=161223.h5")
        specify_mode!(m, mode_file)
    end

    # Use calculated ss18 hessian
    if !calculate_hessian(m)
        hessian_file = joinpath(dataroot, "user", "hessian_vint=161223.h5")
        specify_hessian(m, hessian_file)
    end

    estimate(m)

    # Print tables of estimated parameter moments
    groupings = DSGE.parameter_groupings(m)
    moment_tables(m, groupings = groupings)
end

# Forecast step: produces smoothed histories and shock decompositions
if run_modal_forecast || run_full_forecast

    # what do we want to produce?
    output_vars = [:histpseudo, :forecastpseudo, :shockdecpseudo]

    # conditional type
    cond_type = :none

    # Forecast label: all forecast output filenames will contain this string
    forecast_string = ""

    # Modal forecast
    if run_modal_forecast
        # run modal forecasts and save all draws
        forecast_one(m, :mode, cond_type, output_vars; verbose = :high)

        # compute means and bands
        compute_meansbands(m, :mode, cond_type, output_vars)
    end

    # Full-distribution forecast
    if run_full_forecast
        my_procs = addprocsfcn(nworkers)
        @everywhere using DSGE

        forecast_one(m, :full, cond_type, output_vars; verbose = :high, forecast_string = forecast_string)
        rstar_bands = [0.68, 0.95]
        compute_meansbands(m, :full, cond_type, output_vars; verbose = :high, density_bands = rstar_bands,
                           forecast_string = forecast_string)
        rmprocs(my_procs)

        meansbands_to_matrix(m, :full, cond_type, output_vars; forecast_string = forecast_string)

        # print history means and bands tables to csv
        table_vars = [:ExAnteRealRate, :Forward5YearRealRate, :Forward10YearRealRate,
                      :RealNaturalRate, :Forward5YearRealNaturalRate,
                      :Forward10YearRealNaturalRate, :Forward20YearRealNaturalRate,
                      :Forward30YearRealNaturalRate]
        write_meansbands_tables_all(m, :full, cond_type, [:histpseudo], forecast_string = forecast_string,
                                    vars = table_vars)

        # print shockdec means and bands tables to csv
        if any(x->contains(string(x), "shockdec"), output_vars)
            shockdec_vars = [:RealNaturalRate, :Forward30YearRealNaturalRate]

            write_meansbands_tables_all(m, :full, cond_type, [:shockdecpseudo, :trendpseudo, :dettrendpseudo],
                                        vars = shockdec_vars,
                                        forecast_string = forecast_string)

        end
    end
end

nothing


"""
This is a sample input file for the CloudScat code.  To use it run julia
and type

julia> include("wavelength.jl"); run()

The output file will be based on this file's name so to run different simulations
you can just copy this file, change the configuration and run it as above with
the new file name.  You can also run from the command line as e.g.

> julia wavelength.jl

- - -
2010 Alejandro Luque IAA-CSIC (aluque@iaa.es)

"""

using CloudScat

const co = CloudScat.constants

function run()
    for λ in [337 * co.nano, 777 * co.nano],
        R in [20 * co.micro]
        runone(λ, R)
    end
end

function runone(λ, R)
    params = Params(
        # Number of simulated photons
        # Revert to N = 10000000
        N = 10000000,

        # The wavelength of the scattered light
        λ = λ,

        # The source is defined as a straight line running from source_a
        # to source_b.  Within this line photons are distributed uniformly with
        # initially random, isotropic directions.  To use a point source set
        # source_b = source_a
        source_a = [0, 0, 10 * co.kilo],
        source_b = [0, 0, 12 * co.kilo])

    cth = 15 * co.kilo

    # World geometry
    # Describe the cloud geometry here.  It may consist in combinations of
    # geometrical figures (cylinders, spheres, cones, semi-planes).
    # See cloud_gemoetry.jl for a more detailed description
    #
    cloud = Cylinder(5 * co.kilo,  cth, 0, 0, 20 * co.kilo)

    # Geometry of the full domain.  Photons outside the domain are
    # immediately discarded.
    domain = shapediff(Plane(60 * co.kilo, false), Plane(7 * co.kilo, false))

    # Density of the scattering centers.
    nscat = 100 * co.centi^-3

    composition = Homogeneous(params.λ, nscat, R)

    # Define the full simulation world
    world = World(cloud, domain, composition)

    observers = [Observer(
        # Location of the observer
        position = [0, 0, 400 * co.kilo],

        # Sampling interval
        tsample = 1e-5,

        # Collected samples
        nsamples = 10000,

        # fov of the camera (half of the diagonal fov).
        fov = 40,

        # camera pixels
        pixels = 1024)]

    basename = splitext(splitdir(@__FILE__)[2])[1]

    # The saveto option specifies the output file.  Leave it as it is to base
    # the filename on the name of this file specifying the parameters.
    ofile = ("$(basename)_$(Int(λ/co.nano))nm_" *
             "$(Int(R/co.micro))um.h5")
    CloudScat.main(params, world, observers,
                   saveto=ofile)
end

# Check if the code has been 'included' or run from the shell.  In the latter
# case, run the simulation.
isinteractive() || run()


# Automatically generated using Clang.jl wrap_c

const NFRAMES = 3
const SECRET_NUM = -1234

@cenum(LAYER_TYPE,
    CONVOLUTIONAL = 0,
    DECONVOLUTIONAL = 1,
    CONNECTED = 2,
    MAXPOOL = 3,
    SOFTMAX = 4,
    DETECTION = 5,
    DROPOUT = 6,
    CROP = 7,
    ROUTE = 8,
    COST = 9,
    NORMALIZATION = 10,
    AVGPOOL = 11,
    LOCAL = 12,
    SHORTCUT = 13,
    ACTIVE = 14,
    RNN = 15,
    GRU = 16,
    LSTM = 17,
    CRNN = 18,
    BATCHNORM = 19,
    NETWORK = 20,
    XNOR = 21,
    REGION = 22,
    YOLO = 23,
    ISEG = 24,
    REORG = 25,
    REORG_OLD = 26,
    UPSAMPLE = 27,
    LOGXENT = 28,
    L2NORM = 29,
    BLANK = 30,
)
@cenum(ACTIVATION,
    LOGISTIC = 0,
    RELU = 1,
    RELIE = 2,
    LINEAR = 3,
    RAMP = 4,
    TANH = 5,
    PLSE = 6,
    LEAKY = 7,
    ELU = 8,
    LOGGY = 9,
    STAIR = 10,
    HARDTAN = 11,
    LHTAN = 12,
    SELU = 13,
)
@cenum(COST_TYPE,
    SSE = 0,
    MASKED = 1,
    L1 = 2,
    SEG = 3,
    SMOOTH = 4,
    WGAN = 5,
)

struct tree
    leaf::Ptr{Cint}
    n::Cint
    parent::Ptr{Cint}
    child::Ptr{Cint}
    group::Ptr{Cint}
    name::Ptr{Cstring}
    groups::Cint
    group_size::Ptr{Cint}
    group_offset::Ptr{Cint}
end

struct layer
    type::LAYER_TYPE
    activation::ACTIVATION
    cost_type::COST_TYPE
    forward::Ptr{Cvoid}
    backward::Ptr{Cvoid}
    update::Ptr{Cvoid}
    forward_gpu::Ptr{Cvoid}
    backward_gpu::Ptr{Cvoid}
    update_gpu::Ptr{Cvoid}
    batch_normalize::Cint
    shortcut::Cint
    batch::Cint
    forced::Cint
    flipped::Cint
    inputs::Cint
    outputs::Cint
    nweights::Cint
    nbiases::Cint
    extra::Cint
    truths::Cint
    h::Cint
    w::Cint
    c::Cint
    out_h::Cint
    out_w::Cint
    out_c::Cint
    n::Cint
    max_boxes::Cint
    groups::Cint
    size::Cint
    side::Cint
    stride::Cint
    reverse::Cint
    flatten::Cint
    spatial::Cint
    pad::Cint
    sqrt::Cint
    flip::Cint
    index::Cint
    binary::Cint
    xnor::Cint
    use_bin_output::Cint
    steps::Cint
    hidden::Cint
    truth::Cint
    smooth::Cfloat
    dot::Cfloat
    angle::Cfloat
    jitter::Cfloat
    saturation::Cfloat
    exposure::Cfloat
    shift::Cfloat
    ratio::Cfloat
    learning_rate_scale::Cfloat
    clip::Cfloat
    focal_loss::Cint
    noloss::Cint
    softmax::Cint
    classes::Cint
    coords::Cint
    background::Cint
    rescore::Cint
    objectness::Cint
    does_cost::Cint
    joint::Cint
    noadjust::Cint
    reorg::Cint
    log::Cint
    tanh::Cint
    mask::Ptr{Cint}
    total::Cint
    bflops::Cfloat
    adam::Cint
    B1::Cfloat
    B2::Cfloat
    eps::Cfloat
    t::Cint
    alpha::Cfloat
    beta::Cfloat
    kappa::Cfloat
    coord_scale::Cfloat
    object_scale::Cfloat
    noobject_scale::Cfloat
    mask_scale::Cfloat
    class_scale::Cfloat
    bias_match::Cint
    random::Cint
    ignore_thresh::Cfloat
    truth_thresh::Cfloat
    thresh::Cfloat
    focus::Cfloat
    classfix::Cint
    absolute::Cint
    onlyforward::Cint
    stopbackward::Cint
    dontload::Cint
    dontsave::Cint
    dontloadscales::Cint
    numload::Cint
    temperature::Cfloat
    probability::Cfloat
    scale::Cfloat
    cweights::Cstring
    indexes::Ptr{Cint}
    input_layers::Ptr{Cint}
    input_sizes::Ptr{Cint}
    map::Ptr{Cint}
    counts::Ptr{Cint}
    sums::Ptr{Ptr{Cfloat}}
    rand::Ptr{Cfloat}
    cost::Ptr{Cfloat}
    state::Ptr{Cfloat}
    prev_state::Ptr{Cfloat}
    forgot_state::Ptr{Cfloat}
    forgot_delta::Ptr{Cfloat}
    state_delta::Ptr{Cfloat}
    combine_cpu::Ptr{Cfloat}
    combine_delta_cpu::Ptr{Cfloat}
    concat::Ptr{Cfloat}
    concat_delta::Ptr{Cfloat}
    binary_weights::Ptr{Cfloat}
    biases::Ptr{Cfloat}
    bias_updates::Ptr{Cfloat}
    scales::Ptr{Cfloat}
    scale_updates::Ptr{Cfloat}
    weights::Ptr{Cfloat}
    weight_updates::Ptr{Cfloat}
    align_bit_weights_gpu::Cstring
    mean_arr_gpu::Ptr{Cfloat}
    align_workspace_gpu::Ptr{Cfloat}
    transposed_align_workspace_gpu::Ptr{Cfloat}
    align_workspace_size::Cint
    align_bit_weights::Cstring
    mean_arr::Ptr{Cfloat}
    align_bit_weights_size::Cint
    lda_align::Cint
    new_lda::Cint
    bit_align::Cint
    col_image::Ptr{Cfloat}
    delta::Ptr{Cfloat}
    output::Ptr{Cfloat}
    delta_pinned::Cint
    output_pinned::Cint
    loss::Ptr{Cfloat}
    squared::Ptr{Cfloat}
    norms::Ptr{Cfloat}
    spatial_mean::Ptr{Cfloat}
    mean::Ptr{Cfloat}
    variance::Ptr{Cfloat}
    mean_delta::Ptr{Cfloat}
    variance_delta::Ptr{Cfloat}
    rolling_mean::Ptr{Cfloat}
    rolling_variance::Ptr{Cfloat}
    x::Ptr{Cfloat}
    x_norm::Ptr{Cfloat}
    m::Ptr{Cfloat}
    v::Ptr{Cfloat}
    bias_m::Ptr{Cfloat}
    bias_v::Ptr{Cfloat}
    scale_m::Ptr{Cfloat}
    scale_v::Ptr{Cfloat}
    z_cpu::Ptr{Cfloat}
    r_cpu::Ptr{Cfloat}
    h_cpu::Ptr{Cfloat}
    prev_state_cpu::Ptr{Cfloat}
    temp_cpu::Ptr{Cfloat}
    temp2_cpu::Ptr{Cfloat}
    temp3_cpu::Ptr{Cfloat}
    dh_cpu::Ptr{Cfloat}
    hh_cpu::Ptr{Cfloat}
    prev_cell_cpu::Ptr{Cfloat}
    cell_cpu::Ptr{Cfloat}
    f_cpu::Ptr{Cfloat}
    i_cpu::Ptr{Cfloat}
    g_cpu::Ptr{Cfloat}
    o_cpu::Ptr{Cfloat}
    c_cpu::Ptr{Cfloat}
    dc_cpu::Ptr{Cfloat}
    binary_input::Ptr{Cfloat}
    bin_re_packed_input::Ptr{UInt32}
    t_bit_input::Cstring
    input_layer::Ptr{layer}
    self_layer::Ptr{layer}
    output_layer::Ptr{layer}
    reset_layer::Ptr{layer}
    update_layer::Ptr{layer}
    state_layer::Ptr{layer}
    input_gate_layer::Ptr{layer}
    state_gate_layer::Ptr{layer}
    input_save_layer::Ptr{layer}
    state_save_layer::Ptr{layer}
    input_state_layer::Ptr{layer}
    state_state_layer::Ptr{layer}
    input_z_layer::Ptr{layer}
    state_z_layer::Ptr{layer}
    input_r_layer::Ptr{layer}
    state_r_layer::Ptr{layer}
    input_h_layer::Ptr{layer}
    state_h_layer::Ptr{layer}
    wz::Ptr{layer}
    uz::Ptr{layer}
    wr::Ptr{layer}
    ur::Ptr{layer}
    wh::Ptr{layer}
    uh::Ptr{layer}
    uo::Ptr{layer}
    wo::Ptr{layer}
    uf::Ptr{layer}
    wf::Ptr{layer}
    ui::Ptr{layer}
    wi::Ptr{layer}
    ug::Ptr{layer}
    wg::Ptr{layer}
    softmax_tree::Ptr{tree}
    workspace_size::Cint
end

@cenum(learning_rate_policy,
    CONSTANT = 0,
    STEP = 1,
    EXP = 2,
    POLY = 3,
    STEPS = 4,
    SIG = 5,
    RANDOM = 6,
)

struct network
    n::Cint
    batch::Cint
    seen::Ptr{UInt64}
    t::Ptr{Cint}
    epoch::Cfloat
    subdivisions::Cint
    layers::Ptr{layer}
    output::Ptr{Cfloat}
    policy::learning_rate_policy
    learning_rate::Cfloat
    momentum::Cfloat
    decay::Cfloat
    gamma::Cfloat
    scale::Cfloat
    power::Cfloat
    time_steps::Cint
    step::Cint
    max_batches::Cint
    scales::Ptr{Cfloat}
    steps::Ptr{Cint}
    num_steps::Cint
    burn_in::Cint
    cudnn_half::Cint
    adam::Cint
    B1::Cfloat
    B2::Cfloat
    eps::Cfloat
    inputs::Cint
    outputs::Cint
    truths::Cint
    notruth::Cint
    h::Cint
    w::Cint
    c::Cint
    max_crop::Cint
    min_crop::Cint
    max_ratio::Cfloat
    min_ratio::Cfloat
    center::Cint
    flip::Cint
    angle::Cfloat
    aspect::Cfloat
    exposure::Cfloat
    saturation::Cfloat
    hue::Cfloat
    random::Cint
    track::Cint
    augment_speed::Cint
    try_fix_nan::Cint
    gpu_index::Cint
    hierarchy::Ptr{tree}
    input::Ptr{Cfloat}
    truth::Ptr{Cfloat}
    delta::Ptr{Cfloat}
    workspace::Ptr{Cfloat}
    train::Cint
    index::Cint
    cost::Ptr{Cfloat}
    clip::Cfloat
end

struct network_state
    truth::Ptr{Cfloat}
    input::Ptr{Cfloat}
    delta::Ptr{Cfloat}
    workspace::Ptr{Cfloat}
    train::Cint
    index::Cint
    net::network
end

struct image
    w::Cint
    h::Cint
    c::Cint
    data::Ptr{Cfloat}
end

struct box
    x::Cfloat
    y::Cfloat
    w::Cfloat
    h::Cfloat
end

struct detection
    bbox::box
    classes::Cint
    prob::Ptr{Cfloat}
    mask::Ptr{Cfloat}
    objectness::Cfloat
    sort_class::Cint
end

struct matrix
    rows::Cint
    cols::Cint
    vals::Ptr{Ptr{Cfloat}}
end

struct data
    w::Cint
    h::Cint
    X::matrix
    y::matrix
    shallow::Cint
    num_boxes::Ptr{Cint}
    boxes::Ptr{Ptr{box}}
end

@cenum(data_type,
    CLASSIFICATION_DATA = 0,
    DETECTION_DATA = 1,
    CAPTCHA_DATA = 2,
    REGION_DATA = 3,
    IMAGE_DATA = 4,
    COMPARE_DATA = 5,
    WRITING_DATA = 6,
    SWAG_DATA = 7,
    TAG_DATA = 8,
    OLD_CLASSIFICATION_DATA = 9,
    STUDY_DATA = 10,
    DET_DATA = 11,
    SUPER_DATA = 12,
    LETTERBOX_DATA = 13,
    REGRESSION_DATA = 14,
    SEGMENTATION_DATA = 15,
    INSTANCE_DATA = 16,
    ISEG_DATA = 17,
)

struct load_args
    threads::Cint
    paths::Ptr{Cstring}
    path::Cstring
    n::Cint
    m::Cint
    labels::Ptr{Cstring}
    h::Cint
    w::Cint
    c::Cint
    out_w::Cint
    out_h::Cint
    nh::Cint
    nw::Cint
    num_boxes::Cint
    min::Cint
    max::Cint
    size::Cint
    classes::Cint
    background::Cint
    scale::Cint
    center::Cint
    coords::Cint
    mini_batch::Cint
    track::Cint
    augment_speed::Cint
    jitter::Cfloat
    flip::Cint
    angle::Cfloat
    aspect::Cfloat
    saturation::Cfloat
    exposure::Cfloat
    hue::Cfloat
    d::Ptr{data}
    im::Ptr{image}
    resized::Ptr{image}
    type::data_type
    hierarchy::Ptr{tree}
end

struct metadata
    classes::Cint
    names::Ptr{Cstring}
end

@cenum(IMTYPE,
    PNG = 0,
    BMP = 1,
    TGA = 2,
    JPG = 3,
)
@cenum(BINARY_ACTIVATION,
    MULT = 0,
    ADD = 1,
    SUB = 2,
    DIV = 3,
)

struct update_args
    batch::Cint
    learning_rate::Cfloat
    momentum::Cfloat
    decay::Cfloat
    adam::Cint
    B1::Cfloat
    B2::Cfloat
    eps::Cfloat
    t::Cint
end

struct box_label
    id::Cint
    x::Cfloat
    y::Cfloat
    w::Cfloat
    h::Cfloat
    left::Cfloat
    right::Cfloat
    top::Cfloat
    bottom::Cfloat
end


module util

export Vocab, setdefault!, Report, submittedEarlier, calculateDayTimestamp, DocFreq, updateDocFreq!,isduplicate, get_basic_args, aggregatePositionByFunction, sigmoid, set_doc_len!, generate_ngrams, add_ngram_to_vocab!, vocabLength, aggregatePositionByFuncKeepDim

using ArgParse
using Dates
using JSON
using DataStructures

sigmoid(x)=1.0/(1.0+exp(-x))

struct Vocab
    vocab::Dict{String, UInt32}
    ukn_set::Set{UInt32}
end

function vocabLength(vocab::Vocab)::Int64
    return length(vocab.vocab)
end

function setdefault!(vocab::Vocab, functionName::String, isUkn=false)
    tkn_id = get!(vocab.vocab, functionName, vocabLength(vocab) + 1)

    if isUkn
        push!(vocab.ukn_set, tkn_id)
    end

    return tkn_id
end

mutable struct  ReportStats
    doc_len::Int64
    doc_lens::Vector{Int64}
    weight_sum::Float64
    sorted_idxs::Vector{Vector{UInt16}}
end

mutable struct Report
    id::Int64
    dup_id::Int64
    creation_date::Int64
    creation_day_ts::UInt64
    stacks::Vector{Vector{UInt32}}
    bow::Vector{Pair{UInt32,Float64}}
    positional_bow::Vector{Pair{UInt32, Vector{Int16}}}

    v_bow::Vector{Vector{Pair{UInt32,Float64}}}
    v_positional_bow::Vector{Vector{Pair{UInt32, Vector{Int16}}}}

    stats::ReportStats
end

@inline function Report(id, dup_id, creation_date, creation_day_ts, stacks)
    return Report(id, dup_id, creation_date, creation_day_ts, stacks, Vector{Pair{UInt32,Float64}}(), Vector{Pair{UInt32, Vector{Int16}}}(), Vector{Vector{Pair{UInt32,Float64}}}(), Vector{Vector{Pair{UInt32, Vector{Int16}}}}(), ReportStats(-999999999, Vector{Int64}(), -Inf, Vector{Vector{UInt16}}()))
end

isduplicate(report::Report) = report.id != report.dup_id

submittedEarlier(r1::Report, r2::Report) = r1.creation_date < r2.creation_date || (r1.creation_date == r2.creation_date && r1.id < r2.id)

function calculateDayTimestamp(date)
    return UInt64(floor(date / (24 * 60 * 60)))
end

function set_doc_len!(report::Report)
    doc_len = 0

    for stack in report.stacks
        doc_len += length(stack)
    end

    report.stats.doc_len = doc_len

    return doc_len
end

mutable struct DocFreq
    doc_freq::Vector{Float64}
    nDocs::Float64
    freq_by_stacks::Bool
    static_df_ukn::Bool
    vocab::Vocab

    cache::Vector{Float64}
    previous_n_docs::Float64
end

DocFreq(doc_freq::Vector{Float64}, nDocs::Float64, freq_by_stacks::Bool, static_df_ukn::Bool, vocab::Vocab) = DocFreq(doc_freq, nDocs, freq_by_stacks, static_df_ukn, vocab, Vector{Float64}(), 0)

function updateDocFreq!(docfreqObj::DocFreq, reports)
    for report in reports
        if docfreqObj.freq_by_stacks
            for stack in report.stacks
                for funcId in Set(stack)
                    docfreqObj.doc_freq[funcId] += 1
                end
                docfreqObj.nDocs += 1
            end
        else
            function_set = Set(( funcId for stack in report.stacks for funcId in stack))
            docfreqObj.nDocs += 1

            for funcId in function_set
                docfreqObj.doc_freq[funcId] += 1
            end
        end
    end

    if docfreqObj.static_df_ukn
        for funcId in docfreqObj.vocab.ukn_set
            docfreqObj.doc_freq[funcId]= docfreqObj.nDocs - 1
        end
    end
end

function aggregatePositionByFunction(report::Report)::Vector{Pair{UInt32, Vector{Int16}}}
    bow = Dict{UInt32, Vector{Int16}}()

    for stack in report.stacks
        for (pos, func_id) in enumerate(stack)
            if haskey(bow, func_id)
                push!(bow[func_id], pos)
            else
                bow[func_id] = [pos]
            end
        end
    end

    for positions in values(bow)
        sort!(positions)
    end

    return collect(bow)
end

function aggregatePositionByFuncKeepDim(report::Report)::Vector{Vector{Pair{UInt32, Vector{Int16}}}}
    v_bow = Vector{Vector{Pair{UInt32, Vector{Int16}}}}(undef, length(report.stacks))

    for (idx, stack) in enumerate(report.stacks)
        bow = Dict{UInt32, Vector{Int16}}()
        for (pos, func_id) in enumerate(stack)
            if haskey(bow, func_id)
                push!(bow[func_id], pos)
            else
                bow[func_id] = [pos]
            end
        end

        for positions in values(bow)
            sort!(positions)
        end

        v_bow[idx] = sort!(collect(bow))
    end

    return v_bow
end

function add_ngram_to_vocab!(vocab::Vocab, reportid2report::Dict{Int64, Report}, ngram)
    for report in values(reportid2report)
        for ngram_tkn in generate_ngrams(report.stacks, ngram)
            setdefault!(vocab, ngram_tkn)
        end
    end
end

function generate_ngrams(stacks::Vector{Vector{UInt32}}, ngram)
    ngram_vec = Vector{String}()

    for stack in stacks
        for n=2:ngram
            v = Vector{UInt32}(undef, n)

            for start_idx=1:(length(stack) - n + 1)
                p = 1
                for idx=start_idx:(start_idx + n - 1)
                    v[p] = stack[idx]
                    p+=1
                end

                push!(ngram_vec, join(v, ","))
            end

        end
    end

    return ngram_vec
end

function generate_ngrams(stack::Vector{UInt32}, ngram)
    ngram_vec = Vector{String}()

    for n=2:ngram
        v = Vector{UInt32}(undef, n)

        for start_idx=1:(length(stack) - n + 1)
            p = 1
            for idx=start_idx:(start_idx + n - 1)
                v[p] = stack[idx]
                p+=1
            end

            push!(ngram_vec, join(v, ","))
        end

    end

    return ngram_vec
end

function get_basic_args()
    s = ArgParseSettings()

    @add_arg_table s begin
        "--window", "-w"
            help = "Time window"
            arg_type = Int
            default = -1
        "--max_depth"
            help = ""
            arg_type = Int
            default = 300
        "--filter_recursion"
            help = "Options: none, modani and brodie"
            default = "none"
        "--keep_ukn"
            action = :store_true
        "--aggregate"
            help = "Options: max, avg_query, avg_cand, avg_short, avg_long avg_query_cand"
            default = "max"
        "--static_df_ukn"
            action = :store_true
        "--filter_func"
            help = "Options: none, top_k, top_k_trim"
            default = "none"
        "--filter_func_k"
            help = "Percentage of the k-top functions (values between 0 and 100)"
            arg_type = Float32
            default = Float32(-1.0)
        "--freq_by_stacks"
            action = :store_true
        "--upd_doc_freq"
            action = :store_true
        "--smoothing"
            action = :store_true
        "--interesting_stacks"
            action = :store_true
            help = "Following https://bazaar.launchpad.net/~bgo-maintainers/bugzilla-traceparser/3.4/view/head:/lib/TraceParser/Trace.pm,
                we select the interesting stacks that might contain the problem. This option only works for Gnome."
    end

    s.add_help = false

    return s
end

end